def decode(byte_str, allow_none=False):
    if ((byte_str is None) and allow_none):
        return ''
    if (not isinstance(byte_str, bytes)):
        raise ValueError('The argument {} must be a bytes object.'.format(byte_str))
    if (sys.version_info >= (3, 0)):
        return byte_str.decode('ascii')
    else:
        return byte_str

Make this unicode in Python 3, otherwise leave it as bytes.

Args:
byte_str: The byte string to decode.
allow_none: If true, then we will allow byte_str to be None in which
case we will return an empty string. TODO(rkn): Remove this flag.
This is only here to simplify upgrading to flatbuffers 1.10.0.

Returns:
A byte string in Python 2 and a unicode string in Python 3.


class StableDropout(nn.Module):

    def __init__(self, drop_prob):
        super().__init__()
        self.drop_prob = drop_prob
        self.count = 0
        self.context_stack = None

    def forward(self, x):

        if self.training and self.drop_prob > 0:
            return XDropout.apply(x, self.get_context())
        return x

    def clear_context(self):
        self.count = 0
        self.context_stack = None

    def init_context(self, reuse_mask=True, scale=1):
        if self.context_stack is None:
            self.context_stack = []
        self.count = 0
        for c in self.context_stack:
            c.reuse_mask = reuse_mask
            c.scale = scale

    def get_context(self):
        if self.context_stack is not None:
            if self.count >= len(self.context_stack):
                self.context_stack.append(DropoutContext())
            ctx = self.context_stack[self.count]
            ctx.dropout = self.drop_prob
            self.count += 1
            return ctx
        else:
            return self.drop_prob

Optimized dropout module for stabilizing the training

Args:
drop_prob (float): the dropout probabilities


def persist_perf(run, session, svg_path):
    from benchbuild.utils import schema as s
    with open(svg_path, 'r') as svg_file:
        svg_data = svg_file.read()
        session.add(s.Metadata(name='perf.flamegraph', value=svg_data, run_id=run.id))

Persist the flamegraph in the database.

The flamegraph exists as a SVG image on disk until we persist it in the
database.

Args:
run: The run we attach these perf measurements to.
session: The db transaction we belong to.
svg_path: The path to the SVG file we want to store.


def parse(self, content):

        declarations = self.REGEX_IMPORT_RULE.findall(
            self.remove_comments(content)
        )
        return self.flatten_rules(declarations)

Parse a stylesheet document with a regex (``REGEX_IMPORT_RULE``)
to extract all import rules and return them.

Args:
content (str): A SCSS source.

Returns:
list: Finded paths in import rules.


def round(x, decimals=0):
    if any_symbolic_tensors((x,)):
        return Round(decimals).symbolic_call(x)
    return backend.numpy.round(x, decimals)

Evenly round to the given number of decimals.

Args:
x: Input tensor.
decimals: Number of decimal places to round to. Defaults to `0`.

Returns:
Output tensor.


def _encode_queries(self, program: cfg.Program) -> list[dict[str, Any]]:
    metrics = program.calculate_metrics()
    solvers = metrics.solver_metrics
    enc_queries = []
    query_id = -1
    for solver_idx, solver in enumerate(solvers):
        for query in solver.query_metrics:
            query_id += 1
            steps = []
            for step in query.steps:
                steps.append({'_type': 'QueryStep', 'node': step.node, 'depth': step.depth, 'bindings': step.bindings})
            enc_queries.append({'_type': 'Query', 'solver_idx': solver_idx, 'start_node': query.start_node, 'end_node': query.end_node, 'initial_binding_count': query.initial_binding_count, 'shortcircuited': query.shortcircuited, 'from_cache': query.from_cache, 'steps': steps})
    return enc_queries

Encodes information about solver queries from a Program's metrics.

The queries are numbered in the order they were recorded.

Args:
program: a cfg.Program.

Returns:
A list of dictionaries that correspond to SerializedQuery.


async def _handle_watermark_notification(self, watermark_notification):

        conv_id = watermark_notification.conversation_id.id
        res = parsers.parse_watermark_notification(watermark_notification)
        await self.on_watermark_notification.fire(res)
        try:
            conv = await self._get_or_fetch_conversation(conv_id)
        except exceptions.NetworkError:
            logger.warning(
                'Failed to fetch conversation for watermark notification: %s',
                conv_id
            )
        else:
            await conv.on_watermark_notification.fire(res)

Receive WatermarkNotification and update the conversation.

Args:
watermark_notification: hangouts_pb2.WatermarkNotification instance


def _create_m_objective(w, X):
    (clusters, cells) = w.shape
    genes = X.shape[0]
    w_sum = w.sum(1)

    def objective(m):
        m = m.reshape((X.shape[0], w.shape[0]))
        d = (m.dot(w) + eps)
        temp = (X / d)
        w2 = w.dot(temp.T)
        deriv = (w_sum - w2.T)
        return ((np.sum((d - (X * np.log(d)))) / genes), (deriv.flatten() / genes))
    return objective

Creates an objective function and its derivative for M, given W and X

Args:
w (array): clusters x cells
X (array): genes x cells


def update(self, data=None, timeout=-1, force=True):

        uri = self.data['uri']

        resource = deepcopy(self.data)
        resource.update(data)

        self.data = self._helper.update(resource, uri, force, timeout)

        return self

Updates server profile template.

Args:
data: Data to update the resource.
timeout: Timeout in seconds. Wait for task completion by default. The timeout does not abort the operation
in OneView; it just stops waiting for its completion.
force: Force the update operation.

Returns:
A dict with the updated resource data.


def parse_report_file(input_, nameservers=None, dns_timeout=2.0, strip_attachment_payloads=False, parallel=False):
    if (type(input_) == str):
        file_object = open(input_, 'rb')
    elif (type(input_) == bytes):
        file_object = BytesIO(input_)
    else:
        file_object = input_
    content = file_object.read()
    try:
        report = parse_aggregate_report_file(content, nameservers=nameservers, dns_timeout=dns_timeout, parallel=parallel)
        results = OrderedDict([('report_type', 'aggregate'), ('report', report)])
    except InvalidAggregateReport:
        try:
            sa = strip_attachment_payloads
            results = parse_report_email(content, nameservers=nameservers, dns_timeout=dns_timeout, strip_attachment_payloads=sa, parallel=parallel)
        except InvalidDMARCReport:
            raise InvalidDMARCReport('Not a valid aggregate or forensic report')
    return results

Parses a DMARC aggregate or forensic file at the given path, a
file-like object. or bytes

Args:
input_: A path to a file, a file like object, or bytes
nameservers (list): A list of one or more nameservers to use
(Cloudflare's public DNS resolvers by default)
dns_timeout (float): Sets the DNS timeout in seconds
strip_attachment_payloads (bool): Remove attachment payloads from
forensic report results
parallel (bool): Parallel processing

Returns:
OrderedDict: The parsed DMARC report


def read(self):
    try:
        buf = os.read(self._fd, 8)
    except OSError as e:
        raise LEDError(e.errno, ('Reading LED brightness: ' + e.strerror))
    try:
        os.lseek(self._fd, 0, os.SEEK_SET)
    except OSError as e:
        raise LEDError(e.errno, ('Rewinding LED brightness: ' + e.strerror))
    return int(buf)

Read the brightness of the LED.

Returns:
int: Current brightness.

Raises:
LEDError: if an I/O or OS error occurs.


def call(self, x, y, bias, cache=None):
    q = self.q_dense_layer(x)
    k = self.k_dense_layer(y)
    v = self.v_dense_layer(y)
    if (cache is not None):
        k = tf.concat([cache['k'], k], axis=1)
        v = tf.concat([cache['v'], v], axis=1)
        cache['k'] = k
        cache['v'] = v
    q = self.split_heads(q)
    k = self.split_heads(k)
    v = self.split_heads(v)
    depth = (self.hidden_size
    q *= (depth ** (- 0.5))
    logits = tf.matmul(q, k, transpose_b=True)
    logits += bias
    weights = tf.nn.softmax(logits, name='attention_weights')
    if self.train:
        mlperf_log.transformer_print(key=mlperf_log.MODEL_HP_ATTENTION_DROPOUT, value=self.attention_dropout)
        weights = tf.nn.dropout(weights, (1.0 - self.attention_dropout))
    attention_output = tf.matmul(weights, v)
    attention_output = self.combine_heads(attention_output)
    attention_output = self.output_dense_layer(attention_output)
    return attention_output

Apply attention mechanism to x and y.

Args:
x: a tensor with shape [batch_size, length_x, hidden_size]
y: a tensor with shape [batch_size, length_y, hidden_size]
bias: attention bias that will be added to the result of the dot product.
cache: (Used during prediction) dictionary with tensors containing results
of previous attentions. The dictionary must have the items:
{"k": tensor with shape [batch_size, i, key_channels],
"v": tensor with shape [batch_size, i, value_channels]}
where i is the current decoded length.

Returns:
Attention layer output with shape [batch_size, length_x, hidden_size]


def parse_value(self, text: str) -> Optional[bool]:
    if (text == 'true'):
        return True
    if (text == 'false'):
        return False

Parse boolean value.

Args:
text: String representation of the value.


def mach53(msg):

    d = hex2bin(data(msg))

    if d[23] == '0':
        return None

    mach = bin2int(d[24:33]) * 0.008
    return round(mach, 3)

MACH number, DBS 5,3 message

Args:
msg (String): 28 bytes hexadecimal message

Returns:
float: MACH number


def synchronize_clock(self, offset):

        self.time_offset = offset - self.uptime
        self.is_utc = True

        if self.has_rtc:
            self.stored_offset = self.time_offset

Persistently synchronize the clock to UTC time.

Args:
offset (int): The number of seconds since 1/1/2000 00:00Z


def add_reader(self, fd: IFileLike, callback: typing.Callable[([IFileLike], typing.Any)]) -> None:
    raise NotImplementedError()

Add a file descriptor to the processor and wait for READ.

Args:
fd (IFileLike): Any obect that exposes a 'fileno' method that
returns a valid file descriptor integer.
callback (typing.Callable[[IFileLike], typing.Any]): A function
that consumes the IFileLike object whenever the READ event is
fired.


def handle_enterprise_logistration(backend, user, **kwargs):

    request = backend.strategy.request
    enterprise_customer = get_enterprise_customer_for_running_pipeline(
        request,
        {
            'backend': backend.name,
            'kwargs': kwargs
        }
    )
    if enterprise_customer is None:

        return

    enterprise_customer_user, _ = EnterpriseCustomerUser.objects.update_or_create(
        enterprise_customer=enterprise_customer,
        user_id=user.id
    )
    enterprise_customer_user.update_session(request)

Perform the linking of user in the process of logging to the Enterprise Customer.

Args:
backend: The class handling the SSO interaction (SAML, OAuth, etc)
user: The user object in the process of being logged in with
**kwargs: Any remaining pipeline variables


def _GetResponseClass(self, method_descriptor):

    if method_descriptor.containing_service != self.descriptor:
      raise RuntimeError(
          'GetResponseClass() given method descriptor for wrong service type.')
    return method_descriptor.output_type._concrete_class

Returns the class of the response protocol message.

Args:
method_descriptor: Descriptor of the method for which to return the
response protocol message class.

Returns:
A class that represents the output protocol message of the specified
method.


def __init__(self, finders=None):

        if finders is None:
            self.finders = [LocalPackageFinder(), InstalledPackageFinder()]
        else:
            self.finders = [f() for f in finders]

Initialization method.

Args:
finders (list of classes):
list of package finder classes (not instances) in a specific
order. Default: [LocalPackageFinder, InstalledPackageFinder].


def set_button_map(self, button_map):
    assert (self.finger_count > 0), 'This device does not support tapping'
    return self._libinput.libinput_device_config_tap_set_button_map(self._handle, button_map)

Set the finger number to button number mapping for tap-to-click.

The default mapping on most devices is to have a 1, 2 and 3 finger tap
to map to the left, right and middle button, respectively. A device may
permit changing the button mapping but disallow specific maps. In this
case :attr:`~libinput.constant.ConfigStatus.UNSUPPORTED` is returned,
the caller is expected to handle this case correctly.

Changing the button mapping may not take effect immediately, the device
may wait until it is in a neutral state before applying any changes.

The mapping may be changed when tap-to-click is disabled. The new
mapping takes effect when tap-to-click is enabled in the future.

If :attr:`finger_count` is 0, this method raises :exc:`AssertionError`.

Args:
button_map (~libinput.constant.TapButtonMap): The new
finger-to-button number mapping.
Returns:
~libinput.constant.ConfigStatus: A config status code.
Raises:
AssertionError


def get_public_datasets_and_tokens(self):

        datasets = {}
        tokens = self.get_public_tokens()
        for t in tokens:
            dataset = self.get_token_dataset(t)
            if dataset in datasets:
                datasets[dataset].append(t)
            else:
                datasets[dataset] = [t]
        return datasets

NOTE: VERY SLOW!
Get a dictionary relating key:dataset to value:[tokens] that rely
on that dataset.

Arguments:
None

Returns:
dict: relating key:dataset to value:[tokens]


def _escaped_token_to_subtoken_ids(self, escaped_token):

    return [
        self._subtoken_string_to_id[subtoken]
        for subtoken in self._escaped_token_to_subtoken_strings(escaped_token)
    ]

Converts an escaped token string to a list of subtoken IDs.

Args:
escaped_token: An escaped token as a unicode string.
Returns:
A list of subtoken IDs as integers.


def from_client_secrets_file(cls, client_secrets_file, scopes, **kwargs):

        with open(client_secrets_file, 'r') as json_file:
            client_config = json.load(json_file)

        return cls.from_client_config(client_config, scopes=scopes, **kwargs)

Creates a :class:`Flow` instance from a Google client secrets file.

Args:
client_secrets_file (str): The path to the client secrets .json
file.
scopes (Sequence[str]): The list of scopes to request during the
flow.
kwargs: Any additional parameters passed to
:class:`requests_oauthlib.OAuth2Session`

Returns:
Flow: The constructed Flow instance.


def __init__(self, function_name):

        params = {
            'FUNCTION_NAME': function_name
        }

        super().__init__(
            'int', 'lib_nmsimplex', [],
            resource_filename('mot', 'data/opencl/lib_nmsimplex.cl'),
            var_replace_dict=params)

The NMSimplex algorithm as a reusable library component.

Args:
function_name (str): the name of the evaluation function to call, defaults to 'evaluate'.
This should point to a function with signature:

``double evaluate(local mot_float_type* x, void* data_void);``


def do_decode(cls, obj, obj_type):

        if inspect.isclass(obj_type) and issubclass(
                obj_type, ConjureBeanType
        ):
            return cls.decode_conjure_bean_type(obj, obj_type)

        elif inspect.isclass(obj_type) and issubclass(
                obj_type, ConjureUnionType
        ):
            return cls.decode_conjure_union_type(obj, obj_type)

        elif inspect.isclass(obj_type) and issubclass(
                obj_type, ConjureEnumType
        ):
            return cls.decode_conjure_enum_type(obj, obj_type)

        elif isinstance(obj_type, DictType):
            return cls.decode_dict(obj, obj_type.key_type, obj_type.value_type)

        elif isinstance(obj_type, ListType):
            return cls.decode_list(obj, obj_type.item_type)

        elif isinstance(obj_type, OptionalType):
            return cls.decode_optional(obj, obj_type.item_type)

        return cls.decode_primitive(obj, obj_type)

Decodes json into the specified type

Args:
obj: the json object to decode
element_type: a class object which is the type we're decoding into.


def docx_process_simple_text(text: str, width: int) -> str:

    if width:
        return '\n'.join(textwrap.wrap(text, width=width))
    else:
        return text

Word-wraps text.

Args:
text: text to process
width: width to word-wrap to (or 0 to skip word wrapping)

Returns:
wrapped text


def ParseOptions(cls, options, analysis_plugin):

    if not isinstance(analysis_plugin, tagging.TaggingAnalysisPlugin):
      raise errors.BadConfigObject(
          'Analysis plugin is not an instance of TaggingAnalysisPlugin')

    tagging_file = cls._ParseStringOption(options, 'tagging_file')
    if not tagging_file:
      raise errors.BadConfigOption(
          'Tagging analysis plugin requires a tagging file.')

    tagging_file_path = tagging_file
    if not os.path.isfile(tagging_file_path):

      data_location = getattr(options, 'data_location', None)
      if data_location:
        tagging_file_path = os.path.join(data_location, tagging_file)

    if not os.path.isfile(tagging_file_path):
      raise errors.BadConfigOption(
          'No such tagging file: {0:s}.'.format(tagging_file))

    try:
      analysis_plugin.SetAndLoadTagFile(tagging_file_path)

    except UnicodeDecodeError:
      raise errors.BadConfigOption(
          'Invalid tagging file: {0:s} encoding must be UTF-8.'.format(
              tagging_file))

    except errors.TaggingFileError as exception:
      raise errors.BadConfigOption(
          'Unable to read tagging file: {0:s} with error: {1!s}'.format(
              tagging_file, exception))

Parses and validates options.

Args:
options (argparse.Namespace): parser options.
analysis_plugin (AnalysisPlugin): analysis plugin to configure.

Raises:
BadConfigObject: when the output module object is of the wrong type.
BadConfigOption: when a configuration parameter fails validation.


def has_arg(fn, arg_name):
    if (sys.version_info < (3,)):
        if (isinstance(fn, types.FunctionType) or isinstance(fn, types.MethodType)):
            arg_spec = inspect.getargspec(fn)
        else:
            try:
                arg_spec = inspect.getargspec(fn.__call__)
            except AttributeError:
                return False
        return (arg_name in arg_spec.args)
    elif (sys.version_info < (3, 6)):
        arg_spec = inspect.getfullargspec(fn)
        return ((arg_name in arg_spec.args) or (arg_name in arg_spec.kwonlyargs))
    else:
        try:
            signature = inspect.signature(fn)
        except ValueError:
            signature = inspect.signature(fn.__call__)
        parameter = signature.parameters.get(arg_name)
        if (parameter is None):
            return False
        return (parameter.kind in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY))

Checks if a callable accepts a given keyword argument.

Args:
fn: callable to inspect
arg_name: string, keyword argument name to check

Returns:
bool, whether `fn` accepts a `arg_name` keyword argument.


def _read_file(filename):
    graph_def = graph_pb2.GraphDef()
    if not file_io.file_exists(filename):
        raise IOError(f'File {filename} does not exist.')
    with file_io.FileIO(filename, 'rb') as f:
        file_content = f.read()
    try:
        graph_def.ParseFromString(file_content)
        return graph_def
    except Exception:
        pass
    try:
        text_format.Merge(file_content, graph_def)
    except text_format.ParseError as e:
        raise IOError(f'Cannot parse file {filename}: {str(e)}.')
    return graph_def

Reads a file containing `GraphDef` and returns the protocol buffer.

Args:
filename: `graph_def` filename including the path.

Returns:
A `GraphDef` protocol buffer.

Raises:
IOError: If the file doesn't exist, or cannot be successfully parsed.


def is_locator(self, path, relative=False):
    if (not relative):
        path = self.relpath(path)
    return (path and ('/' not in path.rstrip('/')))

Returns True if path refer to a locator.

Depending the storage, locator may be a bucket or container name,
a hostname, ...

args:
path (str): path or URL.
relative (bool): Path is relative to current root.

Returns:
bool: True if locator.


def build(cls, local_scheduler=True, **task_params):
    luigi.build([cls(**task_params)], local_scheduler=local_scheduler)

Instantiate the task and build it with luigi

Args:
local_scheduler (bool): use a local scheduler (True, default) or a remote scheduler
task_params: parameters to pass to task for instantiation


def FullUpdateFromMap(self, cache, new_map, force_write=False):
    error_count = 0
    if len(new_map) == 0 and (not force_write):
        raise error.EmptyMap('Source map empty during full update, aborting. Use --force-write to override.')
    error_count = cache.WriteMap(map_data=new_map, force_write=force_write)
    if error_count == 0:
        self.WriteModifyTimestamp(new_map.GetModifyTimestamp())
        self.WriteUpdateTimestamp()
    return error_count

Write a new map into the provided cache (overwrites).

Args:
cache: A nss_cache.caches.Cache object.
new_map: A nss_cache.maps.Map object.
force_write: A boolean indicating empty maps are okay to write, defaults
to False which means do not write them.

Returns:
0 if succesful, non-zero indicating number of failures otherwise.

Raises:
EmptyMap: Update is an empty map, not raised if force_write=True.


def key_for_entity_group(cls, key):
    return model.Key(cls.KIND_NAME, cls.ID, parent=key.root())

Return the key for the entity group containing key.

Args:
key: a key for an entity group whose __entity_group__ key you want.

Returns:
The __entity_group__ key for the entity group containing key.


def build_machine(network=None, machine_type=None, preemptible=None, service_account=None, boot_disk_size_gb=None, disks=None, accelerators=None, labels=None, cpu_platform=None, nvidia_driver_version=None):
    return {'network': network, 'machineType': machine_type, 'preemptible': preemptible, 'serviceAccount': service_account, 'bootDiskSizeGb': boot_disk_size_gb, 'disks': disks, 'accelerators': accelerators, 'labels': labels, 'cpuPlatform': cpu_platform, 'nvidiaDriverVersion': nvidia_driver_version}

Build a VirtualMachine object for a Pipeline request.

Args:
network (dict): Network details for the pipeline to run in.
machine_type (str): GCE Machine Type string for the pipeline.
preemptible (bool): Use a preemptible VM for the job.
service_account (dict): Service account configuration for the VM.
boot_disk_size_gb (int): Boot disk size in GB.
disks (list[dict]): List of disks to mount.
accelerators (list[dict]): List of accelerators to attach to the VM.
labels (dict[string, string]): Labels for the VM.
cpu_platform (str): The CPU platform to request.
nvidia_driver_version (str): The NVIDIA driver version to use when attaching
an NVIDIA GPU accelerator.

Returns:
An object representing a VirtualMachine.


def _create_dir_path(self, file_hash, path=None, hash_list=None):
    if (hash_list is None):
        hash_list = list(file_hash)
    if (not hash_list):
        raise IOError('Directory structure is too full!')
    if (not path):
        path = os.path.join(self.path, hash_list.pop(0))
    if (not os.path.exists(path)):
        os.mkdir(path)
        return self._create_dir_path(file_hash=file_hash, path=path, hash_list=hash_list)
    files = os.listdir(path)
    if (file_hash in files):
        return path
    if (len(files) < self.dir_limit):
        return path
    return self._create_dir_path(file_hash=file_hash, path=os.path.join(path, hash_list.pop(0)), hash_list=hash_list)

Create proper filesystem paths for given `file_hash`.

Args:
file_hash (str): Hash of the file for which the path should be
created.
path (str, default None): Recursion argument, don't set this.
hash_list (list, default None): Recursion argument, don't set this.

Returns:
str: Created path.


def prune_neighbors(self):

    def _neighbor_check(neighbors, valid):
        if (not (neighbors == neighbors)):
            return np.nan
        valid_keys = (set(valid) & set(neighbors.keys()))
        d = dict([(k, v) for (k, v) in neighbors.items() if (k in valid_keys)])
        return d
    fixed = self.copy()
    valid = self.get_valid_cell_indecies()
    valid = pd.DataFrame(self).merge(valid, on=self.frame_columns).set_index((self.frame_columns + ['cell_index']))
    valid = valid.apply((lambda x: _neighbor_check(x['neighbors'], x['valid'])), 1).reset_index().rename(columns={0: 'new_neighbors'})
    fixed = fixed.merge(valid, on=(self.frame_columns + ['cell_index'])).drop(columns='neighbors').rename(columns={'new_neighbors': 'neighbors'})
    fixed.microns_per_pixel = self.microns_per_pixel
    fixed.db = self.db
    return fixed

If the CellDataFrame has been subsetted, some of the cell-cell contacts may no longer be part of the the dataset.  This prunes those no-longer existant connections.

Returns:
CellDataFrame: A CellDataFrame with only valid cell-cell contacts


def __init__(self, couplinglist=None):

        self.graph = nx.DiGraph()

        self._dist_matrix = None

        self._qubit_list = None

        if couplinglist is not None:
            for source, target in couplinglist:
                self.add_edge(source, target)

Create coupling graph. By default, the generated coupling has no nodes.

Args:
couplinglist (list or None): An initial coupling graph, specified as
an adjacency list containing couplings, e.g. [[0,1], [0,2], [1,2]].


def add_note(self, note):
    if (type(note) == str):
        return self.update_note({'content': note})
    elif ((type(note) == dict) and ('content' in note)):
        return self.update_note(note)
    else:
        return ('No string or valid note.', (- 1))

Wrapper method to add a note

The method can be passed the note as a dict with the `content`
property set, which is then directly send to the web service for
creation. Alternatively, only the body as string can also be passed. In
this case the parameter is used as `content` for the new note.

Arguments:
- note (dict or string): the note to add

Returns:
A tuple `(note, status)`

- note (dict): the newly created note
- status (int): 0 on success and -1 otherwise


def _GetUrl(self, url_id, cache, database):

    url_cache_results = cache.GetResults('url')
    if not url_cache_results:
      result_set = database.Query(self.URL_CACHE_QUERY)

      cache.CacheQueryResults(
          result_set, 'url', 'id', ('url', 'rev_host'))
      url_cache_results = cache.GetResults('url')

    url, reverse_host = url_cache_results.get(url_id, ['', ''])

    if not url:
      return ''

    hostname = self._ReverseHostname(reverse_host)
    return '{0:s} ({1:s})'.format(url, hostname)

Retrieves an URL from a reference to an entry in the from_visit table.

Args:
url_id (str): identifier of the visited URL.
cache (SQLiteCache): cache.
database (SQLiteDatabase): database.

Returns:
str: URL and hostname.


def map(self, map_fn, desc=None):
    if (desc is None):
        desc = getattr(map_fn, '__name__', '')
    desc = u'map({})'.format(desc)
    return self.transform((lambda xs: (map_fn(x) for x in xs)), desc=desc)

Return a copy of this query, with the values mapped through `map_fn`.

Args:
map_fn (callable): A callable that takes a single argument and returns a new value.

Keyword Args:
desc (str): A description of the mapping transform, for use in log message.
Defaults to the name of the map function.

Returns:
Query


def add_alias(self, alias, name, op=None):
    alias = self.aliases.find_by_name(alias)
    name = self.aliases.find_by_name(name)
    if alias == name:
        return
    elif alias in self and name in self:
        self._merge(alias, name, op)
    elif alias not in self and name not in self:
        self.aliases.merge(alias, name)
    elif alias in self:
        root = self.aliases.merge(alias, name)
        self._copy_item(alias, root)
    elif name in self:
        root = self.aliases.merge(alias, name)
        self._copy_item(name, root)

Alias 'alias' to 'name'.

After aliasing, we will think `alias` and `name`, they represent the same
name. We will merge the values if `op` is provided.

Args:
alias: A string.
name: A string.
op: The function used to merge the values.


def pixel_image(shape, sd=None, init_val=None):
    if ((sd is not None) and (init_val is not None)):
        warnings.warn('`pixel_image` received both an initial value and a sd argument. Ignoring sd in favor of the supplied initial value.')
    sd = (sd or 0.01)
    init_val = (init_val or np.random.normal(size=shape, scale=sd).astype(np.float32))
    return tf.Variable(init_val)

A naive, pixel-based image parameterization.
Defaults to a random initialization, but can take a supplied init_val argument
instead.

Args:
shape: shape of resulting image, [batch, width, height, channels].
sd: standard deviation of param initialization noise.
init_val: an initial value to use instead of a random initialization. Needs
to have the same shape as the supplied shape argument.

Returns:
tensor with shape from first argument.


def fetch(self, order_id, data={}, **kwargs):
    return super(Order, self).fetch(order_id, data, **kwargs)

Fetch Order for given Id

Args:
order_id : Id for which order object has to be retrieved

Returns:
Order dict for given order Id


def print_info(self, capture):
    self.frame_offset += 1
    (ret, frame) = capture.read()
    if ret:
        print('Capture Information')
        print('\tDimensions (HxW): {}x{}'.format(*frame.shape[0:2]))
        print('\tColor channels:   {}'.format((frame.shape[2] if (len(frame.shape) > 2) else 1)))
        print('\tColor range:      {}-{}'.format(np.min(frame), np.max(frame)))
        print('\tdtype:            {}'.format(frame.dtype))
    else:
        print('No source found.')

Prints information about the unprocessed image.

Reads one frame from the source to determine image colors, dimensions
and data types.

Args:
capture: the source to read from.


def sym_descendants(self, where: Optional[Callable[[Any], bool]]=None, option: DescendantQueryOption=DescendantQueryOption.ALL, include_self: bool=False) -> List[Any]:
    descendants = []
    where = where or (lambda x: True)

    def visit(k, v, p):
        del k, p
        if not where(v):
            return TraverseAction.ENTER
        if not include_self and self is v:
            return TraverseAction.ENTER
        if option == DescendantQueryOption.IMMEDIATE:
            descendants.append(v)
            return TraverseAction.CONTINUE
        leaf_descendants = []
        if isinstance(v, Symbolic):
            leaf_descendants = v.sym_descendants(where, option)
        if option is DescendantQueryOption.ALL or not leaf_descendants:
            descendants.append(v)
        descendants.extend(leaf_descendants)
        return TraverseAction.CONTINUE
    traverse(self, visit)
    return descendants

Returns all descendants of specific classes.

Args:
where: Optional callable object as the filter of descendants to return.
option: Descendant query options, indicating whether all matched,
immediate matched or only the matched leaf nodes will be returned.
include_self: If True, `self` will be included in the query, otherwise
only strict descendants are included.

Returns:
A list of objects that match the descendant_cls.


def to_dict(self) -> dict[str, Any]:
    output = copy.deepcopy(self.__dict__)
    output['image_processor_type'] = self.__class__.__name__
    return output

Serializes this instance to a Python dictionary.

Returns:
`Dict[str, Any]`: Dictionary of all the attributes that make up this image processor instance.


def decode_jwt_payload(self, access_token=None):
    c = self.get_credentials()
    jwt = (access_token or c.access_token)
    try:
        (_, payload, _) = jwt.split('.')
        rem = (len(payload) % 4)
        if (rem > 0):
            payload += ('=' * (4 - rem))
        try:
            decoded_jwt = b64decode(payload).decode('utf-8')
        except TypeError as e:
            raise PanCloudError(('Failed to base64 decode JWT: %s' % e))
        else:
            try:
                x = loads(decoded_jwt)
            except ValueError as e:
                raise PanCloudError(('Invalid JSON: %s' % e))
    except (AttributeError, ValueError) as e:
        raise PanCloudError(('Invalid JWT: %s' % e))
    return x

Extract payload field from JWT.

Args:
access_token (str): Access token to decode. Defaults to ``None``.

Returns:
dict: JSON object that contains the claims conveyed by the JWT.


def assemble_common_meta(common_meta_dfs, fields_to_remove, sources, remove_all_metadata_fields, error_report_file):

    all_meta_df, all_meta_df_with_dups = build_common_all_meta_df(common_meta_dfs, fields_to_remove, remove_all_metadata_fields)

    if not all_meta_df.index.is_unique:
        all_report_df = build_mismatched_common_meta_report([x.shape for x in common_meta_dfs],
            sources, all_meta_df, all_meta_df_with_dups)

        unique_duplicate_ids = all_report_df.index.unique()

        if error_report_file is not None:
            all_report_df.to_csv(error_report_file, sep="\t")

        msg = .format(unique_duplicate_ids, all_report_df)
        raise MismatchCommonMetadataConcatException(msg)

    all_meta_df_sorted = all_meta_df.sort_index(axis=0)

    return all_meta_df_sorted

Assemble the common metadata dfs together. Both indices are sorted.
Fields that are not in all the dfs are dropped.

Args:
common_meta_dfs (list of pandas dfs)
fields_to_remove (list of strings): fields to be removed from the
common metadata because they don't agree across files

Returns:
all_meta_df_sorted (pandas df)


def __init__(self, origin):

    super(ShellItemsParser, self).__init__()
    self._origin = origin
    self._path_segments = []

Initializes the parser.

Args:
origin (str): origin of the event.


def search_group_by_id(self, groupID) -> Group:

        for g in self.groups:
            if g.id == groupID:
                return g
        return None

searches a group by given id

Args:
groupID(str): groupID the group to search for

Returns
the group object or None if it couldn't find a group


def run(func, keys, max_procs=None, show_proc=False, affinity=None, **kwargs):
    if (max_procs is None):
        max_procs = cpu_count()
    kw_arr = saturate_kwargs(keys=keys, **kwargs)
    if (len(kw_arr) == 0):
        return
    if isinstance(affinity, int):
        win32process.SetProcessAffinityMask(win32api.GetCurrentProcess(), affinity)
    task_queue = queue.Queue()
    while (len(kw_arr) > 0):
        for _ in range(max_procs):
            if (len(kw_arr) == 0):
                break
            kw = kw_arr.pop(0)
            p = Process(target=func, kwargs=kw)
            p.start()
            sys.stdout.flush()
            task_queue.put(p)
            if show_proc:
                signature = ', '.join([f'{k}={v}' for (k, v) in kw.items()])
                print(f'[{func.__name__}] ({signature})')
        while (not task_queue.empty()):
            p = task_queue.get()
            p.join()

Provide interface for multiprocessing

Args:
func: callable functions
keys: keys in kwargs that want to use process
max_procs: max number of processes
show_proc: whether to show process
affinity: CPU affinity
**kwargs: kwargs for func


def _get_feature_variable_for_type(self, feature_key, variable_key, variable_type, user_id, attributes):
    if (not validator.is_non_empty_string(feature_key)):
        self.logger.error(enums.Errors.INVALID_INPUT_ERROR.format('feature_key'))
        return None
    if (not validator.is_non_empty_string(variable_key)):
        self.logger.error(enums.Errors.INVALID_INPUT_ERROR.format('variable_key'))
        return None
    if (not isinstance(user_id, string_types)):
        self.logger.error(enums.Errors.INVALID_INPUT_ERROR.format('user_id'))
        return None
    if (not self._validate_user_inputs(attributes)):
        return None
    feature_flag = self.config.get_feature_from_key(feature_key)
    if (not feature_flag):
        return None
    variable = self.config.get_variable_for_feature(feature_key, variable_key)
    if (not variable):
        return None
    if (variable.type != variable_type):
        self.logger.warning(('Requested variable type "%s", but variable is of type "%s". Use correct API to retrieve value. Returning None.' % (variable_type, variable.type)))
        return None
    feature_enabled = False
    source_info = {}
    variable_value = variable.defaultValue
    decision = self.decision_service.get_variation_for_feature(feature_flag, user_id, attributes)
    if decision.variation:
        feature_enabled = decision.variation.featureEnabled
        if feature_enabled:
            variable_value = self.config.get_variable_value_for_variation(variable, decision.variation)
            self.logger.info(('Got variable value "%s" for variable "%s" of feature flag "%s".' % (variable_value, variable_key, feature_key)))
        else:
            self.logger.info(('Feature "%s" for variation "%s" is not enabled. Returning the default variable value "%s".' % (feature_key, decision.variation.key, variable_value)))
    else:
        self.logger.info(('User "%s" is not in any variation or rollout rule. Returning default value for variable "%s" of feature flag "%s".' % (user_id, variable_key, feature_key)))
    if (decision.source == enums.DecisionSources.FEATURE_TEST):
        source_info = {'experiment_key': decision.experiment.key, 'variation_key': decision.variation.key}
    try:
        actual_value = self.config.get_typecast_value(variable_value, variable_type)
    except:
        self.logger.error('Unable to cast value. Returning None.')
        actual_value = None
    self.notification_center.send_notifications(enums.NotificationTypes.DECISION, enums.DecisionNotificationTypes.FEATURE_VARIABLE, user_id, (attributes or {}), {'feature_key': feature_key, 'feature_enabled': feature_enabled, 'source': decision.source, 'variable_key': variable_key, 'variable_value': actual_value, 'variable_type': variable_type, 'source_info': source_info})
    return actual_value

Helper method to determine value for a certain variable attached to a feature flag based on type of variable.

Args:
feature_key: Key of the feature whose variable's value is being accessed.
variable_key: Key of the variable whose value is to be accessed.
variable_type: Type of variable which could be one of boolean/double/integer/string.
user_id: ID for user.
attributes: Dict representing user attributes.

Returns:
Value of the variable. None if:
- Feature key is invalid.
- Variable key is invalid.
- Mismatch with type of variable.


def OpenFile(self, windows_path):

    path_spec = self._path_resolver.ResolvePath(windows_path)
    if path_spec is None:
      return None

    return self._file_system.GetFileObjectByPathSpec(path_spec)

Opens the file specificed by the Windows path.

Args:
windows_path (str): Windows path to the file.

Returns:
FileIO: file-like object or None if the file does not exist.


def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: Optional[List[int]]=None) -> List[int]:
    if token_ids_1 is None:
        return self.prefix_tokens + token_ids_0 + self.suffix_tokens
    return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. An MBART-50 sequence has the following format, where `X` represents the sequence:

- `input_ids` (for encoder) `[src_lang_code] X [eos]`
- `labels`: (for decoder) `[tgt_lang_code] X [eos]`

BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
separator.

Args:
token_ids_0 (`List[int]`):
List of IDs to which the special tokens will be added.
token_ids_1 (`List[int]`, *optional*):
Optional second list of IDs for sequence pairs.

Returns:
`List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.


def take_node_screenshot(self, element, screenshot_path):
    from PIL import Image
    'Take a screenshot of a node\n\n        Args:\n            element (object): the proxy_element\n            screenshot_path (str): the path where the screenshot will be saved\n        '
    temp_path = os.path.join(tempdir, screenshot_path)
    el_x = int(element.location['x'])
    el_y = int(element.location['y'])
    el_height = int(element.size['height'])
    el_width = int(element.size['width'])
    if ((el_height == 0) or (el_width == 0)):
        self.debug_log('take_node_screenshot cannot be taken because element width or height equal zero')
        return False
    bounding_box = (el_x, el_y, (el_x + el_width), (el_y + el_height))
    self._driver.save_screenshot(temp_path)
    base_image = Image.open(temp_path)
    cropped_image = base_image.crop(bounding_box)
    base_image = base_image.resize(cropped_image.size)
    base_image.paste(cropped_image, (0, 0))
    base_image.save(screenshot_path)
    '\n        except Exception as e:\n            tb = traceback.format_exc()\n            print unicode(tb)\n            embed()\n        '

Take a screenshot of a node

Args:
element (object): the proxy_element
screenshot_path (str): the path where the screenshot will be saved


def do_command_line(infile: typing.IO[str]) -> int:

    lines = infile.readlines()
    tree = ast.parse(''.join(lines))
    checker = Checker(tree, lines, infile.name)
    checker.load()
    errors = []
    for func in checker.all_funcs(skip_noqa=True):
        try:
            errors = list(func.check_all())
        except ValidationError as error:
            errors = [error.to_aaa()]
        print(func.__str__(errors), end='')
    return len(errors)

Currently a small stub to create an instance of Checker for the passed
``infile`` and run its test functions through linting.

Args:
infile

Returns:
int: Number of flake8 errors raised.


def generate(cls, strategy, **kwargs):

        assert strategy in (enums.STUB_STRATEGY, enums.BUILD_STRATEGY, enums.CREATE_STRATEGY)
        action = getattr(cls, strategy)
        return action(**kwargs)

Generate a new instance.

The instance will be created with the given strategy (one of
BUILD_STRATEGY, CREATE_STRATEGY, STUB_STRATEGY).

Args:
strategy (str): the strategy to use for generating the instance.

Returns:
object: the generated instance


def ParseFileObject(self, parser_mediator, file_object):
    file_header_map = self._GetDataTypeMap('systemd_journal_file_header')
    try:
        (file_header, _) = self._ReadStructureFromFileObject(file_object, 0, file_header_map)
    except (ValueError, errors.ParseError) as exception:
        raise errors.UnableToParseFile('Unable to parse file header with error: {0!s}'.format(exception))
    if (file_header.signature != self._FILE_SIGNATURE):
        raise errors.UnableToParseFile('Invalid file signature.')
    if (file_header.header_size not in self._SUPPORTED_FILE_HEADER_SIZES):
        raise errors.UnableToParseFile('Unsupported file header size: {0:d}.'.format(file_header.header_size))
    data_hash_table_end_offset = (file_header.data_hash_table_offset + file_header.data_hash_table_size)
    field_hash_table_end_offset = (file_header.field_hash_table_offset + file_header.field_hash_table_size)
    self._maximum_journal_file_offset = max(data_hash_table_end_offset, field_hash_table_end_offset)
    entry_object_offsets = self._ParseEntryObjectOffsets(file_object, file_header.entry_array_offset)
    for entry_object_offset in entry_object_offsets:
        if (entry_object_offset == 0):
            continue
        try:
            fields = self._ParseJournalEntry(file_object, entry_object_offset)
        except errors.ParseError as exception:
            parser_mediator.ProduceExtractionWarning('Unable to parse journal entry at offset: 0x{0:08x} with error: {1!s}'.format(entry_object_offset, exception))
            return
        event_data = SystemdJournalEventData()
        event_data.body = fields.get('MESSAGE', None)
        event_data.hostname = fields.get('_HOSTNAME', None)
        event_data.reporter = fields.get('SYSLOG_IDENTIFIER', None)
        if (event_data.reporter and (event_data.reporter != 'kernel')):
            event_data.pid = fields.get('_PID', fields.get('SYSLOG_PID', None))
        date_time = dfdatetime_posix_time.PosixTimeInMicroseconds(timestamp=fields['real_time'])
        event = time_events.DateTimeValuesEvent(date_time, definitions.TIME_DESCRIPTION_WRITTEN)
        parser_mediator.ProduceEventWithEventData(event, event_data)

Parses a Systemd journal file-like object.

Args:
parser_mediator (ParserMediator): parser mediator.
file_object (dfvfs.FileIO): a file-like object.

Raises:
UnableToParseFile: when the header cannot be parsed.


def load_dataset(file_path: str, model: NormalizedModel) -> Dataset:
    xs = []
    ys = array.array('B')
    with open(file_path) as f:
        for row in f:
            cols = row.strip().split('\t')
            if len(cols) < 2:
                continue
            ys.append(cols[0] == '1')
            xs.append(tuple((k in set(cols[1:]) for k in model.features)))
    X = jnp.array(xs) * 2 - 1
    Y = jnp.array(ys)
    return Dataset(X, Y)

Loads a dataset from the given file path.

Args:
file_path: A file path for the encoded data file.
model: A normalized model.

Returns:
A dataset of inputs (X) and outputs (Y).


def _bytestringToValuelist(bytestring, numberOfRegisters):

    _checkInt(numberOfRegisters, minvalue=1, description='number of registers')
    numberOfBytes = _NUMBER_OF_BYTES_PER_REGISTER * numberOfRegisters
    _checkString(bytestring, 'byte string', minlength=numberOfBytes, maxlength=numberOfBytes)

    values = []
    for i in range(numberOfRegisters):
        offset = _NUMBER_OF_BYTES_PER_REGISTER * i
        substring = bytestring[offset : offset + _NUMBER_OF_BYTES_PER_REGISTER]
        values.append(_twoByteStringToNum(substring))

    return values

Convert a bytestring to a list of numerical values.

The bytestring is interpreted as 'unsigned INT16'.

Args:
* bytestring (str): The string from the slave. Length = 2*numberOfRegisters
* numberOfRegisters (int): The number of registers. For error checking.

Returns:
A list of integers.

Raises:
TypeError, ValueError


def _get_music_services_data_xml(soco=None):

        device = soco or discovery.any_soco()
        log.debug("Fetching music services data from %s", device)
        available_services = device.musicServices.ListAvailableServices()
        descriptor_list_xml = available_services[
            'AvailableServiceDescriptorList']
        log.debug("Services descriptor list: %s", descriptor_list_xml)
        return descriptor_list_xml

Fetch the music services data xml from a Sonos device.

Args:
soco (SoCo): a SoCo instance to query. If none is specified, a
random device will be used. Defaults to `None`.

Returns:
str: a string containing the music services data xml


def _bfd_tx(self, **kwargs):

        int_type = kwargs['int_type']
        method_name = 'interface_%s_bfd_interval_min_tx' % int_type
        bfd_tx = getattr(self._interface, method_name)
        config = bfd_tx(**kwargs)
        if kwargs['delete']:
            tag = 'min-tx'
            config.find('.
        return config

Return the BFD minimum transmit interval XML.

You should not use this method.
You probably want `BGP.bfd`.

Args:
min_tx (str): BFD transmit interval in milliseconds (300, 500, etc)
delete (bool): Remove the configuration if ``True``.

Returns:
XML to be passed to the switch.

Raises:
None


def download_file_from_google_drive(file_id, root, filename=None, md5=None):
    import requests
    url = 'https:
    root = os.path.expanduser(root)
    if (not filename):
        filename = file_id
    fpath = os.path.join(root, filename)
    makedir_exist_ok(root)
    if (os.path.isfile(fpath) and check_integrity(fpath, md5)):
        print(('Using downloaded and verified file: ' + fpath))
    else:
        session = requests.Session()
        response = session.get(url, params={'id': file_id}, stream=True)
        token = _get_confirm_token(response)
        if token:
            params = {'id': file_id, 'confirm': token}
            response = session.get(url, params=params, stream=True)
        _save_response_content(response, fpath)

Download a Google Drive file from  and place it in root.

Args:
file_id (str): id of file to be downloaded
root (str): Directory to place downloaded file in
filename (str, optional): Name to save the file under. If None, use the id of the file.
md5 (str, optional): MD5 checksum of the download. If None, do not check


def CopyFromDateTimeString(self, time_string):

    date_time_values = self._CopyDateTimeFromString(time_string)

    year = date_time_values.get('year', 0)
    month = date_time_values.get('month', 0)
    day_of_month = date_time_values.get('day_of_month', 0)
    hours = date_time_values.get('hours', 0)
    minutes = date_time_values.get('minutes', 0)
    seconds = date_time_values.get('seconds', 0)

    microseconds = date_time_values.get('microseconds', 0)
    milliseconds, _ = divmod(
        microseconds, definitions.MICROSECONDS_PER_MILLISECOND)

    if year < 1601 or year > 30827:
      raise ValueError('Unsupported year value: {0:d}.'.format(year))

    self._normalized_timestamp = None
    self._number_of_seconds = self._GetNumberOfSecondsFromElements(
        year, month, day_of_month, hours, minutes, seconds)

    self.year = year
    self.month = month
    self.day_of_month = day_of_month

    self.day_of_week = None
    self.hours = hours
    self.minutes = minutes
    self.seconds = seconds
    self.milliseconds = milliseconds

    self.is_local_time = False

Copies a SYSTEMTIME structure from a date and time string.

Args:
time_string (str): date and time value formatted as:
YYYY-MM-DD hh:mm:ss.######[+-]##:##

Where # are numeric digits ranging from 0 to 9 and the seconds
fraction can be either 3 or 6 digits. The time of day, seconds
fraction and time zone offset are optional. The default time zone
is UTC.

Raises:
ValueError: if the date string is invalid or not supported.


def read_label_file(path):
    labels = []
    for record in textfile.read_separated_lines_generator(path, separator='\t', max_columns=3):
        value = ''
        if (len(record) > 2):
            value = str(record[2])
        labels.append([float(_clean_time(record[0])), float(_clean_time(record[1])), value])
    return labels

Read the labels from an audacity label file.

Args:
path (str): Path to the label file.

Returns:
list: List of labels (start [sec], end [sec], label)

Example::

>>> read_label_file('/path/to/label/file.txt')
[
[0.0, 0.2, 'sie'],
[0.2, 2.2, 'hallo']
]


def default_matrix(self):
    matrix = (c_float * 6)()
    rc = self._libinput.libinput_device_config_calibration_get_default_matrix(self._handle, matrix)
    return (rc, tuple(matrix))

The default calibration matrix for this device.

On most devices, this is the identity matrix. If the udev property
``LIBINPUT_CALIBRATION_MATRIX`` is set on the respective udev device,
that property's value becomes the default matrix, see
`Static device configuration via udev`_.

Returns:
(bool, (float, float, float, float, float, float)): :obj:`False` if
no calibration is set and
the returned matrix is the identity matrix, :obj:`True`
otherwise. :obj:`tuple` representing the first two rows of
a 3x3 matrix as described
in :meth:`config_calibration_set_matrix`.


def get_factors(n):

    def factor(n, i, combi, res):

        while i * i <= n:
            if n % i == 0:
                res += combi + [i, int(n/i)],
                factor(n/i, i, combi+[i], res)
            i += 1
        return res
    return factor(n, 2, [], [])

[summary]

Arguments:
n {[int]} -- [to analysed number]

Returns:
[list of lists] -- [all factors of the number n]


def songs(self, *, uploaded=True, purchased=True):
    if ((not uploaded) and (not purchased)):
        raise ValueError("'uploaded' and 'purchased' cannot both be False.")
    if (purchased and uploaded):
        song_list = []
        for chunk in self.songs_iter(export_type=1):
            song_list.extend(chunk)
    elif purchased:
        song_list = []
        for chunk in self.songs_iter(export_type=2):
            song_list.extend(chunk)
    elif uploaded:
        purchased_songs = []
        for chunk in self.songs_iter(export_type=2):
            purchased_songs.extend(chunk)
        song_list = [song for chunk in self.songs_iter(export_type=1) for song in chunk if (song not in purchased_songs)]
    return song_list

Get a listing of Music Library songs.

Returns:
list: Song dicts.


def find_connected_atoms(struct, tolerance=0.45, ldict=JmolNN().el_radius):
    n_atoms = len(struct.species)
    fc = np.array(struct.frac_coords)
    fc_copy = np.repeat(fc[(:, :, np.newaxis)], 27, axis=2)
    neighbors = np.array(list(itertools.product([0, 1, (- 1)], [0, 1, (- 1)], [0, 1, (- 1)]))).T
    neighbors = np.repeat(neighbors[(np.newaxis, :, :)], 1, axis=0)
    fc_diff = (fc_copy - neighbors)
    species = list(map(str, struct.species))
    for (i, item) in enumerate(species):
        if (not (item in ldict.keys())):
            species[i] = str(Specie.from_string(item).element)
    latmat = struct.lattice.matrix
    connected_matrix = np.zeros((n_atoms, n_atoms))
    for i in range(n_atoms):
        for j in range((i + 1), n_atoms):
            max_bond_length = ((ldict[species[i]] + ldict[species[j]]) + tolerance)
            frac_diff = (fc_diff[j] - fc_copy[i])
            distance_ij = np.dot(latmat.T, frac_diff)
            if (sum((np.linalg.norm(distance_ij, axis=0) < max_bond_length)) > 0):
                connected_matrix[(i, j)] = 1
                connected_matrix[(j, i)] = 1
    return connected_matrix

Finds bonded atoms and returns a adjacency matrix of bonded atoms.

Author: "Gowoon Cheon"
Email: "gcheon@stanford.edu"

Args:
struct (Structure): Input structure
tolerance: length in angstroms used in finding bonded atoms. Two atoms
are considered bonded if (radius of atom 1) + (radius of atom 2) +
(tolerance) < (distance between atoms 1 and 2). Default
value = 0.45, the value used by JMol and Cheon et al.
ldict: dictionary of bond lengths used in finding bonded atoms. Values
from JMol are used as default

Returns:
(np.ndarray): A numpy array of shape (number of atoms, number of atoms);
If any image of atom j is bonded to atom i with periodic boundary
conditions, the matrix element [atom i, atom j] is 1.


def validate_json_against_schema(json_dict, schema, err_msg=None):
    try:
        if isinstance(schema, str):
            schema_name = schema
            schema = _SCHEMAS[schema_name]
            validator = _get_validator(schema_name)
            validator.validate(json_dict)
        else:
            jsonschema.validate(json_dict, schema)
    except jsonschema.ValidationError as err:
        if (err_msg is None):
            err_msg = 'JSON failed validation. Set Qiskit log level to DEBUG for further information.'
        newerr = SchemaValidationError(err_msg)
        newerr.__cause__ = _SummaryValidationError(err)
        logger.debug('%s', _format_causes(err))
        raise newerr

Validates JSON dict against a schema.

Args:
json_dict (dict): JSON to be validated.
schema (dict or str): JSON schema dictionary or the name of one of the
standards schemas in Qiskit to validate against it. The list of
standard schemas is: ``backend_configuration``,
``backend_properties``, ``backend_status``,
``default_pulse_configuration``, ``job_status``, ``qobj``,
``result``.
err_msg (str): Optional error message.

Raises:
SchemaValidationError: Raised if validation fails.


def traverse_preorder(self, leaves=True, internal=True):
    s = deque()
    s.append(self)
    while (len(s) != 0):
        n = s.pop()
        if ((leaves and n.is_leaf()) or (internal and (not n.is_leaf()))):
            (yield n)
        s.extend(n.children)

Perform a preorder traversal starting at this ``Node`` object

Args:
``leaves`` (``bool``): ``True`` to include leaves, otherwise ``False``

``internal`` (``bool``): ``True`` to include internal nodes, otherwise ``False``


def _default_tolerance(dtype):
    if dtype == np.float16:
        return 0.005
    elif dtype in (np.float32, np.complex64):
        return 0.001
    elif dtype in (np.float64, np.complex128):
        return 1e-05
    else:
        return None

Returns a sensible default tolerance for comparing results of a given type.

Args:
dtype: A datatype.


def __init__(self, channel):

    self.Watch = channel.stream_stream(
        '/etcdserverpb.Watch/Watch',
        request_serializer=rpc__pb2.WatchRequest.SerializeToString,
        response_deserializer=rpc__pb2.WatchResponse.FromString,
        )

Constructor.

Args:
channel: A grpc.Channel.


def CheckRValueReference(filename, clean_lines, linenum, nesting_state, error):

  line = clean_lines.elided[linenum]
  match = Match(r'^(.*\S)&&', line)
  if not match:
    match = Match(r'(.*)&&\S', line)
  if (not match) or '(&&)' in line or Search(r'\boperator\s*$', match.group(1)):
    return

  typenames = GetTemplateArgs(clean_lines, linenum)
  and_pos = len(match.group(1))
  if IsRValueType(typenames, clean_lines, nesting_state, linenum, and_pos):
    if not IsRValueAllowed(clean_lines, linenum, typenames):
      error(filename, linenum, 'build/c++11', 3,
            'RValue references are an unapproved C++ feature.')
  else:
    error(filename, linenum, 'whitespace/operators', 3,
          'Missing spaces around &&')

Check for rvalue references.

Args:
filename: The name of the current file.
clean_lines: A CleansedLines instance containing the file.
linenum: The number of the line to check.
nesting_state: A NestingState instance which maintains information about
the current stack of nested blocks being parsed.
error: The function to call with any errors found.


def add(self, index, var):
    import nnabla as nn
    from nnabla.utils.image_utils import imsave
    if ((index != 0) and (((index + 1) % self.interval) != 0)):
        return
    if isinstance(var, nn.Variable):
        data = var.d.copy()
    elif isinstance(var, nn.NdArray):
        data = var.data.copy()
    else:
        assert isinstance(var, np.ndarray)
        data = var.copy()
    assert (data.ndim > 2)
    channels = data.shape[(- 3)]
    data = data.reshape((- 1), *data.shape[(- 3):])
    data = data[:min(data.shape[0], self.num_images)]
    data = self.normalize_method(data)
    if (channels > 3):
        data = data[(:, :3)]
    elif (channels == 2):
        data = np.concatenate([data, np.ones(((data.shape[0], 1) + data.shape[(- 2):]))], axis=1)
    path_tmpl = os.path.join(self.save_dir, '{:06d}-{}.png')
    for j in range(min(self.num_images, data.shape[0])):
        img = data[j].transpose(1, 2, 0)
        if (img.shape[(- 1)] == 1):
            img = img[(..., 0)]
        path = path_tmpl.format(index, '{:03d}'.format(j))
        imsave(path, img)
    if self.verbose:
        logger.info('iter={} {{{}}} are written to {}.'.format(index, self.name, path_tmpl.format(index, '*')))

Add a minibatch of images to the monitor.

Args:
index (int): Index.
var (:obj:`~nnabla.Variable`, :obj:`~nnabla.NdArray`, or :obj:`~numpy.ndarray`):
A minibatch of images with ``(N, ..., C, H, W)`` format.
If C == 2, blue channel is appended with ones. If C > 3,
the array will be sliced to remove C > 3 sub-array.


def query_snl(self, criteria):
    try:
        payload = {'criteria': json.dumps(criteria)}
        response = self.session.post('{}/snl/query'.format(self.preamble), data=payload)
        if (response.status_code in [200, 400]):
            resp = json.loads(response.text)
            if resp['valid_response']:
                if resp.get('warning'):
                    warnings.warn(resp['warning'])
                return resp['response']
            else:
                raise MPRestError(resp['error'])
        raise MPRestError('REST error with status code {} and error {}'.format(response.status_code, response.text))
    except Exception as ex:
        raise MPRestError(str(ex))

Query for submitted SNLs.

.. note::

As of now, this MP REST feature is open only to a select group of
users. Opening up submissions to all users is being planned for
the future.

Args:
criteria (dict): Query criteria.

Returns:
A dict, with a list of submitted SNLs in the "response" key.

Raises:
MPRestError


def register_hooked(self, hooks, func, args_gen=None):
    if (self.hooked is None):
        self.hooked = {}
    if (args_gen is None):
        args_gen = getattr(func, 'call_types', {}).keys
    if (not isinstance(hooks, Sequence)):
        hooks = [hooks]
    for hook_cls in hooks:
        self.hooked[hook_cls] = (func, args_gen)

Register func to be run when any of the hooks are run by parent

Args:
hooks: A Hook class or list of Hook classes of interest
func: The callable that should be run on that Hook
args_gen: Optionally specify the argument names that should be
passed to func. If not given then use func.call_types.keys


def SetName(obj, name):

  precondition.AssertType(name, str)

  if PY2:
    obj.__name__ = name.encode("ascii")
  else:
    obj.__name__ = name

A compatibility wrapper for setting object's name.

See documentation for `GetName` for more information.

Args:
obj: A type or function object to set the name for.
name: A name to set.


def segment_pofile(filename, segments):
    reading_msg = 'Reading {num} entries from {file}'
    writing_msg = 'Writing {num} entries to {file}'
    source_po = polib.pofile(filename)
    LOG.info(reading_msg.format(file=filename, num=len(source_po)))
    remaining_po = copy.deepcopy(source_po)
    remaining_po[:] = []
    segment_po_files = {filename: remaining_po}
    segment_patterns = []
    for (segmentfile, patterns) in segments.items():
        segment_po_files[segmentfile] = copy.deepcopy(remaining_po)
        segment_patterns.extend(((pat, segmentfile) for pat in patterns))
    for msg in source_po:
        msg_segments = set()
        for (occ_file, _) in msg.occurrences:
            for (pat, segment_file) in segment_patterns:
                if fnmatch.fnmatch(occ_file, pat):
                    msg_segments.add(segment_file)
                    break
            else:
                msg_segments.add(filename)
        assert msg_segments
        if (len(msg_segments) == 1):
            segment_file = msg_segments.pop()
            segment_po_files[segment_file].append(msg)
        else:
            remaining_po.append(msg)
    files_written = set()
    for (segment_file, pofile) in segment_po_files.items():
        out_file = (filename.dirname() / segment_file)
        if (not pofile):
            LOG.error('No messages to write to %s, did you run segment twice?', out_file)
        else:
            LOG.info(writing_msg.format(file=out_file, num=len(pofile)))
            pofile.save(out_file)
            files_written.add(out_file)
    return files_written

Segment a .po file using patterns in `segments`.

The .po file at `filename` is read, and the occurrence locations of its
messages are examined.  `segments` is a dictionary: the keys are segment
.po filenames, the values are lists of patterns::

{
'django-studio.po': [
'cms/*',
'some-other-studio-place/*',
],
'django-weird.po': [
'*/weird_*.*',
],
}

If all a message's occurrences match the patterns for a segment, then that
message is written to the new segmented .po file.

Any message that matches no segments, or more than one, is written back to
the original file.

Arguments:
filename (path.path): a path object referring to the original .po file.
segments (dict): specification of the segments to create.

Returns:
a set of path objects, all the segment files written.


def load(png_filename):

    png_filename = os.path.expanduser(png_filename)

    try:
        img = Image.open(png_filename)
    except Exception as e:
        raise ValueError("Could not load file {0} for conversion."
                         .format(png_filename))
        raise

    return numpy.array(img)

Import a png file into a numpy array.

Arguments:
png_filename (str): A string filename of a png datafile

Returns:
A numpy array with data from the png file


def _ParseJournalEntry(self, file_object, file_offset):

    entry_object = self._ParseEntryObject(file_object, file_offset)

    entry_item_map = self._GetDataTypeMap('systemd_journal_entry_item')

    file_offset += 64
    data_end_offset = file_offset + entry_object.data_size - 64

    fields = {'real_time': entry_object.real_time}

    while file_offset < data_end_offset:
      try:
        entry_item, entry_item_data_size = self._ReadStructureFromFileObject(
            file_object, file_offset, entry_item_map)
      except (ValueError, errors.ParseError) as exception:
        raise errors.ParseError((
            'Unable to parse entry item at offset: 0x{0:08x} with error: '
            '{1!s}').format(file_offset, exception))

      file_offset += entry_item_data_size

      if entry_item.object_offset < self._maximum_journal_file_offset:
        raise errors.ParseError(
            'object offset should be after hash tables ({0:d} < {1:d})'.format(
                entry_item.object_offset, self._maximum_journal_file_offset))

      event_data = self._ParseDataObject(file_object, entry_item.object_offset)
      event_string = event_data.decode('utf-8')
      key, value = event_string.split('=', 1)
      fields[key] = value

    return fields

Parses a journal entry.

This method will generate an event per ENTRY object.

Args:
file_object (dfvfs.FileIO): a file-like object.
file_offset (int): offset of the entry object relative to the start
of the file-like object.

Returns:
dict[str, objects]: entry items per key.

Raises:
ParseError: when an object offset is out of bounds.


def _Open(self, path_spec, mode='rb'):

    if not path_spec.HasParent():
      raise errors.PathSpecError(
          'Unsupported path specification without parent.')

    file_object = resolver.Resolver.OpenFileObject(
        path_spec.parent, resolver_context=self._resolver_context)

    try:
      fsnfts_volume = pyfsntfs.volume()
      fsnfts_volume.open_file_object(file_object)
    except:
      file_object.close()
      raise

    self._file_object = file_object
    self._fsntfs_volume = fsnfts_volume

Opens the file system object defined by path specification.

Args:
path_spec (PathSpec): path specification.
mode (Optional[str]): file access mode. The default is 'rb' which
represents read-only binary.

Raises:
AccessError: if the access to open the file was denied.
IOError: if the file system object could not be opened.
PathSpecError: if the path specification is incorrect.
ValueError: if the path specification is invalid.


def format_cert_name(env='', account='', region='', certificate=None):

    cert_name = None

    if certificate:
        if certificate.startswith('arn'):
            LOG.info("Full ARN provided...skipping lookup.")
            cert_name = certificate
        else:
            generated_cert_name = generate_custom_cert_name(env, region, account, certificate)
            if generated_cert_name:
                LOG.info("Found generated certificate %s from template", generated_cert_name)
                cert_name = generated_cert_name
            else:
                LOG.info("Using default certificate name logic")
                cert_name = ('arn:aws:iam::{account}:server-certificate/{name}'.format(
                    account=account, name=certificate))
    LOG.debug('Certificate name: %s', cert_name)

    return cert_name

Format the SSL certificate name into ARN for ELB.

Args:
env (str): Account environment name
account (str): Account number for ARN
region (str): AWS Region.
certificate (str): Name of SSL certificate

Returns:
str: Fully qualified ARN for SSL certificate
None: Certificate is not desired


def seek(self, offset, whence=os.SEEK_SET):

    if not self._is_open:
      raise IOError('Not opened.')

    if self._fsntfs_data_stream:
      self._fsntfs_data_stream.seek(offset, whence)
    else:
      self._fsntfs_file_entry.seek(offset, whence)

Seeks to an offset within the file-like object.

Args:
offset (int): offset to seek to.
whence (Optional(int)): value that indicates whether offset is an absolute
or relative position within the file.

Raises:
IOError: if the seek failed.
OSError: if the seek failed.


def GetProperties(cls, path_spec):

    properties = {}

    for property_name in cls.PROPERTY_NAMES:

      if hasattr(path_spec, property_name):
        properties[property_name] = getattr(path_spec, property_name)

    return properties

Retrieves a dictionary containing the path specification properties.

Args:
path_spec (PathSpec): path specification.

Returns:
dict[str, str]: path specification properties.

Raises:
dict: path specification properties.


def UsesArtifact(self, artifacts):
    if isinstance(artifacts, string_types):
        return (artifacts in self.artifacts)
    else:
        return any((True for artifact in artifacts if (artifact in self.artifacts)))

Determines if the check uses the specified artifact.

Args:
artifacts: Either a single artifact name, or a list of artifact names

Returns:
True if the check uses a specific artifact.


def create(self, project_id=None):

    if not self.exists():
      if project_id is None:
        project_id = self._api.project_id
      try:
        self._info = self._api.buckets_insert(self._name, project_id=project_id)
      except Exception as e:
        raise e
    return self

Creates the bucket.

Args:
project_id: the project in which to create the bucket.
Returns:
The bucket.
Raises:
Exception if there was an error creating the bucket.


def set_attribute(self, key, value):
    if ((not isinstance(key, str)) or (not isinstance(value, str))):
        raise ValueError("The arguments 'key' and 'value' must both be strings. Instead they are {} and {}.".format(key, value))
    self.extra_data[key] = value

Add a key-value pair to the extra_data dict.

This can be used to add attributes that are not available when
ray.profile was called.

Args:
key: The attribute name.
value: The attribute value.


def sum(x, axis=None, keepdims=False):
    if any_symbolic_tensors((x,)):
        return Sum(axis=axis, keepdims=keepdims).symbolic_call(x)
    return backend.numpy.sum(x, axis=axis, keepdims=keepdims)

Sum of a tensor over the given axes.

Args:
x: Input tensor.
axis: Axis or axes along which the sum is computed. The default is to
compute the sum of the flattened tensor.
keepdims: If this is set to `True`, the axes which are reduced are left
in the result as dimensions with size one.

Returns:
Output tensor containing the sum.


def make_tex_table(inputlist, outputfile, close=False, fmt=None,
                   **kwargs):

    output_str = ""
    if fmt is None:
        fmt = {}
    for row in inputlist:
        for key, val in enumerate(row):
            if val is None:
                output_str += r'\text{{{}}}'.format(
                    str(kwargs.get("nonestring", "None"))
                )
            else:

                if np.isscalar(val):
                    temp_str_fmt = "$\\num{{" + fmt.get(
                        key, "{:g}") + "}}$"
                else:
                    temp_str_fmt = fmt.get(key, "{}")
                temp_str = temp_str_fmt.format(val).replace("+", "")
            output_str += temp_str + "&"
        output_str = output_str[:-1]
        output_str += "\\\\\n"
    outputfile.write(output_str)
    if close:
        outputfile.close()

Parse table from inputlist

Args:
inputlist: list
List to parse
outputfile: file
.tex file to write
fmt: dictionary
key: integer
column index starting with 0
values: string
format string. eg "{:g}"
**kwargs:
nonestring: string
string when objecttype is None
Returns:
None


def generate_workflow_description(self):
    if (not self.tasks):
        raise WorkflowError('Workflow contains no tasks, and cannot be executed.')
    self.definition = self.workflow_skeleton()
    if self.batch_values:
        self.definition['batch_values'] = self.batch_values
    all_input_port_values = [t.inputs.__getattribute__(input_port_name).value for t in self.tasks for input_port_name in t.inputs._portnames]
    for task in self.tasks:
        output_multiplex_ports_to_exclude = []
        multiplex_output_port_names = [portname for portname in task.outputs._portnames if task.outputs.__getattribute__(portname).is_multiplex]
        for p in multiplex_output_port_names:
            output_port_reference = ((('source:' + task.name) + ':') + p)
            if (output_port_reference not in all_input_port_values):
                output_multiplex_ports_to_exclude.append(p)
        task_def = task.generate_task_workflow_json(output_multiplex_ports_to_exclude=output_multiplex_ports_to_exclude)
        self.definition['tasks'].append(task_def)
    if self.callback:
        self.definition['callback'] = self.callback
    return self.definition

Generate workflow json for launching the workflow against the gbdx api

Args:
None

Returns:
json string


def id(self, value):

        if value == self._defaults['ai.device.id'] and 'ai.device.id' in self._values:
            del self._values['ai.device.id']
        else:
            self._values['ai.device.id'] = value

The id property.

Args:
value (string). the property value.


def argsort(*args, **kwargs):

    if len(args) == 1 and isinstance(args[0], dict):
        dict_ = args[0]
        index_list = list(dict_.keys())
        value_list = list(dict_.values())
        return sortedby2(index_list, value_list)
    else:
        index_list = list(range(len(args[0])))
        return sortedby2(index_list, *args, **kwargs)

like np.argsort but for lists

Args:
*args: multiple lists to sort by
**kwargs:
reverse (bool): sort order is descending if True else acscending

CommandLine:
python -m utool.util_list argsort

Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_list import *  # NOQA
>>> result = ut.argsort({'a': 3, 'b': 2, 'c': 100})
>>> print(result)


def profile_update_args_v2(self, profile):
    ij = self.load_install_json(profile.get('install_json', 'install.json'))
    if ((profile.get('args', {}).get('app') is None) and (profile.get('args', {}).get('default') is None)):
        _args = profile.pop('args')
        profile['args'] = {}
        profile['args']['app'] = {}
        profile['args']['default'] = {}
        for arg in self.profile_settings_args_install_json(ij, None):
            try:
                profile['args']['app'][arg] = _args.pop(arg)
            except KeyError:
                if self.args.verbose:
                    print('{}{}Input "{}" not found in profile "{}".'.format(c.Style.BRIGHT, c.Fore.YELLOW, arg, profile.get('profile_name')))
        profile['args']['default'] = _args
        print('{}{}Updating args section to v2 schema for profile {}.'.format(c.Style.BRIGHT, c.Fore.YELLOW, profile.get('profile_name')))

Update v1 profile args to v2 schema for args.

.. code-block:: javascript

"args": {
"app": {
"input_strings": "capitalize",
"tc_action": "Capitalize"
}
},
"default": {
"api_access_id": "$env.API_ACCESS_ID",
"api_default_org": "$env.API_DEFAULT_ORG",
},

Args:
profile (dict): The dictionary containting the profile settings.


def _validate_instantiation_options(self, datafile, skip_json_validation):

    if not skip_json_validation and not validator.is_datafile_valid(datafile):
      raise exceptions.InvalidInputException(enums.Errors.INVALID_INPUT_ERROR.format('datafile'))

    if not validator.is_event_dispatcher_valid(self.event_dispatcher):
      raise exceptions.InvalidInputException(enums.Errors.INVALID_INPUT_ERROR.format('event_dispatcher'))

    if not validator.is_logger_valid(self.logger):
      raise exceptions.InvalidInputException(enums.Errors.INVALID_INPUT_ERROR.format('logger'))

    if not validator.is_error_handler_valid(self.error_handler):
      raise exceptions.InvalidInputException(enums.Errors.INVALID_INPUT_ERROR.format('error_handler'))

Helper method to validate all instantiation parameters.

Args:
datafile: JSON string representing the project.
skip_json_validation: Boolean representing whether JSON schema validation needs to be skipped or not.

Raises:
Exception if provided instantiation options are valid.


def get_task_ops(task_type=TaskType.ALG_CTRL):

    try:
      return LearnToExecuteState.TASK_TYPE_OPS[task_type]
    except KeyError:
      raise KeyError("Bad task_type '%s', check config." % task_type)

Returns an operations list based on the specified task index.

Args:
task_type: indicates the task type used.

Returns:
List of the eligible ops.


def _get_object_from_python_path(python_path):
    python_path = python_path.split('.')
    module_path = python_path[:(- 1)]
    object_class = python_path[(- 1)]
    if isinstance(module_path, list):
        module_path = '.'.join(module_path)
    module = import_module(module_path)
    schema = getattr(module, object_class)
    if isclass(schema):
        schema = schema()
    return schema

Method that will fetch a Marshmallow schema from a path to it.

Args:
python_path (str): The string path to the Marshmallow schema.

Returns:
marshmallow.Schema: The schema matching the provided path.

Raises:
TypeError: This is raised if the specified object isn't
a Marshmallow schema.


def __init__(self, location):

        super(ContextFieldExistence, self).__init__(location)
        self.location = location
        self.validate()

Construct a new ContextFieldExistence object for a vertex field from the global context.

Args:
location: Location, specifying where the field was declared. Must point to a vertex.

Returns:
new ContextFieldExistence expression which evaluates to True iff the vertex exists


def to_dict(self) -> Dict[str, Any]:
    output = copy.deepcopy(self.__dict__)
    return output

Serializes this instance to a Python dictionary.

Returns:
Dict[str, Any]: Dictionary of all the attributes that make up this configuration instance.


def __init__(self, server_port, stream_handler_class):
    self._server_port = server_port
    self._stream_handler_class = stream_handler_class
    self._server_lock = threading.Lock()
    self._server_started = False
    self._stop_requested = False
    self._debug_ops_state_change_queue = queue.Queue()
    self._gated_grpc_debug_watches = set()
    self._breakpoints = set()

Constructor.

Args:
server_port: (int) Port number to bind to.
stream_handler_class: A class of the base class
`EventListenerBaseStreamHandler` that will be used to constructor
stream handler objects during `SendEvents` calls.


def put_many(self, items: Iterable[T], context: PipelineContext = None) -> None:

        LOGGER.info("Creating transform generator for items \"{items}\" for sink \"{sink}\"".format(items=items, sink=self._sink))
        transform_generator = (self._transform(data=item, context=context) for item in items)
        LOGGER.info("Putting transform generator for items \"{items}\" into sink \"{sink}\"".format(items=items, sink=self._sink))
        self._sink.put_many(self._store_type, transform_generator, context)

Puts multiple objects of the same type into the data sink. The objects may be transformed into a new type for insertion if necessary.

Args:
items: An iterable (e.g. list) of objects to be inserted into the data sink.
context: The context of the insertions (mutable).


def epoch_to_log_line_timestamp(epoch_time, time_zone=None):
    s, ms = divmod(epoch_time, 1000)
    d = datetime.datetime.fromtimestamp(s, tz=time_zone)
    return d.strftime('%m-%d %H:%M:%S.') + str(ms)

Converts an epoch timestamp in ms to log line timestamp format, which
is readible for humans.

Args:
epoch_time: integer, an epoch timestamp in ms.
time_zone: instance of tzinfo, time zone information.
Using pytz rather than python 3.2 time_zone implementation for
python 2 compatibility reasons.

Returns:
A string that is the corresponding timestamp in log line timestamp
format.


def upload(s3_conn, filepath, s3_path):
    (bucket_name, prefix) = split_s3_path(s3_path)
    bucket = s3_conn.get_bucket(bucket_name)
    filename = os.path.basename(filepath)
    key = boto.s3.key.Key(bucket=bucket, name='{}/{}'.format(prefix, filename))
    logging.info('uploading from %s to %s', filepath, key)
    key.set_contents_from_filename(filepath)

Uploads the given file to s3

Args:
s3_conn: (boto.s3.connection) an s3 connection
filepath (str) the local filename
s3_path (str) the destination path on s3


def fileSave(self, filePath=None, updatePath=False):
    if (not filePath):
        filePath = self.filePath
    if (not os.path.isfile(filePath)):
        print(("Data file '%s' does not exist, will create new file." % filePath))
        if (not os.path.exists(os.path.split(filePath)[0])):
            os.makedirs(os.path.split(filePath)[0])
    dataJsonString = json.dumps(self.data, indent=4, sort_keys=True)
    print(("Writing to file '%s' ... " % filePath), end='', flush=True)
    with open(filePath, 'w') as fileout:
        fileout.write(dataJsonString)
    print('Wrote file!')
    if updatePath:
        self.filePath = filePath

Write the internal JSON data dictionary to a JSON data file.

If no file path is provided, the stored data file path will be used.

Args:
filePath (Optional[str]): A relative or absolute path to a
'.json' file. Defaults to None.
updatePath (Optional[bool]): Specifies whether or not to update
the stored data file path. Defaults to False.


def get_substring_idxs(substr, string):
    return [match.start() for match in re.finditer(substr, string)]

Return a list of indexes of substr. If substr not found, list is
empty.

Arguments:
substr (str): Substring to match.
string (str): String to match in.

Returns:
list of int: Start indices of substr.


def remove_padding(sequence):

  length = sequence.pop('length')
  sequence = tools.nested.map(lambda tensor: tensor[:length], sequence)
  return sequence

Selects the used frames of a sequence, up to its length.

This function does not expect a batch of sequences, but a single sequence.
The sequence must be a dict with `length` key, which will removed from the
result.

Args:
sequence: Nested dict of tensors with time dimension.

Returns:
Nested dict of tensors with padding elements and `length` key removed.


def pprnt(input, return_data=False):
    HEADER = '\x1b[95m'
    OKBLUE = '\x1b[94m'
    OKGREEN = '\x1b[32m'
    WARNING = '\x1b[93m'
    FAIL = '\x1b[91m'
    ENDC = '\x1b[0m'
    BOLD = '\x1b[1m'
    UNDERLINE = '\x1b[4m'
    import json, re
    result = json.dumps(input, sort_keys=True, indent=4)
    result = re.sub('(")(\\w*?_id)(":)', ('\\1%s%s\\2%s\\3' % (BOLD, HEADER, ENDC)), result)
    result = re.sub('(")(\\w*?_set)(":)', ('\\1%s%s\\2%s\\3' % (BOLD, HEADER, ENDC)), result)
    result = re.sub('(\\n *?")(\\w*?)(":)', ('\\1%s%s\\2%s\\3' % (BOLD, OKGREEN, ENDC)), result)
    if (not return_data):
        print(result)
    else:
        return result

Prettier print for nested data

Args:
input: Input data
return_data (bool): Default False. Print outs if False, returns if True.
Returns:
None | Pretty formatted text representation of input data.


def _get_args_and_defaults(args, defaults):

    defaults = defaults or []
    args_and_defaults = [(argument, default) for (argument, default)
                         in zip_longest(args[::-1], defaults[::-1],
                                        fillvalue=NoDefault)]
    return args_and_defaults[::-1]

Return a list of 2-tuples - the argument name and its default value or
a special value that indicates there is no default value.

Args:
args: list of argument name
defaults: tuple of default values


def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str]=None) -> Tuple[str]:
    if not os.path.isdir(save_directory):
        logger.error(f'Vocabulary path ({save_directory}) should be a directory')
        return
    artists_file = os.path.join(save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['artists_file'])
    with open(artists_file, 'w', encoding='utf-8') as f:
        f.write(json.dumps(self.artists_encoder, ensure_ascii=False))
    genres_file = os.path.join(save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['genres_file'])
    with open(genres_file, 'w', encoding='utf-8') as f:
        f.write(json.dumps(self.genres_encoder, ensure_ascii=False))
    lyrics_file = os.path.join(save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['lyrics_file'])
    with open(lyrics_file, 'w', encoding='utf-8') as f:
        f.write(json.dumps(self.lyrics_encoder, ensure_ascii=False))
    return (artists_file, genres_file, lyrics_file)

Saves the tokenizer's vocabulary dictionary to the provided save_directory.

Args:
save_directory (`str`):
A path to the directory where to saved. It will be created if it doesn't exist.

filename_prefix (`Optional[str]`, *optional*):
A prefix to add to the names of the files saved by the tokenizer.


def _convert_tflite_enum_type_to_tf_type(tflite_enum_type):
    tf_type = _MAP_TFLITE_ENUM_TO_TF_TYPES.get(tflite_enum_type)
    if tf_type is None:
        raise ValueError('Unsupported enum {}. The valid map of enum to tf types is : {}'.format(tflite_enum_type, _MAP_TFLITE_ENUM_TO_TF_TYPES))
    return tf_type

Converts tflite enum type (eg: 0) to tf type (eg: tf.float32).

Args:
tflite_enum_type: tflite enum type (eg: 0, that corresponds to float32)

Raises:
ValueError: If an invalid tflite enum type is provided.

Returns:
tf type (eg: tf.float32)


def _get_int_removals_helper(self, spec_amts_oxi, oxid_el, oxid_els, numa):

        oxid_old = min([spec.oxi_state for spec in spec_amts_oxi if spec.symbol == oxid_el.symbol])
        oxid_new = math.floor(oxid_old + 1)

        if oxid_new > oxid_el.max_oxidation_state:
            return numa

        spec_old = Specie(oxid_el.symbol, oxid_old)
        spec_new = Specie(oxid_el.symbol, oxid_new)
        specamt = spec_amts_oxi[spec_old]
        spec_amts_oxi = {sp: amt for sp, amt in spec_amts_oxi.items() if sp != spec_old}
        spec_amts_oxi[spec_new] = specamt
        spec_amts_oxi = Composition(spec_amts_oxi)

        oxi_noA = sum([spec.oxi_state * spec_amts_oxi[spec] for spec in spec_amts_oxi if
                       spec.symbol not in self.cation.symbol])
        a = max(0, -oxi_noA / self.cation_charge)
        numa = numa.union({a})

        if a == 0:
            return numa
        else:
            for oxid_el in oxid_els:
                numa = numa.union(
                    self._get_int_removals_helper(spec_amts_oxi.copy(), oxid_el, oxid_els, numa))
            return numa

This is a helper method for get_removals_int_oxid!

Args:
spec_amts_oxi - a dict of species to their amounts in the structure
oxid_el - the element to oxidize
oxid_els - the full list of elements that might be oxidized
numa - a running set of numbers of A cation at integer oxidation steps
Returns:
a set of numbers A; steps for for oxidizing oxid_el first, then the other oxid_els in this list


def __init__(self, instruments, scripts = None, name=None, settings=None, log_function=None, data_path = None):

        Script.__init__(self, name, settings=settings, scripts=scripts, instruments=instruments, log_function=log_function, data_path = data_path)
        self.data = {'plant_output': deque(maxlen=self.settings['buffer_length']),
                     'control_output': deque(maxlen=self.settings['buffer_length'])}

Example of a script that emits a QT signal for the gui
Args:
name (optional): name of script, if empty same as class name
settings (optional): settings for this script, if empty same as default settings


def do_labels_update(self, info, labels):
    if self.update_label_func:
        self.update_label_func(self.label_name, info, labels)

Updates a dictionary of labels using the assigned update_op_func

Args:
info (:class:`endpoints_management.control.report_request.Info`): the
info instance to update
labels (dict[string[string]]): the labels dictionary

Return:
`True` if desc is supported, otherwise `False`


def convert_tanh(params, w_name, scope_name, inputs, layers, weights, names):
    print('Converting tanh ...')
    if (names == 'short'):
        tf_name = ('TANH' + random_string(4))
    elif (names == 'keep'):
        tf_name = w_name
    else:
        tf_name = (w_name + str(random.random()))
    tanh = keras.layers.Activation('tanh', name=tf_name)
    layers[scope_name] = tanh(layers[inputs[0]])

Convert tanh layer.

Args:
params: dictionary with layer parameters
w_name: name prefix in state_dict
scope_name: pytorch scope name
inputs: pytorch node inputs
layers: dictionary with keras tensors
weights: pytorch state_dict
names: use short names for keras layers


def query(self, watch_key, time_indices=None, slicing=None, mapping=None):
    if (watch_key not in self._tensor_data):
        raise KeyError(('watch_key not found: %s' % watch_key))
    if (time_indices is None):
        time_indices = '-1'
    time_slicing = tensor_helper.parse_time_indices(time_indices)
    all_time_indices = list(range(self._tensor_data[watch_key].num_total()))
    sliced_time_indices = all_time_indices[time_slicing]
    if (not isinstance(sliced_time_indices, list)):
        sliced_time_indices = [sliced_time_indices]
    recombine_and_map = False
    step_mapping = mapping
    if ((len(sliced_time_indices) > 1) and (mapping not in (None,))):
        recombine_and_map = True
        step_mapping = None
    output = []
    for index in sliced_time_indices:
        value = self._tensor_data[watch_key].query(index)[0]
        if ((value is not None) and (not isinstance(value, debug_data.InconvertibleTensorProto))):
            output.append(tensor_helper.array_view(value, slicing=slicing, mapping=step_mapping)[2])
        else:
            output.append(None)
    if recombine_and_map:
        if (mapping == 'image/png'):
            output = tensor_helper.array_to_base64_png(output)
        elif (mapping and (mapping != 'none')):
            logger.warn('Unsupported mapping mode after recomining time steps: %s', mapping)
    return output

Query tensor store for a given watch_key.

Args:
watch_key: The watch key to query.
time_indices: A numpy-style slicing string for time indices. E.g.,
`-1`, `:-2`, `[::2]`. If not provided (`None`), will use -1.
slicing: A numpy-style slicing string for individual time steps.
mapping: An mapping string or a list of them. Supported mappings:
`{None, 'image/png', 'health-pill'}`.

Returns:
The potentially sliced values as a nested list of values or its mapped
format. A `list` of nested `list` of values.

Raises:
ValueError: If the shape of the sliced array is incompatible with mapping
mode. Or if the mapping type is invalid.


def _capture_by_ref(self, graph: Any, lam: Callable[[], Any], key: Hashable=None) -> Any:
    if key is not None and key in self._by_ref_internal:
        return self._by_ref_internal[key]
    if key is None:
        key = len(self._by_ref_internal)
        while key in self._by_ref_internal:
            key += 1
    value_nested = lam()
    capture_trace_type = trace_type.from_value(value_nested)
    ctx = trace_type.InternalPlaceholderContext(graph)
    internal = capture_trace_type.placeholder_value(ctx)

    def lam_fn():
        value = lam()
        return capture_trace_type.to_tensors(value)
    self._by_ref_external[key] = lam_fn
    self._by_ref_internal[key] = internal
    self._by_ref_tracetype[key] = capture_trace_type
    return self._by_ref_internal[key]

Used during tracing process to create/retrive by-ref captures.

Args:
graph: The FuncGraph that captures this tensor.
lam: A callable that takes no arguments and returns tensor captures.
key: A hashable identifier.

Returns:
Tensor from this FuncGraph.


def materialize(self, ref, table_name=None, index_columns=None, logger=None):

        from ambry.library import Library
        assert isinstance(self._library, Library)

        logger.debug('Materializing warehouse partition.\n    partition: {}'.format(ref))
        partition = self._library.partition(ref)

        connection = self._backend._get_connection()

        return self._backend.install(connection, partition, table_name=table_name,
                                     index_columns=index_columns, materialize=True, logger=logger)

Creates materialized table for given partition reference.

Args:
ref (str): id, vid, name or vname of the partition.

Returns:
str: name of the partition table in the database.


def get_proj(prj_code):

    if prj_code in CUSTOM_PRJ:
        proj = pyproj.Proj(CUSTOM_PRJ[prj_code])
    else:
        proj = pyproj.Proj(init=prj_code)
    return proj

Helper method for handling projection codes that are unknown to pyproj

Args:
prj_code (str): an epsg proj code

Returns:
projection: a pyproj projection


def requires_submit(func):

    @functools.wraps(func)
    def _wrapper(self, *args, **kwargs):
        if self._future is None:
            raise JobError("Job not submitted yet!. You have to .submit() first!")
        return func(self, *args, **kwargs)
    return _wrapper

Decorator to ensure that a submit has been performed before
calling the method.

Args:
func (callable): test function to be decorated.

Returns:
callable: the decorated function.


def get(self, addresses):

        with self._lock:
            results = []
            for add in addresses:
                self.validate_read(add)
                results.append(self._get(add))
            return results

Returns the value in this context, or None, for each address in
addresses. Useful for gets on the context manager.

Args:
addresses (list of str): The addresses to return values for, if
within this context.

Returns:
results (list of bytes): The values in state for these addresses.


def get_template_name(env, pipeline_type):
    pipeline_base = 'pipeline/pipeline'
    template_name_format = '{pipeline_base}'
    if env.startswith('prod'):
        template_name_format = (template_name_format + '_{env}')
    else:
        template_name_format = (template_name_format + '_stages')
    if (pipeline_type != 'ec2'):
        template_name_format = (template_name_format + '_{pipeline_type}')
    template_name_format = (template_name_format + '.json.j2')
    template_name = template_name_format.format(pipeline_base=pipeline_base, env=env, pipeline_type=pipeline_type)
    return template_name

Generates the correct template name based on pipeline type

Args:
env (str): environment to generate templates for
pipeline_type (str): Type of pipeline like ec2 or lambda

Returns:
str: Name of template


def inception_v3_arg_scope(weight_decay=0.00004,
                           stddev=0.1,
                           batch_norm_var_collection='moving_vars'):

  batch_norm_params = {

      'decay': 0.9997,

      'epsilon': 0.001,

      'updates_collections': tf.GraphKeys.UPDATE_OPS,

      'variables_collections': {
          'beta': None,
          'gamma': None,
          'moving_mean': [batch_norm_var_collection],
          'moving_variance': [batch_norm_var_collection],
      }
  }

  with slim.arg_scope([slim.conv2d, slim.fully_connected],
                      weights_regularizer=slim.l2_regularizer(weight_decay)):
    with slim.arg_scope([slim.conv2d],
                        weights_initializer=tf.truncated_normal_initializer(stddev=stddev),
                        activation_fn=tf.nn.relu, normalizer_fn=slim.batch_norm,
                        normalizer_params=batch_norm_params) as sc:
      return sc

Defines the default InceptionV3 arg scope.

Args:
weight_decay: The weight decay to use for regularizing the model.
stddev: The standard deviation of the trunctated normal weight initializer.
batch_norm_var_collection: The name of the collection for the batch norm
variables.

Returns:
An `arg_scope` to use for the inception v3 model.


def remove_user(username):
    users = passwd_reader.load_users()
    assert (username in users), ("Username '%s' not found!" % username)
    del users[username]
    passwd_reader.save_users(users)
    home_dir = (settings.DATA_PATH + username)
    if os.path.exists(home_dir):
        shutil.rmtree(home_dir)
    reload_configuration()

Remove user, his home directory and so on..

Args:
username (str): User's name.


def upload_files(self, file_list):

        counter = 0
        files_to_upload = list(set(file_list) - set(self.uploaded_files))
        try:
            for f in files_to_upload:
                with open(config.get_storage_path(f), 'rb') as file_obj:
                    response = config.SESSION.post(config.file_upload_url(), files={'file': file_obj})
                    if response.status_code == 200:
                        response.raise_for_status()
                        self.uploaded_files.append(f)
                        counter += 1
                        config.LOGGER.info("\tUploaded {0} ({count}/{total}) ".format(f, count=counter, total=len(files_to_upload)))
                    else:
                        self.failed_uploads[f] = response._content.decode('utf-8')
        finally:
            config.PROGRESS_MANAGER.set_uploading(self.uploaded_files)

upload_files: uploads files to server
Args:
file_list (str): list of files to upload
Returns: None


def shape_type_conversion(fn):

    def wrapper(instance, input_shape):
        if input_shape is not None:
            input_shape = convert_shapes(input_shape, to_tuples=True)
        output_shape = fn(instance, input_shape)
        if output_shape is not None:
            output_shape = convert_shapes(output_shape, to_tuples=False)
        return output_shape
    return wrapper

Decorator that handles tuple/TensorShape conversion.

Used in `compute_output_shape` and `build`.

Args:
fn: function to wrap.

Returns:
Wrapped function.


def __init__(self, device=''):
    self._resource_handle_value = None
    self._resource_device = device
    self._self_destruction_context = context.eager_mode if context.executing_eagerly() else ops.get_default_graph().as_default

Initialize the `CapturableResource`.

Args:
device: A string indicating a required placement for this resource,
e.g. "CPU" if this resource must be created on a CPU device. A blank
device allows the user to place resource creation, so generally this
should be blank unless the resource only makes sense on one device.