def _parse_arguments(prog, argv):
  """Parses command line arguments.

  Args:
    prog: The path of the program (dsub.py) or an alternate program name to
    display in usage.
    argv: The list of program arguments to parse.

  Returns:
    A Namespace of parsed arguments.
  """
  # Handle version flag and exit if it was passed.
  param_util.handle_version_flag()

  parser = provider_base.create_parser(prog)

  # Add dsub core job submission arguments
  parser.add_argument(
      '--version', '-v', default=False, help='Print the dsub version and exit.')

  parser.add_argument(
      '--unique-job-id',
      default=False,
      action='store_true',
      help="""Experimental: create a unique 32 character UUID for the dsub
          job-id using https://docs.python.org/3/library/uuid.html.
          (default: False)""")
  parser.add_argument(
      '--name',
      help="""Name for the job. Defaults to the script name or
          first token of the --command if specified.""")
  parser.add_argument(
      '--tasks',
      nargs='*',
      action=TaskParamAction,
      help="""Path to a file of tab separated values (TSV) for task parameters.
          The file may be located in the local filesystem or in a Google Cloud
          Storage bucket.

          The first line is a list of column headers specifying an --env,
          --input, --input-recursive, --output or --output-recursive variable,
          and each subsequent line specifies the values for a task.

          Optionally specify tasks from the file to submit. Can take the form
          "m", "m-", or "m-n" where m and n are task numbers starting at 1.
          (default: None)""",
      metavar='FILE M-N')
  parser.add_argument(
      '--image',
      # Defaults to None so we can emit a warning if not specified
      # Will later on be set to DEFAULT_IMAGE
      default=None,
      help="""Image name from Docker Hub, Google Container Repository, or other
          Docker image service. The task must have READ access to the
          image. (default: {})""".format(DEFAULT_IMAGE))
  parser.add_argument(
      '--dry-run',
      default=False,
      action='store_true',
      help='Print the task(s) that would be run and then exit. (default: False)'
  )
  parser.add_argument(
      '--command',
      help="""Command to run inside the job\'s Docker container. This argument
          or the --script argument must be provided.""",
      metavar='COMMAND')
  parser.add_argument(
      '--script',
      help="""Path to a script that is located in the local file system or
          inside a Google Cloud Storage bucket. This script will be run inside
          the job\'s Docker container.  This argument or the --command
          argument must be provided.""",
      metavar='SCRIPT')
  parser.add_argument(
      '--env',
      nargs='*',
      action=param_util.ListParamAction,
      default=[],
      help='Environment variables for the script\'s execution environment',
      metavar='KEY=VALUE')
  parser.add_argument(
      '--label',
      nargs='*',
      action=param_util.ListParamAction,
      default=[],
      help='Labels to associate to the job.',
      metavar='KEY=VALUE')
  parser.add_argument(
      '--input',
      nargs='*',
      action=param_util.ListParamAction,
      default=[],
      help="""Input path arguments to localize into the script's execution
          environment""",
      metavar='KEY=REMOTE_PATH')
  parser.add_argument(
      '--input-recursive',
      nargs='*',
      action=param_util.ListParamAction,
      default=[],
      help="""Input path arguments to localize recursively into the script\'s
          execution environment""",
      metavar='KEY=REMOTE_PATH')
  parser.add_argument(
      '--output',
      nargs='*',
      action=param_util.ListParamAction,
      default=[],
      help="""Output path arguments to de-localize from the script\'s execution
          environment""",
      metavar='KEY=REMOTE_PATH')
  parser.add_argument(
      '--output-recursive',
      nargs='*',
      action=param_util.ListParamAction,
      default=[],
      help="""Output path arguments to de-localize recursively from the script's
          execution environment""",
      metavar='KEY=REMOTE_PATH')
  parser.add_argument(
      '--user',
      '-u',
      help='User submitting the dsub job, defaults to the current OS user.')
  parser.add_argument(
      '--user-project',
      help="""Specify a user project to be billed for all requests to Google
         Cloud Storage (logging, localization, delocalization). This flag exists
         to support accessing Requester Pays buckets (default: None)""")
  parser.add_argument(
      '--mount',
      nargs='*',
      action=param_util.ListParamAction,
      default=[],
      help="""Mount a resource such as a bucket, disk, or directory into your
         Docker container""",
      metavar='KEY=PATH_SPEC')

  # Add dsub job management arguments
  parser.add_argument(
      '--wait',
      action='store_true',
      help='Wait for the job to finish all its tasks. (default: False)')
  parser.add_argument(
      '--retries',
      default=0,
      type=int,
      help='Number of retries to perform on failed tasks. (default: 0)')
  parser.add_argument(
      '--poll-interval',
      default=10,
      type=int,
      help='Polling interval (in seconds) for checking job status '
      'when --wait or --after are set. (default: 10)')
  parser.add_argument(
      '--after',
      nargs='+',
      default=[],
      help='Job ID(s) to wait for before starting this job.')
  parser.add_argument(
      '--skip',
      default=False,
      action='store_true',
      help="""Do not submit the job if all output specified using the --output
          and --output-recursive parameters already exist. Note that wildcard
          and recursive outputs cannot be strictly verified. See the
          documentation for details. (default: False)""")
  parser.add_argument(
      '--summary',
      default=False,
      action='store_true',
      help="""During the --wait loop, display a summary of the results,
          grouped by (job, status). (default: False)""")

  # Add dsub resource requirement arguments
  parser.add_argument(
      '--min-cores',
      type=int,
      help="""Minimum CPU cores for each job. The default is provider-specific.
           The google-v2 provider default is 1 core.
           The local provider does not allocate resources, but uses available
           resources of your machine.""")
  parser.add_argument(
      '--min-ram',
      type=float,
      help="""Minimum RAM per job in GB. The default is provider-specific.
           The google-v2 provider default is 3.75 GB.
           The local provider does not allocate resources, but uses available
           resources of your machine.""")
  parser.add_argument(
      '--disk-size',
      default=job_model.DEFAULT_DISK_SIZE,
      type=int,
      help='Size (in GB) of data disk to attach for each job (default: {})'
      .format(job_model.DEFAULT_DISK_SIZE))

  parser.add_argument(
      '--logging',
      help='Cloud Storage path to send logging output'
      ' (either a folder, or file ending in ".log")')

  # Add provider-specific arguments

  # Shared between the "google-cls-v2" and "google-v2" providers
  google_common = parser.add_argument_group(
      title='google-common',
      description="""Options common to the "google-cls-v2" and "google-v2"
        providers""")
  google_common.add_argument(
      '--project', help='Cloud project ID in which to run the job')
  google_common.add_argument(
      '--boot-disk-size',
      default=job_model.DEFAULT_BOOT_DISK_SIZE,
      type=int,
      help='Size (in GB) of the boot disk (default: {})'.format(
          job_model.DEFAULT_BOOT_DISK_SIZE))
  google_common.add_argument(
      '--preemptible',
      const=param_util.preemptile_param_type(True),
      default=param_util.preemptile_param_type(False),
      nargs='?',  # Be careful if we ever add positional arguments
      type=param_util.preemptile_param_type,
      help="""If --preemptible is given without a number, enables preemptible
          VMs for all attempts for all tasks. If a number value N is used,
          enables preemptible VMs for up to N attempts for each task.
          Defaults to not using preemptible VMs.""")
  google_common.add_argument(
      '--zones', nargs='+', help='List of Google Compute Engine zones.')
  google_common.add_argument(
      '--scopes',
      nargs='+',
      help="""Space-separated scopes for Google Compute Engine instances.
          If unspecified, provider will use '%s'""" % ','.join(
              google_base.DEFAULT_SCOPES))
  google_common.add_argument(
      '--accelerator-type',
      help="""The Compute Engine accelerator type. See
          https://cloud.google.com/compute/docs/gpus/ for supported GPU types.

          Only NVIDIA GPU accelerators are currently supported. If an NVIDIA GPU
          is attached, the required runtime libraries will be made available to
          all containers under /usr/local/nvidia.

          Each version of Container-Optimized OS image (used by the Pipelines
          API) has a default supported NVIDIA GPU driver version. See
          https://cloud.google.com/container-optimized-os/docs/how-to/run-gpus#install

          Note that attaching a GPU increases the worker VM startup time by a
          few minutes. (default: None)""")
  google_common.add_argument(
      '--accelerator-count',
      type=int,
      default=0,
      help="""The number of accelerators of the specified type to attach.
          By specifying this parameter, you will download and install the
          following third-party software onto your job's Compute Engine
          instances: NVIDIA(R) Tesla(R) drivers and NVIDIA(R) CUDA toolkit.
          (default: 0)""")
  google_common.add_argument(
      '--credentials-file',
      type=str,
      help='Path to a local file with JSON credentials for a service account.')
  google_common.add_argument(
      '--regions',
      nargs='+',
      help="""List of Google Compute Engine regions.
          Only one of --zones and --regions may be specified.""")
  google_common.add_argument(
      '--machine-type', help='Provider-specific machine type (default: None)')
  google_common.add_argument(
      '--cpu-platform',
      help="""The CPU platform to request. Supported values can be found at
      https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform
      (default: None)""")
  google_common.add_argument(
      '--network',
      help="""The Compute Engine VPC network name to attach the VM's network
          interface to. The value will be prefixed with global/networks/ unless
          it contains a /, in which case it is assumed to be a fully specified
          network resource URL. (default: None)""")
  google_common.add_argument(
      '--subnetwork',
      help="""The name of the Compute Engine subnetwork to attach the instance
          to. (default: None)""")
  google_common.add_argument(
      '--use-private-address',
      default=False,
      action='store_true',
      help="""If set to true, do not attach a public IP address to the VM.
      (default: False)""")
  google_common.add_argument(
      '--timeout',
      help="""The maximum amount of time to give the task to complete.
          This includes the time spent waiting for a worker to be allocated.
          Time can be listed using a number followed by a unit. Supported units
          are s (seconds), m (minutes), h (hours), d (days), w (weeks). The
          provider-specific default is 7 days. Example: '7d' (7 days).""")
  google_common.add_argument(
      '--log-interval',
      help="""The amount of time to sleep between copies of log files from
          the task to the logging path.
          Time can be listed using a number followed by a unit. Supported units
          are s (seconds), m (minutes), h (hours).
          Example: '5m' (5 minutes). Default is '1m'.""")
  google_common.add_argument(
      '--ssh',
      default=False,
      action='store_true',
      help="""If set to true, start an ssh container in the background
          to allow you to log in using SSH and debug in real time.
          (default: False)""")
  google_common.add_argument('--nvidia-driver-version', help=argparse.SUPPRESS)
  google_common.add_argument(
      '--service-account',
      type=str,
      help="""Email address of the service account to be authorized on the
          Compute Engine VM for each job task. If not specified, the default
          Compute Engine service account for the project will be used.""")
  google_common.add_argument(
      '--disk-type',
      help="""
          The disk type to use for the data disk. Valid values are pd-standard
          pd-ssd and local-ssd. The default value is pd-standard.""")
  google_common.add_argument(
      '--enable-stackdriver-monitoring',
      default=False,
      action='store_true',
      help="""If set to true, enables Stackdriver monitoring on the VM.
              (default: False)""")
  google_common.add_argument(
      '--block-external-network',
      default=False,
      action='store_true',
      help="""If set to true, prevents the container for the user's
          script/command from accessing the external network.
          (default: False)""")

  google_cls_v2 = parser.add_argument_group(
      title='"google-cls-v2" provider options',
      description='See also the "google-common" options listed above')
  google_cls_v2.add_argument(
      '--location',
      default=job_model.DEFAULT_LOCATION,
      help="""Specifies the Google Cloud region to which the pipeline request
        will be sent and where operation metadata will be stored. The associated
        dsub task may be executed in another region if the --regions or --zones
        arguments are specified. (default: {})""".format(
            job_model.DEFAULT_LOCATION))

  args = provider_base.parse_args(
      parser, {
          'google-cls-v2': ['project', 'logging'],
          'google-v2': ['project', 'logging'],
          'test-fails': [],
          'local': ['logging'],
      }, argv)

  if args.provider == 'google-cls-v2':
    _google_cls_v2_parse_arguments(args)
  if args.provider == 'google-v2':
    _google_v2_parse_arguments(args)

  return args


Extract job - global resources requirements from input args .

def _get_job_resources(args):
  """Extract job-global resources requirements from input args.

  Args:
    args: parsed command-line arguments

  Returns:
    Resources object containing the requested resources for the job
  """
  logging = param_util.build_logging_param(
      args.logging) if args.logging else None
  timeout = param_util.timeout_in_seconds(args.timeout)
  log_interval = param_util.log_interval_in_seconds(args.log_interval)

  return job_model.Resources(
      min_cores=args.min_cores,
      min_ram=args.min_ram,
      machine_type=args.machine_type,
      disk_size=args.disk_size,
      disk_type=args.disk_type,
      boot_disk_size=args.boot_disk_size,
      image=args.image,
      regions=args.regions,
      zones=args.zones,
      logging=logging,
      logging_path=None,
      service_account=args.service_account,
      scopes=args.scopes,
      cpu_platform=args.cpu_platform,
      network=args.network,
      subnetwork=args.subnetwork,
      use_private_address=args.use_private_address,
      accelerator_type=args.accelerator_type,
      accelerator_count=args.accelerator_count,
      nvidia_driver_version=None,
      timeout=timeout,
      log_interval=log_interval,
      ssh=args.ssh,
      enable_stackdriver_monitoring=args.enable_stackdriver_monitoring,
      max_retries=args.retries,
      max_preemptible_attempts=args.preemptible,
      block_external_network=args.block_external_network)

def _get_job_resources ( args ) : logging = param_util . build_logging_param ( args . logging ) if args . logging else None timeout = param_util . timeout_in_seconds ( args . timeout ) log_interval = param_util . log_interval_in_seconds ( args . log_interval ) return job_model . Resources ( min_cores = args . min_cores , min_ram = args . min_ram , machine_type = args . machine_type , disk_size = args . disk_size , disk_type = args . disk_type , boot_disk_size = args . boot_disk_size , image = args . image , regions = args . regions , zones = args . zones , logging = logging , logging_path = None , service_account = args . service_account , scopes = args . scopes , cpu_platform = args . cpu_platform , network = args . network , subnetwork = args . subnetwork , use_private_address = args . use_private_address , accelerator_type = args . accelerator_type , accelerator_count = args . accelerator_count , nvidia_driver_version = None , timeout = timeout , log_interval = log_interval , ssh = args . ssh , enable_stackdriver_monitoring = args . enable_stackdriver_monitoring , max_retries = args . retries , max_preemptible_attempts = args . preemptible , block_external_network = args . block_external_network )


Allow provider to extract job - specific metadata from command - line args .

def _get_job_metadata(provider, user_id, job_name, script, task_ids,
                      user_project, unique_job_id):
  """Allow provider to extract job-specific metadata from command-line args.

  Args:
    provider: job service provider
    user_id: user submitting the job
    job_name: name for the job
    script: the script to run
    task_ids: a set of the task-ids for all tasks in the job
    user_project: name of the project to be billed for the request
    unique_job_id: generate a unique job id

  Returns:
    A dictionary of job-specific metadata (such as job id, name, etc.)
  """
  create_time = dsub_util.replace_timezone(datetime.datetime.now(),
                                           dateutil.tz.tzlocal())
  user_id = user_id or dsub_util.get_os_user()
  job_metadata = provider.prepare_job_metadata(script.name, job_name, user_id)
  if unique_job_id:
    job_metadata['job-id'] = uuid.uuid4().hex
  else:
    # Build the job-id. We want the job-id to be expressive while also
    # having a low-likelihood of collisions.
    #
    # For expressiveness, we:
    # * use the job name (truncated at 10 characters).
    # * insert the user-id
    # * add a datetime value
    # To have a high likelihood of uniqueness, the datetime value is out to
    # hundredths of a second.
    #
    # The full job-id is:
    #   <job-name>--<user-id>--<timestamp>
    job_metadata['job-id'] = '%s--%s--%s' % (
        job_metadata['job-name'][:10], job_metadata['user-id'],
        create_time.strftime('%y%m%d-%H%M%S-%f')[:16])

  job_metadata['create-time'] = create_time
  job_metadata['script'] = script
  job_metadata['user-project'] = user_project
  if task_ids:
    job_metadata['task-ids'] = dsub_util.compact_interval_string(list(task_ids))

  return job_metadata


Resolve the logging path from job and task properties .

def _resolve_task_logging(job_metadata, job_resources, task_descriptors):
  """Resolve the logging path from job and task properties.

  Args:
    job_metadata: Job metadata, such as job-id, job-name, and user-id.
    job_resources: Resources specified such as ram, cpu, and logging path.
    task_descriptors: Task metadata, parameters, and resources.

  Resolve the logging path, which may have substitution parameters such as
  job-id, task-id, user-id, and job-name.
  """
  if not job_resources.logging:
    return

  for task_descriptor in task_descriptors:
    logging_uri = provider_base.format_logging_uri(
        job_resources.logging.uri, job_metadata, task_descriptor.task_metadata)
    logging_path = job_model.LoggingParam(logging_uri,
                                          job_resources.logging.file_provider)

    if task_descriptor.task_resources:
      task_descriptor.task_resources = task_descriptor.task_resources._replace(
          logging_path=logging_path)
    else:
      task_descriptor.task_resources = job_model.Resources(
          logging_path=logging_path)


Resolve whether or not to use a preemptible machine .

def _resolve_preemptible(job_resources, task_descriptors):
  """Resolve whether or not to use a preemptible machine.

  Args:
    job_resources: Resources specified such as max_preemptible_attempts.
    task_descriptors: Task metadata, parameters, and resources.
  """
  # Determine if the next attempt should be preemptible
  for task_descriptor in task_descriptors:
    # The original attempt is attempt number 1.
    # The first retry is attempt number 2.
    attempt_number = task_descriptor.task_metadata.get('task-attempt', 1)
    max_preemptible_attempts = job_resources.max_preemptible_attempts
    if max_preemptible_attempts:
      use_preemptible = max_preemptible_attempts.should_use_preemptible(
          attempt_number)
    else:
      use_preemptible = job_model.DEFAULT_PREEMPTIBLE
    task_descriptor.task_resources = task_descriptor.task_resources._replace(
        preemptible=use_preemptible)


Resolve task properties ( such as the logging path ) from job properties .

def _resolve_task_resources(job_metadata, job_resources, task_descriptors):
  """Resolve task properties (such as the logging path) from job properties.

  Args:
    job_metadata: Job metadata, such as job-id, job-name, and user-id.
    job_resources: Resources specified such as ram, cpu, and logging path.
    task_descriptors: Task metadata, parameters, and resources.  This function
      exists to be called at the point that all job properties have been
      validated and resolved. It is also called prior to re-trying a task.
  Right now we resolve two properties: 1) the logging path, which may have
    substitution parameters such as job-id, task-id, task-attempt, user-id, and
    job-name. and 2) preemptible, which depends on how many preemptible attempts
    we have done.
  """
  _resolve_task_logging(job_metadata, job_resources, task_descriptors)
  _resolve_preemptible(job_resources, task_descriptors)


Print status info as we wait for those jobs .

def _wait_after(provider, job_ids, poll_interval, stop_on_failure, summary):
  """Print status info as we wait for those jobs.

  Blocks until either all of the listed jobs succeed,
  or one of them fails.

  Args:
    provider: job service provider
    job_ids: a set of job IDs (string) to wait for
    poll_interval: integer seconds to wait between iterations
    stop_on_failure: whether to stop waiting if one of the tasks fails.
    summary: whether to output summary messages

  Returns:
    Empty list if there was no error,
    a list of error messages from the failed tasks otherwise.
  """

  # Each time through the loop, the job_set is re-set to the jobs remaining to
  # check. Jobs are removed from the list when they complete.
  #
  # We exit the loop when:
  # * No jobs remain are running, OR
  # * stop_on_failure is TRUE AND at least one job returned an error

  # remove NO_JOB
  job_ids_to_check = {j for j in job_ids if j != dsub_util.NO_JOB}
  error_messages = []
  while job_ids_to_check and (not error_messages or not stop_on_failure):
    print('Waiting for: %s.' % (', '.join(job_ids_to_check)))

    # Poll until any remaining jobs have completed
    jobs_left = _wait_for_any_job(provider, job_ids_to_check, poll_interval,
                                  summary)

    # Calculate which jobs just completed
    jobs_completed = job_ids_to_check.difference(jobs_left)

    # Get all tasks for the newly completed jobs
    tasks_completed = provider.lookup_job_tasks({'*'}, job_ids=jobs_completed)

    # We don't want to overwhelm the user with output when there are many
    # tasks per job. So we get a single "dominant" task for each of the
    # completed jobs (one that is representative of the job's fate).
    dominant_job_tasks = _dominant_task_for_jobs(tasks_completed)
    if len(dominant_job_tasks) != len(jobs_completed):
      # print info about the jobs we couldn't find
      # (should only occur for "--after" where the job ID is a typo).
      jobs_found = dsub_util.tasks_to_job_ids(dominant_job_tasks)
      jobs_not_found = jobs_completed.difference(jobs_found)
      for j in jobs_not_found:
        error = '%s: not found' % j
        dsub_util.print_error('  %s' % error)
        error_messages += [error]

    # Print the dominant task for the completed jobs
    for t in dominant_job_tasks:
      job_id = t.get_field('job-id')
      status = t.get_field('task-status')
      print('  %s: %s' % (str(job_id), str(status)))
      if status in ['FAILURE', 'CANCELED']:
        error_messages += [provider.get_tasks_completion_messages([t])]

    job_ids_to_check = jobs_left

  return error_messages


Wait for job and retry any tasks that fail .

def _wait_and_retry(provider, job_id, poll_interval, retries, job_descriptor,
                    summary):
  """Wait for job and retry any tasks that fail.

  Stops retrying an individual task when: it succeeds, is canceled, or has been
  retried "retries" times.

  This function exits when there are no tasks running and there are no tasks
  eligible to be retried.

  Args:
    provider: job service provider
    job_id: a single job ID (string) to wait for
    poll_interval: integer seconds to wait between iterations
    retries: number of retries
    job_descriptor: job descriptor used to originally submit job
    summary: whether to output summary messages

  Returns:
    Empty list if there was no error,
    a list containing an error message from a failed task otherwise.
  """

  while True:
    formatted_tasks = []
    tasks = provider.lookup_job_tasks({'*'}, job_ids=[job_id])

    running_tasks = set()
    completed_tasks = set()
    canceled_tasks = set()
    fully_failed_tasks = set()
    task_fail_count = dict()

    # This is an arbitrary task that is either fully failed or canceled (with
    # preference for the former).
    message_task = None

    task_dict = dict()
    for t in tasks:
      task_id = t.get_field('task-id')
      if task_id is not None:
        task_id = int(task_id)

      task_dict[task_id] = t

      status = t.get_field('task-status')
      if status == 'FAILURE':
        # Could compute this from task-attempt as well.
        task_fail_count[task_id] = task_fail_count.get(task_id, 0) + 1
        if task_fail_count[task_id] > retries:
          fully_failed_tasks.add(task_id)
          message_task = t
      elif status == 'CANCELED':
        canceled_tasks.add(task_id)
        if not message_task:
          message_task = t
      elif status == 'SUCCESS':
        completed_tasks.add(task_id)
      elif status == 'RUNNING':
        running_tasks.add(task_id)

      if summary:
        formatted_tasks.append(
            output_formatter.prepare_row(t, full=False, summary=True))

    if summary:
      formatter = output_formatter.TextOutput(full=False)
      formatter.prepare_and_print_table(formatted_tasks, summary)

    retry_tasks = (
        set(task_fail_count).difference(fully_failed_tasks)
        .difference(running_tasks).difference(completed_tasks)
        .difference(canceled_tasks))

    # job completed.
    if not retry_tasks and not running_tasks:
      # If there are any fully failed tasks, return the completion message of an
      # arbitrary one.
      # If not, but there are canceled tasks, return the completion message of
      # an arbitrary one.
      if message_task:
        return [provider.get_tasks_completion_messages([message_task])]

      # Otherwise successful completion.
      return []

    for task_id in retry_tasks:
      identifier = '{}.{}'.format(job_id, task_id) if task_id else job_id
      print('  {} (attempt {}) failed. Retrying.'.format(
          identifier, task_fail_count[task_id]))
      msg = task_dict[task_id].get_field('status-message')
      print('  Failure message: ' + msg)

      _retry_task(provider, job_descriptor, task_id,
                  task_fail_count[task_id] + 1)

    SLEEP_FUNCTION(poll_interval)


Retry task_id ( numeric id ) assigning it task_attempt .

def _retry_task(provider, job_descriptor, task_id, task_attempt):
  """Retry task_id (numeric id) assigning it task_attempt."""
  td_orig = job_descriptor.find_task_descriptor(task_id)

  new_task_descriptors = [
      job_model.TaskDescriptor({
          'task-id': task_id,
          'task-attempt': task_attempt
      }, td_orig.task_params, td_orig.task_resources)
  ]

  # Update the logging path and preemptible field.
  _resolve_task_resources(job_descriptor.job_metadata,
                          job_descriptor.job_resources, new_task_descriptors)

  provider.submit_job(
      job_model.JobDescriptor(
          job_descriptor.job_metadata, job_descriptor.job_params,
          job_descriptor.job_resources, new_task_descriptors), False)


A list with for each job its dominant task .

def _dominant_task_for_jobs(tasks):
  """A list with, for each job, its dominant task.

  The dominant task is the one that exemplifies its job's
  status. It is either:
  - the first (FAILURE or CANCELED) task, or if none
  - the first RUNNING task, or if none
  - the first SUCCESS task.

  Args:
    tasks: a list of tasks to consider

  Returns:
    A list with, for each job, its dominant task.
  """

  per_job = _group_tasks_by_jobid(tasks)

  ret = []
  for job_id in per_job.keys():
    tasks_in_salience_order = sorted(per_job[job_id], key=_importance_of_task)
    ret.append(tasks_in_salience_order[0])
  return ret

def _dominant_task_for_jobs ( tasks ) : per_job = _group_tasks_by_jobid ( tasks ) ret = [ ] for job_id in per_job . keys ( ) : tasks_in_salience_order = sorted ( per_job [ job_id ] , key = _importance_of_task ) ret . append ( tasks_in_salience_order [ 0 ] ) return ret


A defaultdict with for each job a list of its tasks .

def _group_tasks_by_jobid(tasks):
  """A defaultdict with, for each job, a list of its tasks."""
  ret = collections.defaultdict(list)
  for t in tasks:
    ret[t.get_field('job-id')].append(t)
  return ret

def _group_tasks_by_jobid ( tasks ) : ret = collections . defaultdict ( list ) for t in tasks : ret [ t . get_field ( 'job-id' ) ] . append ( t ) return ret


Tuple ( importance end - time ) . Smaller values are more important .

def _importance_of_task(task):
  """Tuple (importance, end-time). Smaller values are more important."""
  # The status of a job is going to be determined by the roll-up of its tasks.
  # A FAILURE or CANCELED task means the job has FAILED.
  # If none, then any RUNNING task, the job is still RUNNING.
  # If none, then the job status is SUCCESS.
  #
  # Thus the dominant task for each job is one that exemplifies its
  # status:
  #
  # 1- The first (FAILURE or CANCELED) task, or if none
  # 2- The first RUNNING task, or if none
  # 3- The first SUCCESS task.
  importance = {'FAILURE': 0, 'CANCELED': 0, 'RUNNING': 1, 'SUCCESS': 2}
  return (importance[task.get_field('task-status')],
          task.get_field(
              'end-time',
              dsub_util.replace_timezone(datetime.datetime.max,
                                         dateutil.tz.tzlocal())))


Waits until any of the listed jobs is not running .

def _wait_for_any_job(provider, job_ids, poll_interval, summary):
  """Waits until any of the listed jobs is not running.

  In particular, if any of the jobs sees one of its tasks fail,
  we count the whole job as failing (but do not terminate the remaining
  tasks ourselves).

  Args:
    provider: job service provider
    job_ids: a list of job IDs (string) to wait for
    poll_interval: integer seconds to wait between iterations
    summary: whether to output summary messages

  Returns:
    A set of the jobIDs with still at least one running task.
  """
  if not job_ids:
    return
  while True:
    formatted_tasks = []
    tasks = provider.lookup_job_tasks({'*'}, job_ids=job_ids)
    running_jobs = set()
    failed_jobs = set()
    for t in tasks:
      status = t.get_field('task-status')
      job_id = t.get_field('job-id')
      if status in ['FAILURE', 'CANCELED']:
        failed_jobs.add(job_id)
      if status == 'RUNNING':
        running_jobs.add(job_id)

      if summary:
        formatted_tasks.append(
            output_formatter.prepare_row(t, full=False, summary=True))

    if summary:
      formatter = output_formatter.TextOutput(full=False)
      formatter.prepare_and_print_table(formatted_tasks, summary)

    remaining_jobs = running_jobs.difference(failed_jobs)
    if failed_jobs or len(remaining_jobs) != len(job_ids):
      return remaining_jobs
    SLEEP_FUNCTION(poll_interval)

def _wait_for_any_job ( provider , job_ids , poll_interval , summary ) : if not job_ids : return while True : formatted_tasks = [ ] tasks = provider . lookup_job_tasks ( { '*' } , job_ids = job_ids ) running_jobs = set ( ) failed_jobs = set ( ) for t in tasks : status = t . get_field ( 'task-status' ) job_id = t . get_field ( 'job-id' ) if status in [ 'FAILURE' , 'CANCELED' ] : failed_jobs . add ( job_id ) if status == 'RUNNING' : running_jobs . add ( job_id ) if summary : formatted_tasks . append ( output_formatter . prepare_row ( t , full = False , summary = True ) ) if summary : formatter = output_formatter . TextOutput ( full = False ) formatter . prepare_and_print_table ( formatted_tasks , summary ) remaining_jobs = running_jobs . difference ( failed_jobs ) if failed_jobs or len ( remaining_jobs ) != len ( job_ids ) : return remaining_jobs SLEEP_FUNCTION ( poll_interval )


Validates that job and task argument names do not overlap .

def _validate_job_and_task_arguments(job_params, task_descriptors):
  """Validates that job and task argument names do not overlap."""

  if not task_descriptors:
    return

  task_params = task_descriptors[0].task_params

  # The use case for specifying a label or env/input/output parameter on
  # the command-line and also including it in the --tasks file is not obvious.
  # Should the command-line override the --tasks file? Why?
  # Until this use is articulated, generate an error on overlapping names.

  # Check labels
  from_jobs = {label.name for label in job_params['labels']}
  from_tasks = {label.name for label in task_params['labels']}

  intersect = from_jobs & from_tasks
  if intersect:
    raise ValueError(
        'Names for labels on the command-line and in the --tasks file must not '
        'be repeated: {}'.format(','.join(intersect)))

  # Check envs, inputs, and outputs, all of which must not overlap each other
  from_jobs = {
      item.name
      for item in job_params['envs'] | job_params['inputs']
      | job_params['outputs']
  }
  from_tasks = {
      item.name
      for item in task_params['envs'] | task_params['inputs']
      | task_params['outputs']
  }

  intersect = from_jobs & from_tasks
  if intersect:
    raise ValueError(
        'Names for envs, inputs, and outputs on the command-line and in the '
        '--tasks file must not be repeated: {}'.format(','.join(intersect)))


def dsub_main(prog, argv):
  """Main entry point for dsub."""
  # Parse args and validate
  args = _parse_arguments(prog, argv)
  # intent:
  # * dsub tightly controls the output to stdout.
  # * wrap the main body such that output goes to stderr.
  # * only emit the job-id to stdout (which can then be used programmatically).
  with dsub_util.replace_print():
    launched_job = run_main(args)
  print(launched_job.get('job-id', ''))
  return launched_job


Execute job / task submission from command - line arguments .

def run_main(args):
  """Execute job/task submission from command-line arguments."""

  if args.command and args.script:
    raise ValueError('Cannot supply both a --command and --script flag')

  provider_base.check_for_unsupported_flag(args)

  # Set up job parameters and job data from a tasks file or flags.
  input_file_param_util = param_util.InputFileParamUtil(
      DEFAULT_INPUT_LOCAL_PATH)
  output_file_param_util = param_util.OutputFileParamUtil(
      DEFAULT_OUTPUT_LOCAL_PATH)
  mount_param_util = param_util.MountParamUtil(DEFAULT_MOUNT_LOCAL_PATH)

  # Get job arguments from the command line
  job_params = param_util.args_to_job_params(
      args.env, args.label, args.input, args.input_recursive, args.output,
      args.output_recursive, args.mount, input_file_param_util,
      output_file_param_util, mount_param_util)
  # If --tasks is on the command-line, then get task-specific data
  if args.tasks:
    task_descriptors = param_util.tasks_file_to_task_descriptors(
        args.tasks, args.retries, input_file_param_util, output_file_param_util)

    # Validate job data + task data
    _validate_job_and_task_arguments(job_params, task_descriptors)
  else:
    # Create the implicit task
    task_metadata = {'task-id': None}
    if args.retries:
      task_metadata['task-attempt'] = 1
    task_descriptors = [
        job_model.TaskDescriptor(task_metadata, {
            'labels': set(),
            'envs': set(),
            'inputs': set(),
            'outputs': set()
        }, job_model.Resources())
    ]

  # Emit a warning if default image is used
  if args.image is None:
    print('***WARNING: No Docker image specified. The default, '
          '`{}` will be used.'.format(DEFAULT_IMAGE))
    print('***WARNING: For reproducible pipelines, specify an image with the '
          '`--image` flag.')
    args.image = DEFAULT_IMAGE

  return run(
      provider_base.get_provider(
          args, resources, credentials_fn=get_credentials),
      _get_job_resources(args),
      job_params,
      task_descriptors,
      name=args.name,
      dry_run=args.dry_run,
      command=args.command,
      script=args.script,
      user=args.user,
      user_project=args.user_project,
      wait=args.wait,
      retries=args.retries,
      max_preemptible_attempts=args.preemptible,
      poll_interval=args.poll_interval,
      after=args.after,
      skip=args.skip,
      project=args.project,
      location=args.location,
      disable_warning=True,
      unique_job_id=args.unique_job_id,
      summary=args.summary)


Actual dsub body post - stdout - redirection .

def run(provider,
        job_resources,
        job_params,
        task_descriptors,
        name=None,
        dry_run=False,
        command=None,
        script=None,
        user=None,
        user_project=None,
        wait=False,
        retries=0,
        max_preemptible_attempts=None,
        poll_interval=10,
        after=None,
        skip=False,
        project=None,
        location=None,
        disable_warning=False,
        unique_job_id=False,
        summary=False):
  """Actual dsub body, post-stdout-redirection."""
  if not dry_run:
    provider_base.emit_provider_message(provider)

  if not disable_warning:
    raise ValueError('Do not use this unstable API component!')

  if command and script:
    raise ValueError('Cannot supply both a command and script value.')

  if command:
    if name:
      command_name = name
    else:
      command_name = _name_for_command(command)

    # Add the shebang line to ensure the command is treated as Bash
    script = job_model.Script(command_name, '#!/usr/bin/env bash\n' + command)
  elif script:
    # Read the script file
    script_file_contents = dsub_util.load_file(script)
    script = job_model.Script(os.path.basename(script), script_file_contents)
  else:
    raise ValueError('One of --command or a script name must be supplied')

  if retries and not wait:
    raise ValueError('Requesting retries requires requesting wait')

  if summary and not wait:
    raise ValueError('Requesting summary requires requesting wait')

  if max_preemptible_attempts:
    max_preemptible_attempts.validate(retries)

  # The contract with providers and downstream code is that the job_params
  # and task_params contain 'labels', 'envs', 'inputs', and 'outputs'.
  job_model.ensure_job_params_are_complete(job_params)
  job_model.ensure_task_params_are_complete(task_descriptors)

  task_ids = {
      task_descriptor.task_metadata.get('task-id')
      for task_descriptor in task_descriptors
      if task_descriptor.task_metadata.get('task-id') is not None
  }

  # Job and task parameters from the user have been validated.
  # We can now compute some job and task properties, including:
  #  job_metadata such as the job-id, create-time, user-id, etc.
  #  task_resources such as the logging_path (which may include job-id, task-id)
  job_metadata = _get_job_metadata(provider, user, name, script, task_ids,
                                   user_project, unique_job_id)
  _resolve_task_resources(job_metadata, job_resources, task_descriptors)

  # Job and task properties are now all resolved. Begin execution!
  if not dry_run:
    print('Job properties:')
    print('  job-id: %s' % job_metadata['job-id'])
    print('  job-name: %s' % job_metadata['job-name'])
    print('  user-id: %s' % job_metadata['user-id'])

  # Wait for predecessor jobs (if any)
  if after:
    if dry_run:
      print('(Pretend) waiting for: %s.' % after)
    else:
      print('Waiting for predecessor jobs to complete...')
      error_messages = _wait_after(provider, after, poll_interval, True,
                                   summary)
      if error_messages:
        for msg in error_messages:
          dsub_util.print_error(msg)
        raise dsub_errors.PredecessorJobFailureError(
            'One or more predecessor jobs completed but did not succeed.',
            error_messages, None)

  # Launch all the job tasks!
  job_descriptor = job_model.JobDescriptor(job_metadata, job_params,
                                           job_resources, task_descriptors)
  launched_job = provider.submit_job(job_descriptor, skip)

  if not dry_run:
    if launched_job['job-id'] == dsub_util.NO_JOB:
      print('Job output already present, skipping new job submission.')
      return {'job-id': dsub_util.NO_JOB}
    print('Launched job-id: %s' % launched_job['job-id'])
    if launched_job.get('task-id'):
      print('%s task(s)' % len(launched_job['task-id']))
    print('To check the status, run:')
    print("  dstat %s --jobs '%s' --users '%s' --status '*'" %
          (provider_base.get_dstat_provider_args(provider, project, location),
           launched_job['job-id'], launched_job['user-id']))
    print('To cancel the job, run:')
    print("  ddel %s --jobs '%s' --users '%s'" %
          (provider_base.get_ddel_provider_args(provider, project, location),
           launched_job['job-id'], launched_job['user-id']))

  # Poll for job completion
  if wait:
    print('Waiting for job to complete...')

    if retries:
      print('Monitoring for failed tasks to retry...')
      print(
          '*** This dsub process must continue running to retry failed tasks.')
      error_messages = _wait_and_retry(provider, job_metadata['job-id'],
                                       poll_interval, retries, job_descriptor,
                                       summary)
    else:
      error_messages = _wait_after(provider, [job_metadata['job-id']],
                                   poll_interval, False, summary)
    if error_messages:
      for msg in error_messages:
        dsub_util.print_error(msg)
      raise dsub_errors.JobExecutionError(
          'One or more jobs finished with status FAILURE or CANCELED'
          ' during wait.', error_messages, launched_job)

  return launched_job


r Craft a simple command name from the command .

def _name_for_command(command):
  r"""Craft a simple command name from the command.

  The best command strings for this are going to be those where a simple
  command was given; we will use the command to derive the name.

  We won't always be able to figure something out and the caller should just
  specify a "--name" on the command-line.

  For example, commands like "export VAR=val\necho ${VAR}", this function would
  return "export".

  If the command starts space or a comment, then we'll skip to the first code
  we can find.

  If we find nothing, just return "command".

  >>> _name_for_command('samtools index "${BAM}"')
  'samtools'
  >>> _name_for_command('/usr/bin/sort "${INFILE}" > "${OUTFILE}"')
  'sort'
  >>> _name_for_command('# This should be ignored')
  'command'
  >>> _name_for_command('\\\n\\\n# Bad continuations, but ignore.\necho hello.')
  'echo'
  >>> _name_for_command('(uname -a && pwd) # Command begins with non-letter.')
  'uname'
  >>> _name_for_command('my-program.sh # Command with hyphens.')
  'my-program.sh'
  >>> _name_for_command('/home/user/bin/-my-sort # Path with hyphen.')
  'my-sort'

  Arguments:
    command: the user-provided command
  Returns:
    a proposed name for the task.
  """

  lines = command.splitlines()
  for line in lines:
    line = line.strip()
    if line and not line.startswith('#') and line != '\\':
      # Tokenize on whitespace [ \t\n\r\f\v]
      names = re.split(r'\s', line)
      for name in names:
        # Make sure the first character is a letter, number, or underscore
        # Get basename so something like "/usr/bin/sort" becomes just "sort"
        name = re.sub(r'^[^a-zA-Z0-9_]*', '', os.path.basename(name))
        if name:
          return name

  return 'command'


def get_credentials(args):
  """Returns credentials for API requests."""

  # Across dsub, dstat, ddel, defer to the provider for credentials handling
  return provider_base.credentials_from_args(args)


def _parse_arguments():
  """Parses command line arguments.

  Returns:
    A Namespace of parsed arguments.
  """
  # Handle version flag and exit if it was passed.
  param_util.handle_version_flag()

  parser = provider_base.create_parser(sys.argv[0])

  parser.add_argument(
      '--version', '-v', default=False, help='Print the dsub version and exit.')

  parser.add_argument(
      '--jobs',
      '-j',
      required=True,
      nargs='*',
      help='List of job-ids to delete. Use "*" to delete all running jobs.')
  parser.add_argument(
      '--tasks',
      '-t',
      nargs='*',
      help='List of tasks in an array job to delete.')
  parser.add_argument(
      '--users',
      '-u',
      nargs='*',
      default=[],
      help="""Deletes only those jobs which were submitted by the list of users.
          Use "*" to delete jobs of any user.""")
  parser.add_argument(
      '--age',
      help="""Deletes only those jobs newer than the specified age. Ages can be
          listed using a number followed by a unit. Supported units are
          s (seconds), m (minutes), h (hours), d (days), w (weeks).
          For example: '7d' (7 days). Bare numbers are treated as UTC.""")
  parser.add_argument(
      '--label',
      nargs='*',
      action=param_util.ListParamAction,
      default=[],
      help='User labels to match. Tasks returned must match all labels.',
      metavar='KEY=VALUE')

  # Shared between the "google-cls-v2" and "google-v2" providers
  google_common = parser.add_argument_group(
      title='google-common',
      description="""Options common to the "google", "google-cls-v2", and
        "google-v2" providers""")
  google_common.add_argument(
      '--project',
      help='Cloud project ID in which to find and delete the job(s)')

  google_cls_v2 = parser.add_argument_group(
      title='"google-cls-v2" provider options',
      description='See also the "google-common" options listed')
  google_cls_v2.add_argument(
      '--location',
      default=job_model.DEFAULT_LOCATION,
      help="""Specifies the Google Cloud region to which the dsub job was
        submitted. (default: {})""".format(job_model.DEFAULT_LOCATION))

  return provider_base.parse_args(
      parser, {
          'google-cls-v2': ['project'],
          'google-v2': ['project'],
          'test-fails': [],
          'local': [],
      }, sys.argv[1:])


Print the filters used to delete tasks . Use raw flags as arguments .

def _emit_search_criteria(user_ids, job_ids, task_ids, labels):
  """Print the filters used to delete tasks. Use raw flags as arguments."""
  print('Delete running jobs:')
  print('  user:')
  print('    %s\n' % user_ids)
  print('  job-id:')
  print('    %s\n' % job_ids)
  if task_ids:
    print('  task-id:')
    print('    %s\n' % task_ids)
  # Labels are in a LabelParam namedtuple and must be reformated for printing.
  if labels:
    print('  labels:')
    print('    %s\n' % repr(labels))


def ddel_tasks(provider,
               user_ids=None,
               job_ids=None,
               task_ids=None,
               labels=None,
               create_time_min=None,
               create_time_max=None):
  """Kill jobs or job tasks.

  This function separates ddel logic from flag parsing and user output. Users
  of ddel who intend to access the data programmatically should use this.

  Args:
    provider: an instantiated dsub provider.
    user_ids: a set of user ids who "own" the job(s) to delete.
    job_ids: a set of job ids to delete.
    task_ids: a set of task ids to delete.
    labels: a set of LabelParam, each must match the job(s) to be cancelled.
    create_time_min: a timezone-aware datetime value for the earliest create
                     time of a task, inclusive.
    create_time_max: a timezone-aware datetime value for the most recent create
                     time of a task, inclusive.

  Returns:
    list of job ids which were deleted.
  """
  # Delete the requested jobs
  deleted_tasks, error_messages = provider.delete_jobs(
      user_ids, job_ids, task_ids, labels, create_time_min, create_time_max)

  # Emit any errors canceling jobs
  for msg in error_messages:
    print(msg)

  return deleted_tasks

def
ddel_tasks
(
provider
,
user_ids
=
,
job_ids
=
,
task_ids
=
,
labels
=
,
create_time_min
=
,
create_time_max
=
)
:
# Delete the requested jobs
deleted_tasks
,
error_messages
=
provider
.
delete_jobs
(
user_ids
,
job_ids
,
task_ids
,
labels
,
create_time_min
,
create_time_max
)
# Emit any errors canceling jobs
for
msg
in
error_messages
:
print
(
msg
)
return
deleted_tasks


Build a VirtualMachine object for a Pipeline request .

def build_machine(network=None,
                  machine_type=None,
                  preemptible=None,
                  service_account=None,
                  boot_disk_size_gb=None,
                  disks=None,
                  accelerators=None,
                  labels=None,
                  cpu_platform=None,
                  nvidia_driver_version=None,
                  enable_stackdriver_monitoring=None):
  """Build a VirtualMachine object for a Pipeline request.

  Args:
    network (dict): Network details for the pipeline to run in.
    machine_type (str): GCE Machine Type string for the pipeline.
    preemptible (bool): Use a preemptible VM for the job.
    service_account (dict): Service account configuration for the VM.
    boot_disk_size_gb (int): Boot disk size in GB.
    disks (list[dict]): List of disks to mount.
    accelerators (list[dict]): List of accelerators to attach to the VM.
    labels (dict[string, string]): Labels for the VM.
    cpu_platform (str): The CPU platform to request.
    nvidia_driver_version (str): The NVIDIA driver version to use when attaching
      an NVIDIA GPU accelerator.
    enable_stackdriver_monitoring (bool): Enable stackdriver monitoring
      on the VM.

  Returns:
    An object representing a VirtualMachine.
  """
  return {
      'network': network,
      'machineType': machine_type,
      'preemptible': preemptible,
      'serviceAccount': service_account,
      'bootDiskSizeGb': boot_disk_size_gb,
      'disks': disks,
      'accelerators': accelerators,
      'labels': labels,
      'cpuPlatform': cpu_platform,
      'nvidiaDriverVersion': nvidia_driver_version,
      'enableStackdriverMonitoring': enable_stackdriver_monitoring,
  }

def build_machine ( network = None , machine_type = None , preemptible = None , service_account = None , boot_disk_size_gb = None , disks = None , accelerators = None , labels = None , cpu_platform = None , nvidia_driver_version = None , enable_stackdriver_monitoring = None ) : return { 'network' : network , 'machineType' : machine_type , 'preemptible' : preemptible , 'serviceAccount' : service_account , 'bootDiskSizeGb' : boot_disk_size_gb , 'disks' : disks , 'accelerators' : accelerators , 'labels' : labels , 'cpuPlatform' : cpu_platform , 'nvidiaDriverVersion' : nvidia_driver_version , 'enableStackdriverMonitoring' : enable_stackdriver_monitoring , }


Build a Resources object for a Pipeline request .

def build_resources(project=None,
                    regions=None,
                    zones=None,
                    virtual_machine=None):
  """Build a Resources object for a Pipeline request.

  Args:
    project (str): Cloud project for the Pipeline to run in.
    regions (List[str]): List of regions for the pipeline to run in.
    zones (List[str]): List of zones for the pipeline to run in.
    virtual_machine(str): Virtual machine type string.

  Returns:
    An object representing a Resource.
  """

  resources = {
      'regions': regions,
      'zones': zones,
      'virtualMachine': virtual_machine,
  }

  if _API_VERSION == google_v2_versions.V2ALPHA1:
    resources['projectId'] = project

  return resources

def build_resources ( project = None , regions = None , zones = None , virtual_machine = None ) : resources = { 'regions' : regions , 'zones' : zones , 'virtualMachine' : virtual_machine , } if _API_VERSION == google_v2_versions . V2ALPHA1 : resources [ 'projectId' ] = project return resources


Build a Mount object for a Pipeline request .

def build_mount(disk, path, read_only):
  """Build a Mount object for a Pipeline request.

  Args:
    disk (str): Name of disk to mount, as specified in the resources section.
    path (str): Path to mount the disk at inside the container.
    read_only (boolean): If true,  disk is mounted read only in the container.

  Returns:
    An object representing a Mount.
  """

  return {
      'disk': disk,
      'path': path,
      'readOnly': read_only,
  }

def build_mount ( disk , path , read_only ) : return { 'disk' : disk , 'path' : path , 'readOnly' : read_only , }


Build an Action object for a Pipeline request .

def build_action(name=None,
                 image_uri=None,
                 commands=None,
                 entrypoint=None,
                 environment=None,
                 pid_namespace=None,
                 port_mappings=None,
                 mounts=None,
                 labels=None,
                 always_run=None,
                 enable_fuse=None,
                 run_in_background=None,
                 block_external_network=None):
  """Build an Action object for a Pipeline request.

  Args:
    name (str): An optional name for the container.
    image_uri (str): The URI to pull the container image from.
    commands (List[str]): commands and arguments to run inside the container.
    entrypoint (str): overrides the ENTRYPOINT specified in the container.
    environment (dict[str,str]): The environment to pass into the container.
    pid_namespace (str): The PID namespace to run the action inside.
    port_mappings (dict[int, int]): A map of container to host port mappings for
      this container.
    mounts (List): A list of mounts to make available to the action.
    labels (dict[str]): Labels to associate with the action.
    always_run (bool): Action must run even if pipeline has already failed.
    enable_fuse (bool): Enable access to the FUSE device for this action.
    run_in_background (bool): Allow the action to run in the background.
    block_external_network (bool): Prevents the container from accessing the
      external network.

  Returns:
    An object representing an Action resource.
  """

  action = {
      'imageUri': image_uri,
      'commands': commands,
      'entrypoint': entrypoint,
      'environment': environment,
      'pidNamespace': pid_namespace,
      'portMappings': port_mappings,
      'mounts': mounts,
      'labels': labels,
  }

  if _API_VERSION == google_v2_versions.V2ALPHA1:
    action['name'] = name

    # In v2alpha1, the flags are passed as a list of strings
    flags = []
    if always_run:
      flags.append('ALWAYS_RUN')
    if enable_fuse:
      flags.append('ENABLE_FUSE')
    if run_in_background:
      flags.append('RUN_IN_BACKGROUND')
    if block_external_network:
      flags.append('BLOCK_EXTERNAL_NETWORK')

    if flags:
      action['flags'] = flags

  elif _API_VERSION == google_v2_versions.V2BETA:
    action['containerName'] = name

    # In v2beta, the flags are direct members of the action
    action['alwaysRun'] = always_run
    action['enableFuse'] = enable_fuse
    action['runInBackground'] = run_in_background
    action['blockExternalNetwork'] = block_external_network

  else:
    assert False, 'Unexpected version: {}'.format(_API_VERSION)

  return action


Build an Pipeline argument for a Pipeline request .

def build_pipeline(actions, resources, environment, timeout):
  """Build an Pipeline argument for a Pipeline request.

  Args:
    actions (List): A list of actions to execute.
    resources (dict): An object indicating pipeline resources.
    environment (dict[str,str]): The environment to pass into the container.
    timeout (str): A duration in seconds with up to nine fractional digits,
      terminated by 's'.

  Returns:
    An object representing a Pipelines Resource.
  """

  return {
      'actions': actions,
      'resources': resources,
      'environment': environment,
      'timeout': timeout,
  }

def build_pipeline ( actions , resources , environment , timeout ) : return { 'actions' : actions , 'resources' : resources , 'environment' : environment , 'timeout' : timeout , }


Returns a dictionary of metadata fields for the job .

def prepare_job_metadata(self, script, job_name, user_id):
    """Returns a dictionary of metadata fields for the job.

    Call this before calling submit_job.

    The job metadata is a dictionary of values relevant to the job as a whole,
    such as:

    * job-name
    * job-id
    * user-id
    * script
    * dsub-version

    The creation of the job metadata is done by the provider, as the rules
    around these items are provider-specific. For example, one job provider
    might have an 8 character restriction on job names, while another may be
    restricted to lower-case characters.

    The provider *must* set the job-name, job-id, and user-id. They may be used
    by the dsub infrastructure for display and direct return to callers.

    The provider is free to set any other key that it will need in
    submit_job().

    The "job_name" passed in need not be the same 'job-name' value set in the
    returned job metadata, as provider-specific character restrictions may need
    to be applied.

    Args:
      script: path to the job script
      job_name: user-supplied job name, if any
      user_id: user whose jobs to look for
    """
    raise NotImplementedError()

def prepare_job_metadata ( self , script , job_name , user_id ) : raise NotImplementedError ( )


def submit_job(self, job_descriptor, skip_if_output_present):
    """Submit the job to be executed.

    Args:
      job_descriptor (job_model.JobDescriptor): parameters needed to launch all
      job tasks
      skip_if_output_present: (boolean) if true, skip tasks whose output
        is present (see --skip flag for more explanation).

    Returns:
      A dictionary containing the 'user-id', 'job-id', and 'task-id' list.
      For jobs that are not task array jobs, the task-id list should be empty.
      If all tasks were skipped, then the job-id is dsub_lib.NO_JOB.

    Raises:
      ValueError: submit_job may validate any of the parameters and raise
        a value error if any parameter (or specific combination of parameters)
        is not supported by the provider.
    """
    raise NotImplementedError()

def submit_job ( self , job_descriptor , skip_if_output_present ) : raise NotImplementedError ( )


Kills the operations associated with the specified job or job . task .

def delete_jobs(self,
                  user_ids,
                  job_ids,
                  task_ids,
                  labels,
                  create_time_min=None,
                  create_time_max=None):
    """Kills the operations associated with the specified job or job.task.

    Some providers may provide only a "cancel" operation, which terminates the
    task but does not truly "delete" it from the "task list".

    Args:
      user_ids: a set of user ids who "own" the job(s) to delete.
      job_ids: a set of job ids to delete.
      task_ids: a set of task ids to delete.
      labels: a set of LabelParam, each must match the job(s) to be cancelled.
      create_time_min: a timezone-aware datetime value for the earliest create
                       time of a task, inclusive.
      create_time_max: a timezone-aware datetime value for the most recent
                       create time of a task, inclusive.

    Returns:
      (list of tasks canceled,
       for each task that couldn't be canceled, the error message).

      Only tasks that were running are included in the return value.
    """
    raise NotImplementedError()

def delete_jobs ( self , user_ids , job_ids , task_ids , labels , create_time_min = None , create_time_max = None ) : raise NotImplementedError ( )


Return a list of tasks based on the search criteria .

def lookup_job_tasks(self,
                       statuses,
                       user_ids=None,
                       job_ids=None,
                       job_names=None,
                       task_ids=None,
                       task_attempts=None,
                       labels=None,
                       create_time_min=None,
                       create_time_max=None,
                       max_tasks=0,
                       page_size=0):
    """Return a list of tasks based on the search criteria.

    If any of the filters are empty or {'*'}, then no filtering is performed on
    that field. Filtering by both a job id list and job name list is
    unsupported.

    Args:
      statuses: {'*'}, or a set of job status strings to return. Valid
        status strings are 'RUNNING', 'SUCCESS', 'FAILURE', or 'CANCELED'.
      user_ids: a set of ids for the user(s) who launched the job.
      job_ids: a set of job ids to return.
      job_names: a set of job names to return.
      task_ids: a set of specific tasks within the specified job(s) to return.
      task_attempts: a set of specific task attempts within the specified
        tasks(s) to return.
      labels: a list of LabelParam, each must match the job(s) returned.
      create_time_min: a timezone-aware datetime value for the earliest create
                       time of a task, inclusive.
      create_time_max: a timezone-aware datetime value for the most recent
                       create time of a task, inclusive.
      max_tasks: the maximum number of job tasks to return or 0 for no limit.
      page_size: the page size to use for each query to the backend. May be
                 ignored by provider implementations.

    Returns:
      A list of Task objects.

    Raises:
      ValueError: if both a job id list and a job name list are provided
    """
    raise NotImplementedError()

def lookup_job_tasks ( self , statuses , user_ids = None , job_ids = None , job_names = None , task_ids = None , task_attempts = None , labels = None , create_time_min = None , create_time_max = None , max_tasks = 0 , page_size = 0 ) : raise NotImplementedError ( )


List of the error message of each given task .

def get_tasks_completion_messages(self, tasks):
    """List of the error message of each given task."""
    raise NotImplementedError()

def get_tasks_completion_messages ( self , tasks ) : raise NotImplementedError ( )


Return a provider - specific representation of task data .

def raw_task_data(self):
    """Return a provider-specific representation of task data.

    Returns:
      dictionary of task data from the provider.
    """
    raise NotImplementedError()

def raw_task_data ( self ) : raise NotImplementedError ( )


Return a metadata - field for the task .

def get_field(self, field, default=None):
    """Return a metadata-field for the task.

    Not all fields need to be supported by all providers.
    Field identifiers include:

    'job-name', 'job-id', 'task-id', 'task-attempt', 'user-id', 'task-status',
    'error-message', 'create-time', 'start-time', 'end-time', 'inputs',
    'outputs', 'events'

    The following are required by dstat:
    - status: The task status ('RUNNING', 'CANCELED', 'FAILED', 'SUCCESS')
    - status-message: A short message that is displayed in the default
                      dstat output. This should be as concise and useful as
                      possible ("Pending", "Running", "Error: invalid...")
    - status-detail: A longer status message that is displayed in full dstat
                     output. Ideally, this is the last few lines of a log which
                     gives the user enough information that they do not need
                     to go to the log files.

    dstat's short output shows status-message.
    dstat's full output shows status and status-detail

    Args:
      field: one of the choices listed above.
      default: the value to return if no value if found.
    """
    raise NotImplementedError()

def get_field ( self , field , default = None ) : raise NotImplementedError ( )


def set_operations(self, ops):
    """Set the state of the fictional world.

    Args:
     ops: a list of dict, each representing an operation.

    Operations can have the following fields:
       - status: tuple (string,date)
       - user: string
       - job-id: string
       - job-name: string
       - task-id: string
       - task-attempt: integer
       - labels: list<dict>
       - status-message: string
       - error-messages : list of string
    """
    self._operations = [StubTask(o) for o in ops]

def set_operations ( self , ops ) : self . _operations = [ StubTask ( o ) for o in ops ]


Return a list of operations . See base . py for additional detail .

def lookup_job_tasks(self,
                       statuses,
                       user_ids=None,
                       job_ids=None,
                       job_names=None,
                       task_ids=None,
                       task_attempts=None,
                       labels=None,
                       create_time_min=None,
                       create_time_max=None,
                       max_tasks=0):
    """Return a list of operations. See base.py for additional detail."""
    statuses = None if statuses == {'*'} else statuses
    user_ids = None if user_ids == {'*'} else user_ids
    job_ids = None if job_ids == {'*'} else job_ids
    job_names = None if job_names == {'*'} else job_names
    task_ids = None if task_ids == {'*'} else task_ids
    task_attempts = None if task_attempts == {'*'} else task_attempts

    if labels or create_time_min or create_time_max:
      raise NotImplementedError(
          'Lookup by labels and create_time not yet supported by stub.')

    # pylint: disable=g-complex-comprehension
    operations = [
        x for x in self._operations
        if ((not statuses or x.get_field('status', (None, None))[0] in statuses
            ) and (not user_ids or x.get_field('user', None) in user_ids) and
            (not job_ids or x.get_field('job-id', None) in job_ids) and
            (not job_names or x.get_field('job-name', None) in job_names) and
            (not task_ids or x.get_field('task-id', None) in task_ids) and
            (not task_attempts or
             x.get_field('task-attempt', None) in task_attempts))
    ]
    # pylint: enable=g-complex-comprehension

    if max_tasks > 0:
      operations = operations[:max_tasks]
    return operations


Returns a list of zones based on any wildcard input .

def get_zones(input_list):
  """Returns a list of zones based on any wildcard input.

  This function is intended to provide an easy method for producing a list
  of desired zones for a pipeline to run in.

  The Pipelines API default zone list is "any zone". The problem with
  "any zone" is that it can lead to incurring Cloud Storage egress charges
  if the GCE zone selected is in a different region than the GCS bucket.
  See https://cloud.google.com/storage/pricing#network-egress.

  A user with a multi-region US bucket would want to pipelines to run in
  a "us-*" zone.
  A user with a regional bucket in US would want to restrict pipelines to
  run in a zone in that region.

  Rarely does the specific zone matter for a pipeline.

  This function allows for a simple short-hand such as:
     [ "us-*" ]
     [ "us-central1-*" ]
  These examples will expand out to the full list of US and us-central1 zones
  respectively.

  Args:
    input_list: list of zone names/patterns

  Returns:
    A list of zones, with any wildcard zone specifications expanded.
  """
  if not input_list:
    return []

  output_list = []

  for zone in input_list:
    if zone.endswith('*'):
      prefix = zone[:-1]
      output_list.extend([z for z in _ZONES if z.startswith(prefix)])
    else:
      output_list.append(zone)

  return output_list

def get_zones ( input_list ) : if not input_list : return [ ] output_list = [ ] for zone in input_list : if zone . endswith ( '*' ) : prefix = zone [ : - 1 ] output_list . extend ( [ z for z in _ZONES if z . startswith ( prefix ) ] ) else : output_list . append ( zone ) return output_list


Build a set () of standard job and task labels .

def build_pipeline_labels(job_metadata, task_metadata, task_id_pattern=None):
  """Build a set() of standard job and task labels.

  Args:
    job_metadata: Job metadata, such as job-id, job-name, and user-id.
    task_metadata: Task metadata, such as the task-id.
    task_id_pattern: A pattern for the task-id value, such as "task-%d"; the
      original google label values could not be strictly numeric, so "task-"
      was prepended.

  Returns:
    A set of standard dsub Label() objects to attach to a pipeline.
  """
  labels = {
      Label(name, job_metadata[name])
      for name in ['job-name', 'job-id', 'user-id', 'dsub-version']
  }

  task_id = task_metadata.get('task-id')
  if task_id is not None:  # Check for None (as 0 is conceivably valid)
    if task_id_pattern:
      task_id = task_id_pattern % task_id
    labels.add(Label('task-id', str(task_id)))

  task_attempt = task_metadata.get('task-attempt')
  if task_attempt is not None:
    labels.add(Label('task-attempt', str(task_attempt)))

  return labels


Converts the label strings to contain label - appropriate characters .

def prepare_query_label_value(labels):
  """Converts the label strings to contain label-appropriate characters.

  Args:
    labels: A set of strings to be converted.

  Returns:
    A list of converted strings.
  """
  if not labels:
    return None
  return [job_model.convert_to_label_chars(label) for label in labels]

def prepare_query_label_value ( labels ) : if not labels : return None return [ job_model . convert_to_label_chars ( label ) for label in labels ]


Converts a datestamp from RFC3339 UTC to a datetime .

def parse_rfc3339_utc_string(rfc3339_utc_string):
  """Converts a datestamp from RFC3339 UTC to a datetime.

  Args:
    rfc3339_utc_string: a datetime string in RFC3339 UTC "Zulu" format

  Returns:
    A datetime.
  """

  # The timestamp from the Google Operations are all in RFC3339 format, but
  # they are sometimes formatted to millisconds, microseconds, sometimes
  # nanoseconds, and sometimes only seconds:
  # * 2016-11-14T23:05:56Z
  # * 2016-11-14T23:05:56.010Z
  # * 2016-11-14T23:05:56.010429Z
  # * 2016-11-14T23:05:56.010429380Z
  m = re.match(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}).?(\d*)Z',
               rfc3339_utc_string)

  # It would be unexpected to get a different date format back from Google.
  # If we raise an exception here, we can break people completely.
  # Instead, let's just return None and people can report that some dates
  # are not showing up.
  # We might reconsider this approach in the future; it was originally
  # established when dates were only used for display.
  if not m:
    return None

  groups = m.groups()
  if len(groups[6]) not in (0, 3, 6, 9):
    return None

  # Create a UTC datestamp from parsed components
  # 1- Turn components 0-5 from strings to integers
  # 2- If the last component does not exist, set it to 0.
  #    If it does exist, make sure to interpret it as milliseconds.
  g = [int(val) for val in groups[:6]]

  fraction = groups[6]
  if not fraction:
    micros = 0
  elif len(fraction) == 3:
    micros = int(fraction) * 1000
  elif len(fraction) == 6:
    micros = int(fraction)
  elif len(fraction) == 9:
    # When nanoseconds are provided, we round
    micros = int(round(int(fraction) // 1000))
  else:
    assert False, 'Fraction length not 0, 6, or 9: {}'.format(len(fraction))

  try:
    return datetime.datetime(
        g[0], g[1], g[2], g[3], g[4], g[5], micros, tzinfo=pytz.utc)
  except ValueError as e:
    assert False, 'Could not parse RFC3339 datestring: {} exception: {}'.format(
        rfc3339_utc_string, e)


Returns the job - id or job - id . task - id for the operation .

def get_operation_full_job_id(op):
  """Returns the job-id or job-id.task-id for the operation."""
  job_id = op.get_field('job-id')
  task_id = op.get_field('task-id')
  if task_id:
    return '%s.%s' % (job_id, task_id)
  else:
    return job_id

def get_operation_full_job_id ( op ) : job_id = op . get_field ( 'job-id' ) task_id = op . get_field ( 'task-id' ) if task_id : return '%s.%s' % ( job_id , task_id ) else : return job_id


def _cancel_batch(batch_fn, cancel_fn, ops):
  """Cancel a batch of operations.

  Args:
    batch_fn: API-specific batch function.
    cancel_fn: API-specific cancel function.
    ops: A list of operations to cancel.

  Returns:
    A list of operations canceled and a list of error messages.
  """

  # We define an inline callback which will populate a list of
  # successfully canceled operations as well as a list of operations
  # which were not successfully canceled.

  canceled = []
  failed = []

  def handle_cancel_response(request_id, response, exception):
    """Callback for the cancel response."""
    del response  # unused

    if exception:
      # We don't generally expect any failures here, except possibly trying
      # to cancel an operation that is already canceled or finished.
      #
      # If the operation is already finished, provide a clearer message than
      # "error 400: Bad Request".

      msg = 'error %s: %s' % (exception.resp.status, exception.resp.reason)
      if exception.resp.status == FAILED_PRECONDITION_CODE:
        detail = json.loads(exception.content)
        status = detail.get('error', {}).get('status')
        if status == FAILED_PRECONDITION_STATUS:
          msg = 'Not running'

      failed.append({'name': request_id, 'msg': msg})
    else:
      canceled.append({'name': request_id})

    return

  # Set up the batch object
  batch = batch_fn(callback=handle_cancel_response)

  # The callback gets a "request_id" which is the operation name.
  # Build a dict such that after the callback, we can lookup the operation
  # objects by name
  ops_by_name = {}
  for op in ops:
    op_name = op.get_field('internal-id')
    ops_by_name[op_name] = op
    batch.add(cancel_fn(name=op_name, body={}), request_id=op_name)

  # Cancel the operations
  batch.execute()

  # Iterate through the canceled and failed lists to build our return lists
  canceled_ops = [ops_by_name[op['name']] for op in canceled]
  error_messages = []
  for fail in failed:
    op = ops_by_name[fail['name']]
    error_messages.append("Error canceling '%s': %s" %
                          (get_operation_full_job_id(op), fail['msg']))

  return canceled_ops, error_messages


def cancel(batch_fn, cancel_fn, ops):
  """Cancel operations.

  Args:
    batch_fn: API-specific batch function.
    cancel_fn: API-specific cancel function.
    ops: A list of operations to cancel.

  Returns:
    A list of operations canceled and a list of error messages.
  """

  # Canceling many operations one-by-one can be slow.
  # The Pipelines API doesn't directly support a list of operations to cancel,
  # but the requests can be performed in batch.

  canceled_ops = []
  error_messages = []

  max_batch = 256
  total_ops = len(ops)
  for first_op in range(0, total_ops, max_batch):
    batch_canceled, batch_messages = _cancel_batch(
        batch_fn, cancel_fn, ops[first_op:first_op + max_batch])
    canceled_ops.extend(batch_canceled)
    error_messages.extend(batch_messages)

  return canceled_ops, error_messages


def setup_service(api_name, api_version, credentials=None):
  """Configures genomics API client.

  Args:
    api_name: Name of the Google API (for example: "genomics")
    api_version: Version of the API (for example: "v2alpha1")
    credentials: Credentials to be used for the gcloud API calls.

  Returns:
    A configured Google Genomics API client with appropriate credentials.
  """
  # dsub is not a server application, so it is ok to filter this warning.
  warnings.filterwarnings(
      'ignore', 'Your application has authenticated using end user credentials')
  if not credentials:
    credentials, _ = google.auth.default()
  # Set cache_discovery to False because we use google-auth
  # See https://github.com/googleapis/google-api-python-client/issues/299
  return googleapiclient.discovery.build(
      api_name, api_version, cache_discovery=False, credentials=credentials)


def execute(self, api):
    """Executes operation.

    Args:
      api: The base API object

    Returns:
       A response body object
    """
    return api.execute()

def execute ( self , api ) : return api . execute ( )


Create a task name from a job - id task - id and task - attempt .

def _format_task_name(job_id, task_id, task_attempt):
  """Create a task name from a job-id, task-id, and task-attempt.

  Task names are used internally by dsub as well as by the docker task runner.
  The name is formatted as "<job-id>.<task-id>[.task-attempt]". Task names
  follow formatting conventions allowing them to be safely used as a docker
  name.

  Args:
    job_id: (str) the job ID.
    task_id: (str) the task ID.
    task_attempt: (int) the task attempt.

  Returns:
    a task name string.
  """
  docker_name = '%s.%s' % (job_id, 'task' if task_id is None else task_id)

  if task_attempt is not None:
    docker_name += '.' + str(task_attempt)

  # Docker container names must match: [a-zA-Z0-9][a-zA-Z0-9_.-]
  # So 1) prefix it with "dsub-" and 2) change all invalid characters to "-".
  return 'dsub-{}'.format(_convert_suffix_to_docker_chars(docker_name))


Rewrite string so that all characters are valid in a docker name suffix .

def _convert_suffix_to_docker_chars(suffix):
  """Rewrite string so that all characters are valid in a docker name suffix."""
  # Docker container names must match: [a-zA-Z0-9][a-zA-Z0-9_.-]
  accepted_characters = string.ascii_letters + string.digits + '_.-'

  def label_char_transform(char):
    if char in accepted_characters:
      return char
    return '-'

  return ''.join(label_char_transform(c) for c in suffix)


Return a tuple for sorting most recent first .

def _task_sort_function(task):
  """Return a tuple for sorting 'most recent first'."""
  return (task.get_field('create-time'), int(task.get_field('task-id', 0)),
          int(task.get_field('task-attempt', 0)))

def _task_sort_function ( task ) : return ( task . get_field ( 'create-time' ) , int ( task . get_field ( 'task-id' , 0 ) ) , int ( task . get_field ( 'task-attempt' , 0 ) ) )


def _sort_tasks(tasks):
  """Sort tasks by 'most recent first'."""

  # The local provider can launch tasks quickly enough that they end up with
  # the same timestamp.
  #
  # lookup_job_tasks needs to return tasks sorted: "most recent first".
  # Sort the list of tasks by:
  # - create time (descending)
  # - task-id (descending, if any)

  tasks.sort(key=_task_sort_function, reverse=True)


def __init__(self, resources):
    """Run jobs on your local machine.

    Args:
      resources: module providing access to files packaged with dsub
                 (See dsub/libs/resources.py)
    """
    self._operations = []
    self._resources = resources

def __init__ ( self , resources ) : self . _operations = [ ] self . _resources = resources


Determine if the provided time is within the range inclusive .

def _datetime_in_range(self, dt, dt_min=None, dt_max=None):
    """Determine if the provided time is within the range, inclusive."""
    # The pipelines API stores operation create-time with second granularity.
    # We mimic this behavior in the local provider by truncating to seconds.
    dt = dt.replace(microsecond=0)
    if dt_min:
      dt_min = dt_min.replace(microsecond=0)
    else:
      dt_min = dsub_util.replace_timezone(datetime.datetime.min, pytz.utc)
    if dt_max:
      dt_max = dt_max.replace(microsecond=0)
    else:
      dt_max = dsub_util.replace_timezone(datetime.datetime.max, pytz.utc)

    return dt_min <= dt <= dt_max


Return a Task object with this task s info .

def _get_task_from_task_dir(self, job_id, user_id, task_id, task_attempt):
    """Return a Task object with this task's info."""

    # We need to be very careful about how we read and interpret the contents
    # of the task directory. The directory could be changing because a new
    # task is being created. The directory could be changing because a task
    # is ending.
    #
    # If the meta.yaml does not exist, the task does not yet exist.
    # If the meta.yaml exists, it means the task is scheduled. It does not mean
    # it is yet running.
    # If the task.pid file exists, it means that the runner.sh was started.

    task_dir = self._task_directory(job_id, task_id, task_attempt)

    job_descriptor = self._read_task_metadata(task_dir)
    if not job_descriptor:
      return None

    # If we read up an old task, the user-id will not be in the job_descriptor.
    if not job_descriptor.job_metadata.get('user-id'):
      job_descriptor.job_metadata['user-id'] = user_id

    # Get the pid of the runner
    pid = -1
    try:
      with open(os.path.join(task_dir, 'task.pid'), 'r') as f:
        pid = int(f.readline().strip())
    except (IOError, OSError):
      pass

    # Get the script contents
    script = None
    script_name = job_descriptor.job_metadata.get('script-name')
    if script_name:
      script = self._read_script(task_dir, script_name)

    # Read the files written by the runner.sh.
    # For new tasks, these may not have been written yet.
    end_time = self._get_end_time_from_task_dir(task_dir)
    last_update = self._get_last_update_time_from_task_dir(task_dir)
    events = self._get_events_from_task_dir(task_dir)
    status = self._get_status_from_task_dir(task_dir)
    log_detail = self._get_log_detail_from_task_dir(task_dir)

    # If the status file is not yet written, then mark the task as pending
    if not status:
      status = 'RUNNING'
      log_detail = ['Pending']

    return LocalTask(
        task_status=status,
        events=events,
        log_detail=log_detail,
        job_descriptor=job_descriptor,
        end_time=end_time,
        last_update=last_update,
        pid=pid,
        script=script)


def _delocalize_logging_command(self, logging_path, user_project):
    """Returns a command to delocalize logs.

    Args:
      logging_path: location of log files.
      user_project: name of the project to be billed for the request.

    Returns:
      eg. 'gs://bucket/path/myfile' or 'gs://bucket/script-foobar-12'
    """

    # Get the logging prefix (everything up to ".log")
    logging_prefix = os.path.splitext(logging_path.uri)[0]

    # Set the provider-specific mkdir and file copy commands
    if logging_path.file_provider == job_model.P_LOCAL:
      mkdir_cmd = 'mkdir -p "%s"\n' % os.path.dirname(logging_prefix)
      cp_cmd = 'cp'
    elif logging_path.file_provider == job_model.P_GCS:
      mkdir_cmd = ''
      if user_project:
        cp_cmd = 'gsutil -u {} -mq cp'.format(user_project)
      else:
        cp_cmd = 'gsutil -mq cp'
    else:
      assert False

    # Construct the copy command
    copy_logs_cmd = textwrap.dedent("""\
      local cp_cmd="{cp_cmd}"
      local prefix="{prefix}"
    """).format(
        cp_cmd=cp_cmd, prefix=logging_prefix)

    # Build up the command
    body = textwrap.dedent("""\
      {mkdir_cmd}
      {copy_logs_cmd}
    """).format(
        mkdir_cmd=mkdir_cmd, copy_logs_cmd=copy_logs_cmd)

    return body

"""\
      {mkdir_cmd}
      {copy_logs_cmd}
    """


The local dir for staging files for that particular task .

def _task_directory(self, job_id, task_id, task_attempt):
    """The local dir for staging files for that particular task."""
    dir_name = 'task' if task_id is None else str(task_id)
    if task_attempt:
      dir_name = '%s.%s' % (dir_name, task_attempt)
    return self._provider_root() + '/' + job_id + '/' + dir_name

def _task_directory ( self , job_id , task_id , task_attempt ) : dir_name = 'task' if task_id is None else str ( task_id ) if task_attempt : dir_name = '%s.%s' % ( dir_name , task_attempt ) return self . _provider_root ( ) + '/' + job_id + '/' + dir_name


def _split_task_directory(self, task_dir):
    """Return task_id and task_attempt from dir_name."""
    if '.' in task_dir:
      return task_dir.split('.')
    else:
      return task_dir, None

def _split_task_directory ( self , task_dir ) : if '.' in task_dir : return task_dir . split ( '.' ) else : return task_dir , None


Return a dictionary of environment variables for the container .

def _make_environment(self, inputs, outputs, mounts):
    """Return a dictionary of environment variables for the container."""
    env = {}
    env.update(providers_util.get_file_environment_variables(inputs))
    env.update(providers_util.get_file_environment_variables(outputs))
    env.update(providers_util.get_file_environment_variables(mounts))
    return env

def _make_environment ( self , inputs , outputs , mounts ) : env = { } env . update ( providers_util . get_file_environment_variables ( inputs ) ) env . update ( providers_util . get_file_environment_variables ( outputs ) ) env . update ( providers_util . get_file_environment_variables ( mounts ) ) return env


Returns a command that will stage recursive inputs .

def _localize_inputs_recursive_command(self, task_dir, inputs):
    """Returns a command that will stage recursive inputs."""
    data_dir = os.path.join(task_dir, _DATA_SUBDIR)
    provider_commands = [
        providers_util.build_recursive_localize_command(data_dir, inputs,
                                                        file_provider)
        for file_provider in _SUPPORTED_INPUT_PROVIDERS
    ]
    return '\n'.join(provider_commands)

def _localize_inputs_recursive_command ( self , task_dir , inputs ) : data_dir = os . path . join ( task_dir , _DATA_SUBDIR ) provider_commands = [ providers_util . build_recursive_localize_command ( data_dir , inputs , file_provider ) for file_provider in _SUPPORTED_INPUT_PROVIDERS ] return '\n' . join ( provider_commands )


Returns a directory or file path to be the target for gsutil cp .

def _get_input_target_path(self, local_file_path):
    """Returns a directory or file path to be the target for "gsutil cp".

    If the filename contains a wildcard, then the target path must
    be a directory in order to ensure consistency whether the source pattern
    contains one or multiple files.

    Args:
      local_file_path: A full path terminating in a file or a file wildcard.

    Returns:
      The path to use as the "gsutil cp" target.
    """

    path, filename = os.path.split(local_file_path)
    if '*' in filename:
      return path + '/'
    else:
      return local_file_path

def _get_input_target_path ( self , local_file_path ) : path , filename = os . path . split ( local_file_path ) if '*' in filename : return path + '/' else : return local_file_path


Returns a command that will stage inputs .

def _localize_inputs_command(self, task_dir, inputs, user_project):
    """Returns a command that will stage inputs."""
    commands = []
    for i in inputs:
      if i.recursive or not i.value:
        continue

      source_file_path = i.uri
      local_file_path = task_dir + '/' + _DATA_SUBDIR + '/' + i.docker_path
      dest_file_path = self._get_input_target_path(local_file_path)

      commands.append('mkdir -p "%s"' % os.path.dirname(local_file_path))

      if i.file_provider in [job_model.P_LOCAL, job_model.P_GCS]:
        # The semantics that we expect here are implemented consistently in
        # "gsutil cp", and are a bit different than "cp" when it comes to
        # wildcard handling, so use it for both local and GCS:
        #
        # - `cp path/* dest/` will error if "path" has subdirectories.
        # - `cp "path/*" "dest/"` will fail (it expects wildcard expansion
        #   to come from shell).
        if user_project:
          command = 'gsutil -u %s -mq cp "%s" "%s"' % (
              user_project, source_file_path, dest_file_path)
        else:
          command = 'gsutil -mq cp "%s" "%s"' % (source_file_path,
                                                 dest_file_path)
        commands.append(command)

    return '\n'.join(commands)


def _delocalize_outputs_commands(self, task_dir, outputs, user_project):
    """Copy outputs from local disk to GCS."""
    commands = []
    for o in outputs:
      if o.recursive or not o.value:
        continue

      # The destination path is o.uri.path, which is the target directory
      # (rather than o.uri, which includes the filename or wildcard).
      dest_path = o.uri.path
      local_path = task_dir + '/' + _DATA_SUBDIR + '/' + o.docker_path

      if o.file_provider == job_model.P_LOCAL:
        commands.append('mkdir -p "%s"' % dest_path)

      # Use gsutil even for local files (explained in _localize_inputs_command).
      if o.file_provider in [job_model.P_LOCAL, job_model.P_GCS]:
        if user_project:
          command = 'gsutil -u %s -mq cp "%s" "%s"' % (user_project, local_path,
                                                       dest_path)
        else:
          command = 'gsutil -mq cp "%s" "%s"' % (local_path, dest_path)
        commands.append(command)

    return '\n'.join(commands)


Return a provider - specific representation of task data .

def raw_task_data(self):
    """Return a provider-specific representation of task data.

    Returns:
      string of task data from the provider.
    """
    return self._raw._asdict()

def raw_task_data ( self ) : return self . _raw . _asdict ( )


Return the last line ( s ) as a single ( newline delimited ) string .

def _last_lines(value, count):
    """Return the last line(s) as a single (newline delimited) string."""
    if not value:
      return ''

    return '\n'.join(value[-count:])

def _last_lines ( value , count ) : if not value : return '' return '\n' . join ( value [ - count : ] )


Returns the list of regions to use for a pipeline request .

def _get_pipeline_regions(self, regions, zones):
    """Returns the list of regions to use for a pipeline request.

    For v2alpha1, just return the regions value that was set, even if it is
    empty.

    Args:
      regions (str): A space separated list of regions to use for the pipeline.
      zones (str): A space separated list of zones to use for the pipeline.
    """
    return regions or []

def _get_pipeline_regions ( self , regions , zones ) : return regions or [ ]


Returns a provider for job submission requests .

def get_provider(args, resources, credentials_fn=None):
  """Returns a provider for job submission requests."""

  # provider_base has a standard way for dsub, dstat, and ddel to
  # get credentials for each provider, but we allow for overriding
  # that by supplying a custom credentials function.
  if not credentials_fn:
    credentials_fn = credentials_from_args

  provider = getattr(args, 'provider', 'google-v2')

  if provider == 'google-cls-v2':
    return google_cls_v2.GoogleCLSV2JobProvider(
        getattr(args, 'dry_run', False),
        args.project,
        args.location,
        credentials=credentials_fn(args))
  elif provider == 'google-v2':
    return google_v2.GoogleV2JobProvider(
        getattr(args, 'dry_run', False),
        args.project,
        credentials=credentials_fn(args))
  elif provider == 'local':
    return local.LocalJobProvider(resources)
  elif provider == 'test-fails':
    return test_fails.FailsJobProvider()
  else:
    raise ValueError('Unknown provider: ' + provider)


def get_provider_name(provider):
  """Returns the name of a given provider."""
  return PROVIDER_NAME_MAP[provider.__class__]

def get_provider_name ( provider ) : return PROVIDER_NAME_MAP [ provider . __class__ ]


Create an argument parser adding in the list of providers .

def create_parser(prog):
  """Create an argument parser, adding in the list of providers."""
  parser = argparse.ArgumentParser(
      prog=prog, formatter_class=argparse.RawDescriptionHelpFormatter)

  parser.add_argument(
      '--provider',
      default='google-v2',
      choices=['local', 'google-v2', 'google-cls-v2', 'test-fails'],
      help="""Job service provider. Valid values are "google-v2" (Google's
        Pipeline API v2alpha1), "google-cls-v2" (Google's Pipelines API v2beta)
        and "local" (local Docker execution).
        "test-*" providers are for testing purposes only.
        (default: google-v2)""",
      metavar='PROVIDER')

  return parser


Add provider required arguments epilog message parse and validate .

def parse_args(parser, provider_required_args, argv):
  """Add provider required arguments epilog message, parse, and validate."""

  # Add the provider required arguments epilog message
  epilog = 'Provider-required arguments:\n'
  for provider in provider_required_args:
    epilog += '  %s: %s\n' % (provider, provider_required_args[provider])
  parser.epilog = epilog

  # Parse arguments
  args = parser.parse_args(argv)

  # For the selected provider, check the required arguments
  for arg in provider_required_args[args.provider]:
    if not vars(args)[arg]:
      parser.error('argument --%s is required' % arg)

  return args


A string with the arguments to point dstat to the same provider + project .

def get_dstat_provider_args(provider, project, location):
  """A string with the arguments to point dstat to the same provider+project."""
  provider_name = get_provider_name(provider)

  args = []
  if provider_name == 'google-cls-v2':
    args.append('--project %s --location %s' % (project, location))
  elif provider_name == 'google-v2':
    args.append('--project %s' % project)
  elif provider_name == 'local':
    pass
  elif provider_name == 'test-fails':
    pass
  else:
    # New providers should add their dstat required arguments here.
    assert False, 'Provider %s needs get_dstat_provider_args support' % provider

  args.insert(0, '--provider %s' % provider_name)
  return ' '.join(args)


A string with the arguments to point ddel to the same provider + project .

def get_ddel_provider_args(provider_type, project, location):
  """A string with the arguments to point ddel to the same provider+project."""
  # Change this if the two ever diverge.
  return get_dstat_provider_args(provider_type, project, location)

def
get_ddel_provider_args
(
provider_type
,
project
,
location
)
:
# Change this if the two ever diverge.
return
get_dstat_provider_args
(
provider_type
,
project
,
location
)


Raise an error if the provider doesn t support a provided flag .

def check_for_unsupported_flag(args):
  """Raise an error if the provider doesn't support a provided flag."""
  if args.label and args.provider not in [
      'test-fails', 'local', 'google-v2', 'google-cls-v2'
  ]:
    raise ValueError(
        '--label is not supported by the "%s" provider.' % args.provider)


Returns a URI with placeholders replaced by metadata values .

def _format_task_uri(fmt, job_metadata, task_metadata):
  """Returns a URI with placeholders replaced by metadata values."""

  values = {
      'job-id': None,
      'task-id': 'task',
      'job-name': None,
      'user-id': None,
      'task-attempt': None
  }
  for key in values:
    values[key] = task_metadata.get(key) or job_metadata.get(key) or values[key]

  return fmt.format(**values)

def _format_task_uri ( fmt , job_metadata , task_metadata ) : values = { 'job-id' : None , 'task-id' : 'task' , 'job-name' : None , 'user-id' : None , 'task-attempt' : None } for key in values : values [ key ] = task_metadata . get ( key ) or job_metadata . get ( key ) or values [ key ] return fmt . format ( * * values )


Inserts task metadata into the logging URI .

def format_logging_uri(uri, job_metadata, task_metadata):
  """Inserts task metadata into the logging URI.

  The core behavior is inspired by the Google Pipelines API:
    (1) If a the uri ends in ".log", then that is the logging path.
    (2) Otherwise, the uri is treated as "directory" for logs and a filename
        needs to be automatically generated.

  For (1), if the job is a --tasks job, then the {task-id} is inserted
  before ".log".

  For (2), the file name generated is {job-id}, or for --tasks jobs, it is
  {job-id}.{task-id}.

  In both cases .{task-attempt} is inserted before .log for --retries jobs.

  In addition, full task metadata substitution is supported. The URI
  may include substitution strings such as
  "{job-id}", "{task-id}", "{job-name}", "{user-id}", and "{task-attempt}".

  Args:
    uri: User-specified logging URI which may contain substitution fields.
    job_metadata: job-global metadata.
    task_metadata: tasks-specific metadata.

  Returns:
    The logging_uri formatted as described above.
  """

  # If the user specifies any formatting (with curly braces), then use that
  # as the format string unchanged.
  fmt = str(uri)
  if '{' not in fmt:
    if uri.endswith('.log'):
      # URI includes a filename. Trim the extension and just use the prefix.
      fmt = os.path.splitext(uri)[0]
    else:
      # URI is a path to a directory. The job-id becomes the filename prefix.
      fmt = os.path.join(uri, '{job-id}')

    # If this is a task job, add the task-id.
    if task_metadata.get('task-id') is not None:
      fmt += '.{task-id}'

    # If this is a retryable task, add the task-attempt.
    if task_metadata.get('task-attempt') is not None:
      fmt += '.{task-attempt}'

    fmt += '.log'

  return _format_task_uri(fmt, job_metadata, task_metadata)