Get test suite ( Python unit tests only ) .

def unittest_suite():
  """Get test suite (Python unit tests only)."""
  test_loader = unittest.TestLoader()
  test_suite = test_loader.discover('test/unit', pattern='test_*.py')
  return test_suite

def unittest_suite ( ) : test_loader = unittest . TestLoader ( ) test_suite = test_loader . discover ( 'test/unit' , pattern = 'test_*.py' ) return test_suite


Get the dsub version out of the _dsub_version . py source file .

def get_dsub_version():
  """Get the dsub version out of the _dsub_version.py source file.

  Setup.py should not import dsub version from dsub directly since ambiguity in
  import order could lead to an old version of dsub setting the version number.
  Parsing the file directly is simpler than using import tools (whose interface
  varies between python 2.7, 3.4, and 3.5).

  Returns:
    string of dsub version.

  Raises:
    ValueError: if the version is not found.
  """
  filename = os.path.join(os.path.dirname(__file__), 'dsub/_dsub_version.py')
  with open(filename, 'r') as versionfile:
    for line in versionfile:
      if line.startswith('DSUB_VERSION ='):
        # Get the version then strip whitespace and quote characters.
        version = line.partition('=')[2]
        return version.strip().strip('\'"')
  raise ValueError('Could not find version.')


def get_readme_contents():
  """Get the README.md contents."""
  with open('README.md', 'r') as f:
    return f.read()

def get_readme_contents ( ) : with open ( 'README.md' , 'r' ) as f : return f . read ( )


Sys . out replacer by default with stderr .

def replace_print(fileobj=sys.stderr):
  """Sys.out replacer, by default with stderr.

  Use it like this:
  with replace_print_with(fileobj):
    print "hello"  # writes to the file
  print "done"  # prints to stdout

  Args:
    fileobj: a file object to replace stdout.

  Yields:
    The printer.
  """
  printer = _Printer(fileobj)

  previous_stdout = sys.stdout
  sys.stdout = printer
  try:
    yield printer
  finally:
    sys.stdout = previous_stdout

def replace_print ( fileobj = sys . stderr ) : printer = _Printer ( fileobj ) previous_stdout = sys . stdout sys . stdout = printer try : yield printer finally : sys . stdout = previous_stdout


Utility routine to emit messages to stderr .

def print_error(msg):
  """Utility routine to emit messages to stderr."""
  print(msg, file=sys.stderr)

def print_error ( msg ) : print ( msg , file = sys . stderr )


Returns the current OS user this may be different from the dsub user .

def get_os_user():
  """Returns the current OS user, this may be different from the dsub user."""
  return pwd.getpwuid(os.getuid())[0]

def get_os_user ( ) : return pwd . getpwuid ( os . getuid ( ) ) [ 0 ]


Returns the set of job IDs for the given tasks .

def tasks_to_job_ids(task_list):
  """Returns the set of job IDs for the given tasks."""
  return set([t.get_field('job-id') for t in task_list])

def tasks_to_job_ids ( task_list ) : return set ( [ t . get_field ( 'job-id' ) for t in task_list ] )


Compact a list of integers into a comma - separated string of intervals .

def compact_interval_string(value_list):
  """Compact a list of integers into a comma-separated string of intervals.

  Args:
    value_list: A list of sortable integers such as a list of numbers

  Returns:
    A compact string representation, such as "1-5,8,12-15"
  """

  if not value_list:
    return ''

  value_list.sort()

  # Start by simply building up a list of separate contiguous intervals
  interval_list = []
  curr = []
  for val in value_list:
    if curr and (val > curr[-1] + 1):
      interval_list.append((curr[0], curr[-1]))
      curr = [val]
    else:
      curr.append(val)

  if curr:
    interval_list.append((curr[0], curr[-1]))

  # For each interval collapse it down to "first, last" or just "first" if
  # if first == last.
  return ','.join([
      '{}-{}'.format(pair[0], pair[1]) if pair[0] != pair[1] else str(pair[0])
      for pair in interval_list
  ])


Get a storage client using the provided credentials or defaults .

def get_storage_service(credentials):
  """Get a storage client using the provided credentials or defaults."""
  # dsub is not a server application, so it is ok to filter this warning.
  warnings.filterwarnings(
      'ignore', 'Your application has authenticated using end user credentials')
  if credentials is None:
    credentials, _ = google.auth.default()
  # Set cache_discovery to False because we use google-auth
  # See https://github.com/googleapis/google-api-python-client/issues/299
  return googleapiclient.discovery.build(
      'storage', 'v1', credentials=credentials, cache_discovery=False)


def _downloader_next_chunk(downloader):
  """Downloads the next chunk."""
  return downloader.next_chunk()

def _downloader_next_chunk ( downloader ) : return downloader . next_chunk ( )


def _load_file_from_gcs(gcs_file_path, credentials=None):
  """Load context from a text file in gcs.

  Args:
    gcs_file_path: The target file path; should have the 'gs://' prefix.
    credentials: Optional credential to be used to load the file from gcs.

  Returns:
    The content of the text file as a string.
  """
  gcs_service = get_storage_service(credentials)

  bucket_name, object_name = gcs_file_path[len('gs://'):].split('/', 1)
  request = gcs_service.objects().get_media(
      bucket=bucket_name, object=object_name)

  file_handle = io.BytesIO()
  downloader = googleapiclient.http.MediaIoBaseDownload(
      file_handle, request, chunksize=1024 * 1024)
  done = False
  while not done:
    _, done = _downloader_next_chunk(downloader)
  filevalue = file_handle.getvalue()
  if not isinstance(filevalue, str):
    filevalue = filevalue.decode()
  return filevalue

def _load_file_from_gcs ( gcs_file_path , credentials = None ) : gcs_service = get_storage_service ( credentials ) bucket_name , object_name = gcs_file_path [ len ( 'gs://' ) : ] . split ( '/' , 1 ) request = gcs_service . objects ( ) . get_media ( bucket = bucket_name , object = object_name ) file_handle = io . BytesIO ( ) downloader = googleapiclient . http . MediaIoBaseDownload ( file_handle , request , chunksize = 1024 * 1024 ) done = False while not done : _ , done = _downloader_next_chunk ( downloader ) filevalue = file_handle . getvalue ( ) if not isinstance ( filevalue , str ) : filevalue = filevalue . decode ( ) return filevalue


def load_file(file_path, credentials=None):
  """Load a file from either local or gcs.

  Args:
    file_path: The target file path, which should have the prefix 'gs://' if
               to be loaded from gcs.
    credentials: Optional credential to be used to load the file from gcs.

  Returns:
    The contents of the file as a string.
  """
  if file_path.startswith('gs://'):
    return _load_file_from_gcs(file_path, credentials)
  else:
    with open(file_path, 'r') as f:
      return f.read()

def load_file ( file_path , credentials = None ) : if file_path . startswith ( 'gs://' ) : return _load_file_from_gcs ( file_path , credentials ) else : with open ( file_path , 'r' ) as f : return f . read ( )


def _file_exists_in_gcs(gcs_file_path, credentials=None, storage_service=None):
  """Check whether the file exists, in GCS.

  Args:
    gcs_file_path: The target file path; should have the 'gs://' prefix.
    credentials: Optional credential to be used to load the file from gcs.
    storage_service: GCS API service object.

  Returns:
    True if the file's there.
  """
  if storage_service is None:
    storage_service = get_storage_service(credentials)

  bucket_name, object_name = gcs_file_path[len('gs://'):].split('/', 1)
  request = storage_service.objects().get(
      bucket=bucket_name, object=object_name, projection='noAcl')
  try:
    request.execute()
    return True
  except googleapiclient.errors.HttpError:
    return False

def _file_exists_in_gcs ( gcs_file_path , credentials = None , storage_service = None ) : if storage_service is None : storage_service = get_storage_service ( credentials ) bucket_name , object_name = gcs_file_path [ len ( 'gs://' ) : ] . split ( '/' , 1 ) request = storage_service . objects ( ) . get ( bucket = bucket_name , object = object_name , projection = 'noAcl' ) try : request . execute ( ) return True except googleapiclient . errors . HttpError : return False


Check whether there is a GCS object whose name starts with the prefix .

def _prefix_exists_in_gcs(gcs_prefix, credentials=None, storage_service=None):
  """Check whether there is a GCS object whose name starts with the prefix.

  Since GCS doesn't actually have folders, this is how we check instead.

  Args:
    gcs_prefix: The path; should start with 'gs://'.
    credentials: Optional credential to be used to load the file from gcs.
    storage_service: GCS API service object.

  Returns:
    True if the prefix matches at least one object in GCS.

  Raises:
    googleapiclient.errors.HttpError: if it can't talk to the server
  """
  if storage_service is None:
    storage_service = get_storage_service(credentials)

  bucket_name, prefix = gcs_prefix[len('gs://'):].split('/', 1)
  # documentation in
  # https://cloud.google.com/storage/docs/json_api/v1/objects/list
  request = storage_service.objects().list(
      bucket=bucket_name, prefix=prefix, maxResults=1)
  response = request.execute()
  return response.get('items', None)


True iff an object exists matching the input GCS pattern .

def simple_pattern_exists_in_gcs(file_pattern,
                                 credentials=None,
                                 storage_service=None):
  """True iff an object exists matching the input GCS pattern.

  The GCS pattern must be a full object reference or a "simple pattern" that
  conforms to the dsub input and output parameter restrictions:

    * No support for **, ? wildcards or [] character ranges
    * Wildcards may only appear in the file name

  Args:
    file_pattern: eg. 'gs://foo/ba*'
    credentials: Optional credential to be used to load the file from gcs.
    storage_service: GCS API service object.

  Raises:
    ValueError: if file_pattern breaks the rules.

  Returns:
    True iff a file exists that matches that pattern.
  """
  if '*' not in file_pattern:
    return _file_exists_in_gcs(file_pattern, credentials, storage_service)
  if not file_pattern.startswith('gs://'):
    raise ValueError('file name must start with gs://')

  if storage_service is None:
    storage_service = get_storage_service(credentials)

  bucket_name, prefix = file_pattern[len('gs://'):].split('/', 1)
  if '*' in bucket_name:
    raise ValueError('Wildcards may not appear in the bucket name')
  # There is a '*' in prefix because we checked there's one in file_pattern
  # and there isn't one in bucket_name. Hence it must be in prefix.
  assert '*' in prefix
  prefix_no_wildcard = prefix[:prefix.index('*')]
  request = storage_service.objects().list(
      bucket=bucket_name, prefix=prefix_no_wildcard)
  response = request.execute()
  if 'items' not in response:
    return False
  items_list = [i['name'] for i in response['items']]
  return any(fnmatch.fnmatch(i, prefix) for i in items_list)


True if each output contains at least one file or no output specified .

def outputs_are_present(outputs, storage_service=None):
  """True if each output contains at least one file or no output specified."""
  # outputs are OutputFileParam (see param_util.py)

  # If outputs contain a pattern, then there is no way for `dsub` to verify
  # that *all* output is present. The best that `dsub` can do is to verify
  # that *some* output was created for each such parameter.
  for o in outputs:
    if not o.value:
      continue
    if o.recursive:
      if not folder_exists(o.value, storage_service=storage_service):
        return False
    else:
      if not simple_pattern_exists_in_gcs(
          o.value, storage_service=storage_service):
        return False
  return True


Create a JobError to indicate something went wrong .

def __init__(self, message, error_list, launched_job):
    """Create a JobError to indicate something went wrong.

    Args:
        message: user-friendly message
        error_list: what went wrong
        launched_job: if the job is launched, but has errors in
          "--wait"ing on the tasks.
    """
    super(JobError, self).__init__(message)
    self.message = message
    self.error_list = error_list
    self.launched_job = launched_job

def __init__ ( self , message , error_list , launched_job ) : super ( JobError , self ) . __init__ ( message ) self . message = message self . error_list = error_list self . launched_job = launched_job


def _print_error(msg):
  """Utility routine to emit messages to stderr."""
  print(msg, file=sys.stderr)

def _print_error ( msg ) : print ( msg , file = sys . stderr )


Prints an error message if appropriate .

def _print_retry_error(attempt_number, max_attempts, exception):
  """Prints an error message if appropriate."""
  now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
  try:
    status_code = exception.resp.status
  except AttributeError:
    status_code = ''

  if attempt_number % 5 == 0:
    _print_error('{}: Caught exception {} {}'.format(
        now, get_exception_type_string(exception), status_code))
    _print_error('{}: This request is being retried (attempt {} of {}).'.format(
        now, attempt_number, max_attempts))


Returns the full path of the exception .

def get_exception_type_string(exception):
  """Returns the full path of the exception."""
  exception_type_string = str(type(exception))

  # This is expected to look something like
  # "<class 'google.auth.exceptions.RefreshError'>"
  if exception_type_string.startswith(
      "<class '") and exception_type_string.endswith("'>"):
    # Slice off the <class ''> parts
    return exception_type_string[len("<class '"):-len("'>")]
  else:
    # If the exception type looks different than expected,
    # just print out the whole type.
    return exception_type_string


def retry_api_check(retry_state: tenacity.RetryCallState) -> bool:
  """Return True if we should retry.

  False otherwise.

  Args:
    retry_state: A retry state including exception to test for transience.

  Returns:
    True if we should retry. False otherwise.
  """
  exception = retry_state.outcome.exception()
  attempt_number = retry_state.attempt_number
  now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')

  if isinstance(exception, googleapiclient.errors.HttpError):
    if exception.resp.status in TRANSIENT_HTTP_ERROR_CODES:
      _print_retry_error(attempt_number, MAX_API_ATTEMPTS, exception)
      return True

  if isinstance(exception, socket.error):
    if exception.errno in TRANSIENT_SOCKET_ERROR_CODES:
      _print_retry_error(attempt_number, MAX_API_ATTEMPTS, exception)
      return True

  if isinstance(exception, socket.timeout):
    _print_retry_error(attempt_number, MAX_API_ATTEMPTS, exception)
    return True

  if isinstance(exception, google.auth.exceptions.RefreshError):
    _print_retry_error(attempt_number, MAX_API_ATTEMPTS, exception)
    return True

  # For a given installation, this could be a permanent error, but has only
  # been observed as transient.
  if isinstance(exception, ssl.SSLError):
    _print_retry_error(attempt_number, MAX_API_ATTEMPTS, exception)
    return True

  # This has been observed as a transient error:
  #   ServerNotFoundError: Unable to find the server at genomics.googleapis.com
  if isinstance(exception, httplib2.ServerNotFoundError):
    _print_retry_error(attempt_number, MAX_API_ATTEMPTS, exception)
    return True

  # Observed to be thrown transiently from auth libraries which use httplib2
  if isinstance(exception, http.client.ResponseNotReady):
    _print_retry_error(attempt_number, MAX_API_ATTEMPTS, exception)
    return True

  if not exception and attempt_number > 5:
    _print_error('{}: Retry SUCCEEDED'.format(now))

  return False


def retry_auth_check(retry_state: tenacity.RetryCallState) -> bool:
  """Specific check for auth error codes.

  Return True if we should retry.

  False otherwise.
  Args:
    retry_state: A retry state including exception to test for transience.

  Returns:
    True if we should retry. False otherwise.
  """
  exception = retry_state.outcome.exception()
  attempt_number = retry_state.attempt_number
  now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')

  if isinstance(exception, googleapiclient.errors.HttpError):
    if exception.resp.status in HTTP_AUTH_ERROR_CODES:
      _print_retry_error(attempt_number, MAX_AUTH_ATTEMPTS, exception)
      return True

  if not exception and attempt_number > 4:
    _print_error('{}: Retry SUCCEEDED'.format(now))

  return False

def
retry_auth_check
(
retry_state
:
tenacity
.
RetryCallState
)
->
bool
:
exception
=
retry_state
.
outcome
.
exception
(
)
attempt_number
=
retry_state
.
attempt_number
now
=
datetime
.
datetime
.
now
(
)
.
strftime
(
'%Y-%m-%d %H:%M:%S.%f'
)
if
isinstance
(
exception
,
googleapiclient
.
errors
.
HttpError
)
:
if
exception
.
resp
.
status
in
HTTP_AUTH_ERROR_CODES
:
_print_retry_error
(
attempt_number
,
MAX_AUTH_ATTEMPTS
,
exception
)
return
True
if
not
exception
and
attempt_number
>
4
:
_print_error
(
'{}: Retry SUCCEEDED'
.
format
(
now
)
)
return
False


def on_give_up(retry_state: tenacity.RetryCallState) -> None:
  """Called after all retries failed.

  Simply outputs a message and re-raises.

  Args:
    retry_state: info about current retry invocation.

  Returns:
    None.
  """
  exception = retry_state.outcome.exception()
  attempt_number = retry_state.attempt_number
  _print_error('Giving up after {} attempts'.format(attempt_number))
  raise exception

def
on_give_up
(
retry_state
:
tenacity
.
RetryCallState
)
->
:
exception
=
retry_state
.
outcome
.
exception
(
)
attempt_number
=
retry_state
.
attempt_number
_print_error
(
'Giving up after {} attempts'
.
format
(
attempt_number
)
)
raise
exception


Return a dictionary of environment variables for the user container .

def get_file_environment_variables(file_params):
  """Return a dictionary of environment variables for the user container."""
  env = {}
  for param in file_params:
    # We have no cases where the environment variable provided to user
    # scripts have a trailing slash, so be sure to always strip it.
    # The case that this is specifically handling is --input-recursive and
    # --output-recursive variables, which are directory values.
    env[param.name] = os.path.join(
        DATA_MOUNT_POINT, param.docker_path.rstrip('/')) if param.value else ''
  return env


Return a multi - line string with export statements for the variables .

def build_recursive_localize_env(destination, inputs):
  """Return a multi-line string with export statements for the variables.

  Arguments:
    destination: Folder where the data will be put.
                 For example /mnt/data
    inputs: a list of InputFileParam

  Returns:
    a multi-line string with a shell script that sets environment variables
    corresponding to the inputs.
  """
  # pylint: disable=g-complex-comprehension
  export_input_dirs = '\n'.join([
      'export {0}={1}/{2}'.format(var.name, destination.rstrip('/'),
                                  var.docker_path.rstrip('/'))
      for var in inputs
      if var.recursive and var.docker_path
  ])
  # pylint: enable=g-complex-comprehension
  return export_input_dirs


Return a multi - line string with a shell script to copy recursively .

def build_recursive_localize_command(destination, inputs, file_provider):
  """Return a multi-line string with a shell script to copy recursively.

  Arguments:
    destination: Folder where to put the data.
                 For example /mnt/data
    inputs: a list of InputFileParam
    file_provider: file provider string used to filter the output params; the
                 returned command will only apply outputs whose file provider
                 matches this file filter.

  Returns:
    a multi-line string with a shell script that copies the inputs
    recursively from GCS.
  """
  command = _LOCALIZE_COMMAND_MAP[file_provider]
  filtered_inputs = [
      var for var in inputs
      if var.recursive and var.file_provider == file_provider
  ]
  # pylint: disable=g-complex-comprehension
  copy_input_dirs = '\n'.join([
      textwrap.dedent("""
      mkdir -p {data_mount}/{docker_path}
      for ((i = 0; i < 3; i++)); do
        if {command} {source_uri} {data_mount}/{docker_path}; then
          break
        elif ((i == 2)); then
          1>&2 echo "Recursive localization failed."
          exit 1
        fi
      done
      chmod -R o+r {data_mount}/{docker_path}
      """).format(
          command=command,
          source_uri=var.uri,
          data_mount=destination.rstrip('/'),
          docker_path=var.docker_path) for var in filtered_inputs
  ])
  # pylint: enable=g-complex-comprehension
  return copy_input_dirs


def build_recursive_gcs_delocalize_env(source, outputs):
  """Return a multi-line string with export statements for the variables.

  Arguments:
    source: Folder with the data.
            For example /mnt/data
    outputs: a list of OutputFileParam

  Returns:
    a multi-line string with a shell script that sets environment variables
    corresponding to the outputs.
  """
  filtered_outs = [
      var for var in outputs
      if var.recursive and var.file_provider == job_model.P_GCS
  ]
  # pylint: disable=g-complex-comprehension
  return '\n'.join([
      'export {0}={1}/{2}'.format(var.name,
                                  source.rstrip('/'),
                                  var.docker_path.rstrip('/'))
      for var in filtered_outs
  ])


def build_recursive_delocalize_command(source, outputs, file_provider):
  """Return a multi-line string with a shell script to copy recursively.

  Arguments:
    source: Folder with the data.
            For example /mnt/data
    outputs: a list of OutputFileParam.
    file_provider: file provider string used to filter the output params; the
                 returned command will only apply outputs whose file provider
                 matches this file filter.

  Returns:
    a multi-line string with a shell script that copies the inputs
    recursively to GCS.
  """
  command = _LOCALIZE_COMMAND_MAP[file_provider]
  filtered_outputs = [
      var for var in outputs
      if var.recursive and var.file_provider == file_provider
  ]
  # pylint: disable=g-complex-comprehension
  return '\n'.join([
      textwrap.dedent("""
      for ((i = 0; i < 3; i++)); do
        if {command} {data_mount}/{docker_path} {destination_uri}; then
          break
        elif ((i == 2)); then
          1>&2 echo "Recursive de-localization failed."
          exit 1
        fi
      done
      """).format(
          command=command,
          data_mount=source.rstrip('/'),
          docker_path=var.docker_path,
          destination_uri=var.uri) for var in filtered_outputs
  ])


Returns a dict combining job metadata with the task id .

def get_task_metadata(job_metadata, task_id):
  """Returns a dict combining job metadata with the task id."""
  task_metadata = job_metadata.copy()
  task_metadata['task-id'] = task_id

  return task_metadata


def build_mount_env(source, mounts):
  """Return a multi-line string with export statements for the variables.

  Arguments:
    source: Folder with the data. For example /mnt/data
    mounts: a list of MountParam

  Returns:
    a multi-line string with a shell script that sets environment variables
    corresponding to the mounts.
  """
  return '\n'.join([
      'export {0}={1}/{2}'.format(var.name, source.rstrip('/'),
                                  var.docker_path.rstrip('/')) for var in mounts
  ])

def
build_mount_env
(
source
,
mounts
)
:
return
'\n'
.
join
(
[
'export {0}={1}/{2}'
.
format
(
var
.
name
,
source
.
rstrip
(
'/'
)
,
var
.
docker_path
.
rstrip
(
'/'
)
)
for
var
in
mounts
]
)


Returns a dict combining the field for job and task params .

def get_job_and_task_param(job_params, task_params, field):
  """Returns a dict combining the field for job and task params."""
  return job_params.get(field, set()) | task_params.get(field, set())

def get_job_and_task_param ( job_params , task_params , field ) : return job_params . get ( field , set ( ) ) | task_params . get ( field , set ( ) )


Returns a dictionary of metadata fields for the job .

def prepare_job_metadata(script, job_name, user_id):
  """Returns a dictionary of metadata fields for the job."""

  # The name of the job is derived from the job_name and gets set as a
  # 'job-name' label (and so the value must be normalized).
  if not job_name:
    job_name = os.path.basename(script).split('.', 1)[0]
  job_name_value = job_model.convert_to_label_chars(job_name)

  # The user-id will get set as a label
  user_id = job_model.convert_to_label_chars(user_id)

  # Standard version is MAJOR.MINOR(.PATCH). This will convert the version
  # string to "vMAJOR-MINOR(-PATCH)". Example; "0.1.0" -> "v0-1-0".
  version = job_model.convert_to_label_chars('v%s' % DSUB_VERSION)
  return {
      'job-name': job_name_value,
      'user-id': user_id,
      'dsub-version': version,
  }


Validate that the name follows posix conventions for env variables .

def validate_param_name(name, param_type):
  """Validate that the name follows posix conventions for env variables."""
  # http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap03.html#tag_03_235
  #
  # 3.235 Name
  # In the shell command language, a word consisting solely of underscores,
  # digits, and alphabetics from the portable character set.
  if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', name):
    raise ValueError('Invalid %s: %s' % (param_type, name))


Validate that the name is a valid GCS bucket .

def validate_bucket_name(bucket):
  """Validate that the name is a valid GCS bucket."""
  if not bucket.startswith('gs://'):
    raise ValueError(
        'Invalid bucket path "%s". Must start with "gs://".' % bucket)
  bucket_name = bucket[len('gs://'):]
  if not re.search(r'^\w[\w_\.-]{1,61}\w$', bucket_name):
    raise ValueError('Invalid bucket name: %s' % bucket)


Turn the specified name and value into a valid Google label .

def convert_to_label_chars(s):
  """Turn the specified name and value into a valid Google label."""

  # We want the results to be user-friendly, not just functional.
  # So we can't base-64 encode it.
  #   * If upper-case: lower-case it
  #   * If the char is not a standard letter or digit. make it a dash

  # March 2019 note: underscores are now allowed in labels.
  # However, removing the conversion of underscores to dashes here would
  # create inconsistencies between old jobs and new jobs.
  # With existing code, $USER "jane_doe" has a user-id label of "jane-doe".
  # If we remove the conversion, the user-id label for new jobs is "jane_doe".
  # This makes looking up old jobs more complicated.

  accepted_characters = string.ascii_lowercase + string.digits + '-'

  def label_char_transform(char):
    if char in accepted_characters:
      return char
    if char in string.ascii_uppercase:
      return char.lower()
    return '-'

  return ''.join(label_char_transform(c) for c in s)


For the job ensure that each param entry is not None .

def ensure_job_params_are_complete(job_params):
  """For the job, ensure that each param entry is not None."""
  for param in [
      'labels', 'envs', 'inputs', 'outputs', 'mounts', 'input-recursives',
      'output-recursives'
  ]:
    if not job_params.get(param):
      job_params[param] = set()

def ensure_job_params_are_complete ( job_params ) : for param in [ 'labels' , 'envs' , 'inputs' , 'outputs' , 'mounts' , 'input-recursives' , 'output-recursives' ] : if not job_params . get ( param ) : job_params [ param ] = set ( )


For each task ensure that each task param entry is not None .

def ensure_task_params_are_complete(task_descriptors):
  """For each task, ensure that each task param entry is not None."""
  for task_desc in task_descriptors:
    for param in [
        'labels', 'envs', 'inputs', 'outputs', 'input-recursives',
        'output-recursives'
    ]:
      if not task_desc.task_params.get(param):
        task_desc.task_params[param] = set()

def ensure_task_params_are_complete ( task_descriptors ) : for task_desc in task_descriptors : for param in [ 'labels' , 'envs' , 'inputs' , 'outputs' , 'input-recursives' , 'output-recursives' ] : if not task_desc . task_params . get ( param ) : task_desc . task_params [ param ] = set ( )


Return a new dict with any empty items removed .

def _remove_empty_items(d, required):
  """Return a new dict with any empty items removed.

  Note that this is not a deep check. If d contains a dictionary which
  itself contains empty items, those are never checked.

  This method exists to make to_serializable() functions cleaner.
  We could revisit this some day, but for now, the serialized objects are
  stripped of empty values to keep the output YAML more compact.

  Args:
    d: a dictionary
    required: list of required keys (for example, TaskDescriptors always emit
      the "task-id", even if None)

  Returns:
    A dictionary with empty items removed.
  """

  new_dict = {}
  for k, v in d.items():
    if k in required:
      new_dict[k] = v
    elif isinstance(v, int) or v:
      # "if v" would suppress emitting int(0)
      new_dict[k] = v

  return new_dict

def
_remove_empty_items
(
d
,
required
)
:
new_dict
=
{
}
for
k
,
v
in
d
.
items
(
)
:
if
k
in
required
:
new_dict
[
k
]
=
v
elif
isinstance
(
v
,
int
)
or
v
:
# "if v" would suppress emitting int(0)
new_dict
[
k
]
=
v
return
new_dict


Generator that yields a task - specific view of the job .

def task_view_generator(job_descriptor):
  """Generator that yields a task-specific view of the job.

  This generator exists to make it easy for callers to iterate over the tasks
  in a JobDescriptor. Each pass yields a new JobDescriptor with a single task.

  Args:
    job_descriptor: A JobDescriptor with 1 or more tasks.

  Yields:
    A JobDescriptor with a single task.
  """
  for task_descriptor in job_descriptor.task_descriptors:
    jd = JobDescriptor(job_descriptor.job_metadata, job_descriptor.job_params,
                       job_descriptor.job_resources, [task_descriptor])
    yield jd


Raise ValueError if the label is invalid .

def _validate_label(cls, name, value):
    """Raise ValueError if the label is invalid."""
    # Rules for labels are described in:
    #  https://cloud.google.com/compute/docs/labeling-resources#restrictions

    # * Keys and values cannot be longer than 63 characters each.
    # * Keys and values can only contain lowercase letters, numeric characters,
    #   underscores, and dashes.
    # * International characters are allowed.
    # * Label keys must start with a lowercase letter and international
    #   characters are allowed.
    # * Label keys cannot be empty.
    cls._check_label_name(name)
    cls._check_label_value(value)

    # Ensure that reserved labels are not being used.
    if not cls._allow_reserved_keys and name in RESERVED_LABELS:
      raise ValueError('Label flag (%s=...) must not use reserved keys: %r' %
                       (name, list(RESERVED_LABELS)))


Raise ValueError if the label value is invalid .

def _check_label_value(value):
    """Raise ValueError if the label value is invalid."""
    if not value:
      return

    if len(value) > 63:
      raise ValueError(
          'Label values must not be longer than 63 characters: "%s"' % value)

    if not re.match(r'^([-_a-z0-9]*)?$', value):
      raise ValueError(
          'Invalid value for label: "%s". Must contain only lowercase letters, '
          'numeric characters, underscores, and dashes.' % value)


Return a dict populated for serialization ( as YAML / JSON ) .

def to_serializable(self):
    """Return a dict populated for serialization (as YAML/JSON)."""

    task_metadata = self.task_metadata
    task_params = self.task_params
    task_resources = self.task_resources

    # The only required field is the task-id, even if it is None
    task_id = None
    if task_metadata.get('task-id') is not None:
      task_id = str(task_metadata.get('task-id'))

    task = {'task-id': task_id}
    task['create-time'] = task_metadata.get('create-time')
    task['task-attempt'] = task_metadata.get('task-attempt')

    if task_resources.logging_path:
      task['logging-path'] = str(task_resources.logging_path.uri)

    task['labels'] = {var.name: var.value for var in task_params['labels']}

    task['envs'] = {var.name: var.value for var in task_params['envs']}

    task['inputs'] = {
        var.name: var.value
        for var in task_params['inputs']
        if not var.recursive
    }
    task['input-recursives'] = {
        var.name: var.value
        for var in task_params['inputs']
        if var.recursive
    }
    task['outputs'] = {
        var.name: var.value
        for var in task_params['outputs']
        if not var.recursive
    }
    task['output-recursives'] = {
        var.name: var.value
        for var in task_params['outputs']
        if var.recursive
    }

    return _remove_empty_items(task, ['task-id'])


def to_serializable(self):
    """Return a dict populated for serialization (as YAML/JSON)."""

    job_metadata = self.job_metadata
    job_resources = self.job_resources
    job_params = self.job_params
    task_descriptors = self.task_descriptors

    job = {
        'job-id': job_metadata.get('job-id'),
        'job-name': job_metadata.get('job-name'),
        'user-id': job_metadata.get('user-id'),
        'create-time': job_metadata.get('create-time'),
        'dsub-version': job_metadata.get('dsub-version'),
        'user-project': job_metadata.get('user-project'),
        'task-ids': job_metadata.get('task-ids'),
        'script-name': job_metadata['script'].name,
    }

    # logging is specified as a command-line argument and is typically
    # transformed (substituting job-id). The transformed value is saved
    # on a per-task basis as the 'logging-path'.
    if job_resources.logging:
      job['logging'] = str(job_resources.logging.uri)

    job['labels'] = {var.name: var.value for var in job_params['labels']}

    job['envs'] = {var.name: var.value for var in job_params['envs']}

    job['inputs'] = {
        var.name: var.value
        for var in job_params['inputs']
        if not var.recursive
    }
    job['input-recursives'] = {
        var.name: var.value
        for var in job_params['inputs']
        if var.recursive
    }

    job['outputs'] = {
        var.name: var.value
        for var in job_params['outputs']
        if not var.recursive
    }
    job['output-recursives'] = {
        var.name: var.value
        for var in job_params['outputs']
        if var.recursive
    }
    job['mounts'] = {var.name: var.value for var in job_params['mounts']}

    tasks = []
    for task_descriptor in task_descriptors:
      tasks.append(task_descriptor.to_serializable())

    job['tasks'] = tasks

    return _remove_empty_items(job, [])


Return a YAML string representing the job and task data .

def to_yaml(self):
    """Return a YAML string representing the job and task data.

    A provider's internal representation of a dsub task typically does not map
    1-1 to the dsub representation. For example, the Google Genomics Pipeline
    does not natively support "input-recursive" or "output-recursive", so the
    google provider cannot easily reconstruct the user inputs from the
    pipeline's associated Operation object.

    All providers are likely to need a way to reliably serialize job and task-
    related information, either for dstat or for any type of "retry" mechanism
    we might want to build.

    Returns:
      YAML string
    """
    return yaml.dump(self.to_serializable(), default_flow_style=False)

def to_yaml ( self ) : return yaml . dump ( self . to_serializable ( ) , default_flow_style = False )


Populate and return a JobDescriptor from a YAML string .

def from_yaml(cls, yaml_string):
    """Populate and return a JobDescriptor from a YAML string."""
    try:
      job = yaml.full_load(yaml_string)
    except AttributeError:
      # For installations that cannot update their PyYAML version
      job = yaml.load(yaml_string)

    job_metadata = {}
    for key in [
        'job-id', 'job-name', 'task-ids', 'user-id', 'dsub-version',
        'user-project', 'script-name'
    ]:
      if job.get(key) is not None:
        job_metadata[key] = job.get(key)

    # Make sure that create-time string is turned into a datetime
    job_create_time = job.get('create-time')
    cls._set_metadata_create_time(job_metadata, job_create_time)

    job_resources = Resources(logging=job.get('logging'))

    job_params = {}
    job_params['labels'] = cls._label_params_from_dict(job.get('labels', {}))
    job_params['envs'] = cls._env_params_from_dict(job.get('envs', {}))
    job_params['inputs'] = cls._input_file_params_from_dict(
        job.get('inputs', {}), False)
    job_params['input-recursives'] = cls._input_file_params_from_dict(
        job.get('input-recursives', {}), True)
    job_params['outputs'] = cls._output_file_params_from_dict(
        job.get('outputs', {}), False)
    job_params['output-recursives'] = cls._output_file_params_from_dict(
        job.get('output-recursives', {}), True)
    job_params['mounts'] = cls._mount_params_from_dict(job.get('mounts', {}))

    task_descriptors = []
    for task in job.get('tasks', []):
      task_metadata = {'task-id': task.get('task-id')}

      # Old instances of the meta.yaml do not have a task create time.
      create_time = task.get('create-time')
      if create_time:
        cls._set_metadata_create_time(task_metadata, create_time)

      if task.get('task-attempt') is not None:
        task_metadata['task-attempt'] = task.get('task-attempt')

      task_params = {}
      task_params['labels'] = cls._label_params_from_dict(
          task.get('labels', {}))
      task_params['envs'] = cls._env_params_from_dict(task.get('envs', {}))
      task_params['inputs'] = cls._input_file_params_from_dict(
          task.get('inputs', {}), False)
      task_params['input-recursives'] = cls._input_file_params_from_dict(
          task.get('input-recursives', {}), True)
      task_params['outputs'] = cls._output_file_params_from_dict(
          task.get('outputs', {}), False)
      task_params['output-recursives'] = cls._output_file_params_from_dict(
          task.get('output-recursives', {}), True)

      task_resources = Resources(logging_path=task.get('logging-path'))

      task_descriptors.append(
          TaskDescriptor(task_metadata, task_params, task_resources))

    return JobDescriptor(job_metadata, job_params, job_resources,
                         task_descriptors)


Returns the task_descriptor corresponding to task_id .

def find_task_descriptor(self, task_id):
    """Returns the task_descriptor corresponding to task_id."""

    # It is not guaranteed that the index will be task_id - 1 when --tasks is
    # used with a min/max range.
    for task_descriptor in self.task_descriptors:
      if task_descriptor.task_metadata.get('task-id') == task_id:
        return task_descriptor
    return None


Create a new table that is a summary of the input rows .

def prepare_summary_table(rows):
  """Create a new table that is a summary of the input rows.

  All with the same (job-name or job-id, status) go together.

  Args:
    rows: the input rows, a list of dictionaries.

  Returns:
    A new row set of summary information.
  """
  if not rows:
    return []

  # We either group on the job-name (if present) or fall back to the job-id
  key_field = 'job-name'
  if key_field not in rows[0]:
    key_field = 'job-id'

  # Group each of the rows based on (job-name or job-id, status)
  grouped = collections.defaultdict(lambda: collections.defaultdict(lambda: []))
  for row in rows:
    grouped[row.get(key_field, '')][row.get('status', '')] += [row]

  # Now that we have the rows grouped, create a summary table.
  # Use the original table as the driver in order to preserve the order.
  new_rows = []
  for job_key in sorted(grouped.keys()):
    group = grouped[job_key]
    canonical_status = ['RUNNING', 'SUCCESS', 'FAILURE', 'CANCEL']
    # Written this way to ensure that if somehow a new status is introduced,
    # it shows up in our output.
    for status in canonical_status + sorted(group.keys()):
      if status not in group:
        continue
      task_count = len(group[status])
      del group[status]
      if task_count:
        summary_row = collections.OrderedDict()
        summary_row[key_field] = job_key
        summary_row['status'] = status
        summary_row['task-count'] = task_count
        new_rows.append(summary_row)

  return new_rows


return a dict with the task s info ( more if full is set ) .

def prepare_row(task, full, summary):
  """return a dict with the task's info (more if "full" is set)."""

  # Would like to include the Job ID in the default set of columns, but
  # it is a long value and would leave little room for status and update time.

  row_spec = collections.namedtuple('row_spec',
                                    ['key', 'required', 'default_value'])

  # pyformat: disable
  default_columns = [
      row_spec('job-name', True, None),
      row_spec('task-id', False, None),
      row_spec('last-update', True, None),
      row_spec('status-message', True, None)
  ]
  full_columns = default_columns + [
      row_spec('job-id', True, None),
      row_spec('user-id', True, None),
      row_spec('status', True, None),
      row_spec('status-detail', True, None),
      row_spec('task-attempt', False, None),
      row_spec('create-time', True, None),
      row_spec('start-time', True, None),
      row_spec('end-time', True, None),
      row_spec('internal-id', True, None),
      row_spec('logging', True, None),
      row_spec('labels', True, {}),
      row_spec('envs', True, {}),
      row_spec('inputs', True, {}),
      row_spec('input-recursives', False, {}),
      row_spec('outputs', True, {}),
      row_spec('output-recursives', False, {}),
      row_spec('mounts', True, {}),
      row_spec('provider', True, None),
      row_spec('provider-attributes', True, {}),
      row_spec('events', True, []),
      row_spec('user-project', False, None),
      row_spec('dsub-version', False, None),
      row_spec('script-name', False, None),
      row_spec('script', False, None),
  ]
  summary_columns = default_columns + [
      row_spec('job-id', True, None),
      row_spec('user-id', True, None),
      row_spec('status', True, None),
  ]
  # pyformat: enable

  assert not (full and summary), 'Full and summary cannot both be enabled'

  if full:
    columns = full_columns
  elif summary:
    columns = summary_columns
  else:
    columns = default_columns

  row = {}
  for col in columns:
    key, required, default = col

    value = task.get_field(key, default)
    if required or value is not None:
      row[key] = value

  return row


def prepare_output(self, row):
    """Convert types of task fields."""
    date_fields = ['last-update', 'create-time', 'start-time', 'end-time']
    int_fields = ['task-attempt']

    for col in date_fields:
      if col in row:
        row[col] = self.default_format_date(row[col])

    for col in int_fields:
      if col in row and row[col] is not None:
        row[col] = int(row[col])

    return row

def prepare_output ( self , row ) : date_fields = [ 'last-update' , 'create-time' , 'start-time' , 'end-time' ] int_fields = [ 'task-attempt' ] for col in date_fields : if col in row : row [ col ] = self . default_format_date ( row [ col ] ) for col in int_fields : if col in row and row [ col ] is not None : row [ col ] = int ( row [ col ] ) return row


Function to be defined by the derived class to print output .

def print_table(self, table):
    """Function to be defined by the derived class to print output."""
    raise NotImplementedError('print_table method not defined!')

def
print_table
(
self
,
table
)
:
raise
NotImplementedError
(
'print_table method not defined!'
)


Wrapper for prepare_output and print_table .

def prepare_and_print_table(self, rows, summary):
    """Wrapper for prepare_output and print_table."""
    if summary:
      rows = prepare_summary_table(rows)

    table = []
    for row in rows:
      row = self.prepare_output(row)
      table.append(row)
    self.print_table(table)

def prepare_and_print_table ( self , rows , summary ) : if summary : rows = prepare_summary_table ( rows ) table = [ ] for row in rows : row = self . prepare_output ( row ) table . append ( row ) self . print_table ( table )


Return a value for display ; if longer than max length use ellipsis .

def _trim_display_field(self, value, max_length):
    """Return a value for display; if longer than max length, use ellipsis."""
    if not value:
      return ''
    if len(value) > max_length:
      return value[:max_length - 3] + '...'
    return value

def _trim_display_field ( self , value , max_length ) : if not value : return '' if len ( value ) > max_length : return value [ : max_length - 3 ] + '...' return value


Returns a string of comma - delimited key = value pairs .

def _format_pairs(self, values):
    """Returns a string of comma-delimited key=value pairs."""
    return ', '.join(
        '%s=%s' % (key, value) for key, value in sorted(values.items()))

def
_format_pairs
(
self
,
values
)
:
return
', '
.
join
(
'%s=%s'
%
(
key
,
value
)
for
key
,
value
in
sorted
(
values
.
items
(
)
)
)


Presenter to force yaml . dump to use multi - line string style .

def string_presenter(self, dumper, data):
    """Presenter to force yaml.dump to use multi-line string style."""
    if '\n' in data:
      return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
    else:
      return dumper.represent_scalar('tag:yaml.org,2002:str', data)

def string_presenter ( self , dumper , data ) : if '\n' in data : return dumper . represent_scalar ( 'tag:yaml.org,2002:str' , data , style = '|' ) else : return dumper . represent_scalar ( 'tag:yaml.org,2002:str' , data )


Rewrite GCS file paths as required by the rewrite_uris method .

def _gcs_uri_rewriter(raw_uri):
  """Rewrite GCS file paths as required by the rewrite_uris method.

  The GCS rewriter performs no operations on the raw_path and simply returns
  it as the normalized URI. The docker path has the gs:// prefix replaced
  with gs/ so that it can be mounted inside a docker image.

  Args:
    raw_uri: (str) the raw GCS URI, prefix, or pattern.

  Returns:
    normalized: a cleaned version of the uri provided by command line.
    docker_path: the uri rewritten in the format required for mounting inside
                 a docker worker.
  """
  docker_path = raw_uri.replace('gs://', 'gs/', 1)
  return raw_uri, docker_path


Rewrite local file URIs as required by the rewrite_uris method .

def _local_uri_rewriter(raw_uri):
  """Rewrite local file URIs as required by the rewrite_uris method.

  Local file paths, unlike GCS paths, may have their raw URI simplified by
  os.path.normpath which collapses extraneous indirect characters.

  >>> _local_uri_rewriter('/tmp/a_path/../B_PATH/file.txt')
  ('/tmp/B_PATH/file.txt', 'file/tmp/B_PATH/file.txt')
  >>> _local_uri_rewriter('/myhome/./mydir/')
  ('/myhome/mydir/', 'file/myhome/mydir/')

  The local path rewriter will also work to preserve relative paths even
  when creating the docker path. This prevents leaking of information on the
  invoker's system to the remote system. Doing this requires a number of path
  substitutions denoted with the _<rewrite>_ convention.

  >>> _local_uri_rewriter('./../upper_dir/')[1]
  'file/_dotdot_/upper_dir/'
  >>> _local_uri_rewriter('~/localdata/*.bam')[1]
  'file/_home_/localdata/*.bam'

  Args:
    raw_uri: (str) the raw file or directory path.

  Returns:
    normalized: a simplified and/or expanded version of the uri.
    docker_path: the uri rewritten in the format required for mounting inside
                 a docker worker.

  """
  # The path is split into components so that the filename is not rewritten.
  raw_path, filename = os.path.split(raw_uri)
  # Generate the local path that can be resolved by filesystem operations,
  # this removes special shell characters, condenses indirects and replaces
  # any unnecessary prefix.
  prefix_replacements = [('file:///', '/'), ('~/', os.getenv('HOME')), ('./',
                                                                        ''),
                         ('file:/', '/')]
  normed_path = raw_path
  for prefix, replacement in prefix_replacements:
    if normed_path.startswith(prefix):
      normed_path = os.path.join(replacement, normed_path[len(prefix):])
  # Because abspath strips the trailing '/' from bare directory references
  # other than root, this ensures that all directory references end with '/'.
  normed_uri = directory_fmt(os.path.abspath(normed_path))
  normed_uri = os.path.join(normed_uri, filename)

  # Generate the path used inside the docker image;
  #  1) Get rid of extra indirects: /this/./that -> /this/that
  #  2) Rewrite required indirects as synthetic characters.
  #  3) Strip relative or absolute path leading character.
  #  4) Add 'file/' prefix.
  docker_rewrites = [(r'/\.\.', '/_dotdot_'), (r'^\.\.', '_dotdot_'),
                     (r'^~/', '_home_/'), (r'^file:/', '')]
  docker_path = os.path.normpath(raw_path)
  for pattern, replacement in docker_rewrites:
    docker_path = re.sub(pattern, replacement, docker_path)
  docker_path = docker_path.lstrip('./')  # Strips any of '.' './' '/'.
  docker_path = directory_fmt('file/' + docker_path) + filename
  return normed_uri, docker_path


def get_gcs_mounts(mounts):
  """Returns the GCS mounts from mounts."""
  return _get_filtered_mounts(mounts, job_model.GCSMountParam)

def get_gcs_mounts ( mounts ) : return _get_filtered_mounts ( mounts , job_model . GCSMountParam )


Returns the persistent disk mounts from mounts .

def get_persistent_disk_mounts(mounts):
  """Returns the persistent disk mounts from mounts."""
  return _get_filtered_mounts(mounts, job_model.PersistentDiskMountParam)

def get_persistent_disk_mounts ( mounts ) : return _get_filtered_mounts ( mounts , job_model . PersistentDiskMountParam )


def get_local_mounts(mounts):
  """Returns the local mounts from mounts."""
  return _get_filtered_mounts(mounts, job_model.LocalMountParam)

def get_local_mounts ( mounts ) : return _get_filtered_mounts ( mounts , job_model . LocalMountParam )


Helper function to return an appropriate set of mount parameters .

def _get_filtered_mounts(mounts, mount_param_type):
  """Helper function to return an appropriate set of mount parameters."""
  return set([mount for mount in mounts if isinstance(mount, mount_param_type)])

def _get_filtered_mounts ( mounts , mount_param_type ) : return set ( [ mount for mount in mounts if isinstance ( mount , mount_param_type ) ] )


Convenience function simplifies construction of the logging uri .

def build_logging_param(logging_uri, util_class=OutputFileParamUtil):
  """Convenience function simplifies construction of the logging uri."""
  if not logging_uri:
    return job_model.LoggingParam(None, None)
  recursive = not logging_uri.endswith('.log')
  oututil = util_class('')
  _, uri, provider = oututil.parse_uri(logging_uri, recursive)
  if '*' in uri.basename:
    raise ValueError('Wildcards not allowed in logging URI: %s' % uri)
  return job_model.LoggingParam(uri, provider)


Split a string into a pair which can have one empty value .

def split_pair(pair_string, separator, nullable_idx=1):
  """Split a string into a pair, which can have one empty value.

  Args:
    pair_string: The string to be split.
    separator: The separator to be used for splitting.
    nullable_idx: The location to be set to null if the separator is not in the
                  input string. Should be either 0 or 1.

  Returns:
    A list containing the pair.

  Raises:
    IndexError: If nullable_idx is not 0 or 1.
  """

  pair = pair_string.split(separator, 1)
  if len(pair) == 1:
    if nullable_idx == 0:
      return [None, pair[0]]
    elif nullable_idx == 1:
      return [pair[0], None]
    else:
      raise IndexError('nullable_idx should be either 0 or 1.')
  else:
    return pair

def
split_pair
(
pair_string
,
separator
,
nullable_idx
=
1
)
:
pair
=
pair_string
.
split
(
separator
,
1
)
if
len
(
pair
)
==
1
:
if
nullable_idx
==
0
:
return
[
,
pair
[
0
]
]
elif
nullable_idx
==
1
:
return
[
pair
[
0
]
,
]
else
:
raise
IndexError
(
'nullable_idx should be either 0 or 1.'
)
else
:
return
pair


Parse the header from the tasks file into env input output definitions .

def parse_tasks_file_header(header, input_file_param_util,
                            output_file_param_util):
  """Parse the header from the tasks file into env, input, output definitions.

  Elements are formatted similar to their equivalent command-line arguments,
  but with associated values coming from the data rows.

  Environment variables columns are headered as "--env <name>"
  Inputs columns are headered as "--input <name>" with the name optional.
  Outputs columns are headered as "--output <name>" with the name optional.

  For historical reasons, bareword column headers (such as "JOB_ID") are
  equivalent to "--env var_name".

  Args:
    header: Array of header fields
    input_file_param_util: Utility for producing InputFileParam objects.
    output_file_param_util: Utility for producing OutputFileParam objects.

  Returns:
    job_params: A list of EnvParams and FileParams for the environment
    variables, LabelParams, input file parameters, and output file parameters.

  Raises:
    ValueError: If a header contains a ":" and the prefix is not supported.
  """
  job_params = []

  for col in header:

    # Reserve the "-" and "--" namespace.
    # If the column has no leading "-", treat it as an environment variable
    col_type = '--env'
    col_value = col
    if col.startswith('-'):
      col_type, col_value = split_pair(col, ' ', 1)

    if col_type == '--env':
      job_params.append(job_model.EnvParam(col_value))

    elif col_type == '--label':
      job_params.append(job_model.LabelParam(col_value))

    elif col_type == '--input' or col_type == '--input-recursive':
      name = input_file_param_util.get_variable_name(col_value)
      job_params.append(
          job_model.InputFileParam(
              name, recursive=(col_type.endswith('recursive'))))

    elif col_type == '--output' or col_type == '--output-recursive':
      name = output_file_param_util.get_variable_name(col_value)
      job_params.append(
          job_model.OutputFileParam(
              name, recursive=(col_type.endswith('recursive'))))

    else:
      raise ValueError('Unrecognized column header: %s' % col)

  return job_params


def tasks_file_to_task_descriptors(tasks, retries, input_file_param_util,
                                   output_file_param_util):
  """Parses task parameters from a TSV.

  Args:
    tasks: Dict containing the path to a TSV file and task numbers to run
    variables, input, and output parameters as column headings. Subsequent
    lines specify parameter values, one row per job.
    retries: Number of retries allowed.
    input_file_param_util: Utility for producing InputFileParam objects.
    output_file_param_util: Utility for producing OutputFileParam objects.

  Returns:
    task_descriptors: an array of records, each containing the task-id,
    task-attempt, 'envs', 'inputs', 'outputs', 'labels' that defines the set of
    parameters for each task of the job.

  Raises:
    ValueError: If no job records were provided
  """
  task_descriptors = []

  path = tasks['path']
  task_min = tasks.get('min')
  task_max = tasks.get('max')

  # First check for any empty lines
  param_file = dsub_util.load_file(path)
  param_file_lines = param_file.splitlines()
  if any([not line for line in param_file_lines]):
    raise ValueError('Blank line(s) found in {}'.format(path))

  # Set up a Reader that tokenizes the fields
  reader = csv.reader(param_file_lines, delimiter='\t')

  # Read the first line and extract the parameters
  header = next(reader)
  job_params = parse_tasks_file_header(header, input_file_param_util,
                                       output_file_param_util)

  # Build a list of records from the parsed input file
  for row in reader:
    # Tasks are numbered starting at 1 and since the first line of the TSV
    # file is a header, the first task appears on line 2.
    task_id = reader.line_num - 1
    if task_min and task_id < task_min:
      continue
    if task_max and task_id > task_max:
      continue

    if len(row) != len(job_params):
      raise ValueError(
          'Unexpected number of fields {} vs {}: in {} line {}'.format(
              len(row), len(job_params), path, reader.line_num))

    # Each row can contain "envs", "inputs", "outputs"
    envs = set()
    inputs = set()
    outputs = set()
    labels = set()

    for i in range(0, len(job_params)):
      param = job_params[i]
      name = param.name
      if isinstance(param, job_model.EnvParam):
        envs.add(job_model.EnvParam(name, row[i]))

      elif isinstance(param, job_model.LabelParam):
        labels.add(job_model.LabelParam(name, row[i]))

      elif isinstance(param, job_model.InputFileParam):
        inputs.add(
            input_file_param_util.make_param(name, row[i], param.recursive))

      elif isinstance(param, job_model.OutputFileParam):
        outputs.add(
            output_file_param_util.make_param(name, row[i], param.recursive))

    task_descriptors.append(
        job_model.TaskDescriptor({
            'task-id': task_id,
            'task-attempt': 1 if retries else None
        }, {
            'labels': labels,
            'envs': envs,
            'inputs': inputs,
            'outputs': outputs
        }, job_model.Resources()))

  # Ensure that there are jobs to execute (and not just a header)
  if not task_descriptors:
    raise ValueError('No tasks added from %s' % path)

  return task_descriptors


Parse flags of key = value pairs and return a list of argclass .

def parse_pair_args(labels, argclass):
  """Parse flags of key=value pairs and return a list of argclass.

  For pair variables, we need to:
     * split the input into name=value pairs (value optional)
     * Create the EnvParam object

  Args:
    labels: list of 'key' or 'key=value' strings.
    argclass: Container class for args, must instantiate with argclass(k, v).

  Returns:
    list of argclass objects.
  """
  label_data = set()
  for arg in labels:
    name, value = split_pair(arg, '=', nullable_idx=1)
    label_data.add(argclass(name, value))
  return label_data

def parse_pair_args ( labels , argclass ) : label_data = set ( ) for arg in labels : name , value = split_pair ( arg , '=' , nullable_idx = 1 ) label_data . add ( argclass ( name , value ) ) return label_data


Parse env input and output parameters into a job parameters and data .

def args_to_job_params(envs, labels, inputs, inputs_recursive, outputs,
                       outputs_recursive, mounts, input_file_param_util,
                       output_file_param_util, mount_param_util):
  """Parse env, input, and output parameters into a job parameters and data.

  Passing arguments on the command-line allows for launching a single job.
  The env, input, and output arguments encode both the definition of the
  job as well as the single job's values.

  Env arguments are simple name=value pairs.
  Input and output file arguments can contain name=value pairs or just values.
  Either of the following is valid:

    uri
    myfile=uri

  Args:
    envs: list of environment variable job parameters
    labels: list of labels to attach to the tasks
    inputs: list of file input parameters
    inputs_recursive: list of recursive directory input parameters
    outputs: list of file output parameters
    outputs_recursive: list of recursive directory output parameters
    mounts: list of gcs buckets to mount
    input_file_param_util: Utility for producing InputFileParam objects.
    output_file_param_util: Utility for producing OutputFileParam objects.
    mount_param_util: Utility for producing MountParam objects.

  Returns:
    job_params: a dictionary of 'envs', 'inputs', and 'outputs' that defines the
    set of parameters and data for a job.
  """
  # Parse environmental variables and labels.
  env_data = parse_pair_args(envs, job_model.EnvParam)
  label_data = parse_pair_args(labels, job_model.LabelParam)

  # For input files, we need to:
  #   * split the input into name=uri pairs (name optional)
  #   * get the environmental variable name, or automatically set if null.
  #   * create the input file param
  input_data = set()
  for (recursive, args) in ((False, inputs), (True, inputs_recursive)):
    for arg in args:
      name, value = split_pair(arg, '=', nullable_idx=0)
      name = input_file_param_util.get_variable_name(name)
      input_data.add(input_file_param_util.make_param(name, value, recursive))

  # For output files, we need to:
  #   * split the input into name=uri pairs (name optional)
  #   * get the environmental variable name, or automatically set if null.
  #   * create the output file param
  output_data = set()
  for (recursive, args) in ((False, outputs), (True, outputs_recursive)):
    for arg in args:
      name, value = split_pair(arg, '=', 0)
      name = output_file_param_util.get_variable_name(name)
      output_data.add(output_file_param_util.make_param(name, value, recursive))

  mount_data = set()
  for arg in mounts:
    # Mounts can look like `--mount VAR=PATH` or `--mount VAR=PATH {num}`,
    # where num is the size of the disk in Gb. We assume a space is the
    # separator between path and disk size.
    if ' ' in arg:
      key_value_pair, disk_size = arg.split(' ')
      name, value = split_pair(key_value_pair, '=', 1)
      mount_data.add(mount_param_util.make_param(name, value, disk_size))
    else:
      name, value = split_pair(arg, '=', 1)
      mount_data.add(mount_param_util.make_param(name, value, disk_size=None))
  return {
      'envs': env_data,
      'inputs': input_data,
      'outputs': output_data,
      'labels': label_data,
      'mounts': mount_data,
  }


Validate that arguments passed to submit_job have valid file providers .

def validate_submit_args_or_fail(job_descriptor, provider_name, input_providers,
                                 output_providers, logging_providers):
  """Validate that arguments passed to submit_job have valid file providers.

  This utility function takes resources and task data args from `submit_job`
  in the base provider. This function will fail with a value error if any of the
  parameters are not valid. See the following example;

  >>> job_resources = type('', (object,),
  ...    {"logging": job_model.LoggingParam('gs://logtemp', job_model.P_GCS)})()
  >>> job_params={'inputs': set(), 'outputs': set(), 'mounts': set()}
  >>> task_descriptors = [
  ...     job_model.TaskDescriptor(None, {
  ...       'inputs': {
  ...           job_model.FileParam('IN', uri='gs://in/*',
  ...                               file_provider=job_model.P_GCS)},
  ...       'outputs': set()}, None),
  ...     job_model.TaskDescriptor(None, {
  ...       'inputs': set(),
  ...       'outputs': {
  ...           job_model.FileParam('OUT', uri='gs://out/*',
  ...                               file_provider=job_model.P_GCS)}}, None)]
  ...
  >>> validate_submit_args_or_fail(job_model.JobDescriptor(None, job_params,
  ...                              job_resources, task_descriptors),
  ...                              provider_name='MYPROVIDER',
  ...                              input_providers=[job_model.P_GCS],
  ...                              output_providers=[job_model.P_GCS],
  ...                              logging_providers=[job_model.P_GCS])
  ...
  >>> validate_submit_args_or_fail(job_model.JobDescriptor(None, job_params,
  ...                              job_resources, task_descriptors),
  ...                              provider_name='MYPROVIDER',
  ...                              input_providers=[job_model.P_GCS],
  ...                              output_providers=[job_model.P_LOCAL],
  ...                              logging_providers=[job_model.P_GCS])
  Traceback (most recent call last):
       ...
  ValueError: Unsupported output path (gs://out/*) for provider 'MYPROVIDER'.

  Args:
    job_descriptor: instance of job_model.JobDescriptor.
    provider_name: (str) the name of the execution provider.
    input_providers: (string collection) whitelist of file providers for input.
    output_providers: (string collection) whitelist of providers for output.
    logging_providers: (string collection) whitelist of providers for logging.

  Raises:
    ValueError: if any file providers do not match the whitelists.
  """
  job_resources = job_descriptor.job_resources
  job_params = job_descriptor.job_params
  task_descriptors = job_descriptor.task_descriptors

  # Validate logging file provider.
  _validate_providers([job_resources.logging], 'logging', logging_providers,
                      provider_name)

  # Validate job input and output file providers
  _validate_providers(job_params['inputs'], 'input', input_providers,
                      provider_name)
  _validate_providers(job_params['outputs'], 'output', output_providers,
                      provider_name)

  # Validate input and output file providers.
  for task_descriptor in task_descriptors:
    _validate_providers(task_descriptor.task_params['inputs'], 'input',
                        input_providers, provider_name)
    _validate_providers(task_descriptor.task_params['outputs'], 'output',
                        output_providers, provider_name)


def directory_fmt(directory):
  """In ensure that directories end with '/'.

  Frequently we need to ensure that directory paths end with a forward slash.
  Pythons dirname and split functions in the path library treat this
  inconsistently creating this requirement. This function is simple but was
  written to centralize documentation of an often used (and often explained)
  requirement in this codebase.

  >>> os.path.dirname('gs://bucket/folder/file.txt')
  'gs://bucket/folder'
  >>> directory_fmt(os.path.dirname('gs://bucket/folder/file.txt'))
  'gs://bucket/folder/'
  >>> os.path.dirname('/newfile')
  '/'
  >>> directory_fmt(os.path.dirname('/newfile'))
  '/'

  Specifically we need this since copy commands must know whether the
  destination is a directory to function properly. See the following shell
  interaction for an example of the inconsistency. Notice that text files are
  copied as expected but the bam is copied over the directory name.

  Multiple files copy, works as intended in all cases:
      $ touch a.txt b.txt
      $ gsutil cp ./*.txt gs://mybucket/text_dest
      $ gsutil ls gs://mybucket/text_dest/
            0  2017-07-19T21:44:36Z  gs://mybucket/text_dest/a.txt
            0  2017-07-19T21:44:36Z  gs://mybucket/text_dest/b.txt
      TOTAL: 2 objects, 0 bytes (0 B)

  Single file copy fails to copy into a directory:
      $ touch 1.bam
      $ gsutil cp ./*.bam gs://mybucket/bad_dest
      $ gsutil ls gs://mybucket/bad_dest
               0  2017-07-19T21:46:16Z  gs://mybucket/bad_dest
      TOTAL: 1 objects, 0 bytes (0 B)

  Adding a trailing forward slash fixes this:
      $ touch my.sam
      $ gsutil cp ./*.sam gs://mybucket/good_folder
      $ gsutil ls gs://mybucket/good_folder
               0  2017-07-19T21:46:16Z  gs://mybucket/good_folder/my.sam
      TOTAL: 1 objects, 0 bytes (0 B)

  Args:
    directory (str): a uri without an blob or file basename.

  Returns:
    the directory with a trailing slash.
  """
  return directory.rstrip('/') + '/'

def directory_fmt ( directory ) : return directory . rstrip ( '/' ) + '/'


If the -- version flag is passed print version to stdout and exit .

def handle_version_flag():
  """If the --version flag is passed, print version to stdout and exit.

  Within dsub commands, --version should be the highest priority flag.
  This function supplies a repeatable and DRY way of checking for the
  version flag and printing the version. Callers still need to define a version
  flag in the command's flags so that it shows up in help output.
  """
  parser = argparse.ArgumentParser(description='Version parser', add_help=False)
  parser.add_argument('--version', '-v', dest='version', action='store_true')
  parser.set_defaults(version=False)
  args, _ = parser.parse_known_args()
  if args.version:
    print('dsub version: %s' % DSUB_VERSION)
    sys.exit()

def
handle_version_flag
(
)
:
parser
=
argparse
.
ArgumentParser
(
description
=
'Version parser'
,
add_help
=
False
)
parser
.
add_argument
(
'--version'
,
'-v'
,
dest
=
'version'
,
action
=
'store_true'
)
parser
.
set_defaults
(
version
=
False
)
args
,
_
=
parser
.
parse_known_args
(
)
if
args
.
version
:
print
(
'dsub version: %s'
%
DSUB_VERSION
)
sys
.
exit
(
)


Compute the create time ( UTC ) for the list filter .

def age_to_create_time(age, from_time=None):
  """Compute the create time (UTC) for the list filter.

  If the age is an integer value it is treated as a UTC date.
  Otherwise the value must be of the form "<integer><unit>" where supported
  units are s, m, h, d, w (seconds, minutes, hours, days, weeks).

  Args:
    age: A "<integer><unit>" string or integer value.
    from_time:

  Returns:
    A timezone-aware datetime or None if age parameter is empty.
  """

  if not age:
    return None

  if not from_time:
    from_time = dsub_util.replace_timezone(datetime.datetime.now(), tzlocal())

  try:
    last_char = age[-1]

    if last_char == 's':
      return from_time - datetime.timedelta(seconds=int(age[:-1]))
    elif last_char == 'm':
      return from_time - datetime.timedelta(minutes=int(age[:-1]))
    elif last_char == 'h':
      return from_time - datetime.timedelta(hours=int(age[:-1]))
    elif last_char == 'd':
      return from_time - datetime.timedelta(days=int(age[:-1]))
    elif last_char == 'w':
      return from_time - datetime.timedelta(weeks=int(age[:-1]))
    else:
      # If no unit is given treat the age as seconds from epoch, otherwise apply
      # the correct time unit.
      return dsub_util.replace_timezone(
          datetime.datetime.utcfromtimestamp(int(age)), pytz.utc)

  except (ValueError, OverflowError) as e:
    raise ValueError('Unable to parse age string %s: %s' % (age, e))


Convert the timeout duration to seconds .

def _interval_to_seconds(interval, valid_units='smhdw'):
  """Convert the timeout duration to seconds.

  The value must be of the form "<integer><unit>" where supported
  units are s, m, h, d, w (seconds, minutes, hours, days, weeks).

  Args:
    interval: A "<integer><unit>" string.
    valid_units: A list of supported units.

  Returns:
    A string of the form "<integer>s" or None if timeout is empty.
  """
  if not interval:
    return None

  try:
    last_char = interval[-1]

    if last_char == 's' and 's' in valid_units:
      return str(float(interval[:-1])) + 's'
    elif last_char == 'm' and 'm' in valid_units:
      return str(float(interval[:-1]) * 60) + 's'
    elif last_char == 'h' and 'h' in valid_units:
      return str(float(interval[:-1]) * 60 * 60) + 's'
    elif last_char == 'd' and 'd' in valid_units:
      return str(float(interval[:-1]) * 60 * 60 * 24) + 's'
    elif last_char == 'w' and 'w' in valid_units:
      return str(float(interval[:-1]) * 60 * 60 * 24 * 7) + 's'
    else:
      raise ValueError(
          'Unsupported units in interval string %s: %s' % (interval, last_char))

  except (ValueError, OverflowError) as e:
    raise ValueError('Unable to parse interval string %s: %s' % (interval, e))


Wrapper function to create a PreemptibleParam object from argparse .

def preemptile_param_type(preemptible):
  """Wrapper function to create a PreemptibleParam object from argparse."""
  if bool is type(preemptible):
    return PreemptibleParam(preemptible)
  elif str is type(preemptible):
    try:
      return PreemptibleParam(int(preemptible))
    except ValueError:
      raise argparse.ArgumentTypeError(
          'Invalid value {} for --preemptible.'.format(preemptible))
  else:
    raise argparse.ArgumentTypeError(
        'Invalid value {} for --preemptible.'.format(preemptible))

def
preemptile_param_type
(
preemptible
)
:
if
bool
is
type
(
preemptible
)
:
return
PreemptibleParam
(
preemptible
)
elif
str
is
type
(
preemptible
)
:
try
:
return
PreemptibleParam
(
int
(
preemptible
)
)
except
ValueError
:
raise
argparse
.
ArgumentTypeError
(
'Invalid value {} for --preemptible.'
.
format
(
preemptible
)
)
else
:
raise
argparse
.
ArgumentTypeError
(
'Invalid value {} for --preemptible.'
.
format
(
preemptible
)
)


Produce a default variable name if none is specified .

def get_variable_name(self, name):
    """Produce a default variable name if none is specified."""
    if not name:
      name = '%s%s' % (self._auto_prefix, self._auto_index)
      self._auto_index += 1
    return name

def get_variable_name ( self , name ) : if not name : name = '%s%s' % ( self . _auto_prefix , self . _auto_index ) self . _auto_index += 1 return name


Accept a raw uri and return rewritten versions .

def rewrite_uris(self, raw_uri, file_provider):
    """Accept a raw uri and return rewritten versions.

    This function returns a normalized URI and a docker path. The normalized
    URI may have minor alterations meant to disambiguate and prepare for use
    by shell utilities that may require a specific format.

    The docker rewriter makes substantial modifications to the raw URI when
    constructing a docker path, but modifications must follow these rules:
      1) System specific characters are not allowed (ex. indirect paths).
      2) The path, if it is a directory, must end in a forward slash.
      3) The path will begin with the value set in self._relative_path.
      4) The path will have an additional prefix (after self._relative_path) set
         by the file provider-specific rewriter.

    Rewrite output for the docker path:
      >>> out_util = FileParamUtil('AUTO_', 'output')
      >>> out_util.rewrite_uris('gs://mybucket/myfile.txt', job_model.P_GCS)[1]
      'output/gs/mybucket/myfile.txt'
      >>> out_util.rewrite_uris('./data/myfolder/', job_model.P_LOCAL)[1]
      'output/file/data/myfolder/'

    When normalizing the URI for cloud buckets, no rewrites are done. For local
    files, the user directory will be expanded and relative paths will be
    converted to absolute:
      >>> in_util = FileParamUtil('AUTO_', 'input')
      >>> in_util.rewrite_uris('gs://mybucket/gcs_dir/', job_model.P_GCS)[0]
      'gs://mybucket/gcs_dir/'
      >>> in_util.rewrite_uris('/data/./dir_a/../myfile.txt',
      ...   job_model.P_LOCAL)[0]
      '/data/myfile.txt'
      >>> in_util.rewrite_uris('file:///tmp/data/*.bam', job_model.P_LOCAL)[0]
      '/tmp/data/*.bam'

    Args:
      raw_uri: (str) the path component of the raw URI.
      file_provider: a valid provider (contained in job_model.FILE_PROVIDERS).

    Returns:
      normalized: a cleaned version of the uri provided by command line.
      docker_path: the uri rewritten in the format required for mounting inside
                   a docker worker.

    Raises:
      ValueError: if file_provider is not valid.
    """
    if file_provider == job_model.P_GCS:
      normalized, docker_path = _gcs_uri_rewriter(raw_uri)
    elif file_provider == job_model.P_LOCAL:
      normalized, docker_path = _local_uri_rewriter(raw_uri)
    else:
      raise ValueError('File provider not supported: %r' % file_provider)
    return normalized, os.path.join(self._relative_path, docker_path)

def
rewrite_uris
(
self
,
raw_uri
,
file_provider
)
:
if
file_provider
==
job_model
.
P_GCS
:
normalized
,
docker_path
=
_gcs_uri_rewriter
(
raw_uri
)
elif
file_provider
==
job_model
.
P_LOCAL
:
normalized
,
docker_path
=
_local_uri_rewriter
(
raw_uri
)
else
:
raise
ValueError
(
'File provider not supported: %r'
%
file_provider
)
return
normalized
,
os
.
path
.
join
(
self
.
_relative_path
,
docker_path
)


def parse_file_provider(uri):
    """Find the file provider for a URI."""
    providers = {'gs': job_model.P_GCS, 'file': job_model.P_LOCAL}
    # URI scheme detector uses a range up to 30 since none of the IANA
    # registered schemes are longer than this.
    provider_found = re.match(r'^([A-Za-z][A-Za-z0-9+.-]{0,29})://', uri)
    if provider_found:
      prefix = provider_found.group(1).lower()
    else:
      # If no provider is specified in the URI, assume that the local
      # filesystem is being used. Availability and validity of the local
      # file/directory will be checked later.
      prefix = 'file'
    if prefix in providers:
      return providers[prefix]
    else:
      raise ValueError('File prefix not supported: %s://' % prefix)


Do basic validation of the uri return the path and filename .

def _validate_paths_or_fail(uri, recursive):
    """Do basic validation of the uri, return the path and filename."""
    path, filename = os.path.split(uri)

    # dsub could support character ranges ([0-9]) with some more work, but for
    # now we assume that basic asterisk wildcards are sufficient. Reject any URI
    # that includes square brackets or question marks, since we know that
    # if they actually worked, it would be accidental.
    if '[' in uri or ']' in uri:
      raise ValueError(
          'Square bracket (character ranges) are not supported: %s' % uri)
    if '?' in uri:
      raise ValueError('Question mark wildcards are not supported: %s' % uri)

    # Only support file URIs and *filename* wildcards
    # Wildcards at the directory level or "**" syntax would require better
    # support from the Pipelines API *or* doing expansion here and
    # (potentially) producing a series of FileParams, instead of one.
    if '*' in path:
      raise ValueError(
          'Path wildcard (*) are only supported for files: %s' % uri)
    if '**' in filename:
      raise ValueError('Recursive wildcards ("**") not supported: %s' % uri)
    if filename in ('..', '.'):
      raise ValueError('Path characters ".." and "." not supported '
                       'for file names: %s' % uri)

    # Do not allow non-recursive IO to reference directories.
    if not recursive and not filename:
      raise ValueError('Input or output values that are not recursive must '
                       'reference a filename or wildcard: %s' % uri)


Return a valid docker_path uri and file provider from a flag value .

def parse_uri(self, raw_uri, recursive):
    """Return a valid docker_path, uri, and file provider from a flag value."""
    # Assume recursive URIs are directory paths.
    if recursive:
      raw_uri = directory_fmt(raw_uri)
    # Get the file provider, validate the raw URI, and rewrite the path
    # component of the URI for docker and remote.
    file_provider = self.parse_file_provider(raw_uri)
    self._validate_paths_or_fail(raw_uri, recursive)
    uri, docker_uri = self.rewrite_uris(raw_uri, file_provider)
    uri_parts = job_model.UriParts(
        directory_fmt(os.path.dirname(uri)), os.path.basename(uri))
    return docker_uri, uri_parts, file_provider


Return a * FileParam given an input uri .

def make_param(self, name, raw_uri, recursive):
    """Return a *FileParam given an input uri."""
    if not raw_uri:
      return self.param_class(name, None, None, None, recursive, None)
    docker_path, uri_parts, provider = self.parse_uri(raw_uri, recursive)
    return self.param_class(name, raw_uri, docker_path, uri_parts, recursive,
                            provider)

def make_param ( self , name , raw_uri , recursive ) : if not raw_uri : return self . param_class ( name , None , None , None , recursive , None ) docker_path , uri_parts , provider = self . parse_uri ( raw_uri , recursive ) return self . param_class ( name , raw_uri , docker_path , uri_parts , recursive , provider )


Return a valid docker_path from a Google Persistent Disk url .

def _parse_image_uri(self, raw_uri):
    """Return a valid docker_path from a Google Persistent Disk url."""
    # The string replace is so we don't have colons and double slashes in the
    # mount path. The idea is the resulting mount path would look like:
    # /mnt/data/mount/http/www.googleapis.com/compute/v1/projects/...
    docker_uri = os.path.join(self._relative_path,
                              raw_uri.replace('https://', 'https/', 1))
    return docker_uri


Return a valid docker_path for a local file path .

def _parse_local_mount_uri(self, raw_uri):
    """Return a valid docker_path for a local file path."""
    raw_uri = directory_fmt(raw_uri)
    _, docker_path = _local_uri_rewriter(raw_uri)
    local_path = docker_path[len('file'):]
    docker_uri = os.path.join(self._relative_path, docker_path)
    return local_path, docker_uri

def _parse_local_mount_uri ( self , raw_uri ) : raw_uri = directory_fmt ( raw_uri ) _ , docker_path = _local_uri_rewriter ( raw_uri ) local_path = docker_path [ len ( 'file' ) : ] docker_uri = os . path . join ( self . _relative_path , docker_path ) return local_path , docker_uri


Return a valid docker_path for a GCS bucket .

def _parse_gcs_uri(self, raw_uri):
    """Return a valid docker_path for a GCS bucket."""
    # Assume URI is a directory path.
    raw_uri = directory_fmt(raw_uri)
    _, docker_path = _gcs_uri_rewriter(raw_uri)
    docker_uri = os.path.join(self._relative_path, docker_path)
    return docker_uri

def
_parse_gcs_uri
(
self
,
raw_uri
)
:
# Assume URI is a directory path.
raw_uri
=
directory_fmt
(
raw_uri
)
_
,
docker_path
=
_gcs_uri_rewriter
(
raw_uri
)
docker_uri
=
os
.
path
.
join
(
self
.
_relative_path
,
docker_path
)
return
docker_uri


Return a MountParam given a GCS bucket disk image or local path .

def make_param(self, name, raw_uri, disk_size):
    """Return a MountParam given a GCS bucket, disk image or local path."""
    if raw_uri.startswith('https://www.googleapis.com/compute'):
      # Full Image URI should look something like:
      # https://www.googleapis.com/compute/v1/projects/<project>/global/images/
      # But don't validate further, should the form of a valid image URI
      # change (v1->v2, for example)
      docker_path = self._parse_image_uri(raw_uri)
      return job_model.PersistentDiskMountParam(
          name, raw_uri, docker_path, disk_size, disk_type=None)
    elif raw_uri.startswith('file://'):
      local_path, docker_path = self._parse_local_mount_uri(raw_uri)
      return job_model.LocalMountParam(name, raw_uri, docker_path, local_path)
    elif raw_uri.startswith('gs://'):
      docker_path = self._parse_gcs_uri(raw_uri)
      return job_model.GCSMountParam(name, raw_uri, docker_path)
    else:
      raise ValueError(
          'Mount parameter {} must begin with valid prefix.'.format(raw_uri))


Validates that preemptible arguments make sense with retries .

def validate(self, retries):
    """Validates that preemptible arguments make sense with retries."""
    if int is type(self._max_preemptible_attempts):
      if retries < 0 or self._max_preemptible_attempts < 0:
        raise ValueError('--retries and --preemptible may not be negative')

      if self._max_preemptible_attempts >= 1 and not retries:
        # This means user specified a positive preemptible number
        # but didn't specify a retries number
        raise ValueError(
            'Requesting 1 or more preemptible attempts requires setting retries'
        )

      if self._max_preemptible_attempts > retries:
        raise ValueError(
            'Value passed for --preemptible cannot be larger than --retries.')


def get_credentials(args):
  """Returns credentials for API requests."""

  # Across dsub, dstat, ddel, defer to the provider for credentials handling
  return provider_base.credentials_from_args(args)


def _parse_arguments():
  """Parses command line arguments.

  Returns:
    A Namespace of parsed arguments.
  """
  # Handle version flag and exit if it was passed.
  param_util.handle_version_flag()

  parser = provider_base.create_parser(sys.argv[0])

  parser.add_argument(
      '--version', '-v', default=False, help='Print the dsub version and exit.')

  parser.add_argument(
      '--jobs',
      '-j',
      nargs='*',
      help='A list of jobs IDs on which to check status')
  parser.add_argument(
      '--names',
      '-n',
      nargs='*',
      help='A list of job names on which to check status')
  parser.add_argument(
      '--tasks',
      '-t',
      nargs='*',
      help='A list of task IDs on which to check status')
  parser.add_argument(
      '--attempts',
      nargs='*',
      help='A list of task attempts on which to check status')
  parser.add_argument(
      '--users',
      '-u',
      nargs='*',
      default=[],
      help="""Lists only those jobs which were submitted by the list of users.
          Use "*" to list jobs of any user.""")
  parser.add_argument(
      '--status',
      '-s',
      nargs='*',
      default=['RUNNING'],
      choices=['RUNNING', 'SUCCESS', 'FAILURE', 'CANCELED', '*'],
      help="""Lists only those jobs which match the specified status(es).
          Choose from {'RUNNING', 'SUCCESS', 'FAILURE', 'CANCELED'}.
          Use "*" to list jobs of any status.""",
      metavar='STATUS')
  parser.add_argument(
      '--age',
      help="""List only those jobs newer than the specified age. Ages can be
          listed using a number followed by a unit. Supported units are
          s (seconds), m (minutes), h (hours), d (days), w (weeks).
          For example: '7d' (7 days). Bare numbers are treated as UTC.""")
  parser.add_argument(
      '--label',
      nargs='*',
      action=param_util.ListParamAction,
      default=[],
      help='User labels to match. Tasks returned must match all labels.',
      metavar='KEY=VALUE')
  parser.add_argument(
      '--poll-interval',
      default=10,
      type=int,
      help='Polling interval (in seconds) for checking job status '
      'when --wait is set.')
  parser.add_argument(
      '--wait', action='store_true', help='Wait until jobs have all completed.')
  parser.add_argument(
      '--limit',
      default=0,
      type=int,
      help='The maximum number of tasks to list. The default is unlimited.')
  parser.add_argument(
      '--format',
      choices=['text', 'json', 'yaml', 'provider-json'],
      help='Set the output format.')
  output_style = parser.add_mutually_exclusive_group()
  output_style.add_argument(
      '--full',
      '-f',
      action='store_true',
      help='Display output with full task information'
      ' and input parameters.')
  output_style.add_argument(
      '--summary',
      action='store_true',
      help='Display a summary of the results, grouped by (job, status).')

  # Shared between the "google-cls-v2" and "google-v2" providers
  google_common = parser.add_argument_group(
      title='google-common',
      description="""Options common to the "google", "google-cls-v2", and
        "google-v2" providers""")
  google_common.add_argument(
      '--project', help='Cloud project ID in which to find and the job(s)')

  google_cls_v2 = parser.add_argument_group(
      title='"google-cls-v2" provider options',
      description='See also the "google-common" options listed')
  google_cls_v2.add_argument(
      '--location',
      default=job_model.DEFAULT_LOCATION,
      help="""Specifies the Google Cloud region to which the dsub job was
        submitted. (default: {})""".format(job_model.DEFAULT_LOCATION))

  return provider_base.parse_args(
      parser, {
          'google-cls-v2': ['project'],
          'google-v2': ['project'],
          'test-fails': [],
          'local': [],
      }, sys.argv[1:])


Generate jobs as lists of task dicts ready for formatting / output .

def dstat_job_producer(provider,
                       statuses,
                       user_ids=None,
                       job_ids=None,
                       job_names=None,
                       task_ids=None,
                       task_attempts=None,
                       labels=None,
                       create_time_min=None,
                       create_time_max=None,
                       max_tasks=0,
                       full_output=False,
                       summary_output=False,
                       poll_interval=0,
                       raw_format=False):
  """Generate jobs as lists of task dicts ready for formatting/output.

  Args:
    provider: an instantiated dsub provider.
    statuses: a set of status strings that eligible jobs may match.
    user_ids: a set of user strings that eligible jobs may match.
    job_ids: a set of job-id strings eligible jobs may match.
    job_names: a set of job-name strings eligible jobs may match.
    task_ids: a set of task-id strings eligible tasks may match.
    task_attempts: a set of task-attempt strings eligible tasks may match.
    labels: set of LabelParam that all tasks must match.
    create_time_min: a timezone-aware datetime value for the earliest create
                     time of a task, inclusive.
    create_time_max: a timezone-aware datetime value for the most recent create
                     time of a task, inclusive.
    max_tasks: (int) maximum number of tasks to return per dstat job lookup.
    full_output: (bool) return all dsub fields.
    summary_output: (bool) return a summary of the job list.
    poll_interval: (int) wait time between poll events, dstat will poll jobs
                   until all jobs succeed or fail. Set to zero to disable
                   polling and return after the first lookup.
    raw_format: (bool) set True to prevent dsub from normalizing the task dict,
                this defaults to False and should only be set True if a
                provider-specific view of tasks is absolutely required.
                (NB: provider interfaces change over time, no transition path
                will be provided for users depending on this flag).

  Yields:
    lists of task dictionaries - each list representing a dstat poll event.
  """
  some_job_running = True
  while some_job_running:
    # Get a batch of jobs.
    tasks = provider.lookup_job_tasks(
        statuses,
        user_ids=user_ids,
        job_ids=job_ids,
        job_names=job_names,
        task_ids=task_ids,
        task_attempts=task_attempts,
        labels=labels,
        create_time_min=create_time_min,
        create_time_max=create_time_max,
        max_tasks=max_tasks,
        page_size=max_tasks)

    some_job_running = False

    formatted_tasks = []
    for task in tasks:
      if 0 < max_tasks <= len(formatted_tasks):
        break

      # Format tasks as specified.
      if raw_format:
        formatted_tasks.append(task.raw_task_data())
      else:
        formatted_tasks.append(
            output_formatter.prepare_row(task, full_output, summary_output))

      # Determine if any of the jobs are running.
      if task.get_field('task-status') == 'RUNNING':
        some_job_running = True

    # Yield the tasks and determine if the loop should continue.
    yield formatted_tasks
    if poll_interval and some_job_running:
      time.sleep(poll_interval)
    else:
      break


Generate formatted jobs individually in order of create - time .

def lookup_job_tasks(provider,
                     statuses,
                     user_ids=None,
                     job_ids=None,
                     job_names=None,
                     task_ids=None,
                     task_attempts=None,
                     labels=None,
                     create_time_min=None,
                     create_time_max=None,
                     max_tasks=0,
                     page_size=0,
                     summary_output=False):
  """Generate formatted jobs individually, in order of create-time.

  Args:
    provider: an instantiated dsub provider.
    statuses: a set of status strings that eligible jobs may match.
    user_ids: a set of user strings that eligible jobs may match.
    job_ids: a set of job-id strings eligible jobs may match.
    job_names: a set of job-name strings eligible jobs may match.
    task_ids: a set of task-id strings eligible tasks may match.
    task_attempts: a set of task-attempt strings eligible tasks may match.
    labels: set of LabelParam that all tasks must match.
    create_time_min: a timezone-aware datetime value for the earliest create
                     time of a task, inclusive.
    create_time_max: a timezone-aware datetime value for the most recent create
                     time of a task, inclusive.
    max_tasks: (int) maximum number of tasks to return per dstat job lookup.
    page_size: the page size to use for each query to the backend. May be
               ignored by some provider implementations.
    summary_output: (bool) summarize the job list.

  Yields:
    Individual task dictionaries with associated metadata
  """
  tasks_generator = provider.lookup_job_tasks(
      statuses,
      user_ids=user_ids,
      job_ids=job_ids,
      job_names=job_names,
      task_ids=task_ids,
      task_attempts=task_attempts,
      labels=labels,
      create_time_min=create_time_min,
      create_time_max=create_time_max,
      max_tasks=max_tasks,
      page_size=page_size)

  # Yield formatted tasks.
  for task in tasks_generator:
    yield output_formatter.prepare_row(task, True, summary_output)


# Across dsub, dstat, ddel, defer to the provider for credentials handling


If -- use - private - address is enabled ensure the Docker path is for GCR .

def _check_private_address(args):
  """If --use-private-address is enabled, ensure the Docker path is for GCR."""
  if args.use_private_address:
    image = args.image or DEFAULT_IMAGE
    split = image.split('/', 1)
    if len(split) == 1 or not split[0].endswith('gcr.io'):
      raise ValueError(
          '--use-private-address must specify a --image with a gcr.io host')


If -- nvidia - driver - version is set warn that it is ignored .

def _check_nvidia_driver_version(args):
  """If --nvidia-driver-version is set, warn that it is ignored."""
  if args.nvidia_driver_version:
    print('***WARNING: The --nvidia-driver-version flag is deprecated and will '
          'be ignored.')


def _google_cls_v2_parse_arguments(args):
  """Validated google-cls-v2 arguments."""

  # For the google-cls-v2 provider, the addition of the "--location" parameter,
  # along with a default (us-central1), we can just default everything.

  # So we only need to validate that there is not both a region and zone.
  if (args.zones and args.regions):
    raise ValueError('At most one of --regions and --zones may be specified')

  if args.machine_type and (args.min_cores or args.min_ram):
    raise ValueError(
        '--machine-type not supported together with --min-cores or --min-ram.')

  _check_private_address(args)
  _check_nvidia_driver_version(args)


def _google_v2_parse_arguments(args):
  """Validated google-v2 arguments."""
  if (args.zones and args.regions) or (not args.zones and not args.regions):
    raise ValueError('Exactly one of --regions and --zones must be specified')

  if args.machine_type and (args.min_cores or args.min_ram):
    raise ValueError(
        '--machine-type not supported together with --min-cores or --min-ram.')

  _check_private_address(args)
  _check_nvidia_driver_version(args)


def _local_parse_arguments(args):
  """Validated local arguments."""
  if args.user and args.user != dsub_util.get_os_user():
    raise ValueError('If specified, the local provider\'s "--user" flag must '
                     'match the current logged-in user.')