Remove catapult_base from telemetry.

tools/telemetry/catapult_base/dependency_manager/base_config.py

-# Copyright 2015 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import json
-import logging
-import os
-
-from catapult_base import cloud_storage
-from catapult_base.dependency_manager import archive_info
-from catapult_base.dependency_manager import cloud_storage_info
-from catapult_base.dependency_manager import dependency_info
-from catapult_base.dependency_manager import exceptions
-from catapult_base.dependency_manager import local_path_info
-from catapult_base.dependency_manager import uploader
-
-
-class BaseConfig(object):
-  """A basic config class for use with the DependencyManager.
-
-  Initiated with a json file in the following format:
-
-            {  "config_type": "BaseConfig",
-               "dependencies": {
-                 "dep_name1": {
-                   "cloud_storage_base_folder": "base_folder1",
-                   "cloud_storage_bucket": "bucket1",
-                   "file_info": {
-                     "platform1": {
-                        "cloud_storage_hash": "hash_for_platform1",
-                        "download_path": "download_path111",
-                        "version_in_cs": "1.11.1.11."
-                        "local_paths": ["local_path1110", "local_path1111"]
-                      },
-                      "platform2": {
-                        "cloud_storage_hash": "hash_for_platform2",
-                        "download_path": "download_path2",
-                        "local_paths": ["local_path20", "local_path21"]
-                      },
-                      ...
-                   }
-                 },
-                 "dependency_name_2": {
-                    ...
-                 },
-                  ...
-              }
-            }
-
-    Required fields: "dependencies" and "config_type".
-                     Note that config_type must be "BaseConfig"
-
-    Assumptions:
-        "cloud_storage_base_folder" is a top level folder in the given
-          "cloud_storage_bucket" where all of the dependency files are stored
-          at "dependency_name"_"cloud_storage_hash".
-
-        "download_path" and all paths in "local_paths" are relative to the
-          config file's location.
-
-        All or none of the following cloud storage related fields must be
-          included in each platform dictionary:
-          "cloud_storage_hash", "download_path", "cs_remote_path"
-
-        "version_in_cs" is an optional cloud storage field, but is dependent
-          on the above cloud storage related fields.
-
-
-    Also note that platform names are often of the form os_architechture.
-    Ex: "win_AMD64"
-
-    More information on the fields can be found in dependencies_info.py
-  """
-  def __init__(self, file_path, writable=False):
-    """ Initialize a BaseConfig for the DependencyManager.
-
-    Args:
-        writable: False: This config will be used to lookup information.
-                  True: This config will be used to update information.
-
-        file_path: Path to a file containing a json dictionary in the expected
-                   json format for this config class. Base format expected:
-
-                   { "config_type": config_type,
-                     "dependencies": dependencies_dict }
-
-                   config_type: must match the return value of GetConfigType.
-                   dependencies: A dictionary with the information needed to
-                       create dependency_info instances for the given
-                       dependencies.
-
-                   See dependency_info.py for more information.
-    """
-    self._config_path = file_path
-    self._writable = writable
-    self._is_dirty = False
-    self._pending_uploads = []
-    if not self._config_path:
-      raise ValueError('Must supply config file path.')
-    if not os.path.exists(self._config_path):
-      if not writable:
-        raise exceptions.EmptyConfigError(file_path)
-      self._config_data = {}
-      self._WriteConfigToFile(self._config_path, dependencies=self._config_data)
-    else:
-      with open(file_path, 'r') as f:
-        config_data = json.load(f)
-      if not config_data:
-        raise exceptions.EmptyConfigError(file_path)
-      config_type = config_data.pop('config_type', None)
-      if config_type != self.GetConfigType():
-        raise ValueError(
-            'Supplied config_type (%s) is not the expected type (%s) in file '
-            '%s' % (config_type, self.GetConfigType(), file_path))
-      self._config_data = config_data.get('dependencies', {})
-
-  def IterDependencyInfo(self):
-    """ Yields a DependencyInfo for each dependency/platform pair.
-
-    Raises:
-        ReadWriteError: If called when the config is writable.
-        ValueError: If any of the dependencies contain partial information for
-            downloading from cloud_storage. (See dependency_info.py)
-    """
-    if self._writable:
-      raise exceptions.ReadWriteError(
-          'Trying to read dependency info from a  writable config. File for '
-          'config: %s' % self._config_path)
-    base_path = os.path.dirname(self._config_path)
-    for dependency in self._config_data:
-      dependency_dict = self._config_data.get(dependency)
-      platforms_dict = dependency_dict.get('file_info', {})
-      for platform in platforms_dict:
-        platform_info = platforms_dict.get(platform)
-
-        local_info = None
-        local_paths = platform_info.get('local_paths', [])
-        if local_paths:
-          paths = []
-          for path in local_paths:
-            path = self._FormatPath(path)
-            paths.append(os.path.abspath(os.path.join(base_path, path)))
-          local_info = local_path_info.LocalPathInfo(paths)
-
-        cs_info = None
-        cs_bucket = dependency_dict.get('cloud_storage_bucket')
-        cs_base_folder = dependency_dict.get('cloud_storage_base_folder', '')
-        download_path = platform_info.get('download_path')
-        if download_path:
-          download_path = self._FormatPath(download_path)
-          download_path = os.path.abspath(
-              os.path.join(base_path, download_path))
-
-          cs_hash = platform_info.get('cloud_storage_hash')
-          if not cs_hash:
-            raise exceptions.ConfigError(
-                'Dependency %s has cloud storage info on platform %s, but is '
-                'missing a cloud storage hash.', dependency, platform)
-          cs_remote_path = self._CloudStorageRemotePath(
-              dependency, cs_hash, cs_base_folder)
-          version_in_cs = platform_info.get('version_in_cs')
-
-          zip_info = None
-          path_within_archive = platform_info.get('path_within_archive')
-          if path_within_archive:
-            unzip_path = os.path.abspath(
-                os.path.join(os.path.dirname(download_path),
-                             '%s_%s_%s' % (dependency, platform, cs_hash)))
-            zip_info = archive_info.ArchiveInfo(
-                download_path, unzip_path, path_within_archive)
-
-          cs_info = cloud_storage_info.CloudStorageInfo(
-              cs_bucket, cs_hash, download_path, cs_remote_path,
-              version_in_cs=version_in_cs, archive_info=zip_info)
-
-        dep_info = dependency_info.DependencyInfo(
-            dependency, platform, self._config_path, local_path_info=local_info,
-            cloud_storage_info=cs_info)
-        yield dep_info
-
-  @classmethod
-  def GetConfigType(cls):
-    return 'BaseConfig'
-
-  @property
-  def config_path(self):
-    return self._config_path
-
-  def AddCloudStorageDependencyUpdateJob(
-      self, dependency, platform, dependency_path, version=None,
-      execute_job=True):
-    """Update the file downloaded from cloud storage for a dependency/platform.
-
-    Upload a new file to cloud storage for the given dependency and platform
-    pair and update the cloud storage hash and the version for the given pair.
-
-    Example usage:
-      The following should update the default platform for 'dep_name':
-          UpdateCloudStorageDependency('dep_name', 'default', 'path/to/file')
-
-      The following should update both the mac and win platforms for 'dep_name',
-      or neither if either update fails:
-          UpdateCloudStorageDependency(
-              'dep_name', 'mac_x86_64', 'path/to/mac/file', execute_job=False)
-          UpdateCloudStorageDependency(
-              'dep_name', 'win_AMD64', 'path/to/win/file', execute_job=False)
-          ExecuteUpdateJobs()
-
-    Args:
-      dependency: The dependency to update.
-      platform: The platform to update the dependency info for.
-      dependency_path: Path to the new dependency to be used.
-      version: Version of the updated dependency, for checking future updates
-          against.
-      execute_job: True if the config should be written to disk and the file
-          should be uploaded to cloud storage after the update. False if
-          multiple updates should be performed atomically. Must call
-          ExecuteUpdateJobs after all non-executed jobs are added to complete
-          the update.
-
-    Raises:
-      ReadWriteError: If the config was not initialized as writable, or if
-          |execute_job| is True but the config has update jobs still pending
-          execution.
-      ValueError: If no information exists in the config for |dependency| on
-          |platform|.
-    """
-    self._ValidateIsConfigUpdatable(
-        execute_job=execute_job, dependency=dependency, platform=platform)
-    self._is_dirty = True
-    cs_hash = cloud_storage.CalculateHash(dependency_path)
-    if version:
-      self._SetPlatformData(dependency, platform, 'version_in_cs', version)
-    self._SetPlatformData(dependency, platform, 'cloud_storage_hash', cs_hash)
-
-    cs_base_folder = self._GetPlatformData(
-        dependency, platform, 'cloud_storage_base_folder')
-    cs_bucket = self._GetPlatformData(
-        dependency, platform, 'cloud_storage_bucket')
-    cs_remote_path = self._CloudStorageRemotePath(
-        dependency, cs_hash, cs_base_folder)
-    self._pending_uploads.append(uploader.CloudStorageUploader(
-          cs_bucket, cs_remote_path, dependency_path))
-    if execute_job:
-      self.ExecuteUpdateJobs()
-
-  def ExecuteUpdateJobs(self, force=False):
-    """Write all config changes to the config_path specified in __init__.
-
-    Upload all files pending upload and then write the updated config to
-    file. Attempt to remove all uploaded files on failure.
-
-    Args:
-      force: True if files should be uploaded to cloud storage even if a
-          file already exists in the upload location.
-
-    Returns:
-      True: if the config was dirty and the upload succeeded.
-      False: if the config was not dirty.
-
-    Raises:
-      CloudStorageUploadConflictError: If |force| is False and the potential
-          upload location of a file already exists.
-      CloudStorageError: If copying an existing file to the backup location
-          or uploading a new file fails.
-    """
-    self._ValidateIsConfigUpdatable()
-    if not self._is_dirty:
-      logging.info('ExecuteUpdateJobs called on clean config')
-      return False
-    if not self._pending_uploads:
-      logging.debug('No files needing upload.')
-    else:
-      try:
-        for item_pending_upload in self._pending_uploads:
-          item_pending_upload.Upload(force)
-        self._WriteConfigToFile(self._config_path, self._config_data)
-        self._pending_uploads = []
-        self._is_dirty = False
-      except:
-        # Attempt to rollback the update in any instance of failure, even user
-        # interrupt via Ctrl+C; but don't consume the exception.
-        logging.error('Update failed, attempting to roll it back.')
-        for upload_item in reversed(self._pending_uploads):
-          upload_item.Rollback()
-        raise
-    return True
-
-  def GetVersion(self, dependency, platform):
-    """Return the Version information for the given dependency."""
-    return self._GetPlatformData(
-        dependency, platform, data_type='version_in_cs')
-
-  def _SetPlatformData(self, dependency, platform, data_type, data):
-    self._ValidateIsConfigWritable()
-    dependency_dict = self._config_data.get(dependency, {})
-    platform_dict = dependency_dict.get('file_info', {}).get(platform)
-    if not platform_dict:
-      raise ValueError('No platform data for platform %s on dependency %s' %
-                       (platform, dependency))
-    if (data_type == 'cloud_storage_bucket' or
-        data_type == 'cloud_storage_base_folder'):
-      self._config_data[dependency][data_type] = data
-    else:
-      self._config_data[dependency]['file_info'][platform][data_type] = data
-
-  def _GetPlatformData(self, dependency, platform, data_type=None):
-    dependency_dict = self._config_data.get(dependency, {})
-    if not dependency_dict:
-      raise ValueError('Dependency %s is not in config.' % dependency)
-    platform_dict = dependency_dict.get('file_info', {}).get(platform)
-    if not platform_dict:
-      raise ValueError('No platform data for platform %s on dependency %s' %
-                       (platform, dependency))
-    if data_type:
-      if (data_type == 'cloud_storage_bucket' or
-          data_type == 'cloud_storage_base_folder'):
-        return dependency_dict.get(data_type)
-      return platform_dict.get(data_type)
-    return platform_dict
-
-  def _ValidateIsConfigUpdatable(
-      self, execute_job=False, dependency=None, platform=None):
-    self._ValidateIsConfigWritable()
-    if self._is_dirty and execute_job:
-      raise exceptions.ReadWriteError(
-          'A change has already been made to this config. Either call without'
-          'using the execute_job option or first call ExecuteUpdateJobs().')
-    if dependency and not self._config_data.get(dependency):
-      raise ValueError('Cannot update information because dependency %s does '
-                       'not exist.' % dependency)
-    if platform and not self._GetPlatformData(dependency, platform):
-      raise ValueError('No dependency info is available for the given '
-                       'dependency: %s' % dependency)
-
-  def _ValidateIsConfigWritable(self):
-    if not self._writable:
-      raise exceptions.ReadWriteError(
-          'Trying to update the information from a read-only config. '
-          'File for config: %s' % self._config_path)
-
-  @staticmethod
-  def _CloudStorageRemotePath(dependency, cs_hash, cs_base_folder):
-    cs_remote_file = '%s_%s' % (dependency, cs_hash)
-    cs_remote_path = cs_remote_file if not cs_base_folder else (
-        '%s/%s' % (cs_base_folder, cs_remote_file))
-    return cs_remote_path
-
-  @classmethod
-  def _FormatPath(cls, file_path):
-    """ Format |file_path| for the current file system.
-
-    We may be downloading files for another platform, so paths must be
-    downloadable on the current system.
-    """
-    if not file_path:
-      return file_path
-    if os.path.sep != '\\':
-      return file_path.replace('\\', os.path.sep)
-    elif os.path.sep != '/':
-      return file_path.replace('/', os.path.sep)
-    return file_path
-
-  @classmethod
-  def _WriteConfigToFile(cls, file_path, dependencies=None):
-    json_dict = cls._GetJsonDict(dependencies)
-    file_dir = os.path.dirname(file_path)
-    if not os.path.exists(file_dir):
-      os.makedirs(file_dir)
-    with open(file_path, 'w') as outfile:
-      json.dump(
-          json_dict, outfile, indent=2, sort_keys=True, separators=(',', ': '))
-    return json_dict
-
-  @classmethod
-  def _GetJsonDict(cls, dependencies=None):
-    dependencies = dependencies or {}
-    json_dict = {'config_type': cls.GetConfigType(),
-                 'dependencies': dependencies}
-    return json_dict