| Index: tools/telemetry/cloud_storage
|
| diff --git a/tools/telemetry/cloud_storage b/tools/telemetry/cloud_storage
|
| deleted file mode 100755
|
| index edb2dd68028452c75b3081b78425cf8b0ca42873..0000000000000000000000000000000000000000
|
| --- a/tools/telemetry/cloud_storage
|
| +++ /dev/null
|
| @@ -1,200 +0,0 @@
|
| -#!/usr/bin/env python
|
| -# Copyright 2014 The Chromium Authors. All rights reserved.
|
| -# Use of this source code is governed by a BSD-style license that can be
|
| -# found in the LICENSE file.
|
| -
|
| -import argparse
|
| -import logging
|
| -import os
|
| -import subprocess
|
| -import sys
|
| -
|
| -from telemetry.core import util
|
| -from telemetry.internal.util import command_line
|
| -
|
| -sys.path.insert(1, os.path.abspath(os.path.join(
|
| - util.GetCatapultDir(), 'catapult_base')))
|
| -from catapult_base import cloud_storage
|
| -
|
| -
|
| -BUCKETS = {bucket: easy_bucket_name for easy_bucket_name, bucket
|
| - in cloud_storage.BUCKET_ALIASES.iteritems()}
|
| -
|
| -
|
| -def _GetPaths(path):
|
| - root, ext = os.path.splitext(path)
|
| - if ext == '.sha1':
|
| - file_path = root
|
| - hash_path = path
|
| - else:
|
| - file_path = path
|
| - hash_path = path + '.sha1'
|
| - return file_path, hash_path
|
| -
|
| -
|
| -def _FindFilesInCloudStorage(files):
|
| - """Returns a dict of all files and which buckets they're in."""
|
| - # Preprocessing: get the contents of all buckets.
|
| - bucket_contents = {}
|
| - for bucket in BUCKETS:
|
| - try:
|
| - bucket_contents[bucket] = cloud_storage.List(bucket)
|
| - except (cloud_storage.PermissionError, cloud_storage.CredentialsError):
|
| - pass
|
| -
|
| - # Check if each file is in the bucket contents.
|
| - file_buckets = {}
|
| - for path in files:
|
| - file_path, hash_path = _GetPaths(path)
|
| -
|
| - if file_path in file_buckets:
|
| - # Ignore duplicates, if both data and sha1 file were in the file list.
|
| - continue
|
| - if not os.path.exists(hash_path):
|
| - # Probably got some non-Cloud Storage files in the file list. Ignore.
|
| - continue
|
| -
|
| - file_hash = cloud_storage.ReadHash(hash_path)
|
| - file_buckets[file_path] = []
|
| - for bucket in BUCKETS:
|
| - if bucket in bucket_contents and file_hash in bucket_contents[bucket]:
|
| - file_buckets[file_path].append(bucket)
|
| -
|
| - return file_buckets
|
| -
|
| -
|
| -class Ls(command_line.Command):
|
| - """List which bucket each file is in."""
|
| -
|
| - @classmethod
|
| - def AddCommandLineArgs(cls, parser):
|
| - parser.add_argument('-r', '--recursive', action='store_true')
|
| - parser.add_argument('paths', nargs='+')
|
| -
|
| - @classmethod
|
| - def ProcessCommandLineArgs(cls, parser, args):
|
| - for path in args.paths:
|
| - if not os.path.exists(path):
|
| - parser.error('Path not found: %s' % path)
|
| -
|
| - def Run(self, args):
|
| - def GetFilesInPaths(paths, recursive):
|
| - """If path is a dir, yields all files in path, otherwise just yields path.
|
| - If recursive is true, walks subdirectories recursively."""
|
| - for path in paths:
|
| - if not os.path.isdir(path):
|
| - yield path
|
| - continue
|
| -
|
| - if recursive:
|
| - for root, _, filenames in os.walk(path):
|
| - for filename in filenames:
|
| - yield os.path.join(root, filename)
|
| - else:
|
| - for filename in os.listdir(path):
|
| - yield os.path.join(path, filename)
|
| -
|
| - files = _FindFilesInCloudStorage(GetFilesInPaths(args.paths, args.recursive))
|
| -
|
| - if not files:
|
| - print 'No files in Cloud Storage.'
|
| - return
|
| -
|
| - for file_path, buckets in sorted(files.iteritems()):
|
| - if buckets:
|
| - buckets = [BUCKETS[bucket] for bucket in buckets]
|
| - print '%-11s %s' % (','.join(buckets), file_path)
|
| - else:
|
| - print '%-11s %s' % ('not found', file_path)
|
| -
|
| -
|
| -class Mv(command_line.Command):
|
| - """Move files to the given bucket."""
|
| -
|
| - @classmethod
|
| - def AddCommandLineArgs(cls, parser):
|
| - parser.add_argument('files', nargs='+')
|
| - parser.add_argument('bucket', choices=cloud_storage.BUCKET_ALIASES)
|
| -
|
| - @classmethod
|
| - def ProcessCommandLineArgs(cls, parser, args):
|
| - args.bucket = cloud_storage.BUCKET_ALIASES[args.bucket]
|
| -
|
| - def Run(self, args):
|
| - files = _FindFilesInCloudStorage(args.files)
|
| -
|
| - for file_path, buckets in sorted(files.iteritems()):
|
| - if not buckets:
|
| - raise IOError('%s not found in Cloud Storage.' % file_path)
|
| -
|
| - for file_path, buckets in sorted(files.iteritems()):
|
| - if args.bucket in buckets:
|
| - buckets.remove(args.bucket)
|
| - if not buckets:
|
| - logging.info('Skipping %s, no action needed.' % file_path)
|
| - continue
|
| -
|
| - # Move to the target bucket.
|
| - file_hash = cloud_storage.ReadHash(file_path + '.sha1')
|
| - cloud_storage.Move(buckets.pop(), args.bucket, file_hash)
|
| -
|
| - # Delete all additional copies.
|
| - for bucket in buckets:
|
| - cloud_storage.Delete(bucket, file_hash)
|
| -
|
| -
|
| -class Rm(command_line.Command):
|
| - """Remove files from Cloud Storage."""
|
| -
|
| - @classmethod
|
| - def AddCommandLineArgs(cls, parser):
|
| - parser.add_argument('files', nargs='+')
|
| -
|
| - def Run(self, args):
|
| - files = _FindFilesInCloudStorage(args.files)
|
| - for file_path, buckets in sorted(files.iteritems()):
|
| - file_hash = cloud_storage.ReadHash(file_path + '.sha1')
|
| - for bucket in buckets:
|
| - cloud_storage.Delete(bucket, file_hash)
|
| -
|
| -
|
| -class Upload(command_line.Command):
|
| - """Upload files to Cloud Storage."""
|
| -
|
| - @classmethod
|
| - def AddCommandLineArgs(cls, parser):
|
| - parser.add_argument('files', nargs='+')
|
| - parser.add_argument('bucket', choices=cloud_storage.BUCKET_ALIASES)
|
| -
|
| - @classmethod
|
| - def ProcessCommandLineArgs(cls, parser, args):
|
| - args.bucket = cloud_storage.BUCKET_ALIASES[args.bucket]
|
| -
|
| - for path in args.files:
|
| - if not os.path.exists(path):
|
| - parser.error('File not found: %s' % path)
|
| -
|
| - def Run(self, args):
|
| - for file_path in args.files:
|
| - file_hash = cloud_storage.CalculateHash(file_path)
|
| -
|
| - # Create or update the hash file.
|
| - hash_path = file_path + '.sha1'
|
| - with open(hash_path, 'wb') as f:
|
| - f.write(file_hash)
|
| - f.flush()
|
| -
|
| - # Add the data to Cloud Storage.
|
| - cloud_storage.Insert(args.bucket, file_hash, file_path)
|
| -
|
| - # Add the hash file to the branch, for convenience. :)
|
| - subprocess.call(['git', 'add', hash_path])
|
| -
|
| -
|
| -class CloudStorageCommand(command_line.SubcommandCommand):
|
| - commands = (Ls, Mv, Rm, Upload)
|
| -
|
| -
|
| -if __name__ == '__main__':
|
| - logging.getLogger().setLevel(logging.INFO)
|
| - sys.exit(CloudStorageCommand.main())
|
|
|