Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(128)

Side by Side Diff: tools/telemetry/cloud_storage

Issue 556003002: [telemetry] Use consistent bucket naming. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Unit tests. Created 6 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 import argparse 6 import argparse
7 import logging 7 import logging
8 import os 8 import os
9 import subprocess 9 import subprocess
10 import sys 10 import sys
11 11
12 from telemetry.core import command_line 12 from telemetry.core import command_line
13 from telemetry.util import cloud_storage 13 from telemetry.util import cloud_storage
14 14
15 15
16 BUCKET_ALIASES = {
17 'public': cloud_storage.PUBLIC_BUCKET,
18 'partner': cloud_storage.PARTNER_BUCKET,
19 'google-only': cloud_storage.INTERNAL_BUCKET,
20 }
21 BUCKETS = {bucket: easy_bucket_name for easy_bucket_name, bucket 16 BUCKETS = {bucket: easy_bucket_name for easy_bucket_name, bucket
22 in BUCKET_ALIASES.iteritems()} 17 in cloud_storage.BUCKET_ALIASES.iteritems()}
23 18
24 19
25 def _GetPaths(path): 20 def _GetPaths(path):
26 root, ext = os.path.splitext(path) 21 root, ext = os.path.splitext(path)
27 if ext == '.sha1': 22 if ext == '.sha1':
28 file_path = root 23 file_path = root
29 hash_path = path 24 hash_path = path
30 else: 25 else:
31 file_path = path 26 file_path = path
32 hash_path = path + '.sha1' 27 hash_path = path + '.sha1'
(...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after
109 else: 104 else:
110 print '%-11s %s' % ('not found', file_path) 105 print '%-11s %s' % ('not found', file_path)
111 106
112 107
113 class Mv(command_line.Command): 108 class Mv(command_line.Command):
114 """Move files to the given bucket.""" 109 """Move files to the given bucket."""
115 110
116 @classmethod 111 @classmethod
117 def AddCommandLineArgs(cls, parser): 112 def AddCommandLineArgs(cls, parser):
118 parser.add_argument('files', nargs='+') 113 parser.add_argument('files', nargs='+')
119 parser.add_argument('bucket', choices=BUCKET_ALIASES) 114 parser.add_argument('bucket', choices=cloud_storage.BUCKET_ALIASES)
120 115
121 @classmethod 116 @classmethod
122 def ProcessCommandLineArgs(cls, parser, args): 117 def ProcessCommandLineArgs(cls, parser, args):
123 args.bucket = BUCKET_ALIASES[args.bucket] 118 args.bucket = cloud_storage.BUCKET_ALIASES[args.bucket]
124 119
125 def Run(self, args): 120 def Run(self, args):
126 files = _FindFilesInCloudStorage(args.files) 121 files = _FindFilesInCloudStorage(args.files)
127 122
128 for file_path, buckets in sorted(files.iteritems()): 123 for file_path, buckets in sorted(files.iteritems()):
129 if not buckets: 124 if not buckets:
130 raise IOError('%s not found in Cloud Storage.' % file_path) 125 raise IOError('%s not found in Cloud Storage.' % file_path)
131 126
132 for file_path, buckets in sorted(files.iteritems()): 127 for file_path, buckets in sorted(files.iteritems()):
133 if args.bucket in buckets: 128 if args.bucket in buckets:
(...skipping 25 matching lines...) Expand all
159 for bucket in buckets: 154 for bucket in buckets:
160 cloud_storage.Delete(bucket, file_hash) 155 cloud_storage.Delete(bucket, file_hash)
161 156
162 157
163 class Upload(command_line.Command): 158 class Upload(command_line.Command):
164 """Upload files to Cloud Storage.""" 159 """Upload files to Cloud Storage."""
165 160
166 @classmethod 161 @classmethod
167 def AddCommandLineArgs(cls, parser): 162 def AddCommandLineArgs(cls, parser):
168 parser.add_argument('files', nargs='+') 163 parser.add_argument('files', nargs='+')
169 parser.add_argument('bucket', choices=BUCKET_ALIASES) 164 parser.add_argument('bucket', choices=cloud_storage.BUCKET_ALIASES)
170 165
171 @classmethod 166 @classmethod
172 def ProcessCommandLineArgs(cls, parser, args): 167 def ProcessCommandLineArgs(cls, parser, args):
173 args.bucket = BUCKET_ALIASES[args.bucket] 168 args.bucket = cloud_storage.BUCKET_ALIASES[args.bucket]
174 169
175 for path in args.files: 170 for path in args.files:
176 if not os.path.exists(path): 171 if not os.path.exists(path):
177 parser.error('File not found: %s' % path) 172 parser.error('File not found: %s' % path)
178 173
179 def Run(self, args): 174 def Run(self, args):
180 for file_path in args.files: 175 for file_path in args.files:
181 file_hash = cloud_storage.CalculateHash(file_path) 176 file_hash = cloud_storage.CalculateHash(file_path)
182 177
183 # Create or update the hash file. 178 # Create or update the hash file.
184 hash_path = file_path + '.sha1' 179 hash_path = file_path + '.sha1'
185 with open(hash_path, 'wb') as f: 180 with open(hash_path, 'wb') as f:
186 f.write(file_hash) 181 f.write(file_hash)
187 f.flush() 182 f.flush()
188 183
189 # Add the data to Cloud Storage. 184 # Add the data to Cloud Storage.
190 cloud_storage.Insert(args.bucket, file_hash, file_path) 185 cloud_storage.Insert(args.bucket, file_hash, file_path)
191 186
192 # Add the hash file to the branch, for convenience. :) 187 # Add the hash file to the branch, for convenience. :)
193 subprocess.call(['git', 'add', hash_path]) 188 subprocess.call(['git', 'add', hash_path])
194 189
195 190
196 class CloudStorageCommand(command_line.SubcommandCommand): 191 class CloudStorageCommand(command_line.SubcommandCommand):
197 commands = (Ls, Mv, Rm, Upload) 192 commands = (Ls, Mv, Rm, Upload)
198 193
199 194
200 if __name__ == '__main__': 195 if __name__ == '__main__':
201 logging.getLogger().setLevel(logging.INFO) 196 logging.getLogger().setLevel(logging.INFO)
202 sys.exit(CloudStorageCommand.main()) 197 sys.exit(CloudStorageCommand.main())
OLDNEW
« no previous file with comments | « tools/perf/page_sets/PRESUBMIT.py ('k') | tools/telemetry/telemetry/page/page_set_archive_info.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698