Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(386)

Side by Side Diff: tools/telemetry/cloud_storage

Issue 215423004: [telemetry] Minor Cloud Storage script fixes. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/src
Patch Set: Created 6 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 import argparse 6 import argparse
7 import logging 7 import logging
8 import os 8 import os
9 import subprocess 9 import subprocess
10 10
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after
50 if file_path in file_buckets: 50 if file_path in file_buckets:
51 # Ignore duplicates, if both data and sha1 file were in the file list. 51 # Ignore duplicates, if both data and sha1 file were in the file list.
52 continue 52 continue
53 if not os.path.exists(hash_path): 53 if not os.path.exists(hash_path):
54 # Probably got some non-Cloud Storage files in the file list. Ignore. 54 # Probably got some non-Cloud Storage files in the file list. Ignore.
55 continue 55 continue
56 56
57 file_hash = cloud_storage.ReadHash(hash_path) 57 file_hash = cloud_storage.ReadHash(hash_path)
58 file_buckets[file_path] = [] 58 file_buckets[file_path] = []
59 for bucket in BUCKETS: 59 for bucket in BUCKETS:
60 if file_hash in bucket_contents[bucket]: 60 if bucket in bucket_contents and file_hash in bucket_contents[bucket]:
61 file_buckets[file_path].append(bucket) 61 file_buckets[file_path].append(bucket)
62 62
63 return file_buckets 63 return file_buckets
64 64
65 65
66 class Ls(command_line.Command): 66 class Ls(command_line.Command):
67 """List which bucket each file is in.""" 67 """List which bucket each file is in."""
68 68
69 @classmethod 69 @classmethod
70 def AddCommandLineArgs(cls, parser): 70 def AddCommandLineArgs(cls, parser):
71 parser.add_argument('-r', '--recursive', action='store_true') 71 parser.add_argument('-r', '--recursive', action='store_true')
72 parser.add_argument('paths', nargs='+') 72 parser.add_argument('paths', nargs='+')
73 73
74 @classmethod 74 @classmethod
75 def ProcessCommandLineArgs(cls, parser, args): 75 def ProcessCommandLineArgs(cls, parser, args):
76 for path in args.paths: 76 for path in args.paths:
77 if not os.path.exists(path): 77 if not os.path.exists(path):
78 parser.error('Path not found: %s' % path) 78 parser.error('Path not found: %s' % path)
79 79
80 def Run(self, args): 80 def Run(self, args):
81 def GetFilesInPaths(paths, recursive): 81 def GetFilesInPaths(paths, recursive):
82 """If path is a dir, yields all files in path, otherwise just yields path. 82 """If path is a dir, yields all files in path, otherwise just yields path.
83 83
84 If recursive is true, walks subdirectories recursively.""" 84 If recursive is true, walks subdirectories recursively."""
85 for path in paths: 85 for path in paths:
86 if not os.path.isdir(path): 86 if not os.path.isdir(path):
87 yield path 87 yield path
88 return 88 continue
89 89
90 if recursive: 90 if recursive:
91 for root, _, filenames in os.walk(path): 91 for root, _, filenames in os.walk(path):
92 for filename in filenames: 92 for filename in filenames:
93 yield os.path.join(root, filename) 93 yield os.path.join(root, filename)
94 else: 94 else:
95 for filename in os.listdir(path): 95 for filename in os.listdir(path):
96 yield os.path.join(path, filename) 96 yield os.path.join(path, filename)
97 97
98 files = _FindFilesInCloudStorage(GetFilesInPaths(args.paths, args.recursive) ) 98 files = _FindFilesInCloudStorage(GetFilesInPaths(args.paths, args.recursive) )
(...skipping 15 matching lines...) Expand all
114 114
115 @classmethod 115 @classmethod
116 def AddCommandLineArgs(cls, parser): 116 def AddCommandLineArgs(cls, parser):
117 parser.add_argument('files', nargs='+') 117 parser.add_argument('files', nargs='+')
118 parser.add_argument('bucket', choices=BUCKET_ALIASES) 118 parser.add_argument('bucket', choices=BUCKET_ALIASES)
119 119
120 @classmethod 120 @classmethod
121 def ProcessCommandLineArgs(cls, parser, args): 121 def ProcessCommandLineArgs(cls, parser, args):
122 args.bucket = BUCKET_ALIASES[args.bucket] 122 args.bucket = BUCKET_ALIASES[args.bucket]
123 123
124 for path in args.files:
125 _, hash_path = _GetPaths(path)
126 if not os.path.exists(hash_path):
127 parser.error('File not found: %s' % hash_path)
128
129 def Run(self, args): 124 def Run(self, args):
130 files = _FindFilesInCloudStorage(args.files) 125 files = _FindFilesInCloudStorage(args.files)
131 126
132 for file_path, buckets in sorted(files.iteritems()): 127 for file_path, buckets in sorted(files.iteritems()):
133 if not buckets: 128 if not buckets:
134 raise IOError('%s not found in Cloud Storage.' % file_path) 129 raise IOError('%s not found in Cloud Storage.' % file_path)
135 130
136 for file_path, buckets in sorted(files.iteritems()): 131 for file_path, buckets in sorted(files.iteritems()):
132 if args.bucket in buckets:
133 buckets.remove(args.bucket)
134 if not buckets:
135 logging.info('Skipping %s, no action needed.' % file_path)
136 continue
137
137 # Move to the target bucket. 138 # Move to the target bucket.
138 file_hash = cloud_storage.ReadHash(file_path + '.sha1') 139 file_hash = cloud_storage.ReadHash(file_path + '.sha1')
139 cloud_storage.Move(buckets.pop(), args.bucket, file_hash) 140 cloud_storage.Move(buckets.pop(), args.bucket, file_hash)
140 141
141 # Delete all additional copies. 142 # Delete all additional copies.
142 for bucket in buckets: 143 for bucket in buckets:
143 if bucket == args.bucket:
144 continue
145 cloud_storage.Delete(bucket, file_hash) 144 cloud_storage.Delete(bucket, file_hash)
146 145
147 146
148 class Rm(command_line.Command): 147 class Rm(command_line.Command):
149 """Remove files from Cloud Storage.""" 148 """Remove files from Cloud Storage."""
150 149
151 @classmethod 150 @classmethod
152 def AddCommandLineArgs(cls, parser): 151 def AddCommandLineArgs(cls, parser):
153 parser.add_argument('files', nargs='+') 152 parser.add_argument('files', nargs='+')
154 153
155 @classmethod
156 def ProcessCommandLineArgs(cls, parser, args):
157 for path in args.files:
158 _, hash_path = _GetPaths(path)
159 if not os.path.exists(hash_path):
160 parser.error('File not found: %s' % hash_path)
161
162 def Run(self, args): 154 def Run(self, args):
163 files = _FindFilesInCloudStorage(args.files) 155 files = _FindFilesInCloudStorage(args.files)
164 for file_path, buckets in sorted(files.iteritems()): 156 for file_path, buckets in sorted(files.iteritems()):
165 file_hash = cloud_storage.ReadHash(file_path + '.sha1') 157 file_hash = cloud_storage.ReadHash(file_path + '.sha1')
166 for bucket in buckets: 158 for bucket in buckets:
167 cloud_storage.Delete(bucket, file_hash) 159 cloud_storage.Delete(bucket, file_hash)
168 160
169 161
170 class Upload(command_line.Command): 162 class Upload(command_line.Command):
171 """Upload files to Cloud Storage.""" 163 """Upload files to Cloud Storage."""
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after
215 subparser.set_defaults(command=command) 207 subparser.set_defaults(command=command)
216 command.AddCommandLineArgs(subparser) 208 command.AddCommandLineArgs(subparser)
217 209
218 args = parser.parse_args() 210 args = parser.parse_args()
219 args.command.ProcessCommandLineArgs(parser, args) 211 args.command.ProcessCommandLineArgs(parser, args)
220 args.command().Run(args) 212 args.command().Run(args)
221 213
222 214
223 if __name__ == '__main__': 215 if __name__ == '__main__':
224 main() 216 main()
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698