| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 import argparse | 6 import argparse |
| 7 import logging | 7 import logging |
| 8 import os | 8 import os |
| 9 import subprocess | 9 import subprocess |
| 10 | 10 |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 50 if file_path in file_buckets: | 50 if file_path in file_buckets: |
| 51 # Ignore duplicates, if both data and sha1 file were in the file list. | 51 # Ignore duplicates, if both data and sha1 file were in the file list. |
| 52 continue | 52 continue |
| 53 if not os.path.exists(hash_path): | 53 if not os.path.exists(hash_path): |
| 54 # Probably got some non-Cloud Storage files in the file list. Ignore. | 54 # Probably got some non-Cloud Storage files in the file list. Ignore. |
| 55 continue | 55 continue |
| 56 | 56 |
| 57 file_hash = cloud_storage.ReadHash(hash_path) | 57 file_hash = cloud_storage.ReadHash(hash_path) |
| 58 file_buckets[file_path] = [] | 58 file_buckets[file_path] = [] |
| 59 for bucket in BUCKETS: | 59 for bucket in BUCKETS: |
| 60 if file_hash in bucket_contents[bucket]: | 60 if bucket in bucket_contents and file_hash in bucket_contents[bucket]: |
| 61 file_buckets[file_path].append(bucket) | 61 file_buckets[file_path].append(bucket) |
| 62 | 62 |
| 63 return file_buckets | 63 return file_buckets |
| 64 | 64 |
| 65 | 65 |
| 66 class Ls(command_line.Command): | 66 class Ls(command_line.Command): |
| 67 """List which bucket each file is in.""" | 67 """List which bucket each file is in.""" |
| 68 | 68 |
| 69 @classmethod | 69 @classmethod |
| 70 def AddCommandLineArgs(cls, parser): | 70 def AddCommandLineArgs(cls, parser): |
| 71 parser.add_argument('-r', '--recursive', action='store_true') | 71 parser.add_argument('-r', '--recursive', action='store_true') |
| 72 parser.add_argument('paths', nargs='+') | 72 parser.add_argument('paths', nargs='+') |
| 73 | 73 |
| 74 @classmethod | 74 @classmethod |
| 75 def ProcessCommandLineArgs(cls, parser, args): | 75 def ProcessCommandLineArgs(cls, parser, args): |
| 76 for path in args.paths: | 76 for path in args.paths: |
| 77 if not os.path.exists(path): | 77 if not os.path.exists(path): |
| 78 parser.error('Path not found: %s' % path) | 78 parser.error('Path not found: %s' % path) |
| 79 | 79 |
| 80 def Run(self, args): | 80 def Run(self, args): |
| 81 def GetFilesInPaths(paths, recursive): | 81 def GetFilesInPaths(paths, recursive): |
| 82 """If path is a dir, yields all files in path, otherwise just yields path. | 82 """If path is a dir, yields all files in path, otherwise just yields path. |
| 83 | 83 |
| 84 If recursive is true, walks subdirectories recursively.""" | 84 If recursive is true, walks subdirectories recursively.""" |
| 85 for path in paths: | 85 for path in paths: |
| 86 if not os.path.isdir(path): | 86 if not os.path.isdir(path): |
| 87 yield path | 87 yield path |
| 88 return | 88 continue |
| 89 | 89 |
| 90 if recursive: | 90 if recursive: |
| 91 for root, _, filenames in os.walk(path): | 91 for root, _, filenames in os.walk(path): |
| 92 for filename in filenames: | 92 for filename in filenames: |
| 93 yield os.path.join(root, filename) | 93 yield os.path.join(root, filename) |
| 94 else: | 94 else: |
| 95 for filename in os.listdir(path): | 95 for filename in os.listdir(path): |
| 96 yield os.path.join(path, filename) | 96 yield os.path.join(path, filename) |
| 97 | 97 |
| 98 files = _FindFilesInCloudStorage(GetFilesInPaths(args.paths, args.recursive)
) | 98 files = _FindFilesInCloudStorage(GetFilesInPaths(args.paths, args.recursive)
) |
| (...skipping 15 matching lines...) Expand all Loading... |
| 114 | 114 |
| 115 @classmethod | 115 @classmethod |
| 116 def AddCommandLineArgs(cls, parser): | 116 def AddCommandLineArgs(cls, parser): |
| 117 parser.add_argument('files', nargs='+') | 117 parser.add_argument('files', nargs='+') |
| 118 parser.add_argument('bucket', choices=BUCKET_ALIASES) | 118 parser.add_argument('bucket', choices=BUCKET_ALIASES) |
| 119 | 119 |
| 120 @classmethod | 120 @classmethod |
| 121 def ProcessCommandLineArgs(cls, parser, args): | 121 def ProcessCommandLineArgs(cls, parser, args): |
| 122 args.bucket = BUCKET_ALIASES[args.bucket] | 122 args.bucket = BUCKET_ALIASES[args.bucket] |
| 123 | 123 |
| 124 for path in args.files: | |
| 125 _, hash_path = _GetPaths(path) | |
| 126 if not os.path.exists(hash_path): | |
| 127 parser.error('File not found: %s' % hash_path) | |
| 128 | |
| 129 def Run(self, args): | 124 def Run(self, args): |
| 130 files = _FindFilesInCloudStorage(args.files) | 125 files = _FindFilesInCloudStorage(args.files) |
| 131 | 126 |
| 132 for file_path, buckets in sorted(files.iteritems()): | 127 for file_path, buckets in sorted(files.iteritems()): |
| 133 if not buckets: | 128 if not buckets: |
| 134 raise IOError('%s not found in Cloud Storage.' % file_path) | 129 raise IOError('%s not found in Cloud Storage.' % file_path) |
| 135 | 130 |
| 136 for file_path, buckets in sorted(files.iteritems()): | 131 for file_path, buckets in sorted(files.iteritems()): |
| 132 if args.bucket in buckets: |
| 133 buckets.remove(args.bucket) |
| 134 if not buckets: |
| 135 logging.info('Skipping %s, no action needed.' % file_path) |
| 136 continue |
| 137 |
| 137 # Move to the target bucket. | 138 # Move to the target bucket. |
| 138 file_hash = cloud_storage.ReadHash(file_path + '.sha1') | 139 file_hash = cloud_storage.ReadHash(file_path + '.sha1') |
| 139 cloud_storage.Move(buckets.pop(), args.bucket, file_hash) | 140 cloud_storage.Move(buckets.pop(), args.bucket, file_hash) |
| 140 | 141 |
| 141 # Delete all additional copies. | 142 # Delete all additional copies. |
| 142 for bucket in buckets: | 143 for bucket in buckets: |
| 143 if bucket == args.bucket: | |
| 144 continue | |
| 145 cloud_storage.Delete(bucket, file_hash) | 144 cloud_storage.Delete(bucket, file_hash) |
| 146 | 145 |
| 147 | 146 |
| 148 class Rm(command_line.Command): | 147 class Rm(command_line.Command): |
| 149 """Remove files from Cloud Storage.""" | 148 """Remove files from Cloud Storage.""" |
| 150 | 149 |
| 151 @classmethod | 150 @classmethod |
| 152 def AddCommandLineArgs(cls, parser): | 151 def AddCommandLineArgs(cls, parser): |
| 153 parser.add_argument('files', nargs='+') | 152 parser.add_argument('files', nargs='+') |
| 154 | 153 |
| 155 @classmethod | |
| 156 def ProcessCommandLineArgs(cls, parser, args): | |
| 157 for path in args.files: | |
| 158 _, hash_path = _GetPaths(path) | |
| 159 if not os.path.exists(hash_path): | |
| 160 parser.error('File not found: %s' % hash_path) | |
| 161 | |
| 162 def Run(self, args): | 154 def Run(self, args): |
| 163 files = _FindFilesInCloudStorage(args.files) | 155 files = _FindFilesInCloudStorage(args.files) |
| 164 for file_path, buckets in sorted(files.iteritems()): | 156 for file_path, buckets in sorted(files.iteritems()): |
| 165 file_hash = cloud_storage.ReadHash(file_path + '.sha1') | 157 file_hash = cloud_storage.ReadHash(file_path + '.sha1') |
| 166 for bucket in buckets: | 158 for bucket in buckets: |
| 167 cloud_storage.Delete(bucket, file_hash) | 159 cloud_storage.Delete(bucket, file_hash) |
| 168 | 160 |
| 169 | 161 |
| 170 class Upload(command_line.Command): | 162 class Upload(command_line.Command): |
| 171 """Upload files to Cloud Storage.""" | 163 """Upload files to Cloud Storage.""" |
| (...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 215 subparser.set_defaults(command=command) | 207 subparser.set_defaults(command=command) |
| 216 command.AddCommandLineArgs(subparser) | 208 command.AddCommandLineArgs(subparser) |
| 217 | 209 |
| 218 args = parser.parse_args() | 210 args = parser.parse_args() |
| 219 args.command.ProcessCommandLineArgs(parser, args) | 211 args.command.ProcessCommandLineArgs(parser, args) |
| 220 args.command().Run(args) | 212 args.command().Run(args) |
| 221 | 213 |
| 222 | 214 |
| 223 if __name__ == '__main__': | 215 if __name__ == '__main__': |
| 224 main() | 216 main() |
| OLD | NEW |