OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 import argparse | 6 import argparse |
7 import logging | 7 import logging |
8 import os | 8 import os |
9 import subprocess | 9 import subprocess |
10 | 10 |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
50 if file_path in file_buckets: | 50 if file_path in file_buckets: |
51 # Ignore duplicates, if both data and sha1 file were in the file list. | 51 # Ignore duplicates, if both data and sha1 file were in the file list. |
52 continue | 52 continue |
53 if not os.path.exists(hash_path): | 53 if not os.path.exists(hash_path): |
54 # Probably got some non-Cloud Storage files in the file list. Ignore. | 54 # Probably got some non-Cloud Storage files in the file list. Ignore. |
55 continue | 55 continue |
56 | 56 |
57 file_hash = cloud_storage.ReadHash(hash_path) | 57 file_hash = cloud_storage.ReadHash(hash_path) |
58 file_buckets[file_path] = [] | 58 file_buckets[file_path] = [] |
59 for bucket in BUCKETS: | 59 for bucket in BUCKETS: |
60 if file_hash in bucket_contents[bucket]: | 60 if bucket in bucket_contents and file_hash in bucket_contents[bucket]: |
61 file_buckets[file_path].append(bucket) | 61 file_buckets[file_path].append(bucket) |
62 | 62 |
63 return file_buckets | 63 return file_buckets |
64 | 64 |
65 | 65 |
66 class Ls(command_line.Command): | 66 class Ls(command_line.Command): |
67 """List which bucket each file is in.""" | 67 """List which bucket each file is in.""" |
68 | 68 |
69 @classmethod | 69 @classmethod |
70 def AddCommandLineArgs(cls, parser): | 70 def AddCommandLineArgs(cls, parser): |
71 parser.add_argument('-r', '--recursive', action='store_true') | 71 parser.add_argument('-r', '--recursive', action='store_true') |
72 parser.add_argument('paths', nargs='+') | 72 parser.add_argument('paths', nargs='+') |
73 | 73 |
74 @classmethod | 74 @classmethod |
75 def ProcessCommandLineArgs(cls, parser, args): | 75 def ProcessCommandLineArgs(cls, parser, args): |
76 for path in args.paths: | 76 for path in args.paths: |
77 if not os.path.exists(path): | 77 if not os.path.exists(path): |
78 parser.error('Path not found: %s' % path) | 78 parser.error('Path not found: %s' % path) |
79 | 79 |
80 def Run(self, args): | 80 def Run(self, args): |
81 def GetFilesInPaths(paths, recursive): | 81 def GetFilesInPaths(paths, recursive): |
82 """If path is a dir, yields all files in path, otherwise just yields path. | 82 """If path is a dir, yields all files in path, otherwise just yields path. |
83 | 83 |
84 If recursive is true, walks subdirectories recursively.""" | 84 If recursive is true, walks subdirectories recursively.""" |
85 for path in paths: | 85 for path in paths: |
86 if not os.path.isdir(path): | 86 if not os.path.isdir(path): |
87 yield path | 87 yield path |
88 return | 88 continue |
89 | 89 |
90 if recursive: | 90 if recursive: |
91 for root, _, filenames in os.walk(path): | 91 for root, _, filenames in os.walk(path): |
92 for filename in filenames: | 92 for filename in filenames: |
93 yield os.path.join(root, filename) | 93 yield os.path.join(root, filename) |
94 else: | 94 else: |
95 for filename in os.listdir(path): | 95 for filename in os.listdir(path): |
96 yield os.path.join(path, filename) | 96 yield os.path.join(path, filename) |
97 | 97 |
98 files = _FindFilesInCloudStorage(GetFilesInPaths(args.paths, args.recursive)
) | 98 files = _FindFilesInCloudStorage(GetFilesInPaths(args.paths, args.recursive)
) |
(...skipping 15 matching lines...) Expand all Loading... |
114 | 114 |
115 @classmethod | 115 @classmethod |
116 def AddCommandLineArgs(cls, parser): | 116 def AddCommandLineArgs(cls, parser): |
117 parser.add_argument('files', nargs='+') | 117 parser.add_argument('files', nargs='+') |
118 parser.add_argument('bucket', choices=BUCKET_ALIASES) | 118 parser.add_argument('bucket', choices=BUCKET_ALIASES) |
119 | 119 |
120 @classmethod | 120 @classmethod |
121 def ProcessCommandLineArgs(cls, parser, args): | 121 def ProcessCommandLineArgs(cls, parser, args): |
122 args.bucket = BUCKET_ALIASES[args.bucket] | 122 args.bucket = BUCKET_ALIASES[args.bucket] |
123 | 123 |
124 for path in args.files: | |
125 _, hash_path = _GetPaths(path) | |
126 if not os.path.exists(hash_path): | |
127 parser.error('File not found: %s' % hash_path) | |
128 | |
129 def Run(self, args): | 124 def Run(self, args): |
130 files = _FindFilesInCloudStorage(args.files) | 125 files = _FindFilesInCloudStorage(args.files) |
131 | 126 |
132 for file_path, buckets in sorted(files.iteritems()): | 127 for file_path, buckets in sorted(files.iteritems()): |
133 if not buckets: | 128 if not buckets: |
134 raise IOError('%s not found in Cloud Storage.' % file_path) | 129 raise IOError('%s not found in Cloud Storage.' % file_path) |
135 | 130 |
136 for file_path, buckets in sorted(files.iteritems()): | 131 for file_path, buckets in sorted(files.iteritems()): |
| 132 if args.bucket in buckets: |
| 133 buckets.remove(args.bucket) |
| 134 if not buckets: |
| 135 logging.info('Skipping %s, no action needed.' % file_path) |
| 136 continue |
| 137 |
137 # Move to the target bucket. | 138 # Move to the target bucket. |
138 file_hash = cloud_storage.ReadHash(file_path + '.sha1') | 139 file_hash = cloud_storage.ReadHash(file_path + '.sha1') |
139 cloud_storage.Move(buckets.pop(), args.bucket, file_hash) | 140 cloud_storage.Move(buckets.pop(), args.bucket, file_hash) |
140 | 141 |
141 # Delete all additional copies. | 142 # Delete all additional copies. |
142 for bucket in buckets: | 143 for bucket in buckets: |
143 if bucket == args.bucket: | |
144 continue | |
145 cloud_storage.Delete(bucket, file_hash) | 144 cloud_storage.Delete(bucket, file_hash) |
146 | 145 |
147 | 146 |
148 class Rm(command_line.Command): | 147 class Rm(command_line.Command): |
149 """Remove files from Cloud Storage.""" | 148 """Remove files from Cloud Storage.""" |
150 | 149 |
151 @classmethod | 150 @classmethod |
152 def AddCommandLineArgs(cls, parser): | 151 def AddCommandLineArgs(cls, parser): |
153 parser.add_argument('files', nargs='+') | 152 parser.add_argument('files', nargs='+') |
154 | 153 |
155 @classmethod | |
156 def ProcessCommandLineArgs(cls, parser, args): | |
157 for path in args.files: | |
158 _, hash_path = _GetPaths(path) | |
159 if not os.path.exists(hash_path): | |
160 parser.error('File not found: %s' % hash_path) | |
161 | |
162 def Run(self, args): | 154 def Run(self, args): |
163 files = _FindFilesInCloudStorage(args.files) | 155 files = _FindFilesInCloudStorage(args.files) |
164 for file_path, buckets in sorted(files.iteritems()): | 156 for file_path, buckets in sorted(files.iteritems()): |
165 file_hash = cloud_storage.ReadHash(file_path + '.sha1') | 157 file_hash = cloud_storage.ReadHash(file_path + '.sha1') |
166 for bucket in buckets: | 158 for bucket in buckets: |
167 cloud_storage.Delete(bucket, file_hash) | 159 cloud_storage.Delete(bucket, file_hash) |
168 | 160 |
169 | 161 |
170 class Upload(command_line.Command): | 162 class Upload(command_line.Command): |
171 """Upload files to Cloud Storage.""" | 163 """Upload files to Cloud Storage.""" |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
215 subparser.set_defaults(command=command) | 207 subparser.set_defaults(command=command) |
216 command.AddCommandLineArgs(subparser) | 208 command.AddCommandLineArgs(subparser) |
217 | 209 |
218 args = parser.parse_args() | 210 args = parser.parse_args() |
219 args.command.ProcessCommandLineArgs(parser, args) | 211 args.command.ProcessCommandLineArgs(parser, args) |
220 args.command().Run(args) | 212 args.command().Run(args) |
221 | 213 |
222 | 214 |
223 if __name__ == '__main__': | 215 if __name__ == '__main__': |
224 main() | 216 main() |
OLD | NEW |