OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """A git command for managing a local cache of git repositories.""" | 6 """A git command for managing a local cache of git repositories.""" |
7 | 7 |
8 from __future__ import print_function | 8 from __future__ import print_function |
9 import errno | 9 import errno |
10 import logging | 10 import logging |
(...skipping 328 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
339 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'], | 339 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'], |
340 cwd=rundir).strip().splitlines() | 340 cwd=rundir).strip().splitlines() |
341 for spec in fetch_specs: | 341 for spec in fetch_specs: |
342 try: | 342 try: |
343 self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True) | 343 self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True) |
344 except subprocess.CalledProcessError: | 344 except subprocess.CalledProcessError: |
345 logging.warn('Fetch of %s failed' % spec) | 345 logging.warn('Fetch of %s failed' % spec) |
346 if tempdir: | 346 if tempdir: |
347 os.rename(tempdir, self.mirror_path) | 347 os.rename(tempdir, self.mirror_path) |
348 | 348 |
349 def update_bootstrap(self): | 349 def update_bootstrap(self, prune=False): |
350 # The files are named <git number>.zip | 350 # The files are named <git number>.zip |
351 gen_number = subprocess.check_output( | 351 gen_number = subprocess.check_output( |
352 [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip() | 352 [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip() |
353 self.RunGit(['gc']) # Run Garbage Collect to compress packfile. | 353 self.RunGit(['gc']) # Run Garbage Collect to compress packfile. |
354 # Creating a temp file and then deleting it ensures we can use this name. | 354 # Creating a temp file and then deleting it ensures we can use this name. |
355 _, tmp_zipfile = tempfile.mkstemp(suffix='.zip') | 355 _, tmp_zipfile = tempfile.mkstemp(suffix='.zip') |
356 os.remove(tmp_zipfile) | 356 os.remove(tmp_zipfile) |
357 subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path) | 357 subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path) |
358 gsutil = Gsutil(path=self.gsutil_exe, boto_path=None) | 358 gsutil = Gsutil(path=self.gsutil_exe, boto_path=None) |
359 dest_name = 'gs://%s/%s/%s.zip' % ( | 359 gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir) |
360 self.bootstrap_bucket, self.basedir, gen_number) | 360 dest_name = '%s/%s.zip' % (gs_folder, gen_number) |
361 gsutil.call('cp', tmp_zipfile, dest_name) | 361 gsutil.call('cp', tmp_zipfile, dest_name) |
362 os.remove(tmp_zipfile) | 362 os.remove(tmp_zipfile) |
363 | 363 |
| 364 # Remove all other files in the same directory. |
| 365 if prune: |
| 366 _, ls_out, _ = gsutil.check_call('ls', gs_folder) |
| 367 for filename in ls_out.splitlines(): |
| 368 if filename == dest_name: |
| 369 continue |
| 370 gsutil.call('rm', filename) |
| 371 |
364 @staticmethod | 372 @staticmethod |
365 def DeleteTmpPackFiles(path): | 373 def DeleteTmpPackFiles(path): |
366 pack_dir = os.path.join(path, 'objects', 'pack') | 374 pack_dir = os.path.join(path, 'objects', 'pack') |
367 pack_files = [f for f in os.listdir(pack_dir) if | 375 pack_files = [f for f in os.listdir(pack_dir) if |
368 f.startswith('.tmp-') or f.startswith('tmp_pack_')] | 376 f.startswith('.tmp-') or f.startswith('tmp_pack_')] |
369 for f in pack_files: | 377 for f in pack_files: |
370 f = os.path.join(pack_dir, f) | 378 f = os.path.join(pack_dir, f) |
371 try: | 379 try: |
372 os.remove(f) | 380 os.remove(f) |
373 logging.warn('Deleted stale temporary pack file %s' % f) | 381 logging.warn('Deleted stale temporary pack file %s' % f) |
(...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
424 | 432 |
425 | 433 |
426 @subcommand.usage('[url of repo to create a bootstrap zip file]') | 434 @subcommand.usage('[url of repo to create a bootstrap zip file]') |
427 def CMDupdate_bootstrap(parser, args): | 435 def CMDupdate_bootstrap(parser, args): |
428 """Create and uploads a bootstrap tarball.""" | 436 """Create and uploads a bootstrap tarball.""" |
429 # Lets just assert we can't do this on Windows. | 437 # Lets just assert we can't do this on Windows. |
430 if sys.platform.startswith('win'): | 438 if sys.platform.startswith('win'): |
431 print('Sorry, update bootstrap will not work on Windows.', file=sys.stderr) | 439 print('Sorry, update bootstrap will not work on Windows.', file=sys.stderr) |
432 return 1 | 440 return 1 |
433 | 441 |
| 442 parser.add_option('--prune', action='store_true', |
| 443 help='Prune all other cached zipballs of the same repo.') |
| 444 |
434 # First, we need to ensure the cache is populated. | 445 # First, we need to ensure the cache is populated. |
435 populate_args = args[:] | 446 populate_args = args[:] |
436 populate_args.append('--no_bootstrap') | 447 populate_args.append('--no_bootstrap') |
437 CMDpopulate(parser, populate_args) | 448 CMDpopulate(parser, populate_args) |
438 | 449 |
439 # Get the repo directory. | 450 # Get the repo directory. |
440 _, args = parser.parse_args(args) | 451 options, args = parser.parse_args(args) |
441 url = args[0] | 452 url = args[0] |
442 mirror = Mirror(url) | 453 mirror = Mirror(url) |
443 mirror.update_bootstrap() | 454 mirror.update_bootstrap(options.prune) |
444 return 0 | 455 return 0 |
445 | 456 |
446 | 457 |
447 @subcommand.usage('[url of repo to add to or update in cache]') | 458 @subcommand.usage('[url of repo to add to or update in cache]') |
448 def CMDpopulate(parser, args): | 459 def CMDpopulate(parser, args): |
449 """Ensure that the cache has all up-to-date objects for the given repo.""" | 460 """Ensure that the cache has all up-to-date objects for the given repo.""" |
450 parser.add_option('--depth', type='int', | 461 parser.add_option('--depth', type='int', |
451 help='Only cache DEPTH commits of history') | 462 help='Only cache DEPTH commits of history') |
452 parser.add_option('--shallow', '-s', action='store_true', | 463 parser.add_option('--shallow', '-s', action='store_true', |
453 help='Only cache 10000 commits of history') | 464 help='Only cache 10000 commits of history') |
(...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
539 return options, args | 550 return options, args |
540 | 551 |
541 | 552 |
542 def main(argv): | 553 def main(argv): |
543 dispatcher = subcommand.CommandDispatcher(__name__) | 554 dispatcher = subcommand.CommandDispatcher(__name__) |
544 return dispatcher.execute(OptionParser(), argv) | 555 return dispatcher.execute(OptionParser(), argv) |
545 | 556 |
546 | 557 |
547 if __name__ == '__main__': | 558 if __name__ == '__main__': |
548 sys.exit(main(sys.argv[1:])) | 559 sys.exit(main(sys.argv[1:])) |
OLD | NEW |