Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(193)

Side by Side Diff: git_cache.py

Issue 337763005: Allow pruning other files when uploading bootstrap tarball (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/depot_tools
Patch Set: Fix Created 6 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """A git command for managing a local cache of git repositories.""" 6 """A git command for managing a local cache of git repositories."""
7 7
8 from __future__ import print_function 8 from __future__ import print_function
9 import errno 9 import errno
10 import logging 10 import logging
(...skipping 328 matching lines...) Expand 10 before | Expand all | Expand 10 after
339 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'], 339 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
340 cwd=rundir).strip().splitlines() 340 cwd=rundir).strip().splitlines()
341 for spec in fetch_specs: 341 for spec in fetch_specs:
342 try: 342 try:
343 self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True) 343 self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True)
344 except subprocess.CalledProcessError: 344 except subprocess.CalledProcessError:
345 logging.warn('Fetch of %s failed' % spec) 345 logging.warn('Fetch of %s failed' % spec)
346 if tempdir: 346 if tempdir:
347 os.rename(tempdir, self.mirror_path) 347 os.rename(tempdir, self.mirror_path)
348 348
349 def update_bootstrap(self): 349 def update_bootstrap(self, prune=False):
350 # The files are named <git number>.zip 350 # The files are named <git number>.zip
351 gen_number = subprocess.check_output( 351 gen_number = subprocess.check_output(
352 [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip() 352 [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
353 self.RunGit(['gc']) # Run Garbage Collect to compress packfile. 353 self.RunGit(['gc']) # Run Garbage Collect to compress packfile.
354 # Creating a temp file and then deleting it ensures we can use this name. 354 # Creating a temp file and then deleting it ensures we can use this name.
355 _, tmp_zipfile = tempfile.mkstemp(suffix='.zip') 355 _, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
356 os.remove(tmp_zipfile) 356 os.remove(tmp_zipfile)
357 subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path) 357 subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path)
358 gsutil = Gsutil(path=self.gsutil_exe, boto_path=None) 358 gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
359 dest_name = 'gs://%s/%s/%s.zip' % ( 359 gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
360 self.bootstrap_bucket, self.basedir, gen_number) 360 dest_name = '%s/%s.zip' % (gs_folder, gen_number)
361 gsutil.call('cp', tmp_zipfile, dest_name) 361 gsutil.call('cp', tmp_zipfile, dest_name)
362 os.remove(tmp_zipfile) 362 os.remove(tmp_zipfile)
363 363
364 # Remove all other files in the same directory.
365 if prune:
366 _, ls_out, _ = gsutil.check_call('ls', gs_folder)
367 for filename in ls_out.splitlines():
368 if filename == dest_name:
369 continue
370 gsutil.call('rm', filename)
371
364 @staticmethod 372 @staticmethod
365 def DeleteTmpPackFiles(path): 373 def DeleteTmpPackFiles(path):
366 pack_dir = os.path.join(path, 'objects', 'pack') 374 pack_dir = os.path.join(path, 'objects', 'pack')
367 pack_files = [f for f in os.listdir(pack_dir) if 375 pack_files = [f for f in os.listdir(pack_dir) if
368 f.startswith('.tmp-') or f.startswith('tmp_pack_')] 376 f.startswith('.tmp-') or f.startswith('tmp_pack_')]
369 for f in pack_files: 377 for f in pack_files:
370 f = os.path.join(pack_dir, f) 378 f = os.path.join(pack_dir, f)
371 try: 379 try:
372 os.remove(f) 380 os.remove(f)
373 logging.warn('Deleted stale temporary pack file %s' % f) 381 logging.warn('Deleted stale temporary pack file %s' % f)
(...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after
424 432
425 433
426 @subcommand.usage('[url of repo to create a bootstrap zip file]') 434 @subcommand.usage('[url of repo to create a bootstrap zip file]')
427 def CMDupdate_bootstrap(parser, args): 435 def CMDupdate_bootstrap(parser, args):
428 """Create and uploads a bootstrap tarball.""" 436 """Create and uploads a bootstrap tarball."""
429 # Lets just assert we can't do this on Windows. 437 # Lets just assert we can't do this on Windows.
430 if sys.platform.startswith('win'): 438 if sys.platform.startswith('win'):
431 print('Sorry, update bootstrap will not work on Windows.', file=sys.stderr) 439 print('Sorry, update bootstrap will not work on Windows.', file=sys.stderr)
432 return 1 440 return 1
433 441
442 parser.add_option('--prune', action='store_true',
443 help='Prune all other cached zipballs of the same repo.')
444
434 # First, we need to ensure the cache is populated. 445 # First, we need to ensure the cache is populated.
435 populate_args = args[:] 446 populate_args = args[:]
436 populate_args.append('--no_bootstrap') 447 populate_args.append('--no_bootstrap')
437 CMDpopulate(parser, populate_args) 448 CMDpopulate(parser, populate_args)
438 449
439 # Get the repo directory. 450 # Get the repo directory.
440 _, args = parser.parse_args(args) 451 options, args = parser.parse_args(args)
441 url = args[0] 452 url = args[0]
442 mirror = Mirror(url) 453 mirror = Mirror(url)
443 mirror.update_bootstrap() 454 mirror.update_bootstrap(options.prune)
444 return 0 455 return 0
445 456
446 457
447 @subcommand.usage('[url of repo to add to or update in cache]') 458 @subcommand.usage('[url of repo to add to or update in cache]')
448 def CMDpopulate(parser, args): 459 def CMDpopulate(parser, args):
449 """Ensure that the cache has all up-to-date objects for the given repo.""" 460 """Ensure that the cache has all up-to-date objects for the given repo."""
450 parser.add_option('--depth', type='int', 461 parser.add_option('--depth', type='int',
451 help='Only cache DEPTH commits of history') 462 help='Only cache DEPTH commits of history')
452 parser.add_option('--shallow', '-s', action='store_true', 463 parser.add_option('--shallow', '-s', action='store_true',
453 help='Only cache 10000 commits of history') 464 help='Only cache 10000 commits of history')
(...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after
539 return options, args 550 return options, args
540 551
541 552
542 def main(argv): 553 def main(argv):
543 dispatcher = subcommand.CommandDispatcher(__name__) 554 dispatcher = subcommand.CommandDispatcher(__name__)
544 return dispatcher.execute(OptionParser(), argv) 555 return dispatcher.execute(OptionParser(), argv)
545 556
546 557
547 if __name__ == '__main__': 558 if __name__ == '__main__':
548 sys.exit(main(sys.argv[1:])) 559 sys.exit(main(sys.argv[1:]))
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698