Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(78)

Side by Side Diff: git_cache.py

Issue 2031773002: git_cache.py: Clobber git repos if the config is corrupt (Closed) Base URL: https://chromium.googlesource.com/chromium/tools/depot_tools.git@master
Patch Set: subprocess2 -> subprocess Created 4 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """A git command for managing a local cache of git repositories.""" 6 """A git command for managing a local cache of git repositories."""
7 7
8 from __future__ import print_function 8 from __future__ import print_function
9 import errno 9 import errno
10 import logging 10 import logging
(...skipping 20 matching lines...) Expand all
31 try: 31 try:
32 # pylint: disable=E0602 32 # pylint: disable=E0602
33 WinErr = WindowsError 33 WinErr = WindowsError
34 except NameError: 34 except NameError:
35 class WinErr(Exception): 35 class WinErr(Exception):
36 pass 36 pass
37 37
38 class LockError(Exception): 38 class LockError(Exception):
39 pass 39 pass
40 40
41 class RefsHeadsFailedToFetch(Exception): 41 class ClobberNeeded(Exception):
42 pass 42 pass
43 43
44 class Lockfile(object): 44 class Lockfile(object):
45 """Class to represent a cross-platform process-specific lockfile.""" 45 """Class to represent a cross-platform process-specific lockfile."""
46 46
47 def __init__(self, path, timeout=0): 47 def __init__(self, path, timeout=0):
48 self.path = os.path.abspath(path) 48 self.path = os.path.abspath(path)
49 self.timeout = timeout 49 self.timeout = timeout
50 self.lockfile = self.path + ".lock" 50 self.lockfile = self.path + ".lock"
51 self.pid = os.getpid() 51 self.pid = os.getpid()
(...skipping 187 matching lines...) Expand 10 before | Expand all | Expand 10 after
239 env.setdefault('GIT_ASKPASS', 'true') 239 env.setdefault('GIT_ASKPASS', 'true')
240 env.setdefault('SSH_ASKPASS', 'true') 240 env.setdefault('SSH_ASKPASS', 'true')
241 self.print('running "git %s" in "%s"' % (' '.join(cmd), cwd)) 241 self.print('running "git %s" in "%s"' % (' '.join(cmd), cwd))
242 gclient_utils.CheckCallAndFilter([self.git_exe] + cmd, **kwargs) 242 gclient_utils.CheckCallAndFilter([self.git_exe] + cmd, **kwargs)
243 243
244 def config(self, cwd=None): 244 def config(self, cwd=None):
245 if cwd is None: 245 if cwd is None:
246 cwd = self.mirror_path 246 cwd = self.mirror_path
247 247
248 # Don't run git-gc in a daemon. Bad things can happen if it gets killed. 248 # Don't run git-gc in a daemon. Bad things can happen if it gets killed.
249 self.RunGit(['config', 'gc.autodetach', '0'], cwd=cwd) 249 try:
250 self.RunGit(['config', 'gc.autodetach', '0'], cwd=cwd)
251 except subprocess.CalledProcessError:
252 # Hard error, need to clobber.
253 raise ClobberNeeded()
250 254
251 # Don't combine pack files into one big pack file. It's really slow for 255 # Don't combine pack files into one big pack file. It's really slow for
252 # repositories, and there's no way to track progress and make sure it's 256 # repositories, and there's no way to track progress and make sure it's
253 # not stuck. 257 # not stuck.
254 self.RunGit(['config', 'gc.autopacklimit', '0'], cwd=cwd) 258 self.RunGit(['config', 'gc.autopacklimit', '0'], cwd=cwd)
255 259
256 # Allocate more RAM for cache-ing delta chains, for better performance 260 # Allocate more RAM for cache-ing delta chains, for better performance
257 # of "Resolving deltas". 261 # of "Resolving deltas".
258 self.RunGit(['config', 'core.deltaBaseCacheLimit', 262 self.RunGit(['config', 'core.deltaBaseCacheLimit',
259 gclient_utils.DefaultDeltaBaseCacheLimit()], cwd=cwd) 263 gclient_utils.DefaultDeltaBaseCacheLimit()], cwd=cwd)
(...skipping 140 matching lines...) Expand 10 before | Expand all | Expand 10 after
400 fetch_cmd = ['fetch'] + v + d + ['origin'] 404 fetch_cmd = ['fetch'] + v + d + ['origin']
401 fetch_specs = subprocess.check_output( 405 fetch_specs = subprocess.check_output(
402 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'], 406 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
403 cwd=rundir).strip().splitlines() 407 cwd=rundir).strip().splitlines()
404 for spec in fetch_specs: 408 for spec in fetch_specs:
405 try: 409 try:
406 self.print('Fetching %s' % spec) 410 self.print('Fetching %s' % spec)
407 self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True) 411 self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True)
408 except subprocess.CalledProcessError: 412 except subprocess.CalledProcessError:
409 if spec == '+refs/heads/*:refs/heads/*': 413 if spec == '+refs/heads/*:refs/heads/*':
410 raise RefsHeadsFailedToFetch 414 raise ClobberNeeded() # Corrupted cache.
411 logging.warn('Fetch of %s failed' % spec) 415 logging.warn('Fetch of %s failed' % spec)
412 416
413 def populate(self, depth=None, shallow=False, bootstrap=False, 417 def populate(self, depth=None, shallow=False, bootstrap=False,
414 verbose=False, ignore_lock=False, lock_timeout=0): 418 verbose=False, ignore_lock=False, lock_timeout=0):
415 assert self.GetCachePath() 419 assert self.GetCachePath()
416 if shallow and not depth: 420 if shallow and not depth:
417 depth = 10000 421 depth = 10000
418 gclient_utils.safe_makedirs(self.GetCachePath()) 422 gclient_utils.safe_makedirs(self.GetCachePath())
419 423
420 lockfile = Lockfile(self.mirror_path, lock_timeout) 424 lockfile = Lockfile(self.mirror_path, lock_timeout)
421 if not ignore_lock: 425 if not ignore_lock:
422 lockfile.lock() 426 lockfile.lock()
423 427
424 tempdir = None 428 tempdir = None
425 try: 429 try:
426 tempdir = self._ensure_bootstrapped(depth, bootstrap) 430 tempdir = self._ensure_bootstrapped(depth, bootstrap)
427 rundir = tempdir or self.mirror_path 431 rundir = tempdir or self.mirror_path
428 self._fetch(rundir, verbose, depth) 432 self._fetch(rundir, verbose, depth)
429 except RefsHeadsFailedToFetch: 433 except ClobberNeeded:
430 # This is a major failure, we need to clean and force a bootstrap. 434 # This is a major failure, we need to clean and force a bootstrap.
431 gclient_utils.rmtree(rundir) 435 gclient_utils.rmtree(rundir)
432 self.print(GIT_CACHE_CORRUPT_MESSAGE) 436 self.print(GIT_CACHE_CORRUPT_MESSAGE)
433 tempdir = self._ensure_bootstrapped(depth, bootstrap, force=True) 437 tempdir = self._ensure_bootstrapped(depth, bootstrap, force=True)
434 assert tempdir 438 assert tempdir
435 self._fetch(tempdir or self.mirror_path, verbose, depth) 439 self._fetch(tempdir or self.mirror_path, verbose, depth)
436 finally: 440 finally:
437 if tempdir: 441 if tempdir:
438 try: 442 try:
439 if os.path.exists(self.mirror_path): 443 if os.path.exists(self.mirror_path):
(...skipping 285 matching lines...) Expand 10 before | Expand all | Expand 10 after
725 dispatcher = subcommand.CommandDispatcher(__name__) 729 dispatcher = subcommand.CommandDispatcher(__name__)
726 return dispatcher.execute(OptionParser(), argv) 730 return dispatcher.execute(OptionParser(), argv)
727 731
728 732
729 if __name__ == '__main__': 733 if __name__ == '__main__':
730 try: 734 try:
731 sys.exit(main(sys.argv[1:])) 735 sys.exit(main(sys.argv[1:]))
732 except KeyboardInterrupt: 736 except KeyboardInterrupt:
733 sys.stderr.write('interrupted\n') 737 sys.stderr.write('interrupted\n')
734 sys.exit(1) 738 sys.exit(1)
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698