OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """A git command for managing a local cache of git repositories.""" | 6 """A git command for managing a local cache of git repositories.""" |
7 | 7 |
8 from __future__ import print_function | 8 from __future__ import print_function |
9 import errno | 9 import errno |
10 import logging | 10 import logging |
(...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
92 try: | 92 try: |
93 self._make_lockfile() | 93 self._make_lockfile() |
94 except OSError as e: | 94 except OSError as e: |
95 if e.errno == errno.EEXIST: | 95 if e.errno == errno.EEXIST: |
96 raise LockError("%s is already locked" % self.path) | 96 raise LockError("%s is already locked" % self.path) |
97 else: | 97 else: |
98 raise LockError("Failed to create %s (err %s)" % (self.path, e.errno)) | 98 raise LockError("Failed to create %s (err %s)" % (self.path, e.errno)) |
99 | 99 |
100 def unlock(self): | 100 def unlock(self): |
101 """Release the lock.""" | 101 """Release the lock.""" |
102 if not self.is_locked(): | 102 try: |
103 raise LockError("%s is not locked" % self.path) | 103 if not self.is_locked(): |
104 if not self.i_am_locking(): | 104 raise LockError("%s is not locked" % self.path) |
105 raise LockError("%s is locked, but not by me" % self.path) | 105 if not self.i_am_locking(): |
106 self._remove_lockfile() | 106 raise LockError("%s is locked, but not by me" % self.path) |
| 107 self._remove_lockfile() |
| 108 except WinErr: |
| 109 # Windows is unreliable when it comes to file locking. YMMV. |
| 110 pass |
107 | 111 |
108 def break_lock(self): | 112 def break_lock(self): |
109 """Remove the lock, even if it was created by someone else.""" | 113 """Remove the lock, even if it was created by someone else.""" |
110 try: | 114 try: |
111 self._remove_lockfile() | 115 self._remove_lockfile() |
112 return True | 116 return True |
113 except OSError as exc: | 117 except OSError as exc: |
114 if exc.errno == errno.ENOENT: | 118 if exc.errno == errno.ENOENT: |
115 return False | 119 return False |
116 else: | 120 else: |
117 raise | 121 raise |
118 | 122 |
119 def is_locked(self): | 123 def is_locked(self): |
120 """Test if the file is locked by anyone. | 124 """Test if the file is locked by anyone. |
121 | 125 |
122 Note: This method is potentially racy. By the time it returns the lockfile | 126 Note: This method is potentially racy. By the time it returns the lockfile |
123 may have been unlocked, removed, or stolen by some other process. | 127 may have been unlocked, removed, or stolen by some other process. |
124 """ | 128 """ |
125 return os.path.exists(self.lockfile) | 129 return os.path.exists(self.lockfile) |
126 | 130 |
127 def i_am_locking(self): | 131 def i_am_locking(self): |
128 """Test if the file is locked by this process.""" | 132 """Test if the file is locked by this process.""" |
129 return self.is_locked() and self.pid == self._read_pid() | 133 return self.is_locked() and self.pid == self._read_pid() |
130 | 134 |
131 def __enter__(self): | |
132 self.lock() | |
133 return self | |
134 | |
135 def __exit__(self, *_exc): | |
136 # Windows is unreliable when it comes to file locking. YMMV. | |
137 try: | |
138 self.unlock() | |
139 except WinErr: | |
140 pass | |
141 | |
142 | 135 |
143 class Mirror(object): | 136 class Mirror(object): |
144 | 137 |
145 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' | 138 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' |
146 gsutil_exe = os.path.join( | 139 gsutil_exe = os.path.join( |
147 os.path.dirname(os.path.abspath(__file__)), | 140 os.path.dirname(os.path.abspath(__file__)), |
148 'third_party', 'gsutil', 'gsutil') | 141 'third_party', 'gsutil', 'gsutil') |
149 | 142 |
150 def __init__(self, url, refs=None, print_func=None): | 143 def __init__(self, url, refs=None, print_func=None): |
151 self.url = url | 144 self.url = url |
(...skipping 158 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
310 self.print( | 303 self.print( |
311 'Extracting bootstrap zipfile %s failed.\n' | 304 'Extracting bootstrap zipfile %s failed.\n' |
312 'Resuming normal operations.' % filename) | 305 'Resuming normal operations.' % filename) |
313 return False | 306 return False |
314 return True | 307 return True |
315 | 308 |
316 def exists(self): | 309 def exists(self): |
317 return os.path.isfile(os.path.join(self.mirror_path, 'config')) | 310 return os.path.isfile(os.path.join(self.mirror_path, 'config')) |
318 | 311 |
319 def populate(self, depth=None, shallow=False, bootstrap=False, | 312 def populate(self, depth=None, shallow=False, bootstrap=False, |
320 verbose=False): | 313 verbose=False, ignore_lock=False): |
321 assert self.GetCachePath() | 314 assert self.GetCachePath() |
322 if shallow and not depth: | 315 if shallow and not depth: |
323 depth = 10000 | 316 depth = 10000 |
324 gclient_utils.safe_makedirs(self.GetCachePath()) | 317 gclient_utils.safe_makedirs(self.GetCachePath()) |
325 | 318 |
326 v = [] | 319 v = [] |
327 if verbose: | 320 if verbose: |
328 v = ['-v', '--progress'] | 321 v = ['-v', '--progress'] |
329 | 322 |
330 d = [] | 323 d = [] |
331 if depth: | 324 if depth: |
332 d = ['--depth', str(depth)] | 325 d = ['--depth', str(depth)] |
333 | 326 |
334 | 327 |
335 with Lockfile(self.mirror_path): | 328 lockfile = Lockfile(self.mirror_path) |
| 329 if not ignore_lock: |
| 330 lockfile.lock() |
| 331 |
| 332 try: |
336 # Setup from scratch if the repo is new or is in a bad state. | 333 # Setup from scratch if the repo is new or is in a bad state. |
337 tempdir = None | 334 tempdir = None |
338 config_file = os.path.join(self.mirror_path, 'config') | 335 config_file = os.path.join(self.mirror_path, 'config') |
339 pack_dir = os.path.join(self.mirror_path, 'objects', 'pack') | 336 pack_dir = os.path.join(self.mirror_path, 'objects', 'pack') |
340 pack_files = [] | 337 pack_files = [] |
341 if os.path.isdir(pack_dir): | 338 if os.path.isdir(pack_dir): |
342 pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')] | 339 pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')] |
343 | 340 |
344 should_bootstrap = (not os.path.exists(config_file) or | 341 should_bootstrap = (not os.path.exists(config_file) or |
345 len(pack_files) > GC_AUTOPACKLIMIT) | 342 len(pack_files) > GC_AUTOPACKLIMIT) |
(...skipping 27 matching lines...) Expand all Loading... |
373 fetch_specs = subprocess.check_output( | 370 fetch_specs = subprocess.check_output( |
374 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'], | 371 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'], |
375 cwd=rundir).strip().splitlines() | 372 cwd=rundir).strip().splitlines() |
376 for spec in fetch_specs: | 373 for spec in fetch_specs: |
377 try: | 374 try: |
378 self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True) | 375 self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True) |
379 except subprocess.CalledProcessError: | 376 except subprocess.CalledProcessError: |
380 logging.warn('Fetch of %s failed' % spec) | 377 logging.warn('Fetch of %s failed' % spec) |
381 if tempdir: | 378 if tempdir: |
382 os.rename(tempdir, self.mirror_path) | 379 os.rename(tempdir, self.mirror_path) |
| 380 finally: |
| 381 if not ignore_lock: |
| 382 lockfile.unlock() |
383 | 383 |
384 def update_bootstrap(self): | 384 def update_bootstrap(self): |
385 # The files are named <git number>.zip | 385 # The files are named <git number>.zip |
386 gen_number = subprocess.check_output( | 386 gen_number = subprocess.check_output( |
387 [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip() | 387 [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip() |
388 self.RunGit(['gc']) # Run Garbage Collect to compress packfile. | 388 self.RunGit(['gc']) # Run Garbage Collect to compress packfile. |
389 # Creating a temp file and then deleting it ensures we can use this name. | 389 # Creating a temp file and then deleting it ensures we can use this name. |
390 _, tmp_zipfile = tempfile.mkstemp(suffix='.zip') | 390 _, tmp_zipfile = tempfile.mkstemp(suffix='.zip') |
391 os.remove(tmp_zipfile) | 391 os.remove(tmp_zipfile) |
392 subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path) | 392 subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path) |
(...skipping 97 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
490 def CMDpopulate(parser, args): | 490 def CMDpopulate(parser, args): |
491 """Ensure that the cache has all up-to-date objects for the given repo.""" | 491 """Ensure that the cache has all up-to-date objects for the given repo.""" |
492 parser.add_option('--depth', type='int', | 492 parser.add_option('--depth', type='int', |
493 help='Only cache DEPTH commits of history') | 493 help='Only cache DEPTH commits of history') |
494 parser.add_option('--shallow', '-s', action='store_true', | 494 parser.add_option('--shallow', '-s', action='store_true', |
495 help='Only cache 10000 commits of history') | 495 help='Only cache 10000 commits of history') |
496 parser.add_option('--ref', action='append', | 496 parser.add_option('--ref', action='append', |
497 help='Specify additional refs to be fetched') | 497 help='Specify additional refs to be fetched') |
498 parser.add_option('--no_bootstrap', action='store_true', | 498 parser.add_option('--no_bootstrap', action='store_true', |
499 help='Don\'t bootstrap from Google Storage') | 499 help='Don\'t bootstrap from Google Storage') |
| 500 parser.add_option('--ignore_locks', action='store_true', |
| 501 help='Don\'t try to lock repository') |
500 | 502 |
501 options, args = parser.parse_args(args) | 503 options, args = parser.parse_args(args) |
502 if not len(args) == 1: | 504 if not len(args) == 1: |
503 parser.error('git cache populate only takes exactly one repo url.') | 505 parser.error('git cache populate only takes exactly one repo url.') |
504 url = args[0] | 506 url = args[0] |
505 | 507 |
506 mirror = Mirror(url, refs=options.ref) | 508 mirror = Mirror(url, refs=options.ref) |
507 kwargs = { | 509 kwargs = { |
508 'verbose': options.verbose, | 510 'verbose': options.verbose, |
509 'shallow': options.shallow, | 511 'shallow': options.shallow, |
510 'bootstrap': not options.no_bootstrap, | 512 'bootstrap': not options.no_bootstrap, |
| 513 'ignore_lock': options.ignore_locks, |
511 } | 514 } |
512 if options.depth: | 515 if options.depth: |
513 kwargs['depth'] = options.depth | 516 kwargs['depth'] = options.depth |
514 mirror.populate(**kwargs) | 517 mirror.populate(**kwargs) |
515 | 518 |
516 | 519 |
517 @subcommand.usage('[url of repo to unlock, or -a|--all]') | 520 @subcommand.usage('[url of repo to unlock, or -a|--all]') |
518 def CMDunlock(parser, args): | 521 def CMDunlock(parser, args): |
519 """Unlock one or all repos if their lock files are still around.""" | 522 """Unlock one or all repos if their lock files are still around.""" |
520 parser.add_option('--force', '-f', action='store_true', | 523 parser.add_option('--force', '-f', action='store_true', |
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
581 return options, args | 584 return options, args |
582 | 585 |
583 | 586 |
584 def main(argv): | 587 def main(argv): |
585 dispatcher = subcommand.CommandDispatcher(__name__) | 588 dispatcher = subcommand.CommandDispatcher(__name__) |
586 return dispatcher.execute(OptionParser(), argv) | 589 return dispatcher.execute(OptionParser(), argv) |
587 | 590 |
588 | 591 |
589 if __name__ == '__main__': | 592 if __name__ == '__main__': |
590 sys.exit(main(sys.argv[1:])) | 593 sys.exit(main(sys.argv[1:])) |
OLD | NEW |