OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """A git command for managing a local cache of git repositories.""" | 6 """A git command for managing a local cache of git repositories.""" |
7 | 7 |
8 from __future__ import print_function | 8 from __future__ import print_function |
9 import errno | 9 import errno |
10 import logging | 10 import logging |
(...skipping 10 matching lines...) Expand all Loading... |
21 import gclient_utils | 21 import gclient_utils |
22 import subcommand | 22 import subcommand |
23 | 23 |
24 try: | 24 try: |
25 # pylint: disable=E0602 | 25 # pylint: disable=E0602 |
26 WinErr = WindowsError | 26 WinErr = WindowsError |
27 except NameError: | 27 except NameError: |
28 class WinErr(Exception): | 28 class WinErr(Exception): |
29 pass | 29 pass |
30 | 30 |
| 31 |
31 class LockError(Exception): | 32 class LockError(Exception): |
32 pass | 33 pass |
33 | 34 |
34 | 35 |
35 class Lockfile(object): | 36 class Lockfile(object): |
36 """Class to represent a cross-platform process-specific lockfile.""" | 37 """Class to represent a cross-platform process-specific lockfile.""" |
37 | 38 |
38 def __init__(self, path): | 39 def __init__(self, path): |
39 self.path = os.path.abspath(path) | 40 self.path = os.path.abspath(path) |
40 self.lockfile = self.path + ".lock" | 41 self.lockfile = self.path + ".lock" |
(...skipping 103 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
144 'third_party', 'gsutil', 'gsutil') | 145 'third_party', 'gsutil', 'gsutil') |
145 bootstrap_bucket = 'chromium-git-cache' | 146 bootstrap_bucket = 'chromium-git-cache' |
146 | 147 |
147 def __init__(self, url, refs=None, print_func=None): | 148 def __init__(self, url, refs=None, print_func=None): |
148 self.url = url | 149 self.url = url |
149 self.refs = refs or [] | 150 self.refs = refs or [] |
150 self.basedir = self.UrlToCacheDir(url) | 151 self.basedir = self.UrlToCacheDir(url) |
151 self.mirror_path = os.path.join(self.GetCachePath(), self.basedir) | 152 self.mirror_path = os.path.join(self.GetCachePath(), self.basedir) |
152 self.print = print_func or print | 153 self.print = print_func or print |
153 | 154 |
| 155 @classmethod |
| 156 def from_repo(cls, path=None): |
| 157 """Returns Mirror if path is in a cached repo, else None.""" |
| 158 args = ['-C', path] if path else [] |
| 159 args = [cls.git_exe] + args + ['rev-parse', '--git-dir'] |
| 160 git_path = subprocess.check_output(args).strip() |
| 161 alt_path = os.path.join(git_path, 'objects', 'info', 'alternates') |
| 162 |
| 163 if os.path.exists(alt_path): |
| 164 with open(alt_path, 'rb') as alt: |
| 165 mirror_path = alt.read().strip() |
| 166 mirror_path = os.path.dirname(mirror_path) |
| 167 cache_path = os.path.dirname(mirror_path) |
| 168 if os.path.exists(mirror_path): |
| 169 url = subprocess.check_output( |
| 170 [cls.git_exe, '-C', mirror_path, 'config', 'remote.origin.url'] |
| 171 ).strip() |
| 172 |
| 173 # TODO(iannucci): cache_path should NOT be a class attribute. Maybe |
| 174 # a `default_cache_path`, but not the actual path. |
| 175 cls.SetCachePath(cache_path) |
| 176 |
| 177 return cls(url) |
| 178 |
154 @staticmethod | 179 @staticmethod |
155 def UrlToCacheDir(url): | 180 def UrlToCacheDir(url): |
156 """Convert a git url to a normalized form for the cache dir path.""" | 181 """Convert a git url to a normalized form for the cache dir path.""" |
157 parsed = urlparse.urlparse(url) | 182 parsed = urlparse.urlparse(url) |
158 norm_url = parsed.netloc + parsed.path | 183 norm_url = parsed.netloc + parsed.path |
159 if norm_url.endswith('.git'): | 184 if norm_url.endswith('.git'): |
160 norm_url = norm_url[:-len('.git')] | 185 norm_url = norm_url[:-len('.git')] |
161 return norm_url.replace('-', '--').replace('/', '-').lower() | 186 return norm_url.replace('-', '--').replace('/', '-').lower() |
162 | 187 |
163 @staticmethod | 188 @staticmethod |
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
215 self.RunGit(['config', '--replace-all', 'remote.origin.fetch', | 240 self.RunGit(['config', '--replace-all', 'remote.origin.fetch', |
216 '+refs/heads/*:refs/heads/*'], cwd=cwd) | 241 '+refs/heads/*:refs/heads/*'], cwd=cwd) |
217 for ref in self.refs: | 242 for ref in self.refs: |
218 ref = ref.lstrip('+').rstrip('/') | 243 ref = ref.lstrip('+').rstrip('/') |
219 if ref.startswith('refs/'): | 244 if ref.startswith('refs/'): |
220 refspec = '+%s:%s' % (ref, ref) | 245 refspec = '+%s:%s' % (ref, ref) |
221 else: | 246 else: |
222 refspec = '+refs/%s/*:refs/%s/*' % (ref, ref) | 247 refspec = '+refs/%s/*:refs/%s/*' % (ref, ref) |
223 self.RunGit(['config', '--add', 'remote.origin.fetch', refspec], cwd=cwd) | 248 self.RunGit(['config', '--add', 'remote.origin.fetch', refspec], cwd=cwd) |
224 | 249 |
225 def bootstrap_repo(self, directory): | 250 def bootstrap_repo(self, directory, verbose): |
226 """Bootstrap the repo from Google Stroage if possible.""" | 251 """Bootstrap the repo from Google Stroage if possible.""" |
227 | 252 |
228 python_fallback = False | 253 python_fallback = False |
229 if sys.platform.startswith('win') and not self.FindExecutable('7z'): | 254 if sys.platform.startswith('win') and not self.FindExecutable('7z'): |
230 python_fallback = True | 255 python_fallback = True |
231 elif sys.platform.startswith('darwin'): | 256 elif sys.platform.startswith('darwin'): |
232 # The OSX version of unzip doesn't support zip64. | 257 # The OSX version of unzip doesn't support zip64. |
233 python_fallback = True | 258 python_fallback = True |
234 elif not self.FindExecutable('unzip'): | 259 elif not self.FindExecutable('unzip'): |
235 python_fallback = True | 260 python_fallback = True |
236 | 261 |
237 gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir) | 262 gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir) |
238 gsutil = Gsutil( | 263 gsutil = Gsutil( |
239 self.gsutil_exe, boto_path=os.devnull, bypass_prodaccess=True) | 264 self.gsutil_exe, boto_path=os.devnull, bypass_prodaccess=True) |
240 # Get the most recent version of the zipfile. | 265 # Get the most recent version of the zipfile. |
241 _, ls_out, _ = gsutil.check_call('ls', gs_folder) | 266 _, ls_out, _ = gsutil.check_call('ls', gs_folder) |
242 ls_out_sorted = sorted(ls_out.splitlines()) | 267 ls_out_sorted = sorted(ls_out.splitlines()) |
243 if not ls_out_sorted: | 268 if not ls_out_sorted: |
244 # This repo is not on Google Storage. | 269 # This repo is not on Google Storage. |
245 return False | 270 return False |
246 latest_checkout = ls_out_sorted[-1] | 271 latest_checkout = ls_out_sorted[-1] |
247 | 272 |
248 # Download zip file to a temporary directory. | 273 # Download zip file to a temporary directory. |
249 try: | 274 try: |
250 tempdir = tempfile.mkdtemp() | 275 tempdir = tempfile.mkdtemp() |
251 self.print('Downloading %s' % latest_checkout) | 276 if not verbose: |
252 code, out, err = gsutil.check_call('cp', latest_checkout, tempdir) | 277 self.print('Downloading %s' % latest_checkout) |
| 278 code, out, err = gsutil.check_call('cp', latest_checkout, tempdir, |
| 279 verbose=verbose) |
253 if code: | 280 if code: |
254 self.print('%s\n%s' % (out, err)) | 281 self.print('%s\n%s' % (out, err)) |
255 return False | 282 return False |
256 filename = os.path.join(tempdir, latest_checkout.split('/')[-1]) | 283 filename = os.path.join(tempdir, latest_checkout.split('/')[-1]) |
257 | 284 |
258 # Unpack the file with 7z on Windows, unzip on linux, or fallback. | 285 # Unpack the file with 7z on Windows, unzip on linux, or fallback. |
259 if not python_fallback: | 286 if not python_fallback: |
260 if sys.platform.startswith('win'): | 287 if sys.platform.startswith('win'): |
261 cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename] | 288 cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename] |
262 else: | 289 else: |
(...skipping 17 matching lines...) Expand all Loading... |
280 self.print( | 307 self.print( |
281 'Extracting bootstrap zipfile %s failed.\n' | 308 'Extracting bootstrap zipfile %s failed.\n' |
282 'Resuming normal operations.' % filename) | 309 'Resuming normal operations.' % filename) |
283 return False | 310 return False |
284 return True | 311 return True |
285 | 312 |
286 def exists(self): | 313 def exists(self): |
287 return os.path.isfile(os.path.join(self.mirror_path, 'config')) | 314 return os.path.isfile(os.path.join(self.mirror_path, 'config')) |
288 | 315 |
289 def populate(self, depth=None, shallow=False, bootstrap=False, | 316 def populate(self, depth=None, shallow=False, bootstrap=False, |
290 verbose=False): | 317 verbose=False, fetch_specs=()): |
291 if shallow and not depth: | 318 if shallow and not depth: |
292 depth = 10000 | 319 depth = 10000 |
293 gclient_utils.safe_makedirs(self.GetCachePath()) | 320 gclient_utils.safe_makedirs(self.GetCachePath()) |
294 | 321 |
295 v = [] | 322 v = [] |
296 if verbose: | 323 if verbose: |
297 v = ['-v', '--progress'] | 324 v = ['-v', '--progress'] |
298 | 325 |
299 d = [] | 326 d = [] |
300 if depth: | 327 if depth: |
301 d = ['--depth', str(depth)] | 328 d = ['--depth', str(depth)] |
302 | 329 |
303 | 330 |
304 with Lockfile(self.mirror_path): | 331 with Lockfile(self.mirror_path): |
305 # Setup from scratch if the repo is new or is in a bad state. | 332 # Setup from scratch if the repo is new or is in a bad state. |
306 tempdir = None | 333 tempdir = None |
307 if not os.path.exists(os.path.join(self.mirror_path, 'config')): | 334 if not os.path.exists(os.path.join(self.mirror_path, 'config')): |
308 gclient_utils.rmtree(self.mirror_path) | 335 gclient_utils.rmtree(self.mirror_path) |
309 tempdir = tempfile.mkdtemp( | 336 tempdir = tempfile.mkdtemp( |
310 suffix=self.basedir, dir=self.GetCachePath()) | 337 suffix=self.basedir, dir=self.GetCachePath()) |
311 bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir) | 338 bootstrapped = (not depth and bootstrap and |
| 339 self.bootstrap_repo(tempdir, verbose)) |
312 if not bootstrapped: | 340 if not bootstrapped: |
313 self.RunGit(['init', '--bare'], cwd=tempdir) | 341 self.RunGit(['init', '--bare'], cwd=tempdir) |
314 else: | 342 else: |
315 if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')): | 343 if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')): |
316 logging.warn( | 344 logging.warn( |
317 'Shallow fetch requested, but repo cache already exists.') | 345 'Shallow fetch requested, but repo cache already exists.') |
318 d = [] | 346 d = [] |
319 | 347 |
320 rundir = tempdir or self.mirror_path | 348 rundir = tempdir or self.mirror_path |
321 self.config(rundir) | 349 self.config(rundir) |
322 fetch_cmd = ['fetch'] + v + d + ['origin'] | 350 fetch_cmd = ['fetch'] + v + d + ['origin'] |
323 fetch_specs = subprocess.check_output( | 351 fetch_specs = fetch_specs or subprocess.check_output( |
324 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'], | 352 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'], |
325 cwd=rundir).strip().splitlines() | 353 cwd=rundir).strip().splitlines() |
326 for spec in fetch_specs: | 354 for spec in fetch_specs: |
327 try: | 355 try: |
328 self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True) | 356 self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True) |
329 except subprocess.CalledProcessError: | 357 except subprocess.CalledProcessError: |
330 logging.warn('Fetch of %s failed' % spec) | 358 logging.warn('Fetch of %s failed' % spec) |
331 if tempdir: | 359 if tempdir: |
332 os.rename(tempdir, self.mirror_path) | 360 os.rename(tempdir, self.mirror_path) |
333 | 361 |
(...skipping 173 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
507 return options, args | 535 return options, args |
508 | 536 |
509 | 537 |
510 def main(argv): | 538 def main(argv): |
511 dispatcher = subcommand.CommandDispatcher(__name__) | 539 dispatcher = subcommand.CommandDispatcher(__name__) |
512 return dispatcher.execute(OptionParser(), argv) | 540 return dispatcher.execute(OptionParser(), argv) |
513 | 541 |
514 | 542 |
515 if __name__ == '__main__': | 543 if __name__ == '__main__': |
516 sys.exit(main(sys.argv[1:])) | 544 sys.exit(main(sys.argv[1:])) |
OLD | NEW |