OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """A git command for managing a local cache of git repositories.""" | 6 """A git command for managing a local cache of git repositories.""" |
7 | 7 |
8 from __future__ import print_function | 8 from __future__ import print_function |
9 import errno | 9 import errno |
10 import logging | 10 import logging |
11 import optparse | 11 import optparse |
12 import os | 12 import os |
13 import tempfile | 13 import tempfile |
14 import subprocess | 14 import subprocess |
15 import sys | 15 import sys |
16 import urlparse | 16 import urlparse |
17 | 17 |
18 from download_from_google_storage import Gsutil | 18 from download_from_google_storage import Gsutil |
19 import gclient_utils | 19 import gclient_utils |
20 import subcommand | 20 import subcommand |
21 | 21 |
22 try: | 22 try: |
23 # pylint: disable=E0602 | 23 # pylint: disable=E0602 |
24 WinErr = WindowsError | 24 WinErr = WindowsError |
25 except NameError: | 25 except NameError: |
26 class WinErr(Exception): | 26 class WinErr(Exception): |
27 pass | 27 pass |
28 | 28 |
29 | |
29 class LockError(Exception): | 30 class LockError(Exception): |
30 pass | 31 pass |
31 | 32 |
32 | 33 |
33 class Lockfile(object): | 34 class Lockfile(object): |
34 """Class to represent a cross-platform process-specific lockfile.""" | 35 """Class to represent a cross-platform process-specific lockfile.""" |
35 | 36 |
36 def __init__(self, path): | 37 def __init__(self, path): |
37 self.path = os.path.abspath(path) | 38 self.path = os.path.abspath(path) |
38 self.lockfile = self.path + ".lock" | 39 self.lockfile = self.path + ".lock" |
(...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
128 'third_party', 'gsutil', 'gsutil') | 129 'third_party', 'gsutil', 'gsutil') |
129 bootstrap_bucket = 'chromium-git-cache' | 130 bootstrap_bucket = 'chromium-git-cache' |
130 | 131 |
131 def __init__(self, url, refs=None, print_func=None): | 132 def __init__(self, url, refs=None, print_func=None): |
132 self.url = url | 133 self.url = url |
133 self.refs = refs or [] | 134 self.refs = refs or [] |
134 self.basedir = self.UrlToCacheDir(url) | 135 self.basedir = self.UrlToCacheDir(url) |
135 self.mirror_path = os.path.join(self.GetCachePath(), self.basedir) | 136 self.mirror_path = os.path.join(self.GetCachePath(), self.basedir) |
136 self.print = print_func or print | 137 self.print = print_func or print |
137 | 138 |
139 @classmethod | |
140 def from_repo(cls, path=None): | |
141 """Returns Mirror if path is in a cached repo, else None.""" | |
142 args = ['-C', path] if path else [] | |
143 args = [cls.git_exe] + args + ['rev-parse', '--git-dir'] | |
144 git_path = subprocess.check_output(args).strip() | |
145 alt_path = os.path.join(git_path, 'objects', 'info', 'alternates') | |
146 | |
147 if os.path.exists(alt_path): | |
148 with open(alt_path, 'rb') as alt: | |
149 mirror_path = alt.read().strip() | |
150 mirror_path = os.path.dirname(mirror_path) | |
151 cache_path = os.path.dirname(mirror_path) | |
152 if os.path.exists(mirror_path): | |
153 url = subprocess.check_output( | |
154 [cls.git_exe, '-C', mirror_path, 'config', 'remote.origin.url'] | |
155 ).strip() | |
156 | |
157 # TODO(iannucci): cache_path should NOT be a class attribute. Maybe | |
158 # a `default_cache_path`, but not the actual path. | |
159 cls.SetCachePath(cache_path) | |
160 | |
161 return cls(url) | |
162 | |
138 @staticmethod | 163 @staticmethod |
139 def UrlToCacheDir(url): | 164 def UrlToCacheDir(url): |
140 """Convert a git url to a normalized form for the cache dir path.""" | 165 """Convert a git url to a normalized form for the cache dir path.""" |
141 parsed = urlparse.urlparse(url) | 166 parsed = urlparse.urlparse(url) |
142 norm_url = parsed.netloc + parsed.path | 167 norm_url = parsed.netloc + parsed.path |
143 if norm_url.endswith('.git'): | 168 if norm_url.endswith('.git'): |
144 norm_url = norm_url[:-len('.git')] | 169 norm_url = norm_url[:-len('.git')] |
145 return norm_url.replace('-', '--').replace('/', '-').lower() | 170 return norm_url.replace('-', '--').replace('/', '-').lower() |
146 | 171 |
147 @staticmethod | 172 @staticmethod |
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
199 self.RunGit(['config', '--replace-all', 'remote.origin.fetch', | 224 self.RunGit(['config', '--replace-all', 'remote.origin.fetch', |
200 '+refs/heads/*:refs/heads/*'], cwd=cwd) | 225 '+refs/heads/*:refs/heads/*'], cwd=cwd) |
201 for ref in self.refs: | 226 for ref in self.refs: |
202 ref = ref.lstrip('+').rstrip('/') | 227 ref = ref.lstrip('+').rstrip('/') |
203 if ref.startswith('refs/'): | 228 if ref.startswith('refs/'): |
204 refspec = '+%s:%s' % (ref, ref) | 229 refspec = '+%s:%s' % (ref, ref) |
205 else: | 230 else: |
206 refspec = '+refs/%s/*:refs/%s/*' % (ref, ref) | 231 refspec = '+refs/%s/*:refs/%s/*' % (ref, ref) |
207 self.RunGit(['config', '--add', 'remote.origin.fetch', refspec], cwd=cwd) | 232 self.RunGit(['config', '--add', 'remote.origin.fetch', refspec], cwd=cwd) |
208 | 233 |
209 def bootstrap_repo(self, directory): | 234 def bootstrap_repo(self, directory, verbose): |
210 """Bootstrap the repo from Google Stroage if possible. | 235 """Bootstrap the repo from Google Stroage if possible. |
211 | 236 |
212 Requires 7z on Windows and Unzip on Linux/Mac. | 237 Requires 7z on Windows and Unzip on Linux/Mac. |
213 """ | 238 """ |
214 if sys.platform.startswith('win'): | 239 if sys.platform.startswith('win'): |
215 if not self.FindExecutable('7z'): | 240 if not self.FindExecutable('7z'): |
216 self.print(''' | 241 self.print(''' |
217 Cannot find 7z in the path. If you want git cache to be able to bootstrap from | 242 Cannot find 7z in the path. If you want git cache to be able to bootstrap from |
218 Google Storage, please install 7z from: | 243 Google Storage, please install 7z from: |
219 | 244 |
(...skipping 15 matching lines...) Expand all Loading... | |
235 _, ls_out, _ = gsutil.check_call('ls', gs_folder) | 260 _, ls_out, _ = gsutil.check_call('ls', gs_folder) |
236 ls_out_sorted = sorted(ls_out.splitlines()) | 261 ls_out_sorted = sorted(ls_out.splitlines()) |
237 if not ls_out_sorted: | 262 if not ls_out_sorted: |
238 # This repo is not on Google Storage. | 263 # This repo is not on Google Storage. |
239 return False | 264 return False |
240 latest_checkout = ls_out_sorted[-1] | 265 latest_checkout = ls_out_sorted[-1] |
241 | 266 |
242 # Download zip file to a temporary directory. | 267 # Download zip file to a temporary directory. |
243 try: | 268 try: |
244 tempdir = tempfile.mkdtemp() | 269 tempdir = tempfile.mkdtemp() |
245 self.print('Downloading %s' % latest_checkout) | 270 if not verbose: |
agable
2014/04/18 00:17:08
print if *not* verbose??
iannucci
2014/04/28 21:05:28
Yeah, otherwise this doubles up with the gsutil ou
| |
246 code, out, err = gsutil.check_call('cp', latest_checkout, tempdir) | 271 self.print('Downloading %s' % latest_checkout) |
272 code, out, err = gsutil.check_call('cp', latest_checkout, tempdir, | |
273 verbose=verbose) | |
247 if code: | 274 if code: |
248 self.print('%s\n%s' % (out, err)) | 275 self.print('%s\n%s' % (out, err)) |
249 return False | 276 return False |
250 filename = os.path.join(tempdir, latest_checkout.split('/')[-1]) | 277 filename = os.path.join(tempdir, latest_checkout.split('/')[-1]) |
251 | 278 |
252 # Unpack the file with 7z on Windows, or unzip everywhere else. | 279 # Unpack the file with 7z on Windows, or unzip everywhere else. |
253 if sys.platform.startswith('win'): | 280 if sys.platform.startswith('win'): |
254 cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename] | 281 cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename] |
255 else: | 282 else: |
256 cmd = ['unzip', filename, '-d', directory] | 283 cmd = ['unzip', filename, '-d', directory] |
257 retcode = subprocess.call(cmd) | 284 retcode = subprocess.call(cmd) |
258 finally: | 285 finally: |
259 # Clean up the downloaded zipfile. | 286 # Clean up the downloaded zipfile. |
260 gclient_utils.rmtree(tempdir) | 287 gclient_utils.rmtree(tempdir) |
261 | 288 |
262 if retcode: | 289 if retcode: |
263 self.print( | 290 self.print( |
264 'Extracting bootstrap zipfile %s failed.\n' | 291 'Extracting bootstrap zipfile %s failed.\n' |
265 'Resuming normal operations.' % filename) | 292 'Resuming normal operations.' % filename) |
266 return False | 293 return False |
267 return True | 294 return True |
268 | 295 |
269 def exists(self): | 296 def exists(self): |
270 return os.path.isfile(os.path.join(self.mirror_path, 'config')) | 297 return os.path.isfile(os.path.join(self.mirror_path, 'config')) |
271 | 298 |
272 def populate(self, depth=None, shallow=False, bootstrap=False, | 299 def populate(self, depth=None, shallow=False, bootstrap=False, |
273 verbose=False): | 300 verbose=False, fetch_specs=()): |
agable
2014/04/18 00:17:08
None
iannucci
2014/04/28 21:05:28
Er... why? it's iterable and immutable... that's s
| |
274 if shallow and not depth: | 301 if shallow and not depth: |
275 depth = 10000 | 302 depth = 10000 |
276 gclient_utils.safe_makedirs(self.GetCachePath()) | 303 gclient_utils.safe_makedirs(self.GetCachePath()) |
277 | 304 |
278 v = [] | 305 v = [] |
279 if verbose: | 306 if verbose: |
280 v = ['-v', '--progress'] | 307 v = ['-v', '--progress'] |
281 | 308 |
282 d = [] | 309 d = [] |
283 if depth: | 310 if depth: |
284 d = ['--depth', str(depth)] | 311 d = ['--depth', str(depth)] |
285 | 312 |
286 | 313 |
287 with Lockfile(self.mirror_path): | 314 with Lockfile(self.mirror_path): |
288 # Setup from scratch if the repo is new or is in a bad state. | 315 # Setup from scratch if the repo is new or is in a bad state. |
289 tempdir = None | 316 tempdir = None |
290 if not os.path.exists(os.path.join(self.mirror_path, 'config')): | 317 if not os.path.exists(os.path.join(self.mirror_path, 'config')): |
291 gclient_utils.rmtree(self.mirror_path) | 318 gclient_utils.rmtree(self.mirror_path) |
292 tempdir = tempfile.mkdtemp( | 319 tempdir = tempfile.mkdtemp( |
293 suffix=self.basedir, dir=self.GetCachePath()) | 320 suffix=self.basedir, dir=self.GetCachePath()) |
294 bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir) | 321 bootstrapped = (not depth and bootstrap and |
322 self.bootstrap_repo(tempdir, verbose)) | |
295 if not bootstrapped: | 323 if not bootstrapped: |
296 self.RunGit(['init', '--bare'], cwd=tempdir) | 324 self.RunGit(['init', '--bare'], cwd=tempdir) |
297 else: | 325 else: |
298 if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')): | 326 if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')): |
299 logging.warn( | 327 logging.warn( |
300 'Shallow fetch requested, but repo cache already exists.') | 328 'Shallow fetch requested, but repo cache already exists.') |
301 d = [] | 329 d = [] |
302 | 330 |
303 rundir = tempdir or self.mirror_path | 331 rundir = tempdir or self.mirror_path |
304 self.config(rundir) | 332 self.config(rundir) |
305 fetch_cmd = ['fetch'] + v + d + ['origin'] | 333 fetch_cmd = ['fetch'] + v + d + ['origin'] |
306 fetch_specs = subprocess.check_output( | 334 fetch_specs = fetch_specs or subprocess.check_output( |
307 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'], | 335 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'], |
308 cwd=rundir).strip().splitlines() | 336 cwd=rundir).strip().splitlines() |
309 for spec in fetch_specs: | 337 for spec in fetch_specs: |
310 try: | 338 try: |
311 self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True) | 339 self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True) |
312 except subprocess.CalledProcessError: | 340 except subprocess.CalledProcessError: |
313 logging.warn('Fetch of %s failed' % spec) | 341 logging.warn('Fetch of %s failed' % spec) |
314 if tempdir: | 342 if tempdir: |
315 os.rename(tempdir, self.mirror_path) | 343 os.rename(tempdir, self.mirror_path) |
316 | 344 |
(...skipping 173 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
490 return options, args | 518 return options, args |
491 | 519 |
492 | 520 |
493 def main(argv): | 521 def main(argv): |
494 dispatcher = subcommand.CommandDispatcher(__name__) | 522 dispatcher = subcommand.CommandDispatcher(__name__) |
495 return dispatcher.execute(OptionParser(), argv) | 523 return dispatcher.execute(OptionParser(), argv) |
496 | 524 |
497 | 525 |
498 if __name__ == '__main__': | 526 if __name__ == '__main__': |
499 sys.exit(main(sys.argv[1:])) | 527 sys.exit(main(sys.argv[1:])) |
OLD | NEW |