Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1)

Side by Side Diff: git_cache.py

Issue 313933005: Add internal buckets for bootstrapping internal repos (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/depot_tools
Patch Set: Created 6 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """A git command for managing a local cache of git repositories.""" 6 """A git command for managing a local cache of git repositories."""
7 7
8 from __future__ import print_function 8 from __future__ import print_function
9 import errno 9 import errno
10 import logging 10 import logging
(...skipping 125 matching lines...) Expand 10 before | Expand all | Expand 10 after
136 except WinErr: 136 except WinErr:
137 pass 137 pass
138 138
139 139
140 class Mirror(object): 140 class Mirror(object):
141 141
142 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' 142 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git'
143 gsutil_exe = os.path.join( 143 gsutil_exe = os.path.join(
144 os.path.dirname(os.path.abspath(__file__)), 144 os.path.dirname(os.path.abspath(__file__)),
145 'third_party', 'gsutil', 'gsutil') 145 'third_party', 'gsutil', 'gsutil')
146 bootstrap_bucket = 'chromium-git-cache'
147 146
148 def __init__(self, url, refs=None, print_func=None): 147 def __init__(self, url, refs=None, print_func=None):
149 self.url = url 148 self.url = url
150 self.refs = refs or [] 149 self.refs = refs or []
151 self.basedir = self.UrlToCacheDir(url) 150 self.basedir = self.UrlToCacheDir(url)
152 self.mirror_path = os.path.join(self.GetCachePath(), self.basedir) 151 self.mirror_path = os.path.join(self.GetCachePath(), self.basedir)
153 self.print = print_func or print 152 self.print = print_func or print
154 153
154 def bootstrap_bucket(self):
agable 2014/06/05 00:45:27 make it an @property?
155 if 'chrome-internal' in self.url:
156 return 'chrome-git-cache'
157 else:
158 return 'chromium-git-cache'
159
155 @classmethod 160 @classmethod
156 def FromPath(cls, path): 161 def FromPath(cls, path):
157 return cls(cls.CacheDirToUrl(path)) 162 return cls(cls.CacheDirToUrl(path))
158 163
159 @staticmethod 164 @staticmethod
160 def UrlToCacheDir(url): 165 def UrlToCacheDir(url):
161 """Convert a git url to a normalized form for the cache dir path.""" 166 """Convert a git url to a normalized form for the cache dir path."""
162 parsed = urlparse.urlparse(url) 167 parsed = urlparse.urlparse(url)
163 norm_url = parsed.netloc + parsed.path 168 norm_url = parsed.netloc + parsed.path
164 if norm_url.endswith('.git'): 169 if norm_url.endswith('.git'):
(...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after
238 243
239 python_fallback = False 244 python_fallback = False
240 if sys.platform.startswith('win') and not self.FindExecutable('7z'): 245 if sys.platform.startswith('win') and not self.FindExecutable('7z'):
241 python_fallback = True 246 python_fallback = True
242 elif sys.platform.startswith('darwin'): 247 elif sys.platform.startswith('darwin'):
243 # The OSX version of unzip doesn't support zip64. 248 # The OSX version of unzip doesn't support zip64.
244 python_fallback = True 249 python_fallback = True
245 elif not self.FindExecutable('unzip'): 250 elif not self.FindExecutable('unzip'):
246 python_fallback = True 251 python_fallback = True
247 252
248 gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir) 253 gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket(), self.basedir)
agable 2014/06/05 00:45:27 Then this could go away
249 gsutil = Gsutil( 254 gsutil = Gsutil(
250 self.gsutil_exe, boto_path=os.devnull, bypass_prodaccess=True) 255 self.gsutil_exe, boto_path=os.devnull, bypass_prodaccess=True)
251 # Get the most recent version of the zipfile. 256 # Get the most recent version of the zipfile.
252 _, ls_out, _ = gsutil.check_call('ls', gs_folder) 257 _, ls_out, _ = gsutil.check_call('ls', gs_folder)
253 ls_out_sorted = sorted(ls_out.splitlines()) 258 ls_out_sorted = sorted(ls_out.splitlines())
254 if not ls_out_sorted: 259 if not ls_out_sorted:
255 # This repo is not on Google Storage. 260 # This repo is not on Google Storage.
256 return False 261 return False
257 latest_checkout = ls_out_sorted[-1] 262 latest_checkout = ls_out_sorted[-1]
258 263
(...skipping 87 matching lines...) Expand 10 before | Expand all | Expand 10 after
346 # The files are named <git number>.zip 351 # The files are named <git number>.zip
347 gen_number = subprocess.check_output( 352 gen_number = subprocess.check_output(
348 [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip() 353 [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
349 self.RunGit(['gc']) # Run Garbage Collect to compress packfile. 354 self.RunGit(['gc']) # Run Garbage Collect to compress packfile.
350 # Creating a temp file and then deleting it ensures we can use this name. 355 # Creating a temp file and then deleting it ensures we can use this name.
351 _, tmp_zipfile = tempfile.mkstemp(suffix='.zip') 356 _, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
352 os.remove(tmp_zipfile) 357 os.remove(tmp_zipfile)
353 subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path) 358 subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path)
354 gsutil = Gsutil(path=self.gsutil_exe, boto_path=None) 359 gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
355 dest_name = 'gs://%s/%s/%s.zip' % ( 360 dest_name = 'gs://%s/%s/%s.zip' % (
356 self.bootstrap_bucket, self.basedir, gen_number) 361 self.bootstrap_bucket(), self.basedir, gen_number)
357 gsutil.call('cp', tmp_zipfile, dest_name) 362 gsutil.call('cp', tmp_zipfile, dest_name)
358 os.remove(tmp_zipfile) 363 os.remove(tmp_zipfile)
359 364
360 365
361 @staticmethod 366 @staticmethod
362 def BreakLocks(path): 367 def BreakLocks(path):
363 did_unlock = False 368 did_unlock = False
364 lf = Lockfile(path) 369 lf = Lockfile(path)
365 if lf.break_lock(): 370 if lf.break_lock():
366 did_unlock = True 371 did_unlock = True
(...skipping 156 matching lines...) Expand 10 before | Expand all | Expand 10 after
523 return options, args 528 return options, args
524 529
525 530
526 def main(argv): 531 def main(argv):
527 dispatcher = subcommand.CommandDispatcher(__name__) 532 dispatcher = subcommand.CommandDispatcher(__name__)
528 return dispatcher.execute(OptionParser(), argv) 533 return dispatcher.execute(OptionParser(), argv)
529 534
530 535
531 if __name__ == '__main__': 536 if __name__ == '__main__':
532 sys.exit(main(sys.argv[1:])) 537 sys.exit(main(sys.argv[1:]))
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698