bot_update: make location of git cache configurable

recipe_modules/bot_update/resources/bot_update.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # TODO(hinoka): Use logging.
 
 import cStringIO
 import codecs
 import collections
@@ skipping 334 matching lines @@
 
 
 # How many times to try before giving up.
 ATTEMPTS = 5
 
 # Find deps2git
 DEPS2GIT_DIR_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git')
 DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py')
 S2G_INTERNAL_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git_internal',
                               'svn_to_git_internal.py')
-
-# ../../cache_dir aka /b/build/slave/cache_dir
 GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py')
-CACHE_DIR = path.join(SLAVE_DIR, 'cache_dir')
-# Because we print CACHE_DIR out into a .gclient file, and then later run
-# eval() on it, backslashes need to be escaped, otherwise "E:\b\build" gets
-# parsed as "E:[\x08][\x08]uild".
-if sys.platform.startswith('win'):
-  CACHE_DIR = CACHE_DIR.replace('\\', '\\\\')
 
 # Find the patch tool.
 if sys.platform.startswith('win'):
   if not BUILD_INTERNAL_DIR:
     print 'Warning: could not find patch tool because there is no '
     print 'build_internal present.'
     PATCH_TOOL = None
   else:
     PATCH_TOOL = path.join(BUILD_INTERNAL_DIR, 'tools', 'patch.EXE')
 else:
@@ skipping 144 matching lines @@
   else:
     git_executable = 'git'
     # On windows, subprocess doesn't fuzzy-match 'git' to 'git.bat', so we
     # have to do it explicitly. This is better than passing shell=True.
     if sys.platform.startswith('win'):
       git_executable += '.bat'
     cmd = (git_executable,) + args
   return call(*cmd, **kwargs)
 
 
-def get_gclient_spec(solutions, target_os, target_os_only):
+def get_gclient_spec(solutions, target_os, target_os_only, git_cache_dir):
   return GCLIENT_TEMPLATE % {
       'solutions': pprint.pformat(solutions, indent=4),
-      'cache_dir': '"%s"' % CACHE_DIR,
+      'cache_dir': '"%s"' % git_cache_dir,
       'target_os': ('\ntarget_os=%s' % target_os) if target_os else '',
       'target_os_only': '\ntarget_os_only=%s' % target_os_only
   }
 
 
 def check_enabled(master, builder, slave):
   if master in ENABLED_MASTERS:
     return True
   builder_list = ENABLED_BUILDERS.get(master)
   if builder_list and builder in builder_list:
@@ skipping 151 matching lines @@
     if scm_dirname != '*':
       prefix = '%s detected in checkout, ' % scm_dirname
 
     for filename in os.listdir(build_dir):
       deletion_target = path.join(build_dir, filename)
       print '%sdeleting %s...' % (prefix, deletion_target),
       remove(deletion_target)
       print 'done'
 
 
-def gclient_configure(solutions, target_os, target_os_only):
+def gclient_configure(solutions, target_os, target_os_only, git_cache_dir):
   """Should do the same thing as gclient --spec='...'."""
   with codecs.open('.gclient', mode='w', encoding='utf-8') as f:
-    f.write(get_gclient_spec(solutions, target_os, target_os_only))
+    f.write(get_gclient_spec(
+        solutions, target_os, target_os_only, git_cache_dir))
 
 
 def gclient_sync(with_branch_heads, shallow):
   # We just need to allocate a filename.
   fd, gclient_output_file = tempfile.mkstemp(suffix='.json')
   os.close(fd)
   gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
   cmd = [gclient_bin, 'sync', '--verbose', '--reset', '--force',
          '--ignore_locks', '--output-json', gclient_output_file,
          '--nohooks', '--noprehooks', '--delete_unversioned_trees']
@@ skipping 118 matching lines @@
   last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base)
   merge_base_ref = git('merge-base', last_known_deps_ref,
                        last_known_deps_git_ref, cwd=repo_base).strip()
 
   # If the merge base of the last DEPS and last .DEPS.git file is not
   # equivilent to the hash of the last DEPS file, that means the DEPS file
   # was committed after the last .DEPS.git file.
   return last_known_deps_ref != merge_base_ref
 
 
-def ensure_deps2git(solution, shallow):
+def ensure_deps2git(solution, shallow, git_cache_dir):
   repo_base = path.join(os.getcwd(), solution['name'])
   deps_file = path.join(repo_base, 'DEPS')
   deps_git_file = path.join(repo_base, '.DEPS.git')
   if (not git('ls-files', 'DEPS', cwd=repo_base).strip() or
       not git('ls-files', '.DEPS.git', cwd=repo_base).strip()):
     return
 
   print 'Checking if %s is newer than %s' % (deps_file, deps_git_file)
   if not need_to_run_deps2git(repo_base, deps_file, deps_git_file):
     return
 
   print '===DEPS file modified, need to run deps2git==='
   cmd = [sys.executable, DEPS2GIT_PATH,
          '--workspace', os.getcwd(),
-         '--cache_dir', CACHE_DIR,
+         '--cache_dir', git_cache_dir,
          '--deps', deps_file,
          '--out', deps_git_file]
   if 'chrome-internal.googlesource' in solution['url']:
     cmd.extend(['--extra-rules', S2G_INTERNAL_PATH])
   if shallow:
     cmd.append('--shallow')
   call(*cmd)
 
 
 def emit_log_lines(name, lines):
@@ skipping 54 matching lines @@
       # rev_num is really a svn revision number, convert it into a git hash.
       git_ref = get_git_hash(int(revision), branch, folder_name)
     else:
       # rev_num is actually a git hash or ref, we can just use it.
       git_ref = revision
     git('checkout', '--force', git_ref, cwd=folder_name)
   else:
     ref = branch if branch.startswith('refs/') else 'origin/%s' % branch
     git('checkout', '--force', ref, cwd=folder_name)
 
-def git_checkout(solutions, revisions, shallow, refs):
+def git_checkout(solutions, revisions, shallow, refs, git_cache_dir):
   build_dir = os.getcwd()
   # Before we do anything, break all git_cache locks.
-  if path.isdir(CACHE_DIR):
-    git('cache', 'unlock', '-vv', '--force', '--all', '--cache-dir', CACHE_DIR)
-    for item in os.listdir(CACHE_DIR):
-      filename = os.path.join(CACHE_DIR, item)
+  if path.isdir(git_cache_dir):
+    git('cache', 'unlock', '-vv', '--force', '--all',
+        '--cache-dir', git_cache_dir)
+    for item in os.listdir(git_cache_dir):
+      filename = os.path.join(git_cache_dir, item)
       if item.endswith('.lock'):
         raise Exception('%s exists after cache unlock' % filename)
   first_solution = True
   for sln in solutions:
     # This is so we can loop back and try again if we need to wait for the
     # git mirrors to update from SVN.
     done = False
     tries_left = 60
     while not done:
       name = sln['name']
       url = sln['url']
       if url == CHROMIUM_SRC_URL or url + '.git' == CHROMIUM_SRC_URL:
         # Experiments show there's little to be gained from
         # a shallow clone of src.
         shallow = False
       sln_dir = path.join(build_dir, name)
       s = ['--shallow'] if shallow else []
       populate_cmd = (['cache', 'populate', '--ignore_locks', '-v',
-                       '--cache-dir', CACHE_DIR] + s + [url])
+                       '--cache-dir', git_cache_dir] + s + [url])
       for ref in refs:
         populate_cmd.extend(['--ref', ref])
       git(*populate_cmd)
       mirror_dir = git(
-          'cache', 'exists', '--quiet', '--cache-dir', CACHE_DIR, url).strip()
+          'cache', 'exists', '--quiet',
+          '--cache-dir', git_cache_dir, url).strip()
       clone_cmd = (
           'clone', '--no-checkout', '--local', '--shared', mirror_dir, sln_dir)
 
       try:
         if not path.isdir(sln_dir):
           git(*clone_cmd)
         else:
           git('remote', 'set-url', 'origin', mirror_dir, cwd=sln_dir)
           git('fetch', 'origin', cwd=sln_dir)
         for ref in refs:
@@ skipping 317 matching lines @@
       continue
     # TODO(hinoka): Catch SVNRevisionNotFound error maybe?
     git('fetch', 'origin', cwd=deps_name)
     force_revision(deps_name, revision)
 
 
 def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
                     patch_root, issue, patchset, patch_url, rietveld_server,
                     gerrit_repo, gerrit_ref, revision_mapping,
                     apply_issue_email_file, apply_issue_key_file, buildspec,
-                    gyp_env, shallow, runhooks, refs):
+                    gyp_env, shallow, runhooks, refs, git_cache_dir):
   # Get a checkout of each solution, without DEPS or hooks.
   # Calling git directly because there is no way to run Gclient without
   # invoking DEPS.
   print 'Fetching Git checkout'
 
-  git_ref = git_checkout(solutions, revisions, shallow, refs)
+  git_ref = git_checkout(solutions, revisions, shallow, refs, git_cache_dir)
 
   patches = None
   if patch_url:
     patches = get_svn_patch(patch_url)
 
   already_patched = []
   patch_root = patch_root or ''
   for solution in solutions:
     if (patch_root == solution['name'] or
         solution['name'].startswith(patch_root + '/')):
       relative_root = solution['name'][len(patch_root) + 1:]
       target = '/'.join([relative_root, 'DEPS']).lstrip('/')
       if patches:
         apply_svn_patch(patch_root, patches, whitelist=[target])
         already_patched.append(target)
       elif issue:
         apply_rietveld_issue(issue, patchset, patch_root, rietveld_server,
                              revision_mapping, git_ref, apply_issue_email_file,
                              apply_issue_key_file, whitelist=[target])
         already_patched.append(target)
 
   if not buildspec:
     # Run deps2git if there is a DEPS change after the last .DEPS.git commit.
     for solution in solutions:
-      ensure_deps2git(solution, shallow)
+      ensure_deps2git(solution, shallow, git_cache_dir)
 
   # Ensure our build/ directory is set up with the correct .gclient file.
-  gclient_configure(solutions, target_os, target_os_only)
+  gclient_configure(solutions, target_os, target_os_only, git_cache_dir)
 
   # Let gclient do the DEPS syncing.
   # The branch-head refspec is a special case because its possible Chrome
   # src, which contains the branch-head refspecs, is DEPSed in.
   gclient_output = gclient_sync(buildspec or BRANCH_HEADS_REFSPEC in refs,
                                 shallow)
 
   # Now that gclient_sync has finished, we should revert any .DEPS.git so that
   # presubmit doesn't complain about it being modified.
   if (not buildspec and
@@ skipping 16 matching lines @@
   elif issue:
     apply_rietveld_issue(issue, patchset, patch_root, rietveld_server,
                          revision_mapping, git_ref, apply_issue_email_file,
                          apply_issue_key_file, blacklist=already_patched)
   elif gerrit_ref:
     apply_gerrit_ref(gerrit_repo, gerrit_ref, patch_root)
 
   # Reset the deps_file point in the solutions so that hooks get run properly.
   for sln in solutions:
     sln['deps_file'] = sln.get('deps_file', 'DEPS').replace('.DEPS.git', 'DEPS')
-  gclient_configure(solutions, target_os, target_os_only)
+  gclient_configure(solutions, target_os, target_os_only, git_cache_dir)
 
   return gclient_output
 
 
 def parse_revisions(revisions, root):
   """Turn a list of revision specs into a nice dictionary.
 
   We will always return a dict with {root: something}.  By default if root
   is unspecified, or if revisions is [], then revision will be assigned 'HEAD'
   """
@@ skipping 102 matching lines @@
                    help='Bypass disk detection and never shallow clone. '
                         'Does not override the --shallow flag')
   parse.add_option('--no_runhooks', action='store_true',
                    help='Do not run hooks on official builder.')
   parse.add_option('--refs', action='append',
                    help='Also fetch this refspec for the main solution(s). '
                         'Eg. +refs/branch-heads/*')
   parse.add_option('--with_branch_heads', action='store_true',
                     help='Always pass --with_branch_heads to gclient.  This '
                           'does the same thing as --refs +refs/branch-heads/*')
+  parse.add_option('--git-cache-dir', default=path.join(SLAVE_DIR, 'cache_dir'),
+                   help='Path to git cache directory.')
 
 
   options, args = parse.parse_args()
 
   if not options.refs:
     options.refs = []
 
   if options.with_branch_heads:
     options.refs.append(BRANCH_HEADS_REFSPEC)
     del options.with_branch_heads
 
   try:
     if options.revision_mapping_file:
       if options.revision_mapping:
         print ('WARNING: Ignoring --revision_mapping: --revision_mapping_file '
                'was set at the same time as --revision_mapping?')
       with open(options.revision_mapping_file, 'r') as f:
         options.revision_mapping = json.load(f)
     elif options.revision_mapping:
       options.revision_mapping = json.loads(options.revision_mapping)
   except Exception as e:
     print (
         'WARNING: Caught execption while parsing revision_mapping*: %s'
         % (str(e),)
     )
 
+  # Because we print CACHE_DIR out into a .gclient file, and then later run
+  # eval() on it, backslashes need to be escaped, otherwise "E:\b\build" gets
+  # parsed as "E:[\x08][\x08]uild".
+  if sys.platform.startswith('win'):
+    options.git_cache_dir = options.git_cache_dir.replace('\\', '\\\\')
+
   return options, args
 
 
 def prepare(options, git_slns, active):
   """Prepares the target folder before we checkout."""
   dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
   # If we're active now, but the flag file doesn't exist (we weren't active
   # last run) or vice versa, blow away all checkouts.
   if bool(active) != bool(check_flag(options.flag_file)):
     ensure_no_checkout(dir_names, '*')
@@ skipping 65 matching lines @@
           apply_issue_email_file=options.apply_issue_email_file,
           apply_issue_key_file=options.apply_issue_key_file,
 
           # For official builders.
           buildspec=buildspec,
           gyp_env=options.gyp_env,
           runhooks=not options.no_runhooks,
 
           # Finally, extra configurations such as shallowness of the clone.
           shallow=options.shallow,
-          refs=options.refs)
+          refs=options.refs,
+          git_cache_dir=options.git_cache_dir)
       gclient_output = ensure_checkout(**checkout_parameters)
     except GclientSyncFailed:
       print 'We failed gclient sync, lets delete the checkout and retry.'
       ensure_no_checkout(dir_names, '*')
       gclient_output = ensure_checkout(**checkout_parameters)
   except PatchFailed as e:
     if options.output_json:
       # Tell recipes information such as root, got_revision, etc.
       emit_json(options.output_json,
                 did_run=True,
@@ skipping 125 matching lines @@
   except Exception:
     # Unexpected failure.
     emit_flag(options.flag_file)
     raise
   else:
     emit_flag(options.flag_file)
 
 
 if __name__ == '__main__':
   sys.exit(main())