Bot Update support on official builders

scripts/slave/bot_update.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import codecs
 import copy
 import cStringIO
 import ctypes
 import json
@@ skipping 13 matching lines @@
 import os.path as path
 
 from common import chromium_utils
 
 
 CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src.git'
 
 RECOGNIZED_PATHS = {
     # If SVN path matches key, the entire URL is rewritten to the Git url.
     '/chrome/trunk/src':
-       CHROMIUM_SRC_URL,
+        CHROMIUM_SRC_URL,
     '/chrome-internal/trunk/src-internal':
-        'https://chrome-internal.googlesource.com/chrome/src-internal.git'
+        'https://chrome-internal.googlesource.com/chrome/src-internal.git',
 }
 
+# Official builds use buildspecs, so this is a special case.
+BUILDSPEC_RE = r'^/chrome-internal/trunk/tools/buildspec/build/(.*)$'
+GIT_BUILDSPEC_PATH = ('https://chrome-internal.googlesource.com/chrome/tools/'
+                      'buildspec')
+
+# This is the repository that the buildspecs2git cron job mirrors
+# all buildspecs into. When we see an svn buildspec, we rely on the
+# buildspecs2git cron job to produce the git version of the buildspec.
+GIT_BUILDSPEC_REPO = (
+    'https://chrome-internal.googlesource.com/chrome/tools/git_buildspecs')
+
 # Copied from scripts/recipes/chromium.py.
 GOT_REVISION_MAPPINGS = {
     '/chrome/trunk/src': {
         'src/': 'got_revision',
         'src/native_client/': 'got_nacl_revision',
         'src/tools/swarm_client/': 'got_swarm_client_revision',
         'src/tools/swarming_client/': 'got_swarming_client_revision',
         'src/third_party/WebKit/': 'got_webkit_revision',
         'src/third_party/webrtc/': 'got_webrtc_revision',
     }
 }
 
 
-
 BOT_UPDATE_MESSAGE = """
 What is the "Bot Update" step?
 ==============================
 
 This step ensures that the source checkout on the bot (e.g. Chromium's src/ and
 its dependencies) is checked out in a consistent state. This means that all of
 the necessary repositories are checked out, no extra repositories are checked
 out, and no locally modified files are present.
 
 These actions used to be taken care of by the "gclient revert" and "update"
@@ skipping 36 matching lines @@
 
 
 GCLIENT_TEMPLATE = """solutions = %(solutions)s
 
 cache_dir = %(cache_dir)s
 %(target_os)s
 """
 
 # IMPORTANT: If you're trying to enable a RECIPE bot, you'll need to
 # edit recipe_modules/bot_update/api.py instead.
-ENABLED_MASTERS = ['chromium.git']
+ENABLED_MASTERS = ['chromium.git', 'chrome_git']
 ENABLED_BUILDERS = {
     'tryserver.chromium': ['linux_rel_alt'],
 }
 ENABLED_SLAVES = {
     # This is enabled on a bot-to-bot basis to ensure that we don't have
     # bots that have mixed configs.
     'chromium.fyi': [
         'build1-m1',  # Chromium Builder / Chromium Builder (dbg)
         'vm928-m1',   # Chromium Linux Buildrunner
         'vm859-m1',   # Chromium Linux Redux
@@ skipping 175 matching lines @@
 
 
 def solutions_to_git(input_solutions):
   """Modifies urls in solutions to point at Git repos.
 
   returns: (git solution, svn root of first solution) tuple.
   """
   assert input_solutions
   solutions = copy.deepcopy(input_solutions)
   first_solution = True
+  buildspec_name = False
   for solution in solutions:
     original_url = solution['url']
     parsed_url = urlparse.urlparse(original_url)
     parsed_path = parsed_url.path
+
+    # Rewrite SVN urls into Git urls.
+    buildspec_m = re.match(BUILDSPEC_RE, parsed_path)
+    if first_solution and buildspec_m:
+      solution['url'] = GIT_BUILDSPEC_PATH
+      buildspec_name = buildspec_m.group(1)
+    elif parsed_path in RECOGNIZED_PATHS:
+      solution['url'] = RECOGNIZED_PATHS[parsed_path]
+    else:
+      print 'Warning: path %s not recognized' % parsed_path
+
     if first_solution:
       root = parsed_path
       first_solution = False
-    if parsed_path in RECOGNIZED_PATHS:
-      solution['url'] = RECOGNIZED_PATHS[parsed_path]
-    else:
-      print 'Warning: path %s not recognized' % parsed_path
-    if solution.get('deps_file', 'DEPS') == 'DEPS':
-      solution['deps_file'] = '.DEPS.git'
+
+
     solution['managed'] = False
     # We don't want gclient to be using a safesync URL. Instead it should
     # using the lkgr/lkcr branch/tags.
     if 'safesync_url' in solution:
       print 'Removing safesync url %s from %s' % (solution['safesync_url'],
                                                   parsed_path)
       del solution['safesync_url']
-  return solutions, root
+  return solutions, root, buildspec_name
 
 
 def ensure_no_checkout(dir_names, scm_dirname):
   """Ensure that there is no undesired checkout under build/.
 
   If there is an incorrect checkout under build/, then
   move build/ to build.dead/
   This function will check each directory in dir_names.
 
   scm_dirname is expected to be either ['.svn', '.git']
@@ skipping 17 matching lines @@
         chromium_utils.RemoveFile(deletion_target)
       print 'done'
 
 
 def gclient_configure(solutions, target_os):
   """Should do the same thing as gclient --spec='...'."""
   with codecs.open('.gclient', mode='w', encoding='utf-8') as f:
     f.write(get_gclient_spec(solutions, target_os))
 
 
-def gclient_sync(output_json):
+def gclient_sync(output_json, buildspec_name):
   gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
-  call(gclient_bin, 'sync', '--verbose', '--reset', '--force',
-       '--nohooks', '--noprehooks', '--output-json', output_json)
+  cmd = [gclient_bin, 'sync', '--verbose', '--reset', '--force',
+         '--output-json', output_json]
+  if buildspec_name:
+    cmd += ['--with_branch_heads']
+  else:
+    cmd += ['--nohooks', '--noprehooks']
+  call(*cmd)
   with open(output_json) as f:
     return json.load(f)
 
 
 def create_less_than_or_equal_regex(number):
   """ Return a regular expression to test whether an integer less than or equal
       to 'number' is present in a given string.
   """
 
   # In three parts, build a regular expression that match any numbers smaller
@@ skipping 92 matching lines @@
 def _last_commit_for_file(filename, repo_base):
   cmd = ['log', '--format=%H', '--max-count=1', '--', filename]
   return git(*cmd, cwd=repo_base).strip()
 
 
 def need_to_run_deps2git(repo_base, deps_file, deps_git_file):
   """Checks to see if we need to run deps2git.
 
   Returns True if there was a DEPS change after the last .DEPS.git update.
   """
+  print 'Checking if %s exists' % deps_git_file
   if not path.isfile(deps_git_file):
     # .DEPS.git doesn't exist but DEPS does? We probably want to generate one.
+    print 'it exists!'
     return True
 
   last_known_deps_ref = _last_commit_for_file(deps_file, repo_base)
   last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base)
   merge_base_ref = git('merge-base', last_known_deps_ref,
                        last_known_deps_git_ref, cwd=repo_base).strip()
 
   # If the merge base of the last DEPS and last .DEPS.git file is not
   # equivilent to the hash of the last DEPS file, that means the DEPS file
   # was committed after the last .DEPS.git file.
   return last_known_deps_ref != merge_base_ref
 
 
+def get_git_buildspec(version):
+  """Get the git buildspec of a version, return its contents.
+
+  The contents are returned instead of the file so that we can check the
+  repository into a temp directory and confine the cleanup logic here.
+  """
+  git('cache', 'populate', '-v', '--cache-dir', CACHE_DIR, GIT_BUILDSPEC_REPO)
+  mirror_dir = git(
+      'cache', 'exists', '--cache-dir', CACHE_DIR, GIT_BUILDSPEC_REPO).strip()
+  TOTAL_TRIES = 30

[agable, 2014/04/22 21:38:03]

How'd you arrive at this number? 2.5 minutes?

[Ryan Tseng, 2014/04/22 21:46:42]

Yep, its pretty arbitrary.  afaik the cron job is pretty fast, and if its not
there by 2 minutes somethings probably wrong.

+  for tries in range(TOTAL_TRIES):
+    try:
+      return git('show', 'master:%s/DEPS' % version, cwd=mirror_dir)
+    except SubprocessFailed:
+      if tries < TOTAL_TRIES - 1:
+        print 'Buildspec for %s not committed yet, waiting 5 seconds...'
+        time.sleep(5)
+        git('cache', 'populate', '-v', '--cache-dir',
+            CACHE_DIR, GIT_BUILDSPEC_REPO)
+      else:
+        print >> sys.stderr, '%s not found, ' % version,
+        print >> sys.stderr, 'the buildspec2git cron job probably is not ',
+        print >> sys.stderr, 'running correctly, please contact mmoss@.'
+        raise
+
+
+def buildspecs2git(sln_dir, buildspec_name):
+  """This is like deps2git, but for buildspecs.
+
+  Because buildspecs are vastly different than normal DEPS files, we cannot
+  use deps2git.py to generate git versions of the git DEPS.  Fortunately
+  we don't have buildspec trybots, and there is already a service that
+  generates git DEPS for every buildspec commit already, so we can leverage
+  that service so that we don't need to run buildspec2git.py serially.
+
+  This checks the commit message of the current DEPS file for the release
+  number, waits in a busy loop for the coorisponding .DEPS.git file to be
+  committed into the git_buildspecs repository.
+  """
+  repo_base = path.join(os.getcwd(), sln_dir)
+  deps_file = path.join(repo_base, 'build', buildspec_name, 'DEPS')
+  deps_git_file = path.join(repo_base, 'build', buildspec_name, '.DEPS.git')
+  deps_log = git('log', '-1', '--format=%B', deps_file, cwd=repo_base)
+  m = re.search(r'Buildspec for\s+version (\d+\.\d+\.\d+\.\d+)', deps_log)
+  version = m.group(1)
+  git_buildspec = get_git_buildspec(version)
+  with open(deps_git_file, 'wb') as f:
+    f.write(git_buildspec)
+
+
 def ensure_deps2git(sln_dir, shallow):
   repo_base = path.join(os.getcwd(), sln_dir)
   deps_file = path.join(repo_base, 'DEPS')
   deps_git_file = path.join(repo_base, '.DEPS.git')
+  print 'Checking if %s is newer than %s' % (deps_file, deps_git_file)
   if not path.isfile(deps_file):
     return
 
   if not need_to_run_deps2git(repo_base, deps_file, deps_git_file):
     return
 
   print '===DEPS file modified, need to run deps2git==='
   # Magic to get deps2git to work with internal DEPS.
   shutil.copyfile(S2G_INTERNAL_FROM_PATH, S2G_INTERNAL_DEST_PATH)
 
   # TODO(hinoka): This might need to be smarter if we need to deal with
   #               DEPS changes that are in an internal repository.
-  repo_type = 'internal' if 'internal' in sln_dir else 'public'
+  repo_type = 'public'
+  if sln_dir in ['src-internal']:
+    repo_type = 'internal'
   cmd = [sys.executable, DEPS2GIT_PATH,
          '-t', repo_type,
          '--cache_dir=%s' % CACHE_DIR,
          '--deps=%s' % deps_file,
          '--out=%s' % deps_git_file]
   if shallow:
     cmd.append('--shallow')
   call(*cmd)
 
 
@@ skipping 247 matching lines @@
     'slave': slave or 'Not specified',
     'recipe': recipe_force,
   },
   # Print to stderr so that it shows up red on win/mac.
   print ACTIVATED_MESSAGE if active else NOT_ACTIVATED_MESSAGE
 
   # Parse, munipulate, and print the gclient solutions.
   specs = {}
   exec(options.specs, specs)
   svn_solutions = specs.get('solutions', [])
-  git_solutions, svn_root = solutions_to_git(svn_solutions)
+  git_solutions, svn_root, buildspec_name = solutions_to_git(svn_solutions)
   solutions_printer(git_solutions)
 
   dir_names = [sln.get('name') for sln in svn_solutions if 'name' in sln]
   # If we're active now, but the flag file doesn't exist (we weren't active last
   # run) or vice versa, blow away all checkouts.
   if bool(active) != bool(check_flag(options.flag_file)):
     ensure_no_checkout(dir_names, '*')
   if options.output_json:
     # Make sure we tell recipes that we didn't run if the script exits here.
     emit_json(options.output_json, did_run=active)
@@ skipping 31 matching lines @@
   # If either patch_url or issue is passed in, then we need to apply a patch.
   if options.patch_url:
     # patch_url takes precidence since its only passed in on gcl try/git try.
     apply_issue_svn(options.root, options.patch_url)
   elif options.issue:
     apply_issue_rietveld(options.issue, options.patchset, options.root,
                          options.rietveld_server, options.revision_mapping,
                          git_ref)
 
   # Run deps2git if there is a DEPS commit after the last .DEPS.git commit.
-  ensure_deps2git(options.root, options.shallow)
+  if buildspec_name:
+    buildspecs2git(options.root, buildspec_name)
+  else:
+    ensure_deps2git(options.root, options.shallow)
 
   # Ensure our build/ directory is set up with the correct .gclient file.
   gclient_configure(git_solutions, specs.get('target_os', []))
 
   # Let gclient do the DEPS syncing.  Also we can get "got revision" data
   # from gclient by passing in --output-json.  In our case, we can just reuse
   # the temp file that
   _, gclient_output_file = tempfile.mkstemp(suffix='.json')
-  gclient_output = gclient_sync(gclient_output_file)
+  gclient_output = gclient_sync(gclient_output_file, buildspec_name)
 
   # If we're fed an svn revision number as --revision, then our got_revision
   # output should be in svn revs.  Otherwise it'll be in git hashes.
   use_svn_rev = (options.revision and options.revision.isdigit() and
                  len(options.revision) < 40)
   # Take care of got_revisions outputs.
   revision_mapping = get_revision_mapping(svn_root, options.revision_mapping)
   got_revisions = parse_got_revision(gclient_output, revision_mapping,
                                      use_svn_rev)
 
   if options.output_json:
     # Tell recipes information such as root, got_revision, etc.
     emit_json(options.output_json,
               did_run=True,
               root=options.root,
               step_text=step_text,
               properties=got_revisions)
   else:
     # If we're not on recipes, tell annotator about our got_revisions.
     emit_properties(got_revisions)
 
 
 if __name__ == '__main__':
   sys.exit(main())