Bot Update support on official builders

scripts/slave/bot_update.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import codecs
 import copy
 import cStringIO
 import ctypes
 import json
@@ skipping 15 matching lines @@
 from common import chromium_utils
 
 
 CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src.git'
 
 RECOGNIZED_PATHS = {
     # If SVN path matches key, the entire URL is rewritten to the Git url.
     '/chrome/trunk/src':
        CHROMIUM_SRC_URL,
     '/chrome-internal/trunk/src-internal':
-        'https://chrome-internal.googlesource.com/chrome/src-internal.git'
+        'https://chrome-internal.googlesource.com/chrome/src-internal.git',
+    '/chrome-internal/trunk/tools/buildspec/build/chrome-official':
+        'https://chrome-internal.googlesource.com/chrome/tools/buildspec',
 }
 
+CUSTOM_DEPS_FILE = {
+    '/chrome-internal/trunk/tools/buildspec/build/chrome-official':
+        'build/chrome-official/DEPS'
+}
+
+# Solution names that use git_buildspecs to resolve .DEPS.git.
+BUILDSPEC_SOLUTIONS = [
+  'chrome-official',

[agable, 2014/04/18 03:45:07]

I think this needs to be more than chrome-official: there's also
chrome-official-win, chrome-official-win-asan, etc. Every subdirectory of
chrome/tools/buildspec/build is a possible value.

[Ryan Tseng, 2014/04/18 18:42:38]

Done.

+]
+
+GIT_BUILDSPEC_REPO = (
+    'https://chrome-internal.googlesource.com/chrome/tools/git_buildspecs')
+
 # Copied from scripts/recipes/chromium.py.
 GOT_REVISION_MAPPINGS = {
     '/chrome/trunk/src': {
         'src/': 'got_revision',
         'src/native_client/': 'got_nacl_revision',
         'src/tools/swarm_client/': 'got_swarm_client_revision',
         'src/tools/swarming_client/': 'got_swarming_client_revision',
         'src/third_party/WebKit/': 'got_webkit_revision',
         'src/third_party/webrtc/': 'got_webrtc_revision',
     }
@@ skipping 50 matching lines @@
 
 
 GCLIENT_TEMPLATE = """solutions = %(solutions)s
 
 cache_dir = %(cache_dir)s
 %(target_os)s
 """
 
 # IMPORTANT: If you're trying to enable a RECIPE bot, you'll need to
 # edit recipe_modules/bot_update/api.py instead.
-ENABLED_MASTERS = ['chromium.git']
+ENABLED_MASTERS = ['chromium.git', 'chrome_git']
 ENABLED_BUILDERS = {
     'tryserver.chromium': ['linux_rel_alt'],
 }
 ENABLED_SLAVES = {
     # This is enabled on a bot-to-bot basis to ensure that we don't have
     # bots that have mixed configs.
     'chromium.fyi': [
         'build1-m1',  # Chromium Builder / Chromium Builder (dbg)
         'vm928-m1',   # Chromium Linux Buildrunner
         'vm859-m1',   # Chromium Linux Redux
@@ skipping 181 matching lines @@
   """
   assert input_solutions
   solutions = copy.deepcopy(input_solutions)
   first_solution = True
   for solution in solutions:
     original_url = solution['url']
     parsed_url = urlparse.urlparse(original_url)
     parsed_path = parsed_url.path
     if first_solution:
       root = parsed_path
+      deps_file = CUSTOM_DEPS_FILE.get(parsed_path, 'DEPS')
       first_solution = False
+
+    # Rewrite SVN urls into Git urls.
     if parsed_path in RECOGNIZED_PATHS:
       solution['url'] = RECOGNIZED_PATHS[parsed_path]
     else:
       print 'Warning: path %s not recognized' % parsed_path
-    if solution.get('deps_file', 'DEPS') == 'DEPS':
-      solution['deps_file'] = '.DEPS.git'
+
+    if parsed_path in CUSTOM_DEPS_FILE:
+      # For some bots (eg. official), we want to specify what DEPS file
+      # to look at.
+      solution['deps_file'] = CUSTOM_DEPS_FILE[parsed_path]
+
+    if solution.get('deps_file', 'DEPS').endswith('DEPS'):
+      # Point .DEPS.git is the git version of the DEPS file.
+      solution['deps_file'] = '.DEPS.git'.join(
+          solution['deps_file'].rsplit('DEPS', 1))
+
     solution['managed'] = False
     # We don't want gclient to be using a safesync URL. Instead it should
     # using the lkgr/lkcr branch/tags.
     if 'safesync_url' in solution:
       print 'Removing safesync url %s from %s' % (solution['safesync_url'],
                                                   parsed_path)
       del solution['safesync_url']
-  return solutions, root
+  return solutions, root, deps_file
 
 
 def ensure_no_checkout(dir_names, scm_dirname):
   """Ensure that there is no undesired checkout under build/.
 
   If there is an incorrect checkout under build/, then
   move build/ to build.dead/
   This function will check each directory in dir_names.
 
   scm_dirname is expected to be either ['.svn', '.git']
@@ skipping 17 matching lines @@
         chromium_utils.RemoveFile(deletion_target)
       print 'done'
 
 
 def gclient_configure(solutions, target_os):
   """Should do the same thing as gclient --spec='...'."""
   with codecs.open('.gclient', mode='w', encoding='utf-8') as f:
     f.write(get_gclient_spec(solutions, target_os))
 
 
-def gclient_sync(output_json):
+def gclient_sync(output_json, buildspec_mode):
   gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
-  call(gclient_bin, 'sync', '--verbose', '--reset', '--force',
-       '--nohooks', '--noprehooks', '--output-json', output_json)
+  cmd = [gclient_bin, 'sync', '--verbose', '--reset', '--force',
+         '--output-json', output_json]
+  if buildspec_mode:
+    cmd += ['--with_branch_heads']
+  else:
+    cmd += ['--nohooks', '--noprehooks']
+  call(*cmd)
   with open(output_json) as f:
     return json.load(f)
 
 
 def create_less_than_or_equal_regex(number):
   """ Return a regular expression to test whether an integer less than or equal
       to 'number' is present in a given string.
   """
 
   # In three parts, build a regular expression that match any numbers smaller
@@ skipping 92 matching lines @@
 def _last_commit_for_file(filename, repo_base):
   cmd = ['log', '--format=%H', '--max-count=1', '--', filename]
   return git(*cmd, cwd=repo_base).strip()
 
 
 def need_to_run_deps2git(repo_base, deps_file, deps_git_file):
   """Checks to see if we need to run deps2git.
 
   Returns True if there was a DEPS change after the last .DEPS.git update.
   """
+  print 'Checking if %s exists' % deps_git_file
   if not path.isfile(deps_git_file):
     # .DEPS.git doesn't exist but DEPS does? We probably want to generate one.
+    print 'it exists!'
     return True
 
   last_known_deps_ref = _last_commit_for_file(deps_file, repo_base)
   last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base)
   merge_base_ref = git('merge-base', last_known_deps_ref,
                        last_known_deps_git_ref, cwd=repo_base).strip()
 
   # If the merge base of the last DEPS and last .DEPS.git file is not
   # equivilent to the hash of the last DEPS file, that means the DEPS file
   # was committed after the last .DEPS.git file.
   return last_known_deps_ref != merge_base_ref
 
 
-def ensure_deps2git(sln_dir, shallow):
+def get_git_buildspec(version):
+  """Get the git buildspec of a version, return its contents.
+
+  The contents are returned instead of the file so that we can check the
+  repository into a temp directory and confine the cleanup logic here."""
+  git('cache', 'populate', '-v', '--cache-dir', CACHE_DIR, GIT_BUILDSPEC_REPO)
+  mirror_dir = git(
+      'cache', 'exists', '--cache-dir', CACHE_DIR, GIT_BUILDSPEC_REPO).strip()
+  tempdir = tempfile.mkdtemp()
+  git('clone', mirror_dir, tempdir)
+  while True:
+    if path.isdir(path.join(tempdir, version)):
+      with open(path.join(tempdir, version, 'DEPS'), 'rb') as f:
+        git_buildspec = f.read()
+      chromium_utils.RemoveDirectory(tempdir)
+      return git_buildspec
+    print 'Buildspec for %s not committed yet, waiting 5 seconds...'
+    time.sleep(5)
+    git('cache', 'populate', '-v', '--cache-dir',
+        CACHE_DIR, GIT_BUILDSPEC_REPO)
+    git('fetch', 'origin' , cwd=tempdir)
+
+
+def buildspecs2git(repo_base, deps_file, deps_git_file):
+  """This is like deps2git, but for buildspecs.
+
+  Because buildspecs are vastly different than normal DEPS files, we cannot
+  use deps2git.py to generate git versions of the git DEPS.  Fortunately
+  we don't have buildspec trybots, and there is already a service that
+  generates git DEPS for every buildspec commit already, so we can leverage
+  that service so that we don't need to run buildspec2git.py serially.
+
+  This checks the commit message of the current DEPS file for the release
+  number, waits in a busy loop for the coorisponding .DEPS.git file to be
+  committed into the git_buildspecs repository."""
+  deps_log = git('log', '-1', deps_file, cwd=repo_base)
+  m = re.search(r'Buildspec for\s+version (\d+\.\d+\.\d+\.\d+)', deps_log)
+  version = m.group(1)
+  git_buildspec = get_git_buildspec(version)
+  with open(deps_git_file, 'wb') as f:
+    f.write(git_buildspec)
+
+
+def ensure_deps2git(sln_dir, deps_file, shallow, buildspec_mode):
   repo_base = path.join(os.getcwd(), sln_dir)
-  deps_file = path.join(repo_base, 'DEPS')
-  deps_git_file = path.join(repo_base, '.DEPS.git')
+  deps_file = path.join(repo_base, deps_file)
+  deps_git_file = path.join(path.dirname(deps_file), '.DEPS.git')
+  if buildspec_mode:
+    return buildspecs2git(repo_base, deps_file, deps_git_file)
+  print 'Checking if %s is newer than %s' % (deps_file, deps_git_file)
   if not path.isfile(deps_file):
     return
 
   if not need_to_run_deps2git(repo_base, deps_file, deps_git_file):
     return
 
   print '===DEPS file modified, need to run deps2git==='
   # Magic to get deps2git to work with internal DEPS.
   shutil.copyfile(S2G_INTERNAL_FROM_PATH, S2G_INTERNAL_DEST_PATH)
 
   # TODO(hinoka): This might need to be smarter if we need to deal with
   #               DEPS changes that are in an internal repository.
-  repo_type = 'internal' if 'internal' in sln_dir else 'public'
+  repo_type = 'public'
+  if sln_dir in ['src-internal', 'chrome-official']:
+    repo_type = 'internal'
   cmd = [sys.executable, DEPS2GIT_PATH,
          '-t', repo_type,
          '--cache_dir=%s' % CACHE_DIR,
          '--deps=%s' % deps_file,
          '--out=%s' % deps_git_file]
   if shallow:
     cmd.append('--shallow')
   call(*cmd)
 
 
@@ skipping 247 matching lines @@
     'slave': slave or 'Not specified',
     'recipe': recipe_force,
   },
   # Print to stderr so that it shows up red on win/mac.
   print ACTIVATED_MESSAGE if active else NOT_ACTIVATED_MESSAGE
 
   # Parse, munipulate, and print the gclient solutions.
   specs = {}
   exec(options.specs, specs)
   svn_solutions = specs.get('solutions', [])
-  git_solutions, svn_root = solutions_to_git(svn_solutions)
+  git_solutions, svn_root, deps_file = solutions_to_git(svn_solutions)
   solutions_printer(git_solutions)
 
   dir_names = [sln.get('name') for sln in svn_solutions if 'name' in sln]
   # If we're active now, but the flag file doesn't exist (we weren't active last
   # run) or vice versa, blow away all checkouts.
   if bool(active) != bool(check_flag(options.flag_file)):
     ensure_no_checkout(dir_names, '*')
   if options.output_json:
     # Make sure we tell recipes that we didn't run if the script exits here.
     emit_json(options.output_json, did_run=active)
@@ skipping 31 matching lines @@
   # If either patch_url or issue is passed in, then we need to apply a patch.
   if options.patch_url:
     # patch_url takes precidence since its only passed in on gcl try/git try.
     apply_issue_svn(options.root, options.patch_url)
   elif options.issue:
     apply_issue_rietveld(options.issue, options.patchset, options.root,
                          options.rietveld_server, options.revision_mapping,
                          git_ref)
 
   # Run deps2git if there is a DEPS commit after the last .DEPS.git commit.
-  ensure_deps2git(options.root, options.shallow)
+  buildspec_mode = options.root in BUILDSPEC_SOLUTIONS
+  ensure_deps2git(options.root, deps_file, options.shallow, buildspec_mode)
 
   # Ensure our build/ directory is set up with the correct .gclient file.
   gclient_configure(git_solutions, specs.get('target_os', []))
 
   # Let gclient do the DEPS syncing.  Also we can get "got revision" data
   # from gclient by passing in --output-json.  In our case, we can just reuse
   # the temp file that
   _, gclient_output_file = tempfile.mkstemp(suffix='.json')
-  gclient_output = gclient_sync(gclient_output_file)
+  gclient_output = gclient_sync(gclient_output_file, buildspec_mode)
 
   # If we're fed an svn revision number as --revision, then our got_revision
   # output should be in svn revs.  Otherwise it'll be in git hashes.
   use_svn_rev = (options.revision and options.revision.isdigit() and
                  len(options.revision) < 40)
   # Take care of got_revisions outputs.
   revision_mapping = get_revision_mapping(svn_root, options.revision_mapping)
   got_revisions = parse_got_revision(gclient_output, revision_mapping,
                                      use_svn_rev)
 
   if options.output_json:
     # Tell recipes information such as root, got_revision, etc.
     emit_json(options.output_json,
               did_run=True,
               root=options.root,
               step_text=step_text,
               properties=got_revisions)
   else:
     # If we're not on recipes, tell annotator about our got_revisions.
     emit_properties(got_revisions)
 
 
 if __name__ == '__main__':
   sys.exit(main())