Bot Update support on official builders

scripts/slave/bot_update.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import codecs
 import copy
 import cStringIO
 import ctypes
 import json
@@ skipping 10 matching lines @@
 import urllib2
 import urlparse
 
 import os.path as path
 
 from common import chromium_utils
 
 
 CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src.git'
 
+# Solution names that use git_buildspecs to resolve .DEPS.git.
+BUILDSPEC_SOLUTIONS = [
+  'chrome-official',

[agable, 2014/04/18 19:09:33]

Is there any way these could be regex'd? 'chrome-official*'?

[Ryan Tseng, 2014/04/21 22:10:33]

Hm, maybe we can regex "/chrome-internal/trunk/tools/buildspec/build/(.*)" and
turn on buildspec mode.

[Ryan Tseng, 2014/04/21 22:10:33]

Done.

+  'chrome-official-android',
+  'chrome-official-aura',
+  'chrome-official-chromeos',
+  'chrome-official-ios',
+  'chrome-official-linux',
+  'chrome-official-mac',
+  'chrome-official-win',
+  'chrome-official-win-asan',
+  'chrome-official-win64',
+  'chrome-official-wincf',
+]
+
 RECOGNIZED_PATHS = {
     # If SVN path matches key, the entire URL is rewritten to the Git url.
     '/chrome/trunk/src':
-       CHROMIUM_SRC_URL,
+        CHROMIUM_SRC_URL,
     '/chrome-internal/trunk/src-internal':
-        'https://chrome-internal.googlesource.com/chrome/src-internal.git'
+        'https://chrome-internal.googlesource.com/chrome/src-internal.git',
 }
+# Official builders trigger on changes to buildspecs, and therefore the
+# gclient solution they get are always in a very specific folder.
+# We need to add all of them into our set of recognized paths.
+GIT_BUILDSPEC_PATH = ('https://chrome-internal.googlesource.com/chrome/tools/'
+                      'buildspec')
+for build_name in BUILDSPEC_SOLUTIONS:
+  RECOGNIZED_PATHS['/chrome-internal/trunk/tools/buildspec/build/%s'
+                   % build_name] = GIT_BUILDSPEC_PATH
+
+# In addition, because buildspecs are in a very specific directory that isn't
+# mirrored into Git as its own repo, but rather the parent of the solution
+# directory is mirrored into Git, we have to tell gclient to use the DEPS
+# thats farther down in the Git directory structure.
+CUSTOM_DEPS_FILE = {
+    '/chrome-internal/trunk/tools/buildspec/build/%s' % sln_name:
+        'build/%s/DEPS' % sln_name
+    for sln_name in BUILDSPEC_SOLUTIONS
+}
+
+# This is the repository that the buildspec -> git deps cron job mirrors
+# all buildspecs into. When we see an svn buildspec, we rely on the
+# buildspec -> git cron job to produce the git version of the buildspec.

[agable, 2014/04/18 19:09:33]

buildspec2git

[Ryan Tseng, 2014/04/21 22:10:33]

Done.

+GIT_BUILDSPEC_REPO = (
+    'https://chrome-internal.googlesource.com/chrome/tools/git_buildspecs')
 
 # Copied from scripts/recipes/chromium.py.
 GOT_REVISION_MAPPINGS = {
     '/chrome/trunk/src': {
         'src/': 'got_revision',
         'src/native_client/': 'got_nacl_revision',
         'src/tools/swarm_client/': 'got_swarm_client_revision',
         'src/tools/swarming_client/': 'got_swarming_client_revision',
         'src/third_party/WebKit/': 'got_webkit_revision',
         'src/third_party/webrtc/': 'got_webrtc_revision',
@@ skipping 51 matching lines @@
 
 
 GCLIENT_TEMPLATE = """solutions = %(solutions)s
 
 cache_dir = %(cache_dir)s
 %(target_os)s
 """
 
 # IMPORTANT: If you're trying to enable a RECIPE bot, you'll need to
 # edit recipe_modules/bot_update/api.py instead.
-ENABLED_MASTERS = ['chromium.git']
+ENABLED_MASTERS = ['chromium.git', 'chrome_git']

[agable, 2014/04/18 19:09:33]

Have we made sure that the chrome_git waterfall is now running as close to a
clone of the chrome.official as we can? It needs to be not doing all its other
crazy git stuff.

[Ryan Tseng, 2014/04/21 22:10:33]

Yep

 ENABLED_BUILDERS = {
     'tryserver.chromium': ['linux_rel_alt'],
 }
 ENABLED_SLAVES = {
     # This is enabled on a bot-to-bot basis to ensure that we don't have
     # bots that have mixed configs.
     'chromium.fyi': [
         'build1-m1',  # Chromium Builder / Chromium Builder (dbg)
         'vm928-m1',   # Chromium Linux Buildrunner
         'vm859-m1',   # Chromium Linux Redux
@@ skipping 181 matching lines @@
   """
   assert input_solutions
   solutions = copy.deepcopy(input_solutions)
   first_solution = True
   for solution in solutions:
     original_url = solution['url']
     parsed_url = urlparse.urlparse(original_url)
     parsed_path = parsed_url.path
     if first_solution:
       root = parsed_path
+      deps_file = CUSTOM_DEPS_FILE.get(parsed_path, 'DEPS')
       first_solution = False
+
+    # Rewrite SVN urls into Git urls.
     if parsed_path in RECOGNIZED_PATHS:
       solution['url'] = RECOGNIZED_PATHS[parsed_path]
     else:
       print 'Warning: path %s not recognized' % parsed_path
-    if solution.get('deps_file', 'DEPS') == 'DEPS':
-      solution['deps_file'] = '.DEPS.git'
+
+    if parsed_path in CUSTOM_DEPS_FILE:
+      # For some bots (eg. official), we want to specify what DEPS file
+      # to look at.
+      solution['deps_file'] = CUSTOM_DEPS_FILE[parsed_path]
+
+    if solution.get('deps_file', 'DEPS').endswith('DEPS'):
+      # Point .DEPS.git is the git version of the DEPS file.
+      solution['deps_file'] = '.DEPS.git'.join(
+          solution['deps_file'].rsplit('DEPS', 1))

[agable, 2014/04/18 19:09:33]

wat. '.DEPS.git'.join()!?

[Ryan Tseng, 2014/04/21 22:10:33]

rsplit('DEPS') will remove the rightmost 'DEPS', and turn it into a ['asdf/',
''], and running '.DEPS.git'.join() on that list will produce 'asdf/.DEPS.git'. 

+
     solution['managed'] = False
     # We don't want gclient to be using a safesync URL. Instead it should
     # using the lkgr/lkcr branch/tags.
     if 'safesync_url' in solution:
       print 'Removing safesync url %s from %s' % (solution['safesync_url'],
                                                   parsed_path)
       del solution['safesync_url']
-  return solutions, root
+  return solutions, root, deps_file
 
 
 def ensure_no_checkout(dir_names, scm_dirname):
   """Ensure that there is no undesired checkout under build/.
 
   If there is an incorrect checkout under build/, then
   move build/ to build.dead/
   This function will check each directory in dir_names.
 
   scm_dirname is expected to be either ['.svn', '.git']
@@ skipping 17 matching lines @@
         chromium_utils.RemoveFile(deletion_target)
       print 'done'
 
 
 def gclient_configure(solutions, target_os):
   """Should do the same thing as gclient --spec='...'."""
   with codecs.open('.gclient', mode='w', encoding='utf-8') as f:
     f.write(get_gclient_spec(solutions, target_os))
 
 
-def gclient_sync(output_json):
+def gclient_sync(output_json, buildspec_mode):
   gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
-  call(gclient_bin, 'sync', '--verbose', '--reset', '--force',
-       '--nohooks', '--noprehooks', '--output-json', output_json)
+  cmd = [gclient_bin, 'sync', '--verbose', '--reset', '--force',
+         '--output-json', output_json]
+  if buildspec_mode:
+    cmd += ['--with_branch_heads']
+  else:
+    cmd += ['--nohooks', '--noprehooks']
+  call(*cmd)
   with open(output_json) as f:
     return json.load(f)
 
 
 def create_less_than_or_equal_regex(number):
   """ Return a regular expression to test whether an integer less than or equal
       to 'number' is present in a given string.
   """
 
   # In three parts, build a regular expression that match any numbers smaller
@@ skipping 92 matching lines @@
 def _last_commit_for_file(filename, repo_base):
   cmd = ['log', '--format=%H', '--max-count=1', '--', filename]
   return git(*cmd, cwd=repo_base).strip()
 
 
 def need_to_run_deps2git(repo_base, deps_file, deps_git_file):
   """Checks to see if we need to run deps2git.
 
   Returns True if there was a DEPS change after the last .DEPS.git update.
   """
+  print 'Checking if %s exists' % deps_git_file
   if not path.isfile(deps_git_file):
     # .DEPS.git doesn't exist but DEPS does? We probably want to generate one.
+    print 'it exists!'
     return True
 
   last_known_deps_ref = _last_commit_for_file(deps_file, repo_base)
   last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base)
   merge_base_ref = git('merge-base', last_known_deps_ref,
                        last_known_deps_git_ref, cwd=repo_base).strip()
 
   # If the merge base of the last DEPS and last .DEPS.git file is not
   # equivilent to the hash of the last DEPS file, that means the DEPS file
   # was committed after the last .DEPS.git file.
   return last_known_deps_ref != merge_base_ref
 
 
-def ensure_deps2git(sln_dir, shallow):
+def get_git_buildspec(version):
+  """Get the git buildspec of a version, return its contents.
+
+  The contents are returned instead of the file so that we can check the
+  repository into a temp directory and confine the cleanup logic here."""

[agable, 2014/04/18 19:09:33]

nit: close docstring quotes on next line

[Ryan Tseng, 2014/04/21 22:10:33]

Done.

+  git('cache', 'populate', '-v', '--cache-dir', CACHE_DIR, GIT_BUILDSPEC_REPO)
+  mirror_dir = git(
+      'cache', 'exists', '--cache-dir', CACHE_DIR, GIT_BUILDSPEC_REPO).strip()
+  tempdir = tempfile.mkdtemp()
+  git('clone', mirror_dir, tempdir)

[agable, 2014/04/18 19:09:33]

Shouldn't need the tempdir or the clone. Once you have the cache, you can just
run 'git show' against the right path to get the file contents without ever
having to check them out.

[Ryan Tseng, 2014/04/21 22:10:33]

Ohh interesting, done.

+  while True:
+    if path.isdir(path.join(tempdir, version)):
+      with open(path.join(tempdir, version, 'DEPS'), 'rb') as f:
+        git_buildspec = f.read()
+      chromium_utils.RemoveDirectory(tempdir)
+      return git_buildspec
+    print 'Buildspec for %s not committed yet, waiting 5 seconds...'
+    time.sleep(5)

[agable, 2014/04/18 19:09:33]

Would be wise to have an upper limit on this, so it doesn't sleep forever (and
buildbot will never kill it because it prints output every 5 seconds).

[Ryan Tseng, 2014/04/21 22:10:33]

Done.

+    git('cache', 'populate', '-v', '--cache-dir',
+        CACHE_DIR, GIT_BUILDSPEC_REPO)
+    git('fetch', 'origin' , cwd=tempdir)
+
+
+def buildspecs2git(repo_base, deps_file, deps_git_file):
+  """This is like deps2git, but for buildspecs.
+
+  Because buildspecs are vastly different than normal DEPS files, we cannot
+  use deps2git.py to generate git versions of the git DEPS.  Fortunately
+  we don't have buildspec trybots, and there is already a service that
+  generates git DEPS for every buildspec commit already, so we can leverage
+  that service so that we don't need to run buildspec2git.py serially.
+
+  This checks the commit message of the current DEPS file for the release
+  number, waits in a busy loop for the coorisponding .DEPS.git file to be
+  committed into the git_buildspecs repository."""

[agable, 2014/04/18 19:09:33]

nit: close docstring quotes on next line.

[Ryan Tseng, 2014/04/21 22:10:33]

Done.

+  deps_log = git('log', '-1', deps_file, cwd=repo_base)

[agable, 2014/04/18 19:09:33]

you could pass a format argument to git-log so that it only prints the commit
message, and not any of the other metadata.

[Ryan Tseng, 2014/04/21 22:10:33]

Done.

+  m = re.search(r'Buildspec for\s+version (\d+\.\d+\.\d+\.\d+)', deps_log)
+  version = m.group(1)
+  git_buildspec = get_git_buildspec(version)
+  with open(deps_git_file, 'wb') as f:
+    f.write(git_buildspec)
+
+
+def ensure_deps2git(sln_dir, deps_file, shallow, buildspec_mode):
   repo_base = path.join(os.getcwd(), sln_dir)
-  deps_file = path.join(repo_base, 'DEPS')
-  deps_git_file = path.join(repo_base, '.DEPS.git')
+  deps_file = path.join(repo_base, deps_file)
+  deps_git_file = path.join(path.dirname(deps_file), '.DEPS.git')
+  if buildspec_mode:
+    return buildspecs2git(repo_base, deps_file, deps_git_file)
+  print 'Checking if %s is newer than %s' % (deps_file, deps_git_file)
   if not path.isfile(deps_file):
     return
 
   if not need_to_run_deps2git(repo_base, deps_file, deps_git_file):
     return
 
   print '===DEPS file modified, need to run deps2git==='
   # Magic to get deps2git to work with internal DEPS.
   shutil.copyfile(S2G_INTERNAL_FROM_PATH, S2G_INTERNAL_DEST_PATH)
 
   # TODO(hinoka): This might need to be smarter if we need to deal with
   #               DEPS changes that are in an internal repository.
-  repo_type = 'internal' if 'internal' in sln_dir else 'public'
+  repo_type = 'public'
+  if sln_dir in ['src-internal', 'chrome-official']:

[agable, 2014/04/18 19:09:33]

hard-coding chrome-official again

[Ryan Tseng, 2014/04/21 22:10:33]

Removed, this actually isn't needed

+    repo_type = 'internal'
   cmd = [sys.executable, DEPS2GIT_PATH,
          '-t', repo_type,
          '--cache_dir=%s' % CACHE_DIR,
          '--deps=%s' % deps_file,
          '--out=%s' % deps_git_file]
   if shallow:
     cmd.append('--shallow')
   call(*cmd)
 
 
@@ skipping 247 matching lines @@
     'slave': slave or 'Not specified',
     'recipe': recipe_force,
   },
   # Print to stderr so that it shows up red on win/mac.
   print ACTIVATED_MESSAGE if active else NOT_ACTIVATED_MESSAGE
 
   # Parse, munipulate, and print the gclient solutions.
   specs = {}
   exec(options.specs, specs)
   svn_solutions = specs.get('solutions', [])
-  git_solutions, svn_root = solutions_to_git(svn_solutions)
+  git_solutions, svn_root, deps_file = solutions_to_git(svn_solutions)
   solutions_printer(git_solutions)
 
   dir_names = [sln.get('name') for sln in svn_solutions if 'name' in sln]
   # If we're active now, but the flag file doesn't exist (we weren't active last
   # run) or vice versa, blow away all checkouts.
   if bool(active) != bool(check_flag(options.flag_file)):
     ensure_no_checkout(dir_names, '*')
   if options.output_json:
     # Make sure we tell recipes that we didn't run if the script exits here.
     emit_json(options.output_json, did_run=active)
@@ skipping 31 matching lines @@
   # If either patch_url or issue is passed in, then we need to apply a patch.
   if options.patch_url:
     # patch_url takes precidence since its only passed in on gcl try/git try.
     apply_issue_svn(options.root, options.patch_url)
   elif options.issue:
     apply_issue_rietveld(options.issue, options.patchset, options.root,
                          options.rietveld_server, options.revision_mapping,
                          git_ref)
 
   # Run deps2git if there is a DEPS commit after the last .DEPS.git commit.
-  ensure_deps2git(options.root, options.shallow)
+  buildspec_mode = options.root in BUILDSPEC_SOLUTIONS
+  ensure_deps2git(options.root, deps_file, options.shallow, buildspec_mode)
 
   # Ensure our build/ directory is set up with the correct .gclient file.
   gclient_configure(git_solutions, specs.get('target_os', []))
 
   # Let gclient do the DEPS syncing.  Also we can get "got revision" data
   # from gclient by passing in --output-json.  In our case, we can just reuse
   # the temp file that
   _, gclient_output_file = tempfile.mkstemp(suffix='.json')
-  gclient_output = gclient_sync(gclient_output_file)
+  gclient_output = gclient_sync(gclient_output_file, buildspec_mode)
 
   # If we're fed an svn revision number as --revision, then our got_revision
   # output should be in svn revs.  Otherwise it'll be in git hashes.
   use_svn_rev = (options.revision and options.revision.isdigit() and
                  len(options.revision) < 40)
   # Take care of got_revisions outputs.
   revision_mapping = get_revision_mapping(svn_root, options.revision_mapping)
   got_revisions = parse_got_revision(gclient_output, revision_mapping,
                                      use_svn_rev)
 
   if options.output_json:
     # Tell recipes information such as root, got_revision, etc.
     emit_json(options.output_json,
               did_run=True,
               root=options.root,
               step_text=step_text,
               properties=got_revisions)
   else:
     # If we're not on recipes, tell annotator about our got_revisions.
     emit_properties(got_revisions)
 
 
 if __name__ == '__main__':
   sys.exit(main())