Bot Update support on official builders

scripts/slave/bot_update.py

Index: scripts/slave/bot_update.py
diff --git a/scripts/slave/bot_update.py b/scripts/slave/bot_update.py
index 5f6753b529fe04ccd46601f7a962f0fe322a781d..d8c100c6d09959856c84e00f613e929884dfbeb3 100755
--- a/scripts/slave/bot_update.py
+++ b/scripts/slave/bot_update.py
@@ -28,14 +28,53 @@ from common import chromium_utils
 
 CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src.git'
 
+# Solution names that use git_buildspecs to resolve .DEPS.git.
+BUILDSPEC_SOLUTIONS = [
+  'chrome-official',

[agable, 2014/04/18 19:09:33]

Is there any way these could be regex'd? 'chrome-official*'?

[Ryan Tseng, 2014/04/21 22:10:33]

Hm, maybe we can regex "/chrome-internal/trunk/tools/buildspec/build/(.*)" and
turn on buildspec mode.

[Ryan Tseng, 2014/04/21 22:10:33]

Done.

+  'chrome-official-android',
+  'chrome-official-aura',
+  'chrome-official-chromeos',
+  'chrome-official-ios',
+  'chrome-official-linux',
+  'chrome-official-mac',
+  'chrome-official-win',
+  'chrome-official-win-asan',
+  'chrome-official-win64',
+  'chrome-official-wincf',
+]
+
 RECOGNIZED_PATHS = {
     # If SVN path matches key, the entire URL is rewritten to the Git url.
     '/chrome/trunk/src':
-       CHROMIUM_SRC_URL,
+        CHROMIUM_SRC_URL,
     '/chrome-internal/trunk/src-internal':
-        'https://chrome-internal.googlesource.com/chrome/src-internal.git'
+        'https://chrome-internal.googlesource.com/chrome/src-internal.git',
+}
+# Official builders trigger on changes to buildspecs, and therefore the
+# gclient solution they get are always in a very specific folder.
+# We need to add all of them into our set of recognized paths.
+GIT_BUILDSPEC_PATH = ('https://chrome-internal.googlesource.com/chrome/tools/'
+                      'buildspec')
+for build_name in BUILDSPEC_SOLUTIONS:
+  RECOGNIZED_PATHS['/chrome-internal/trunk/tools/buildspec/build/%s'
+                   % build_name] = GIT_BUILDSPEC_PATH
+
+# In addition, because buildspecs are in a very specific directory that isn't
+# mirrored into Git as its own repo, but rather the parent of the solution
+# directory is mirrored into Git, we have to tell gclient to use the DEPS
+# thats farther down in the Git directory structure.
+CUSTOM_DEPS_FILE = {
+    '/chrome-internal/trunk/tools/buildspec/build/%s' % sln_name:
+        'build/%s/DEPS' % sln_name
+    for sln_name in BUILDSPEC_SOLUTIONS
 }
 
+# This is the repository that the buildspec -> git deps cron job mirrors
+# all buildspecs into. When we see an svn buildspec, we rely on the
+# buildspec -> git cron job to produce the git version of the buildspec.

[agable, 2014/04/18 19:09:33]

buildspec2git

[Ryan Tseng, 2014/04/21 22:10:33]

Done.

+GIT_BUILDSPEC_REPO = (
+    'https://chrome-internal.googlesource.com/chrome/tools/git_buildspecs')
+
 # Copied from scripts/recipes/chromium.py.
 GOT_REVISION_MAPPINGS = {
     '/chrome/trunk/src': {
@@ -106,7 +145,7 @@ cache_dir = %(cache_dir)s
 
 # IMPORTANT: If you're trying to enable a RECIPE bot, you'll need to
 # edit recipe_modules/bot_update/api.py instead.
-ENABLED_MASTERS = ['chromium.git']
+ENABLED_MASTERS = ['chromium.git', 'chrome_git']

[agable, 2014/04/18 19:09:33]

Have we made sure that the chrome_git waterfall is now running as close to a
clone of the chrome.official as we can? It needs to be not doing all its other
crazy git stuff.

[Ryan Tseng, 2014/04/21 22:10:33]

Yep

 ENABLED_BUILDERS = {
     'tryserver.chromium': ['linux_rel_alt'],
 }
@@ -308,13 +347,25 @@ def solutions_to_git(input_solutions):
     parsed_path = parsed_url.path
     if first_solution:
       root = parsed_path
+      deps_file = CUSTOM_DEPS_FILE.get(parsed_path, 'DEPS')
       first_solution = False
+
+    # Rewrite SVN urls into Git urls.
     if parsed_path in RECOGNIZED_PATHS:
       solution['url'] = RECOGNIZED_PATHS[parsed_path]
     else:
       print 'Warning: path %s not recognized' % parsed_path
-    if solution.get('deps_file', 'DEPS') == 'DEPS':
-      solution['deps_file'] = '.DEPS.git'
+
+    if parsed_path in CUSTOM_DEPS_FILE:
+      # For some bots (eg. official), we want to specify what DEPS file
+      # to look at.
+      solution['deps_file'] = CUSTOM_DEPS_FILE[parsed_path]
+
+    if solution.get('deps_file', 'DEPS').endswith('DEPS'):
+      # Point .DEPS.git is the git version of the DEPS file.
+      solution['deps_file'] = '.DEPS.git'.join(
+          solution['deps_file'].rsplit('DEPS', 1))

[agable, 2014/04/18 19:09:33]

wat. '.DEPS.git'.join()!?

[Ryan Tseng, 2014/04/21 22:10:33]

rsplit('DEPS') will remove the rightmost 'DEPS', and turn it into a ['asdf/',
''], and running '.DEPS.git'.join() on that list will produce 'asdf/.DEPS.git'. 

+
     solution['managed'] = False
     # We don't want gclient to be using a safesync URL. Instead it should
     # using the lkgr/lkcr branch/tags.
@@ -322,7 +373,7 @@ def solutions_to_git(input_solutions):
       print 'Removing safesync url %s from %s' % (solution['safesync_url'],
                                                   parsed_path)
       del solution['safesync_url']
-  return solutions, root
+  return solutions, root, deps_file
 
 
 def ensure_no_checkout(dir_names, scm_dirname):
@@ -360,10 +411,15 @@ def gclient_configure(solutions, target_os):
     f.write(get_gclient_spec(solutions, target_os))
 
 
-def gclient_sync(output_json):
+def gclient_sync(output_json, buildspec_mode):
   gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
-  call(gclient_bin, 'sync', '--verbose', '--reset', '--force',
-       '--nohooks', '--noprehooks', '--output-json', output_json)
+  cmd = [gclient_bin, 'sync', '--verbose', '--reset', '--force',
+         '--output-json', output_json]
+  if buildspec_mode:
+    cmd += ['--with_branch_heads']
+  else:
+    cmd += ['--nohooks', '--noprehooks']
+  call(*cmd)
   with open(output_json) as f:
     return json.load(f)
 
@@ -476,8 +532,10 @@ def need_to_run_deps2git(repo_base, deps_file, deps_git_file):
 
   Returns True if there was a DEPS change after the last .DEPS.git update.
   """
+  print 'Checking if %s exists' % deps_git_file
   if not path.isfile(deps_git_file):
     # .DEPS.git doesn't exist but DEPS does? We probably want to generate one.
+    print 'it exists!'
     return True
 
   last_known_deps_ref = _last_commit_for_file(deps_file, repo_base)
@@ -491,10 +549,56 @@ def need_to_run_deps2git(repo_base, deps_file, deps_git_file):
   return last_known_deps_ref != merge_base_ref
 
 
-def ensure_deps2git(sln_dir, shallow):
+def get_git_buildspec(version):
+  """Get the git buildspec of a version, return its contents.
+
+  The contents are returned instead of the file so that we can check the
+  repository into a temp directory and confine the cleanup logic here."""

[agable, 2014/04/18 19:09:33]

nit: close docstring quotes on next line

[Ryan Tseng, 2014/04/21 22:10:33]

Done.

+  git('cache', 'populate', '-v', '--cache-dir', CACHE_DIR, GIT_BUILDSPEC_REPO)
+  mirror_dir = git(
+      'cache', 'exists', '--cache-dir', CACHE_DIR, GIT_BUILDSPEC_REPO).strip()
+  tempdir = tempfile.mkdtemp()
+  git('clone', mirror_dir, tempdir)

[agable, 2014/04/18 19:09:33]

Shouldn't need the tempdir or the clone. Once you have the cache, you can just
run 'git show' against the right path to get the file contents without ever
having to check them out.

[Ryan Tseng, 2014/04/21 22:10:33]

Ohh interesting, done.

+  while True:
+    if path.isdir(path.join(tempdir, version)):
+      with open(path.join(tempdir, version, 'DEPS'), 'rb') as f:
+        git_buildspec = f.read()
+      chromium_utils.RemoveDirectory(tempdir)
+      return git_buildspec
+    print 'Buildspec for %s not committed yet, waiting 5 seconds...'
+    time.sleep(5)

[agable, 2014/04/18 19:09:33]

Would be wise to have an upper limit on this, so it doesn't sleep forever (and
buildbot will never kill it because it prints output every 5 seconds).

[Ryan Tseng, 2014/04/21 22:10:33]

Done.

+    git('cache', 'populate', '-v', '--cache-dir',
+        CACHE_DIR, GIT_BUILDSPEC_REPO)
+    git('fetch', 'origin' , cwd=tempdir)
+
+
+def buildspecs2git(repo_base, deps_file, deps_git_file):
+  """This is like deps2git, but for buildspecs.
+
+  Because buildspecs are vastly different than normal DEPS files, we cannot
+  use deps2git.py to generate git versions of the git DEPS.  Fortunately
+  we don't have buildspec trybots, and there is already a service that
+  generates git DEPS for every buildspec commit already, so we can leverage
+  that service so that we don't need to run buildspec2git.py serially.
+
+  This checks the commit message of the current DEPS file for the release
+  number, waits in a busy loop for the coorisponding .DEPS.git file to be
+  committed into the git_buildspecs repository."""

[agable, 2014/04/18 19:09:33]

nit: close docstring quotes on next line.

[Ryan Tseng, 2014/04/21 22:10:33]

Done.

+  deps_log = git('log', '-1', deps_file, cwd=repo_base)

[agable, 2014/04/18 19:09:33]

you could pass a format argument to git-log so that it only prints the commit
message, and not any of the other metadata.

[Ryan Tseng, 2014/04/21 22:10:33]

Done.

+  m = re.search(r'Buildspec for\s+version (\d+\.\d+\.\d+\.\d+)', deps_log)
+  version = m.group(1)
+  git_buildspec = get_git_buildspec(version)
+  with open(deps_git_file, 'wb') as f:
+    f.write(git_buildspec)
+
+
+def ensure_deps2git(sln_dir, deps_file, shallow, buildspec_mode):
   repo_base = path.join(os.getcwd(), sln_dir)
-  deps_file = path.join(repo_base, 'DEPS')
-  deps_git_file = path.join(repo_base, '.DEPS.git')
+  deps_file = path.join(repo_base, deps_file)
+  deps_git_file = path.join(path.dirname(deps_file), '.DEPS.git')
+  if buildspec_mode:
+    return buildspecs2git(repo_base, deps_file, deps_git_file)
+  print 'Checking if %s is newer than %s' % (deps_file, deps_git_file)
   if not path.isfile(deps_file):
     return
 
@@ -507,7 +611,9 @@ def ensure_deps2git(sln_dir, shallow):
 
   # TODO(hinoka): This might need to be smarter if we need to deal with
   #               DEPS changes that are in an internal repository.
-  repo_type = 'internal' if 'internal' in sln_dir else 'public'
+  repo_type = 'public'
+  if sln_dir in ['src-internal', 'chrome-official']:

[agable, 2014/04/18 19:09:33]

hard-coding chrome-official again

[Ryan Tseng, 2014/04/21 22:10:33]

Removed, this actually isn't needed

+    repo_type = 'internal'
   cmd = [sys.executable, DEPS2GIT_PATH,
          '-t', repo_type,
          '--cache_dir=%s' % CACHE_DIR,
@@ -775,7 +881,7 @@ def main():
   specs = {}
   exec(options.specs, specs)
   svn_solutions = specs.get('solutions', [])
-  git_solutions, svn_root = solutions_to_git(svn_solutions)
+  git_solutions, svn_root, deps_file = solutions_to_git(svn_solutions)
   solutions_printer(git_solutions)
 
   dir_names = [sln.get('name') for sln in svn_solutions if 'name' in sln]
@@ -827,7 +933,8 @@ def main():
                          git_ref)
 
   # Run deps2git if there is a DEPS commit after the last .DEPS.git commit.
-  ensure_deps2git(options.root, options.shallow)
+  buildspec_mode = options.root in BUILDSPEC_SOLUTIONS
+  ensure_deps2git(options.root, deps_file, options.shallow, buildspec_mode)
 
   # Ensure our build/ directory is set up with the correct .gclient file.
   gclient_configure(git_solutions, specs.get('target_os', []))
@@ -836,7 +943,7 @@ def main():
   # from gclient by passing in --output-json.  In our case, we can just reuse
   # the temp file that
   _, gclient_output_file = tempfile.mkstemp(suffix='.json')
-  gclient_output = gclient_sync(gclient_output_file)
+  gclient_output = gclient_sync(gclient_output_file, buildspec_mode)
 
   # If we're fed an svn revision number as --revision, then our got_revision
   # output should be in svn revs.  Otherwise it'll be in git hashes.