Bot Update support on official builders

scripts/slave/bot_update.py

Index: scripts/slave/bot_update.py
diff --git a/scripts/slave/bot_update.py b/scripts/slave/bot_update.py
index 5f6753b529fe04ccd46601f7a962f0fe322a781d..8fe825cff67cab3c25223b38876af2a2ef4e64dc 100755
--- a/scripts/slave/bot_update.py
+++ b/scripts/slave/bot_update.py
@@ -33,9 +33,24 @@ RECOGNIZED_PATHS = {
     '/chrome/trunk/src':
        CHROMIUM_SRC_URL,
     '/chrome-internal/trunk/src-internal':
-        'https://chrome-internal.googlesource.com/chrome/src-internal.git'
+        'https://chrome-internal.googlesource.com/chrome/src-internal.git',
+    '/chrome-internal/trunk/tools/buildspec/build/chrome-official':
+        'https://chrome-internal.googlesource.com/chrome/tools/buildspec',
 }
 
+CUSTOM_DEPS_FILE = {
+    '/chrome-internal/trunk/tools/buildspec/build/chrome-official':
+        'build/chrome-official/DEPS'
+}
+
+# Solution names that use git_buildspecs to resolve .DEPS.git.
+BUILDSPEC_SOLUTIONS = [
+  'chrome-official',

[agable, 2014/04/18 03:45:07]

I think this needs to be more than chrome-official: there's also
chrome-official-win, chrome-official-win-asan, etc. Every subdirectory of
chrome/tools/buildspec/build is a possible value.

[Ryan Tseng, 2014/04/18 18:42:38]

Done.

+]
+
+GIT_BUILDSPEC_REPO = (
+    'https://chrome-internal.googlesource.com/chrome/tools/git_buildspecs')
+
 # Copied from scripts/recipes/chromium.py.
 GOT_REVISION_MAPPINGS = {
     '/chrome/trunk/src': {
@@ -106,7 +121,7 @@ cache_dir = %(cache_dir)s
 
 # IMPORTANT: If you're trying to enable a RECIPE bot, you'll need to
 # edit recipe_modules/bot_update/api.py instead.
-ENABLED_MASTERS = ['chromium.git']
+ENABLED_MASTERS = ['chromium.git', 'chrome_git']
 ENABLED_BUILDERS = {
     'tryserver.chromium': ['linux_rel_alt'],
 }
@@ -308,13 +323,25 @@ def solutions_to_git(input_solutions):
     parsed_path = parsed_url.path
     if first_solution:
       root = parsed_path
+      deps_file = CUSTOM_DEPS_FILE.get(parsed_path, 'DEPS')
       first_solution = False
+
+    # Rewrite SVN urls into Git urls.
     if parsed_path in RECOGNIZED_PATHS:
       solution['url'] = RECOGNIZED_PATHS[parsed_path]
     else:
       print 'Warning: path %s not recognized' % parsed_path
-    if solution.get('deps_file', 'DEPS') == 'DEPS':
-      solution['deps_file'] = '.DEPS.git'
+
+    if parsed_path in CUSTOM_DEPS_FILE:
+      # For some bots (eg. official), we want to specify what DEPS file
+      # to look at.
+      solution['deps_file'] = CUSTOM_DEPS_FILE[parsed_path]
+
+    if solution.get('deps_file', 'DEPS').endswith('DEPS'):
+      # Point .DEPS.git is the git version of the DEPS file.
+      solution['deps_file'] = '.DEPS.git'.join(
+          solution['deps_file'].rsplit('DEPS', 1))
+
     solution['managed'] = False
     # We don't want gclient to be using a safesync URL. Instead it should
     # using the lkgr/lkcr branch/tags.
@@ -322,7 +349,7 @@ def solutions_to_git(input_solutions):
       print 'Removing safesync url %s from %s' % (solution['safesync_url'],
                                                   parsed_path)
       del solution['safesync_url']
-  return solutions, root
+  return solutions, root, deps_file
 
 
 def ensure_no_checkout(dir_names, scm_dirname):
@@ -360,10 +387,15 @@ def gclient_configure(solutions, target_os):
     f.write(get_gclient_spec(solutions, target_os))
 
 
-def gclient_sync(output_json):
+def gclient_sync(output_json, buildspec_mode):
   gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
-  call(gclient_bin, 'sync', '--verbose', '--reset', '--force',
-       '--nohooks', '--noprehooks', '--output-json', output_json)
+  cmd = [gclient_bin, 'sync', '--verbose', '--reset', '--force',
+         '--output-json', output_json]
+  if buildspec_mode:
+    cmd += ['--with_branch_heads']
+  else:
+    cmd += ['--nohooks', '--noprehooks']
+  call(*cmd)
   with open(output_json) as f:
     return json.load(f)
 
@@ -476,8 +508,10 @@ def need_to_run_deps2git(repo_base, deps_file, deps_git_file):
 
   Returns True if there was a DEPS change after the last .DEPS.git update.
   """
+  print 'Checking if %s exists' % deps_git_file
   if not path.isfile(deps_git_file):
     # .DEPS.git doesn't exist but DEPS does? We probably want to generate one.
+    print 'it exists!'
     return True
 
   last_known_deps_ref = _last_commit_for_file(deps_file, repo_base)
@@ -491,10 +525,56 @@ def need_to_run_deps2git(repo_base, deps_file, deps_git_file):
   return last_known_deps_ref != merge_base_ref
 
 
-def ensure_deps2git(sln_dir, shallow):
+def get_git_buildspec(version):
+  """Get the git buildspec of a version, return its contents.
+
+  The contents are returned instead of the file so that we can check the
+  repository into a temp directory and confine the cleanup logic here."""
+  git('cache', 'populate', '-v', '--cache-dir', CACHE_DIR, GIT_BUILDSPEC_REPO)
+  mirror_dir = git(
+      'cache', 'exists', '--cache-dir', CACHE_DIR, GIT_BUILDSPEC_REPO).strip()
+  tempdir = tempfile.mkdtemp()
+  git('clone', mirror_dir, tempdir)
+  while True:
+    if path.isdir(path.join(tempdir, version)):
+      with open(path.join(tempdir, version, 'DEPS'), 'rb') as f:
+        git_buildspec = f.read()
+      chromium_utils.RemoveDirectory(tempdir)
+      return git_buildspec
+    print 'Buildspec for %s not committed yet, waiting 5 seconds...'
+    time.sleep(5)
+    git('cache', 'populate', '-v', '--cache-dir',
+        CACHE_DIR, GIT_BUILDSPEC_REPO)
+    git('fetch', 'origin' , cwd=tempdir)
+
+
+def buildspecs2git(repo_base, deps_file, deps_git_file):
+  """This is like deps2git, but for buildspecs.
+
+  Because buildspecs are vastly different than normal DEPS files, we cannot
+  use deps2git.py to generate git versions of the git DEPS.  Fortunately
+  we don't have buildspec trybots, and there is already a service that
+  generates git DEPS for every buildspec commit already, so we can leverage
+  that service so that we don't need to run buildspec2git.py serially.
+
+  This checks the commit message of the current DEPS file for the release
+  number, waits in a busy loop for the coorisponding .DEPS.git file to be
+  committed into the git_buildspecs repository."""
+  deps_log = git('log', '-1', deps_file, cwd=repo_base)
+  m = re.search(r'Buildspec for\s+version (\d+\.\d+\.\d+\.\d+)', deps_log)
+  version = m.group(1)
+  git_buildspec = get_git_buildspec(version)
+  with open(deps_git_file, 'wb') as f:
+    f.write(git_buildspec)
+
+
+def ensure_deps2git(sln_dir, deps_file, shallow, buildspec_mode):
   repo_base = path.join(os.getcwd(), sln_dir)
-  deps_file = path.join(repo_base, 'DEPS')
-  deps_git_file = path.join(repo_base, '.DEPS.git')
+  deps_file = path.join(repo_base, deps_file)
+  deps_git_file = path.join(path.dirname(deps_file), '.DEPS.git')
+  if buildspec_mode:
+    return buildspecs2git(repo_base, deps_file, deps_git_file)
+  print 'Checking if %s is newer than %s' % (deps_file, deps_git_file)
   if not path.isfile(deps_file):
     return
 
@@ -507,7 +587,9 @@ def ensure_deps2git(sln_dir, shallow):
 
   # TODO(hinoka): This might need to be smarter if we need to deal with
   #               DEPS changes that are in an internal repository.
-  repo_type = 'internal' if 'internal' in sln_dir else 'public'
+  repo_type = 'public'
+  if sln_dir in ['src-internal', 'chrome-official']:
+    repo_type = 'internal'
   cmd = [sys.executable, DEPS2GIT_PATH,
          '-t', repo_type,
          '--cache_dir=%s' % CACHE_DIR,
@@ -775,7 +857,7 @@ def main():
   specs = {}
   exec(options.specs, specs)
   svn_solutions = specs.get('solutions', [])
-  git_solutions, svn_root = solutions_to_git(svn_solutions)
+  git_solutions, svn_root, deps_file = solutions_to_git(svn_solutions)
   solutions_printer(git_solutions)
 
   dir_names = [sln.get('name') for sln in svn_solutions if 'name' in sln]
@@ -827,7 +909,8 @@ def main():
                          git_ref)
 
   # Run deps2git if there is a DEPS commit after the last .DEPS.git commit.
-  ensure_deps2git(options.root, options.shallow)
+  buildspec_mode = options.root in BUILDSPEC_SOLUTIONS
+  ensure_deps2git(options.root, deps_file, options.shallow, buildspec_mode)
 
   # Ensure our build/ directory is set up with the correct .gclient file.
   gclient_configure(git_solutions, specs.get('target_os', []))
@@ -836,7 +919,7 @@ def main():
   # from gclient by passing in --output-json.  In our case, we can just reuse
   # the temp file that
   _, gclient_output_file = tempfile.mkstemp(suffix='.json')
-  gclient_output = gclient_sync(gclient_output_file)
+  gclient_output = gclient_sync(gclient_output_file, buildspec_mode)
 
   # If we're fed an svn revision number as --revision, then our got_revision
   # output should be in svn revs.  Otherwise it'll be in git hashes.