Bot update!

scripts/slave/bot_update.py

Index: scripts/slave/bot_update.py
diff --git a/scripts/slave/bot_update.py b/scripts/slave/bot_update.py
index 2e16e4cd7c7366ff710d7bdea1866cd7e8eaee5e..c55ea04e18932d0d15cfab76df2c7a684b810d93 100644
--- a/scripts/slave/bot_update.py
+++ b/scripts/slave/bot_update.py
@@ -3,13 +3,21 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-
+import codecs
 import copy
+import json
 import optparse
 import os
+import pprint
+import shutil
+import socket
+import subprocess
 import sys
+import time
 import urlparse
 
+import os.path as path
+
 
 RECOGNIZED_PATHS = {
     # If SVN path matches key, the entire URL is rewritten to the Git url.
@@ -51,10 +59,14 @@ This step does nothing. You actually want to look at the "update" step.
 
 """
 
+
+GCLIENT_TEMPLATE = """solutions = %(solutions)s
+
+cache_dir = %(cache_dir)s
+"""
+
 ENABLED_MASTERS = ['chromium.git']
-# Master: Builders dict.
 ENABLED_BUILDERS = {}
-# Master: Slaves dict.
 ENABLED_SLAVES = {}
 
 # Disabled filters get run AFTER enabled filters, so for example if a builder
@@ -63,6 +75,63 @@ ENABLED_SLAVES = {}
 DISABLED_BUILDERS = {}
 DISABLED_SLAVES = {}
 
+# How many times to retry failed subprocess calls.
+RETRIES = 3
+
+SCRIPTS_PATH = path.dirname(path.dirname(path.abspath(__file__)))
+DEPS2GIT_DIR_PATH = path.join(SCRIPTS_PATH, 'tools', 'deps2git')
+DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py')
+S2G_INTERNAL_FROM_PATH = path.join(SCRIPTS_PATH, 'tools', 'deps2git_internal',
+                                   'svn_to_git_internal.py')
+S2G_INTERNAL_DEST_PATH = path.join(DEPS2GIT_DIR_PATH, 'svn_to_git_internal.py')
+
+CACHE_DIR = os.path.join(os.path.dirname(os.path.abspath(os.getcwd())),
+                         'cache_dir')
+
+
+class SubprocessFailed(Exception):
+  pass
+
+
+def call(*args):
+  """Interactive subprocess call."""
+  tries = 0
+  while tries < RETRIES:
+    tries_msg = '(retry #%d)' % tries if tries else ''
+    print '===Running %s%s===' % (' '.join(args), tries_msg)
+    start_time = time.time()
+    proc = subprocess.Popen(args, stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+    # This is here because passing 'sys.stdout' into stdout for proc will
+    # produce out of order output.
+    while True:
+      buf = proc.stdout.read(1)
+      if not buf:
+        break
+      sys.stdout.write(buf)
+    code = proc.wait()

[agable, 2014/02/07 04:47:40]

retcode

[Ryan Tseng, 2014/02/07 21:23:07]

I feel more comfortable waiting for the process, because the pipe could
potentially close before the process actually completes.

+    if not code:

[agable, 2014/02/07 04:47:40]

elapsed = ((time.time() - start_time) / 60.0)

reused elapsed in prints below

[Ryan Tseng, 2014/02/07 21:23:07]

Done.

+      print '===Succeeded in %.1f mins===' % ((time.time() - start_time) / 60.0)
+      print
+      break
+    print '===Failed in %.1f mins===' % ((time.time() - start_time) / 60.0)
+    print
+    tries += 1
+
+  if code:
+    raise SubprocessFailed('%s failed with code %d in %s' %
+                           (' '.join(args), code, os.getcwd()))
+  return code
+
+
+def get_gclient_spec(solutions):
+  for sln in solutions:
+    sln['deps_file'] = '.DEPS.git'

[agable, 2014/02/07 04:47:40]

solutions_to_git already does this, I wouldn't do it here.

[Ryan Tseng, 2014/02/07 21:23:07]

oops you're right.  Done.

+  return GCLIENT_TEMPLATE % {
+      'solutions': pprint.pformat(solutions, indent=4),
+      'cache_dir': '"%s"' % CACHE_DIR
+  }
+
 
 def check_enabled(master, builder, slave):
   if master in ENABLED_MASTERS:
@@ -73,6 +142,9 @@ def check_enabled(master, builder, slave):
   slave_list = ENABLED_SLAVES.get(master)
   if slave_list and slave in slave_list:
     return True
+  if os.environ.get('TESTING_MASTER') == 'localhost':

[agable, 2014/02/07 04:47:40]

nope nope nope

[Ryan Tseng, 2014/02/07 21:23:07]

herp de derp.  Done.

+    # Magic conditional to get bot_update to always run locally.
+    return True
   return False
 
 
@@ -88,7 +160,8 @@ def check_disabled(master, builder, slave):
 
 
 def check_valid_host(master, builder, slave):
-  return False
+  return (check_enabled(master, builder, slave)
+          and not check_disabled(master, builder, slave))
 
 
 def solutions_printer(solutions):
@@ -99,6 +172,10 @@ def solutions_printer(solutions):
     name = solution.get('name')
     url = solution.get('url')
     print '%s (%s)' % (name, url)
+    if solution.get('deps_file'):
+      print '  Dependencies file is %s' % solution['deps_file']
+    if 'managed' in solution:
+      print '  Managed mode is %s' % ('ON' if solution['managed'] else 'OFF')
     custom_vars = solution.get('custom_vars')
     if custom_vars:
       print '  Custom Variables:'
@@ -112,10 +189,6 @@ def solutions_printer(solutions):
           print '    %s -> %s' % (deps_name, deps_value)
         else:
           print '    %s: Ignore' % deps_name
-    if solution.get('deps_file'):
-      print '  Dependencies file is %s' % solution['deps_file']
-    if 'managed' in solution:
-      print '  Managed mode is %s' % ('ON' if solution['managed'] else 'OFF')
     print
 
 
@@ -136,49 +209,105 @@ def solutions_to_git(input_solutions):
   return solutions
 
 
-def ensure_no_git_checkout():
+def ensure_no_checkout(dir_names, scm_filename):
   """Ensure that there is no git checkout under build/.
 
-  If there is a git checkout under build/, then move build/ to build.dead/
+  If there is an incorrect checkout under build/, then
+  move build/ to build.dead/
+  This function will check each directory in dir_names.
+
+  scm_filename is expected to be either ['.svn', '.git']

[agable, 2014/02/07 04:47:40]

scm_dirname?

[Ryan Tseng, 2014/02/07 21:23:07]

Done.

   """
-  pass
+  assert scm_filename in ['.svn', '.git']
+  has_checkout = any(map(lambda dir_name: path.exists(
+      path.join(os.getcwd(), dir_name, scm_filename)), dir_names))
 
+  if has_checkout:
+    # cd .. && rm -rf ./build && mkdir ./build && cd build
+    build_dir = os.getcwd()
 
-def ensure_no_svn_checkout():
-  """Ensure that there is no svn checkout under build/.
+    os.chdir(path.dirname(os.getcwd()))

[agable, 2014/02/07 04:47:40]

Can't this be done without chdir? rmtree should only care about open files, not
processes who think their pwd is build/.

[Ryan Tseng, 2014/02/07 21:23:07]

Discussed offline, but this needs to be here.

+    print '%s detected in checking, deleting %s...' % (scm_filename, build_dir),
+    shutil.rmtree(build_dir)
+    print 'done'
+    os.mkdir(build_dir)
+    os.chdir(build_dir)
 
-  If there is a svn checkout under build/, then move build/ to build.dead/
-  """
-  pass
 
 
 def gclient_configure(solutions):
-  pass
-
+  """Should do the same thing as gclient --spec='...'."""
+  with codecs.open('.gclient', mode='w', encoding='utf-8') as f:
+    f.write(get_gclient_spec(solutions))
 
-def gclient_shallow_sync():
-  pass
 
+def gclient_sync():
+  call('gclient', 'sync', '--verbose', '--reset', '--force',
+       '--nohooks', '--noprehooks')
+
+
+def get_git_hash(revision, dir_name):
+  match = "^git-svn-id: [^ ]*@%d" % revision
+  cmd = ['git', 'log', '--grep', match, '--format=%H', dir_name]
+  return subprocess.check_output(cmd).strip() or None
+
+
+def deps2git(sln_dirs):
+  for sln_dir in sln_dirs:
+    deps_file = path.join(os.getcwd(), sln_dir, 'DEPS')
+    deps_git_file = path.join(os.getcwd(), sln_dir, '.DEPS.git')
+    if not path.isfile(deps_file):
+      return
+    # Do we have a better way of doing this....?
+    repo_type = 'internal' if 'internal' in sln_dir else 'public'
+    # TODO(hinoka): This will be what populates the git caches on the first
+    #               run for all of the bots.  Since deps2git is single threaded,
+    #               all of this will run in a single threaded context and be
+    #               super slow.  Should make deps2git multithreaded.
+    call(sys.executable, DEPS2GIT_PATH, '-t', repo_type,
+         '--cache_dir=%s' % CACHE_DIR,
+         '--deps=%s' % deps_file, '--out=%s' % deps_git_file)
+
+
+def git_checkout(solutions, revision):
+  build_dir = os.getcwd()
+  for sln in solutions:
+    name = sln['name']
+    url = sln['url']
+    sln_dir = path.join(build_dir, name)
+    if not path.isdir(sln_dir):
+      call('git', 'clone', url, sln_dir)
+
+    # Clean out .DEPS.git changes first.
+    call('git', '-C', sln_dir, 'reset', '--hard')
+    call('git', '-C', sln_dir, 'clean', '-df')
+    call('git', '-C', sln_dir, 'pull', 'origin', 'master')
+    if revision and revision.lower() != 'head':
+      root, rev_num = revision.split('@')
+      git_hash = get_git_hash(rev_num, name) if root == name else None
+      call('git', '-C', sln_dir, 'checkout', git_hash)

[agable, 2014/02/07 04:47:40]

does call('git', '-C', sln_dir, 'checkout', None) just checkout HEAD anyway? Not
sure what subprocess.Popen's behavior is on 'None' args. Also, this is a really
silent fallback, which I'm not sure is good.

[Ryan Tseng, 2014/02/07 21:23:07]

oops fixed (by not even checking for that conditional)

+    else:
+      call('git', '-C', sln_dir, 'checkout', 'origin/master')
 
-def git_pull_and_clean():
-  pass
+    # Force another clean, because why not.
+    call('git', '-C', sln_dir, 'clean', '-df')
 
 
 def apply_issue(issue, patchset, root, server):
   pass
 
 
-def deps2git():
-  pass
+def delete_flag(flag_file):
+  """Remove bot update flag."""
+  if os.path.exists(flag_file):
+    os.remove(flag_file)
 
 
-def gclient_sync():
-  pass
-
-
-def deposit_bot_update_flag():
+def emit_flag(flag_file):
   """Deposit a bot update flag on the system to tell gclient not to run."""
-  pass
+  print 'Emitting flag file at %s' % flag_file
+  with open(flag_file, 'wb') as f:
+    f.write('Success!')
 
 
 def parse_args():
@@ -195,6 +324,10 @@ def parse_args():
                    help='Bypass check to see if we want to be run. '
                         'Should ONLY be used locally.')
   parse.add_option('-e', '--revision-mapping')
+  parse.add_option('-v', '--revision')
+  parse.add_option('-b', '--build_dir', default=os.getcwd())
+  parse.add_option('-g', '--flag_file', default=path.join(os.getcwd(),
+                                                          'update.flag'))
 
   return parse.parse_args()
 
@@ -202,8 +335,9 @@ def parse_args():
 def main():
   # Get inputs.
   options, _ = parse_args()
+  delete_flag(options.flag_file)

[agable, 2014/02/07 04:47:40]

I think we should only delete the flag if we're not supposed to be active. If
this script somehow fails in the middle (between deleting and re-adding the flag
file) we still don't want old processes to take over -- that'll just make a
mess. It should fail, and fail hard.

[Ryan Tseng, 2014/02/07 21:23:07]

sgtm.  Moved delete/emit flag down next to ensure_no_checkout on line 365

   builder = os.environ.get('BUILDBOT_BUILDERNAME', None)
-  slave = os.environ.get('BUILDBOT_SLAVENAME', None)
+  slave = os.environ.get('BUILDBOT_SLAVENAME', socket.getfqdn().split('.')[0])
   master = options.master
 
   # Check if this script should activate or not.
@@ -220,21 +354,36 @@ def main():
 
   # Parse, munipulate, and print the gclient solutions.
   specs = {}
-  exec(options.specs, specs)  # TODO(hinoka): LOL this is terrible.
-  solutions = specs.get('solutions', [])
-  git_solutions = solutions_to_git(solutions)
+  exec(options.specs, specs)
+  svn_solutions = specs.get('solutions', [])
+  git_solutions = solutions_to_git(svn_solutions)
   solutions_printer(git_solutions)
 
-  # Do the checkout.
-  # TODO(hinoka): Uncomment these once they're implemented.
-  # ensure_no_svn_checkout()
-  # gclient_configure(git_solutions)
-  # gclient_shallow_sync()
-  # git_pull_and_clean()
+  # Cleanup svn checkout if active, otherwise remove git checkout and exit.
+  dir_names = [sln.get('name') for sln in svn_solutions if 'name' in sln]
+  if active:
+    ensure_no_checkout(dir_names, '.svn')
+  else:
+    ensure_no_checkout(dir_names, '.git')
+    return
+
+  # Get a checkout of each solution, without DEPS or hooks.
+  # Calling git directory because there is no way to run Gclient without
+  # invoking DEPS.
+  print 'Fetching Git checkout'
+  git_checkout(git_solutions, options.revision)
+
+  # TODO(hinoka): This must be implemented before we can turn this on for TS.
   # if options.issue:
   #   apply_issue(options.issue, options.patchset, options.root, options.server)
-  # deps2git()
-  # gclient_sync()
+
+  # Magic to get deps2git to work with internal DEPS.
+  shutil.copyfile(S2G_INTERNAL_FROM_PATH, S2G_INTERNAL_DEST_PATH)
+  deps2git(dir_names)
+
+  gclient_configure(git_solutions)
+  gclient_sync()
+  emit_flag(options.flag_file)
 
 
 if __name__ == '__main__':