Bot update!

scripts/slave/bot_update.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-
+import codecs
 import copy
+import json
 import optparse
 import os
+import pprint
+import shutil
+import socket
+import subprocess
 import sys
+import time
 import urlparse
 
+import os.path as path
+
 
 RECOGNIZED_PATHS = {
     # If SVN path matches key, the entire URL is rewritten to the Git url.
     '/chrome/trunk/src':
         'https://chromium.googlesource.com/chromium/src.git',
     '/chrome-internal/trunk/src-internal':
         'https://chrome-internal.googlesource.com/chrome/src-internal.git'
 }
 
 
@@ skipping 21 matching lines @@
 The bot will perform a Git checkout in this step.
 The "gclient revert" and "update" steps are no-ops.
 
 """
 
 NOT_ACTIVATED_MESSAGE = """INACTIVE.
 This step does nothing. You actually want to look at the "update" step.
 
 """
 
+
+GCLIENT_TEMPLATE = """solutions = %(solutions)s
+
+cache_dir = %(cache_dir)s
+"""
+
 ENABLED_MASTERS = ['chromium.git']
-# Master: Builders dict.
 ENABLED_BUILDERS = {}
-# Master: Slaves dict.
 ENABLED_SLAVES = {}
 
 # Disabled filters get run AFTER enabled filters, so for example if a builder
 # config is enabled, but a bot on that builder is disabled, that bot will
 # be disabled.
 DISABLED_BUILDERS = {}
 DISABLED_SLAVES = {}
 
+# How many times to retry failed subprocess calls.
+RETRIES = 3
+
+SCRIPTS_PATH = path.dirname(path.dirname(path.abspath(__file__)))
+DEPS2GIT_DIR_PATH = path.join(SCRIPTS_PATH, 'tools', 'deps2git')
+DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py')
+S2G_INTERNAL_FROM_PATH = path.join(SCRIPTS_PATH, 'tools', 'deps2git_internal',
+                                   'svn_to_git_internal.py')
+S2G_INTERNAL_DEST_PATH = path.join(DEPS2GIT_DIR_PATH, 'svn_to_git_internal.py')
+
+# ../../cache_dir aka /b/build/slave/cache_dir
+CACHE_DIR = os.path.join(

[agable, 2014/02/07 22:19:06]

could do this relative to SCRIPTS_DIR. Probably should, actually.

Another good idiom is

THIS_DIR=path.abspath(__file__)
SCRIPTS_DIR = stuff(THIS_DIR)
CACHE_DIR = stuff(THIS_DIR)

etc.

[iannucci, 2014/02/07 22:24:09]

* THIS_DIR = path.abspath(path.dirname(__file__))

[Ryan Tseng, 2014/02/07 22:27:50]

Done.

+    os.path.dirname(os.path.dirname(os.path.abspath(os.getcwd()))),
+    'cache_dir')
+
+
+class SubprocessFailed(Exception):
+  pass
+
+
+def call(*args):
+  """Interactive subprocess call."""
+  tries = 0
+  while tries < RETRIES:
+    tries_msg = '(retry #%d)' % tries if tries else ''
+    print '===Running %s%s===' % (' '.join(args), tries_msg)
+    start_time = time.time()
+    proc = subprocess.Popen(args, stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+    # This is here because passing 'sys.stdout' into stdout for proc will
+    # produce out of order output.
+    while True:
+      buf = proc.stdout.read(1)
+      if not buf:
+        break
+      sys.stdout.write(buf)
+    code = proc.wait()
+    elapsed_time = ((time.time() - start_time) / 60.0)
+    if not code:
+      print '===Succeeded in %.1f mins===' % elapsed_time
+      print
+      break
+    print '===Failed in %.1f mins===' % elapsed_time
+    print
+    tries += 1
+
+  if code:
+    raise SubprocessFailed('%s failed with code %d in %s' %
+                           (' '.join(args), code, os.getcwd()))
+  return code
+
+
+def get_gclient_spec(solutions):
+  return GCLIENT_TEMPLATE % {
+      'solutions': pprint.pformat(solutions, indent=4),
+      'cache_dir': '"%s"' % CACHE_DIR
+  }
+
 
 def check_enabled(master, builder, slave):
   if master in ENABLED_MASTERS:
     return True
   builder_list = ENABLED_BUILDERS.get(master)
   if builder_list and builder in builder_list:
     return True
   slave_list = ENABLED_SLAVES.get(master)
   if slave_list and slave in slave_list:
     return True
   return False
 
 
 def check_disabled(master, builder, slave):
   """Returns True if disabled, False if not disabled."""
   builder_list = DISABLED_BUILDERS.get(master)
   if builder_list and builder in builder_list:
     return True
   slave_list = DISABLED_SLAVES.get(master)
   if slave_list and slave in slave_list:
     return True
   return False
 
 
 def check_valid_host(master, builder, slave):
-  return False
+  return (check_enabled(master, builder, slave)
+          and not check_disabled(master, builder, slave))
 
 
 def solutions_printer(solutions):
   """Prints gclient solution to stdout."""
   print 'Gclient Solutions'
   print '================='
   for solution in solutions:
     name = solution.get('name')
     url = solution.get('url')
     print '%s (%s)' % (name, url)
+    if solution.get('deps_file'):
+      print '  Dependencies file is %s' % solution['deps_file']
+    if 'managed' in solution:
+      print '  Managed mode is %s' % ('ON' if solution['managed'] else 'OFF')
     custom_vars = solution.get('custom_vars')
     if custom_vars:
       print '  Custom Variables:'
       for var_name, var_value in sorted(custom_vars.iteritems()):
         print '    %s = %s' % (var_name, var_value)
     custom_deps = solution.get('custom_deps')
     if 'custom_deps' in solution:
       print '  Custom Dependencies:'
       for deps_name, deps_value in sorted(custom_deps.iteritems()):
         if deps_value:
           print '    %s -> %s' % (deps_name, deps_value)
         else:
           print '    %s: Ignore' % deps_name
-    if solution.get('deps_file'):
-      print '  Dependencies file is %s' % solution['deps_file']
-    if 'managed' in solution:
-      print '  Managed mode is %s' % ('ON' if solution['managed'] else 'OFF')
     print
 
 
 def solutions_to_git(input_solutions):
   """Modifies urls in solutions to point at Git repos."""
   solutions = copy.deepcopy(input_solutions)
   for solution in solutions:
     original_url = solution['url']
     parsed_url = urlparse.urlparse(original_url)
     path = parsed_url.path
     if path in RECOGNIZED_PATHS:
       solution['url'] = RECOGNIZED_PATHS[path]
     else:
       print 'Warning: path %s not recognized' % path
     if solution.get('deps_file', 'DEPS') == 'DEPS':
       solution['deps_file'] = '.DEPS.git'
     solution['managed'] = False
   return solutions
 
 
-def ensure_no_git_checkout():
+def ensure_no_checkout(dir_names, scm_dirname):
   """Ensure that there is no git checkout under build/.
 
-  If there is a git checkout under build/, then move build/ to build.dead/
+  If there is an incorrect checkout under build/, then
+  move build/ to build.dead/
+  This function will check each directory in dir_names.
+
+  scm_dirname is expected to be either ['.svn', '.git']
   """
-  pass
+  assert scm_dirname in ['.svn', '.git']
+  has_checkout = any(map(lambda dir_name: path.exists(
+      path.join(os.getcwd(), dir_name, scm_dirname)), dir_names))
 
+  if has_checkout:
+    # cd .. && rm -rf ./build && mkdir ./build && cd build
+    build_dir = os.getcwd()
 
-def ensure_no_svn_checkout():
-  """Ensure that there is no svn checkout under build/.
+    os.chdir(path.dirname(os.getcwd()))
+    print '%s detected in checking, deleting %s...' % (scm_dirname, build_dir),
+    shutil.rmtree(build_dir)
+    print 'done'
+    os.mkdir(build_dir)
+    os.chdir(build_dir)
 
-  If there is a svn checkout under build/, then move build/ to build.dead/
-  """
-  pass
 
 
 def gclient_configure(solutions):
-  pass
+  """Should do the same thing as gclient --spec='...'."""
+  with codecs.open('.gclient', mode='w', encoding='utf-8') as f:
+    f.write(get_gclient_spec(solutions))
 
 
-def gclient_shallow_sync():
-  pass
+def gclient_sync():
+  call('gclient', 'sync', '--verbose', '--reset', '--force',
+       '--nohooks', '--noprehooks')
 
 
-def git_pull_and_clean():
-  pass
+def get_git_hash(revision, dir_name):
+  match = "^git-svn-id: [^ ]*@%d" % revision
+  cmd = ['git', 'log', '--grep', match, '--format=%H', dir_name]
+  return subprocess.check_output(cmd).strip() or None
+
+
+def deps2git(sln_dirs):
+  for sln_dir in sln_dirs:
+    deps_file = path.join(os.getcwd(), sln_dir, 'DEPS')
+    deps_git_file = path.join(os.getcwd(), sln_dir, '.DEPS.git')
+    if not path.isfile(deps_file):
+      return
+    # Do we have a better way of doing this....?
+    repo_type = 'internal' if 'internal' in sln_dir else 'public'
+    # TODO(hinoka): This will be what populates the git caches on the first
+    #               run for all of the bots.  Since deps2git is single threaded,
+    #               all of this will run in a single threaded context and be
+    #               super slow.  Should make deps2git multithreaded.
+    call(sys.executable, DEPS2GIT_PATH, '-t', repo_type,
+         '--cache_dir=%s' % CACHE_DIR,
+         '--deps=%s' % deps_file, '--out=%s' % deps_git_file)
+
+
+def git_checkout(solutions, revision):
+  build_dir = os.getcwd()
+  # Revision only applies to the first solution.
+  first_solution = True
+  for sln in solutions:
+    name = sln['name']
+    url = sln['url']
+    sln_dir = path.join(build_dir, name)
+    if not path.isdir(sln_dir):
+      call('git', 'clone', url, sln_dir)
+
+    # Clean out .DEPS.git changes first.
+    call('git', '-C', sln_dir, 'reset', '--hard')
+    call('git', '-C', sln_dir, 'clean', '-df')
+    call('git', '-C', sln_dir, 'pull', 'origin', 'master')
+    # TODO(hinoka): We probably have to make use of revision mapping.
+    if first_solution and revision and revision.lower() != 'head':
+      if revision and revision.isdigit() and len(revision) < 40:
+        # rev_num is really a svn revision number, convert it into a git hash.
+        git_hash = get_git_hash(revision, name)
+      else:
+        # rev_num is actually a git hash or ref, we can just use it.
+        git_hash = revision

[agable, 2014/02/07 22:19:06]

Since we're willing to use refs (HEAD, master, my-funky-dev-branch, etc) this
should probably be called git_ref, not git_hash.

[Ryan Tseng, 2014/02/07 22:27:50]

Done.

+      call('git', '-C', sln_dir, 'checkout', git_hash)
+    else:
+      call('git', '-C', sln_dir, 'checkout', 'origin/master')
+
+    # Force another clean, because why not.
+    call('git', '-C', sln_dir, 'clean', '-df')

[agable, 2014/02/07 22:19:06]

Only do this if you really think it's necessary. Shouldn't do any extra work
since this step will already take long enough as it is.

[iannucci, 2014/02/07 22:24:09]

Isn't this necessary because there could be a patch applied? Though apply_issue
currently actually commits the patch...

Oh, yeah, since apply_issue commits the patch, we should also make sure to set
user.name and user.email on the root repo.

That said, if apply_issue really commits it, then we won't need clean because
there will never be any untracked stuff.

[Ryan Tseng, 2014/02/07 22:27:50]

The one on line 283 is probably enough. I'll remove this.

+
+    first_solution = False
 
 
 def apply_issue(issue, patchset, root, server):
   pass
 
 
-def deps2git():
-  pass
+def delete_flag(flag_file):
+  """Remove bot update flag."""
+  if os.path.exists(flag_file):
+    os.remove(flag_file)
 
 
-def gclient_sync():
-  pass
-
-
-def deposit_bot_update_flag():
+def emit_flag(flag_file):
   """Deposit a bot update flag on the system to tell gclient not to run."""
-  pass
+  print 'Emitting flag file at %s' % flag_file
+  with open(flag_file, 'wb') as f:
+    f.write('Success!')
 
 
 def parse_args():
   parse = optparse.OptionParser()
 
   parse.add_option('-i', '--issue', help='Issue number to patch from.')
   parse.add_option('-p', '--patchset',
                    help='Patchset from issue to patch from, if applicable.')
   parse.add_option('-r', '--root', help='Repository root.')
   parse.add_option('-c', '--server', help='Rietveld server.')
   parse.add_option('-s', '--specs', help='Gcilent spec.')
   parse.add_option('-m', '--master', help='Master name.')
   parse.add_option('-f', '--force', action='store_true',
                    help='Bypass check to see if we want to be run. '
                         'Should ONLY be used locally.')
+  # TODO(hinoka): We don't actually use this

[agable, 2014/02/07 22:19:06]

Then can we remove it? Or does that require a master side change too?

[iannucci, 2014/02/07 22:24:09]

We probably shouldn't rely on this being a parameter.

In fact, I think MANY of these parameters are not needed/wanted, and should be
derived slaveside from mastername/buildername instead.

Otherwise we require the master configuration to pass them correctly.......

[Ryan Tseng, 2014/02/07 22:27:50]

Rephrased.  I mean we need to use this at some point, but it is yet to be
implemented.  Before its implemented anything that depends on the behavior of
revision mapping will be incorrect.

   parse.add_option('-e', '--revision-mapping')
+  parse.add_option('-v', '--revision')
+  parse.add_option('-b', '--build_dir', default=os.getcwd())
+  parse.add_option('-g', '--flag_file', default=path.join(os.getcwd(),
+                                                          'update.flag'))
 
   return parse.parse_args()
 
 
 def main():
   # Get inputs.
   options, _ = parse_args()
   builder = os.environ.get('BUILDBOT_BUILDERNAME', None)
-  slave = os.environ.get('BUILDBOT_SLAVENAME', None)
+  slave = os.environ.get('BUILDBOT_SLAVENAME', socket.getfqdn().split('.')[0])
   master = options.master
+  if not options.revision:
+    options.revision = os.environ.get('BUILDBOT_REVISION')
 
   # Check if this script should activate or not.
   active = check_valid_host(master, builder, slave) or options.force
 
   # Print helpful messages to tell devs whats going on.
   print BOT_UPDATE_MESSAGE % {
     'master': master,
     'builder': builder,
     'slave': slave,
   },
   # Print to stderr so that it shows up red on win/mac.
   print ACTIVATED_MESSAGE if active else NOT_ACTIVATED_MESSAGE
 
   # Parse, munipulate, and print the gclient solutions.
   specs = {}
-  exec(options.specs, specs)  # TODO(hinoka): LOL this is terrible.
-  solutions = specs.get('solutions', [])
-  git_solutions = solutions_to_git(solutions)
+  exec(options.specs, specs)
+  svn_solutions = specs.get('solutions', [])
+  git_solutions = solutions_to_git(svn_solutions)
   solutions_printer(git_solutions)
 
-  # Do the checkout.
-  # TODO(hinoka): Uncomment these once they're implemented.
-  # ensure_no_svn_checkout()
-  # gclient_configure(git_solutions)
-  # gclient_shallow_sync()
-  # git_pull_and_clean()
+  # Cleanup svn checkout if active, otherwise remove git checkout and exit.
+  dir_names = [sln.get('name') for sln in svn_solutions if 'name' in sln]
+  if active:
+    ensure_no_checkout(dir_names, '.svn')
+    emit_flag(options.flag_file)
+  else:
+    ensure_no_checkout(dir_names, '.git')
+    delete_flag(options.flag_file)
+    return
+
+  # Get a checkout of each solution, without DEPS or hooks.
+  # Calling git directory because there is no way to run Gclient without
+  # invoking DEPS.
+  print 'Fetching Git checkout'
+  git_checkout(git_solutions, options.revision)
+
+  # TODO(hinoka): This must be implemented before we can turn this on for TS.
   # if options.issue:
   #   apply_issue(options.issue, options.patchset, options.root, options.server)
-  # deps2git()
-  # gclient_sync()
+
+  # Magic to get deps2git to work with internal DEPS.
+  shutil.copyfile(S2G_INTERNAL_FROM_PATH, S2G_INTERNAL_DEST_PATH)
+  deps2git(dir_names)
+
+  gclient_configure(git_solutions)
+  gclient_sync()
 
 
 if __name__ == '__main__':
   sys.exit(main())