Bot update!

scripts/slave/bot_update.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-
+import codecs
 import copy
 import optparse
 import os
+import pprint
+import shutil
+import socket
+import subprocess
 import sys
+import time
 import urlparse
 
+import os.path as path
+
 
 RECOGNIZED_PATHS = {
     # If SVN path matches key, the entire URL is rewritten to the Git url.
     '/chrome/trunk/src':
         'https://chromium.googlesource.com/chromium/src.git',
     '/chrome-internal/trunk/src-internal':
         'https://chrome-internal.googlesource.com/chrome/src-internal.git'
 }
 
 
@@ skipping 21 matching lines @@
 The bot will perform a Git checkout in this step.
 The "gclient revert" and "update" steps are no-ops.
 
 """
 
 NOT_ACTIVATED_MESSAGE = """INACTIVE.
 This step does nothing. You actually want to look at the "update" step.
 
 """
 
+
+GCLIENT_TEMPLATE = """solutions = %(solutions)s
+
+cache_dir = %(cache_dir)s
+"""
+
 ENABLED_MASTERS = ['chromium.git']
-# Master: Builders dict.
 ENABLED_BUILDERS = {}
-# Master: Slaves dict.
 ENABLED_SLAVES = {}
 
 # Disabled filters get run AFTER enabled filters, so for example if a builder
 # config is enabled, but a bot on that builder is disabled, that bot will
 # be disabled.
 DISABLED_BUILDERS = {}
 DISABLED_SLAVES = {}
 
+# How many times to retry failed subprocess calls.
+RETRIES = 3
+
+SCRIPTS_PATH = path.dirname(path.dirname(path.abspath(__file__)))
+DEPS2GIT_DIR_PATH = path.join(SCRIPTS_PATH, 'tools', 'deps2git')
+DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py')
+S2G_INTERNAL_FROM_PATH = path.join(SCRIPTS_PATH, 'tools', 'deps2git_internal',
+                                   'svn_to_git_internal.py')
+S2G_INTERNAL_DEST_PATH = path.join(DEPS2GIT_DIR_PATH, 'svn_to_git_internal.py')
+
+# ../../cache_dir aka /b/build/slave/cache_dir
+THIS_DIR = path.abspath(os.getcwd())
+BUILDER_DIR = path.dirname(THIS_DIR)
+SLAVE_DIR = path.dirname(BUILDER_DIR)
+CACHE_DIR = path.join(SLAVE_DIR, 'cache_dir')
+
+
+class SubprocessFailed(Exception):
+  pass
+
+
+def call(*args):
+  """Interactive subprocess call."""
+  tries = 0
+  while tries < RETRIES:

[iannucci, 2014/02/08 01:31:31]

could also do

for try in xrange(RETRIES):
   ...

[Ryan Tseng, 2014/02/08 01:52:21]

Done.

+    tries_msg = '(retry #%d)' % tries if tries else ''
+    print '===Running %s%s===' % (' '.join(args), tries_msg)
+    start_time = time.time()
+    proc = subprocess.Popen(args, stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+    # This is here because passing 'sys.stdout' into stdout for proc will
+    # produce out of order output.
+    while True:
+      buf = proc.stdout.read(1)

[iannucci, 2014/02/08 01:31:31]

can't we just do readline() calls?

[Ryan Tseng, 2014/02/08 01:52:21]

And rely on the system native newline semantics playing nicely with python? :/
I'd rather have as little things in the python stdout pipe as i can.

[iannucci, 2014/02/08 03:37:21]

oh, I see you're just copying bytes... how does that prevent weird interleaving?

[Ryan Tseng, 2014/02/10 23:11:28]

It doesn't but unless we do multiple calls at a time there shouldn't be any
weird interleaving...?

+      if not buf:
+        break
+      sys.stdout.write(buf)
+    code = proc.wait()
+    elapsed_time = ((time.time() - start_time) / 60.0)
+    if not code:
+      print '===Succeeded in %.1f mins===' % elapsed_time
+      print
+      break

[iannucci, 2014/02/08 01:31:31]

just do a return here

[Ryan Tseng, 2014/02/08 01:52:21]

Done.

+    print '===Failed in %.1f mins===' % elapsed_time
+    print
+    tries += 1
+
+  if code:

[iannucci, 2014/02/08 01:31:31]

then you don't need this check here

[Ryan Tseng, 2014/02/08 01:52:21]

Done.

+    raise SubprocessFailed('%s failed with code %d in %s' %
+                           (' '.join(args), code, os.getcwd()))

[iannucci, 2014/02/08 01:31:31]

... after XX tries

?

[Ryan Tseng, 2014/02/08 01:52:21]

Done.

+  return code

[iannucci, 2014/02/08 01:31:31]

don't need this here either

[Ryan Tseng, 2014/02/08 01:52:21]

Done.

+
+
+def get_gclient_spec(solutions):
+  return GCLIENT_TEMPLATE % {
+      'solutions': pprint.pformat(solutions, indent=4),
+      'cache_dir': '"%s"' % CACHE_DIR
+  }
+
 
 def check_enabled(master, builder, slave):
   if master in ENABLED_MASTERS:
     return True
   builder_list = ENABLED_BUILDERS.get(master)
   if builder_list and builder in builder_list:
     return True
   slave_list = ENABLED_SLAVES.get(master)
   if slave_list and slave in slave_list:
     return True
   return False
 
 
 def check_disabled(master, builder, slave):
   """Returns True if disabled, False if not disabled."""
   builder_list = DISABLED_BUILDERS.get(master)
   if builder_list and builder in builder_list:
     return True
   slave_list = DISABLED_SLAVES.get(master)
   if slave_list and slave in slave_list:
     return True
   return False
 
 
 def check_valid_host(master, builder, slave):
-  return False
+  return (check_enabled(master, builder, slave)
+          and not check_disabled(master, builder, slave))
 
 
 def solutions_printer(solutions):
   """Prints gclient solution to stdout."""
   print 'Gclient Solutions'
   print '================='
   for solution in solutions:
     name = solution.get('name')
     url = solution.get('url')
     print '%s (%s)' % (name, url)
+    if solution.get('deps_file'):
+      print '  Dependencies file is %s' % solution['deps_file']
+    if 'managed' in solution:
+      print '  Managed mode is %s' % ('ON' if solution['managed'] else 'OFF')
     custom_vars = solution.get('custom_vars')
     if custom_vars:
       print '  Custom Variables:'
       for var_name, var_value in sorted(custom_vars.iteritems()):
         print '    %s = %s' % (var_name, var_value)
     custom_deps = solution.get('custom_deps')
     if 'custom_deps' in solution:
       print '  Custom Dependencies:'
       for deps_name, deps_value in sorted(custom_deps.iteritems()):
         if deps_value:
           print '    %s -> %s' % (deps_name, deps_value)
         else:
           print '    %s: Ignore' % deps_name
-    if solution.get('deps_file'):
-      print '  Dependencies file is %s' % solution['deps_file']
-    if 'managed' in solution:
-      print '  Managed mode is %s' % ('ON' if solution['managed'] else 'OFF')
     print
 
 
 def solutions_to_git(input_solutions):
   """Modifies urls in solutions to point at Git repos."""
   solutions = copy.deepcopy(input_solutions)
   for solution in solutions:
     original_url = solution['url']
     parsed_url = urlparse.urlparse(original_url)
-    path = parsed_url.path
-    if path in RECOGNIZED_PATHS:
-      solution['url'] = RECOGNIZED_PATHS[path]
+    parsed_path = parsed_url.path
+    if parsed_path in RECOGNIZED_PATHS:
+      solution['url'] = RECOGNIZED_PATHS[parsed_path]
     else:
-      print 'Warning: path %s not recognized' % path
+      print 'Warning: path %s not recognized' % parsed_path
     if solution.get('deps_file', 'DEPS') == 'DEPS':
       solution['deps_file'] = '.DEPS.git'
     solution['managed'] = False
   return solutions
 
 
-def ensure_no_git_checkout():
+def ensure_no_checkout(dir_names, scm_dirname):
   """Ensure that there is no git checkout under build/.
 
-  If there is a git checkout under build/, then move build/ to build.dead/
+  If there is an incorrect checkout under build/, then
+  move build/ to build.dead/
+  This function will check each directory in dir_names.
+
+  scm_dirname is expected to be either ['.svn', '.git']
   """
-  pass
+  assert scm_dirname in ['.svn', '.git']
+  has_checkout = any(map(lambda dir_name: path.exists(
+      path.join(os.getcwd(), dir_name, scm_dirname)), dir_names))
 
+  if has_checkout:
+    # cd .. && rm -rf ./build && mkdir ./build && cd build
+    build_dir = os.getcwd()
 
-def ensure_no_svn_checkout():
-  """Ensure that there is no svn checkout under build/.
+    os.chdir(path.dirname(os.getcwd()))
+    print '%s detected in checking, deleting %s...' % (scm_dirname, build_dir),

[iannucci, 2014/02/08 01:31:31]

checking? checkout?

[Ryan Tseng, 2014/02/08 01:52:21]

Done.

+    shutil.rmtree(build_dir)
+    print 'done'
+    os.mkdir(build_dir)
+    os.chdir(build_dir)
 
-  If there is a svn checkout under build/, then move build/ to build.dead/
-  """
-  pass
 
 
 def gclient_configure(solutions):
-  pass
+  """Should do the same thing as gclient --spec='...'."""
+  with codecs.open('.gclient', mode='w', encoding='utf-8') as f:
+    f.write(get_gclient_spec(solutions))
 
 
-def gclient_shallow_sync():
-  pass
+def gclient_sync():
+  call('gclient', 'sync', '--verbose', '--reset', '--force',
+       '--nohooks', '--noprehooks')
 
 
-def git_pull_and_clean():
-  pass
+def get_git_hash(revision, dir_name):
+  match = "^git-svn-id: [^ ]*@%d" % revision
+  cmd = ['git', 'log', '--grep', match, '--format=%H', dir_name]
+  return subprocess.check_output(cmd).strip() or None
+
+
+def deps2git(sln_dirs):
+  for sln_dir in sln_dirs:
+    deps_file = path.join(os.getcwd(), sln_dir, 'DEPS')
+    deps_git_file = path.join(os.getcwd(), sln_dir, '.DEPS.git')
+    if not path.isfile(deps_file):
+      return
+    # Do we have a better way of doing this....?
+    repo_type = 'internal' if 'internal' in sln_dir else 'public'
+    # TODO(hinoka): This will be what populates the git caches on the first
+    #               run for all of the bots.  Since deps2git is single threaded,
+    #               all of this will run in a single threaded context and be
+    #               super slow.  Should make deps2git multithreaded.
+    call(sys.executable, DEPS2GIT_PATH, '-t', repo_type,
+         '--cache_dir=%s' % CACHE_DIR,
+         '--deps=%s' % deps_file, '--out=%s' % deps_git_file)
+
+
+def git_checkout(solutions, revision):
+  build_dir = os.getcwd()
+  # Revision only applies to the first solution.
+  first_solution = True
+  for sln in solutions:
+    name = sln['name']
+    url = sln['url']
+    sln_dir = path.join(build_dir, name)
+    if not path.isdir(sln_dir):
+      call('git', 'clone', url, sln_dir)
+
+    # Clean out .DEPS.git changes first.
+    call('git', '-C', sln_dir, 'reset', '--hard')
+    call('git', '-C', sln_dir, 'clean', '-df')
+    call('git', '-C', sln_dir, 'pull', 'origin', 'master')
+    # TODO(hinoka): We probably have to make use of revision mapping.
+    if first_solution and revision and revision.lower() != 'head':
+      if revision and revision.isdigit() and len(revision) < 40:
+        # rev_num is really a svn revision number, convert it into a git hash.
+        git_ref = get_git_hash(revision, name)
+      else:
+        # rev_num is actually a git hash or ref, we can just use it.
+        git_ref = revision
+      call('git', '-C', sln_dir, 'checkout', git_ref)
+    else:
+      call('git', '-C', sln_dir, 'checkout', 'origin/master')
+
+    first_solution = False
 
 
 def apply_issue(issue, patchset, root, server):
   pass
 
 
-def deps2git():
-  pass
+def delete_flag(flag_file):
+  """Remove bot update flag."""
+  if os.path.exists(flag_file):
+    os.remove(flag_file)
 
 
-def gclient_sync():
-  pass
-
-
-def deposit_bot_update_flag():
+def emit_flag(flag_file):
   """Deposit a bot update flag on the system to tell gclient not to run."""
-  pass
+  print 'Emitting flag file at %s' % flag_file
+  with open(flag_file, 'wb') as f:
+    f.write('Success!')
 
 
 def parse_args():
   parse = optparse.OptionParser()
 
   parse.add_option('-i', '--issue', help='Issue number to patch from.')
   parse.add_option('-p', '--patchset',
                    help='Patchset from issue to patch from, if applicable.')
   parse.add_option('-r', '--root', help='Repository root.')
   parse.add_option('-c', '--server', help='Rietveld server.')

[iannucci, 2014/02/08 01:31:31]

--server isn't really descriptive. Should be --rietveld_server or something like
that.

[Ryan Tseng, 2014/02/08 01:52:21]

Done.

   parse.add_option('-s', '--specs', help='Gcilent spec.')
   parse.add_option('-m', '--master', help='Master name.')
   parse.add_option('-f', '--force', action='store_true',
                    help='Bypass check to see if we want to be run. '
                         'Should ONLY be used locally.')
+  # TODO(hinoka): We don't actually use this yet, we should factor this in.
   parse.add_option('-e', '--revision-mapping')
+  parse.add_option('-v', '--revision')
+  parse.add_option('-b', '--build_dir', default=os.getcwd())
+  parse.add_option('-g', '--flag_file', default=path.join(os.getcwd(),

[iannucci, 2014/02/08 01:31:31]

These short options aren't super-meaningful... wdyt about just removing them all
so that nothing starts depending on them?

[Ryan Tseng, 2014/02/08 01:52:21]

But I like short option names :(

I guess thats okay if bot_update.py should only ever be used by bots.

+                                                          'update.flag'))
 
   return parse.parse_args()
 
 
 def main():
   # Get inputs.
   options, _ = parse_args()
   builder = os.environ.get('BUILDBOT_BUILDERNAME', None)
-  slave = os.environ.get('BUILDBOT_SLAVENAME', None)
+  slave = os.environ.get('BUILDBOT_SLAVENAME', socket.getfqdn().split('.')[0])

[iannucci, 2014/02/08 01:31:31]

urk.... We need CLI options for these so we can use this in recipes without
setting the env vars.

why don't we just pass these as normal options anyway? they should just be
%(mastername:-)s and %(buildername:-)s and %(slavename:-)s right?

[Ryan Tseng, 2014/02/08 01:52:21]

We get them as free always in the environ as part of annotated command, I don't
see the need for the duplication?

[iannucci, 2014/02/08 03:37:21]

1) They assume buildbot
2) Passing data via env vars when it's avoidable is a great way to shoot
yourself in various sensitive parts of your anatomy.

   master = options.master
+  if not options.revision:
+    options.revision = os.environ.get('BUILDBOT_REVISION')
 
   # Check if this script should activate or not.
   active = check_valid_host(master, builder, slave) or options.force
 
   # Print helpful messages to tell devs whats going on.
   print BOT_UPDATE_MESSAGE % {
     'master': master,
     'builder': builder,
     'slave': slave,
   },
   # Print to stderr so that it shows up red on win/mac.
   print ACTIVATED_MESSAGE if active else NOT_ACTIVATED_MESSAGE
 
   # Parse, munipulate, and print the gclient solutions.
   specs = {}
-  exec(options.specs, specs)  # TODO(hinoka): LOL this is terrible.
-  solutions = specs.get('solutions', [])
-  git_solutions = solutions_to_git(solutions)
+  exec(options.specs, specs)
+  svn_solutions = specs.get('solutions', [])
+  git_solutions = solutions_to_git(svn_solutions)
   solutions_printer(git_solutions)
 
-  # Do the checkout.
-  # TODO(hinoka): Uncomment these once they're implemented.
-  # ensure_no_svn_checkout()
-  # gclient_configure(git_solutions)
-  # gclient_shallow_sync()
-  # git_pull_and_clean()
+  # Cleanup svn checkout if active, otherwise remove git checkout and exit.
+  dir_names = [sln.get('name') for sln in svn_solutions if 'name' in sln]
+  if active:
+    ensure_no_checkout(dir_names, '.svn')
+    emit_flag(options.flag_file)
+  else:
+    ensure_no_checkout(dir_names, '.git')
+    delete_flag(options.flag_file)
+    return
+
+  # Get a checkout of each solution, without DEPS or hooks.
+  # Calling git directory because there is no way to run Gclient without
+  # invoking DEPS.
+  print 'Fetching Git checkout'
+  git_checkout(git_solutions, options.revision)
+
+  # TODO(hinoka): This must be implemented before we can turn this on for TS.
   # if options.issue:
   #   apply_issue(options.issue, options.patchset, options.root, options.server)
-  # deps2git()
-  # gclient_sync()
+
+  # Magic to get deps2git to work with internal DEPS.
+  shutil.copyfile(S2G_INTERNAL_FROM_PATH, S2G_INTERNAL_DEST_PATH)
+  deps2git(dir_names)
+
+  gclient_configure(git_solutions)
+  gclient_sync()
 
 
 if __name__ == '__main__':
   sys.exit(main())