Add bot_update_lite into recipes

recipe_modules/bot_update_lite/resources/bot_update.py

Index: recipe_modules/bot_update_lite/resources/bot_update.py
diff --git a/recipe_modules/bot_update/resources/bot_update.py b/recipe_modules/bot_update_lite/resources/bot_update.py
similarity index 80%
copy from recipe_modules/bot_update/resources/bot_update.py
copy to recipe_modules/bot_update_lite/resources/bot_update.py
index 906d9099f7be77c9a4b3acff3fe5a1f48ab2a53b..19b7015a2f03d480d016b6539fcc3c2b5cb7ffc8 100755
--- a/recipe_modules/bot_update/resources/bot_update.py
+++ b/recipe_modules/bot_update_lite/resources/bot_update.py
@@ -1,10 +1,8 @@
 #!/usr/bin/env python
-# Copyright 2014 The Chromium Authors. All rights reserved.
+# Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-# TODO(hinoka): Use logging.
-
 import cStringIO
 import codecs
 import collections
@@ -73,16 +71,6 @@ ROOT_DIR = path.dirname(BUILD_DIR)
 
 DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..'))
 
-BUILD_INTERNAL_DIR = check_dir(
-    'build_internal', [
-        path.join(ROOT_DIR, 'build_internal'),
-        path.join(ROOT_DIR,      # .recipe_deps
-                  path.pardir,   # slave
-                  path.pardir,   # scripts
-                  path.pardir),  # build_internal
-    ])
-
-
 CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com'
 CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git'
 
@@ -168,39 +156,9 @@ GOT_REVISION_MAPPINGS = {
 
 
 BOT_UPDATE_MESSAGE = """
-What is the "Bot Update" step?
-==============================
-
-This step ensures that the source checkout on the bot (e.g. Chromium's src/ and
-its dependencies) is checked out in a consistent state. This means that all of
-the necessary repositories are checked out, no extra repositories are checked
-out, and no locally modified files are present.
-
-These actions used to be taken care of by the "gclient revert" and "update"
-steps. However, those steps are known to be buggy and occasionally flaky. This
-step has two main advantages over them:
-  * it only operates in Git, so the logic can be clearer and cleaner; and
-  * it is a slave-side script, so its behavior can be modified without
-    restarting the master.
-
-Why Git, you ask? Because that is the direction that the Chromium project is
-heading. This step is an integral part of the transition from using the SVN repo
-at chrome/trunk/src to using the Git repo src.git. Please pardon the dust while
-we fully convert everything to Git. This message will get out of your way
-eventually, and the waterfall will be a happier place because of it.
-
-This step can be activated or deactivated independently on every builder on
-every master. When it is active, the "gclient revert" and "update" steps become
-no-ops. When it is inactive, it prints this message, cleans up after itself, and
-lets everything else continue as though nothing has changed. Eventually, when
-everything is stable enough, this step will replace them entirely.
+Bot Update Debugging Information
 
 Debugging information:
-(master/builder/slave may be unspecified on recipes)
-master: %(master)s
-builder: %(builder)s
-slave: %(slave)s
-forced by recipes: %(recipe)s
 CURRENT_DIR: %(CURRENT_DIR)s
 BUILDER_DIR: %(BUILDER_DIR)s
 SLAVE_DIR: %(SLAVE_DIR)s
@@ -208,19 +166,7 @@ THIS_DIR: %(THIS_DIR)s
 SCRIPTS_DIR: %(SCRIPTS_DIR)s
 BUILD_DIR: %(BUILD_DIR)s
 ROOT_DIR: %(ROOT_DIR)s
-DEPOT_TOOLS_DIR: %(DEPOT_TOOLS_DIR)s
-bot_update.py is:"""
-
-ACTIVATED_MESSAGE = """ACTIVE.
-The bot will perform a Git checkout in this step.
-The "gclient revert" and "update" steps are no-ops.
-
-"""
-
-NOT_ACTIVATED_MESSAGE = """INACTIVE.
-This step does nothing. You actually want to look at the "update" step.
-
-"""
+DEPOT_TOOLS_DIR: %(DEPOT_TOOLS_DIR)s"""
 
 
 GCLIENT_TEMPLATE = """solutions = %(solutions)s
@@ -231,139 +177,15 @@ cache_dir = r%(cache_dir)s
 """
 
 
-internal_data = {}
-if BUILD_INTERNAL_DIR:
-  local_vars = {}
-  try:
-    execfile(os.path.join(
-        BUILD_INTERNAL_DIR, 'scripts', 'slave', 'bot_update_cfg.py'),
-        local_vars)
-  except Exception:
-    # Same as if BUILD_INTERNAL_DIR didn't exist in the first place.
-    print 'Warning: unable to read internal configuration file.'
-    print 'If this is an internal bot, this step may be erroneously inactive.'
-  internal_data = local_vars
-
-RECOGNIZED_PATHS = {
-    # If SVN path matches key, the entire URL is rewritten to the Git url.
-    '/chrome/trunk/src':
-        CHROMIUM_SRC_URL,
-    '/chrome/trunk/src/tools/cros.DEPS':
-        CHROMIUM_GIT_HOST + '/chromium/src/tools/cros.DEPS.git',
-}
-RECOGNIZED_PATHS.update(internal_data.get('RECOGNIZED_PATHS', {}))
-
-ENABLED_MASTERS = [
-    'bot_update.always_on',
-    'chromium.android',
-    'chromium.angle',
-    'chromium.chrome',
-    'chromium.chromedriver',
-    'chromium.chromiumos',
-    'chromium',
-    'chromium.fyi',
-    'chromium.goma',
-    'chromium.gpu',
-    'chromium.gpu.fyi',
-    'chromium.infra',
-    'chromium.infra.cron',
-    'chromium.linux',
-    'chromium.lkgr',
-    'chromium.mac',
-    'chromium.memory',
-    'chromium.memory.fyi',
-    'chromium.perf',
-    'chromium.perf.fyi',
-    'chromium.swarm',
-    'chromium.webkit',
-    'chromium.webrtc',
-    'chromium.webrtc.fyi',
-    'chromium.win',
-    'client.catapult',
-    'client.drmemory',
-    'client.mojo',
-    'client.nacl',
-    'client.nacl.ports',
-    'client.nacl.sdk',
-    'client.nacl.toolchain',
-    'client.pdfium',
-    'client.skia',
-    'client.skia.fyi',
-    'client.v8',
-    'client.v8.branches',
-    'client.v8.fyi',
-    'client.v8.ports',
-    'client.webrtc',
-    'client.webrtc.fyi',
-    'tryserver.blink',
-    'tryserver.client.catapult',
-    'tryserver.client.mojo',
-    'tryserver.chromium.android',
-    'tryserver.chromium.angle',
-    'tryserver.chromium.linux',
-    'tryserver.chromium.mac',
-    'tryserver.chromium.perf',
-    'tryserver.chromium.win',
-    'tryserver.infra',
-    'tryserver.nacl',
-    'tryserver.v8',
-    'tryserver.webrtc',
-]
-ENABLED_MASTERS += internal_data.get('ENABLED_MASTERS', [])
-
-ENABLED_BUILDERS = {
-    'client.dart.fyi': [
-        'v8-linux-release',
-        'v8-mac-release',
-        'v8-win-release',
-    ],
-    'client.dynamorio': [
-        'linux-v8-dr',
-    ],
-}
-ENABLED_BUILDERS.update(internal_data.get('ENABLED_BUILDERS', {}))
-
-ENABLED_SLAVES = {}
-ENABLED_SLAVES.update(internal_data.get('ENABLED_SLAVES', {}))
-
-# Disabled filters get run AFTER enabled filters, so for example if a builder
-# config is enabled, but a bot on that builder is disabled, that bot will
-# be disabled.
-DISABLED_BUILDERS = {}
-DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {}))
-
-DISABLED_SLAVES = {}
-DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {}))
-
-# These masters work only in Git, meaning for got_revision, always output
-# a git hash rather than a SVN rev.
-GIT_MASTERS = [
-    'client.v8',
-    'client.v8.branches',
-    'client.v8.ports',
-    'tryserver.v8',
-]
-GIT_MASTERS += internal_data.get('GIT_MASTERS', [])
-
-
 # How many times to try before giving up.
 ATTEMPTS = 5
 
-# Find deps2git
-DEPS2GIT_DIR_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git')
-DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py')
-S2G_INTERNAL_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git_internal',
-                              'svn_to_git_internal.py')
 GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py')
 
 # Find the patch tool.
 if sys.platform.startswith('win'):
-  if not BUILD_INTERNAL_DIR:
-    print 'Warning: could not find patch tool because there is no '
-    print 'build_internal present.'
-    PATCH_TOOL = None
-  else:
-    PATCH_TOOL = path.join(BUILD_INTERNAL_DIR, 'tools', 'patch.EXE')
+  # TODO(hinoka): Check this in, is required for perf builders.
+  PATCH_TOOL = path.join(THIS_DIR, 'patch.EXE')
 else:
   PATCH_TOOL = '/usr/bin/patch'
 
@@ -395,11 +217,6 @@ class InvalidDiff(Exception):
   pass
 
 
-class Inactive(Exception):
-  """Not really an exception, just used to exit early cleanly."""
-  pass
-
-
 RETRY = object()
 OK = object()
 FAIL = object()
@@ -528,34 +345,6 @@ def get_gclient_spec(solutions, target_os, target_os_only, git_cache_dir):
   }
 
 
-def check_enabled(master, builder, slave):
-  if master in ENABLED_MASTERS:
-    return True
-  builder_list = ENABLED_BUILDERS.get(master)
-  if builder_list and builder in builder_list:
-    return True
-  slave_list = ENABLED_SLAVES.get(master)
-  if slave_list and slave in slave_list:
-    return True
-  return False
-
-
-def check_disabled(master, builder, slave):
-  """Returns True if disabled, False if not disabled."""
-  builder_list = DISABLED_BUILDERS.get(master)
-  if builder_list and builder in builder_list:
-    return True
-  slave_list = DISABLED_SLAVES.get(master)
-  if slave_list and slave in slave_list:
-    return True
-  return False
-
-
-def check_valid_host(master, builder, slave):
-  return (check_enabled(master, builder, slave)
-          and not check_disabled(master, builder, slave))
-
-
 def maybe_ignore_revision(revision, buildspec):
   """Handle builders that don't care what buildbot tells them to build.
 
@@ -809,59 +598,6 @@ def get_git_hash(revision, branch, sln_dir):
                             (revision, sln_dir))
 
 
-def _last_commit_for_file(filename, repo_base):
-  cmd = ['log', '--format=%H', '--max-count=1', '--', filename]
-  return git(*cmd, cwd=repo_base).strip()
-
-
-def need_to_run_deps2git(repo_base, deps_file, deps_git_file):
-  """Checks to see if we need to run deps2git.
-
-  Returns True if there was a DEPS change after the last .DEPS.git update
-  or if DEPS has local modifications.
-  """
-  # See if DEPS is dirty
-  deps_file_status = git(
-      'status', '--porcelain', deps_file, cwd=repo_base).strip()
-  if deps_file_status and deps_file_status.startswith('M '):
-    return True
-
-  last_known_deps_ref = _last_commit_for_file(deps_file, repo_base)
-  last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base)
-  merge_base_ref = git('merge-base', last_known_deps_ref,
-                       last_known_deps_git_ref, cwd=repo_base).strip()
-
-  # If the merge base of the last DEPS and last .DEPS.git file is not
-  # equivilent to the hash of the last DEPS file, that means the DEPS file
-  # was committed after the last .DEPS.git file.
-  return last_known_deps_ref != merge_base_ref
-
-
-def ensure_deps2git(solution, shallow, git_cache_dir):
-  repo_base = path.join(os.getcwd(), solution['name'])
-  deps_file = path.join(repo_base, 'DEPS')
-  deps_git_file = path.join(repo_base, '.DEPS.git')
-  if (not git('ls-files', 'DEPS', cwd=repo_base).strip() or
-      not git('ls-files', '.DEPS.git', cwd=repo_base).strip()):
-    return
-
-  print 'Checking if %s is newer than %s' % (deps_file, deps_git_file)
-  if not need_to_run_deps2git(repo_base, deps_file, deps_git_file):
-    return
-
-  print '===DEPS file modified, need to run deps2git==='
-  cmd = [sys.executable, DEPS2GIT_PATH,
-         '--workspace', os.getcwd(),
-         '--cache_dir', git_cache_dir,
-         '--deps', deps_file,
-         '--out', deps_git_file]
-  if 'chrome-internal.googlesource' in solution['url']:
-    cmd.extend(['--extra-rules', S2G_INTERNAL_PATH])
-  if shallow:
-    cmd.append('--shallow')
-  call(*cmd)
-
-
 def emit_log_lines(name, lines):
   for line in lines.splitlines():
     print '@@@STEP_LOG_LINE@%s@%s@@@' % (name, line)
@@ -1007,16 +743,6 @@ def git_checkout(solutions, revisions, shallow, refs, git_cache_dir):
   return git_ref
 
 
-def _download(url):
-  """Fetch url and return content, with retries for flake."""
-  for attempt in xrange(ATTEMPTS):
-    try:
-      return urllib2.urlopen(url).read()
-    except Exception:
-      if attempt == ATTEMPTS - 1:
-        raise
-
-
 def parse_diff(diff):
   """Takes a unified diff and returns a list of diffed files and their diffs.
 
@@ -1126,10 +852,6 @@ def apply_rietveld_issue(issue, patchset, root, server, _rev_map, _revision,
     for item in blacklist:
       cmd.extend(['--blacklist', item])
 
-  # TODO(kjellander): Remove this hack when http://crbug.com/611808 is fixed.
-  if root == 'src/third_party/webrtc':
-    cmd.extend(['--extra_patchlevel=1'])
-
   # Only try once, since subsequent failures hide the real failure.
   try:
     call(*cmd, tries=1)
@@ -1139,8 +861,6 @@ def apply_rietveld_issue(issue, patchset, root, server, _rev_map, _revision,
 def apply_gerrit_ref(gerrit_repo, gerrit_ref, root, gerrit_reset):
   gerrit_repo = gerrit_repo or 'origin'
   assert gerrit_ref
-  print '===Applying gerrit ref==='
-  print 'Repo is %r, ref is %r, root is %r' % (gerrit_repo, gerrit_ref, root)
   try:
     base_rev = git('rev-parse', 'HEAD', cwd=root).strip()
     git('retry', 'fetch', gerrit_repo, gerrit_ref, cwd=root, tries=1)
@@ -1317,17 +1037,13 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
   if patch_url:
     patches = get_svn_patch(patch_url)
 
-  print '===Processing patch solutions==='
   already_patched = []
   patch_root = patch_root or ''
-  print 'Patch root is %r' % patch_root
   for solution in solutions:
-    print 'Processing solution %r' % solution['name']
     if (patch_root == solution['name'] or
         solution['name'].startswith(patch_root + '/')):
       relative_root = solution['name'][len(patch_root) + 1:]
       target = '/'.join([relative_root, 'DEPS']).lstrip('/')
-      print '  relative root is %r, target is %r' % (relative_root, target)
       if patches:
         apply_svn_patch(patch_root, patches, whitelist=[target])
         already_patched.append(target)
@@ -1337,11 +1053,6 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
                              apply_issue_key_file, whitelist=[target])
         already_patched.append(target)
 
-  if not buildspec:
-    # Run deps2git if there is a DEPS change after the last .DEPS.git commit.
-    for solution in solutions:
-      ensure_deps2git(solution, shallow, git_cache_dir)
-
   # Ensure our build/ directory is set up with the correct .gclient file.
   gclient_configure(solutions, target_os, target_os_only, git_cache_dir)
 
@@ -1445,8 +1156,6 @@ def parse_args():
                    help='--private-key-file option passthrough for '
                         'apply_patch.py.')
   parse.add_option('--patch_url', help='Optional URL to SVN patch.')
-  parse.add_option('--root', dest='patch_root',
-                   help='DEPRECATED: Use --patch_root.')
   parse.add_option('--patch_root', help='Directory to patch on top of.')
   parse.add_option('--rietveld_server',
                    default='codereview.chromium.org',
@@ -1457,10 +1166,6 @@ def parse_args():
   parse.add_option('--gerrit_no_reset', action='store_true',
                    help='Bypass calling reset after applying a gerrit ref.')
   parse.add_option('--specs', help='Gcilent spec.')
-  parse.add_option('--master', help='Master name.')
-  parse.add_option('-f', '--force', action='store_true',
-                   help='Bypass check to see if we want to be run. '
-                        'Should ONLY be used locally or by smart recipes.')
   parse.add_option('--revision_mapping',
                    help='{"path/to/repo/": "property_name"}')
   parse.add_option('--revision_mapping_file',
@@ -1476,14 +1181,7 @@ def parse_args():
                         'set to <branch>:<revision>.')
   parse.add_option('--output_manifest', action='store_true',
                    help=('Add manifest json to the json output.'))
-  parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0],
-                   help='Hostname of the current machine, '
-                   'used for determining whether or not to activate.')
-  parse.add_option('--builder_name', help='Name of the builder, '
-                   'used for determining whether or not to activate.')
   parse.add_option('--build_dir', default=os.getcwd())
-  parse.add_option('--flag_file', default=path.join(os.getcwd(),
-                                                    'update.flag'))
   parse.add_option('--shallow', action='store_true',
                    help='Use shallow clones for cache repositories.')
   parse.add_option('--gyp_env', action='append', default=[],
@@ -1542,25 +1240,14 @@ def parse_args():
   return options, args
 
 
-def prepare(options, git_slns, active):
+def prepare(options, git_slns):
   """Prepares the target folder before we checkout."""
   dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
-  # If we're active now, but the flag file doesn't exist (we weren't active
-  # last run) or vice versa, blow away all checkouts.
-  if bool(active) != bool(check_flag(options.flag_file)):
+  emit_json(options.output_json, did_run=True)
+  if options.clobber:
     ensure_no_checkout(dir_names, '*')
-  if options.output_json:
-    # Make sure we tell recipes that we didn't run if the script exits here.
-    emit_json(options.output_json, did_run=active)
-  if active:
-    if options.clobber:
-      ensure_no_checkout(dir_names, '*')
-    else:
-      ensure_no_checkout(dir_names, '.svn')
-    emit_flag(options.flag_file)
   else:
-    delete_flag(options.flag_file)
-    raise Inactive  # This is caught in main() and we exit cleanly.
+    ensure_no_checkout(dir_names, '.svn')
 
   # Do a shallow checkout if the disk is less than 100GB.
   total_disk_space, free_disk_space = get_total_disk_space()
@@ -1571,8 +1258,6 @@ def prepare(options, git_slns, active):
   step_text = '[%dGB/%dGB used (%d%%)]' % (used_disk_space_gb,
                                            total_disk_space_gb,
                                            percent_used)
-  if not options.output_json:
-    print '@@@STEP_TEXT@%s@@@' % step_text
   if not options.shallow:
     options.shallow = (total_disk_space < SHALLOW_CLONE_THRESHOLD
                        and not options.no_shallow)
@@ -1587,7 +1272,7 @@ def prepare(options, git_slns, active):
   return revisions, step_text
 
 
-def checkout(options, git_slns, specs, buildspec, master,
+def checkout(options, git_slns, specs, buildspec,
              svn_root, revisions, step_text):
   first_sln = git_slns[0]['name']
   dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
@@ -1650,9 +1335,6 @@ def checkout(options, git_slns, specs, buildspec, master,
       print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text
     raise
 
-  # Revision is an svn revision, unless it's a git master.
-  use_svn_rev = master not in GIT_MASTERS
-
   # Take care of got_revisions outputs.
   revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {}))
   if options.revision_mapping:
@@ -1664,8 +1346,7 @@ def checkout(options, git_slns, specs, buildspec, master,
   if not revision_mapping:
     revision_mapping[first_sln] = 'got_revision'
 
-  got_revisions = parse_got_revision(gclient_output, revision_mapping,
-                                     use_svn_rev)
+  got_revisions = parse_got_revision(gclient_output, revision_mapping)
 
   if not got_revisions:
     # TODO(hinoka): We should probably bail out here, but in the interest
@@ -1690,22 +1371,9 @@ def checkout(options, git_slns, specs, buildspec, master,
     emit_properties(got_revisions)
 
 
-def print_help_text(force, output_json, active, master, builder, slave):
+def print_help_text():
   """Print helpful messages to tell devs whats going on."""
-  if force and output_json:
-    recipe_force = 'Forced on by recipes'
-  elif active and output_json:
-    recipe_force = 'Off by recipes, but forced on by bot update'
-  elif not active and output_json:
-    recipe_force = 'Forced off by recipes'
-  else:
-    recipe_force = 'N/A. Was not called by recipes'
-
   print BOT_UPDATE_MESSAGE % {
-    'master': master or 'Not specified',
-    'builder': builder or 'Not specified',
-    'slave': slave or 'Not specified',
-    'recipe': recipe_force,
     'CURRENT_DIR': CURRENT_DIR,
     'BUILDER_DIR': BUILDER_DIR,
     'SLAVE_DIR': SLAVE_DIR,
@@ -1714,23 +1382,15 @@ def print_help_text(force, output_json, active, master, builder, slave):
     'BUILD_DIR': BUILD_DIR,
     'ROOT_DIR': ROOT_DIR,
     'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR,
-  },
-  print ACTIVATED_MESSAGE if active else NOT_ACTIVATED_MESSAGE
+  }
 
 
 def main():
   # Get inputs.
   options, _ = parse_args()
-  builder = options.builder_name
-  slave = options.slave_name
-  master = options.master
-
-  # Check if this script should activate or not.
-  active = check_valid_host(master, builder, slave) or options.force or False
 
   # Print a helpful message to tell developers whats going on with this step.
-  print_help_text(
-      options.force, options.output_json, active, master, builder, slave)
+  print_help_text()
 
   # Parse, munipulate, and print the gclient solutions.
   specs = {}
@@ -1743,13 +1403,10 @@ def main():
 
   try:
     # Dun dun dun, the main part of bot_update.
-    revisions, step_text = prepare(options, git_slns, active)
-    checkout(options, git_slns, specs, buildspec, master, svn_root, revisions,
+    revisions, step_text = prepare(options, git_slns)
+    checkout(options, git_slns, specs, buildspec, svn_root, revisions,
              step_text)
 
-  except Inactive:
-    # Not active, should count as passing.
-    pass
   except PatchFailed as e:
     emit_flag(options.flag_file)
     # Return a specific non-zero exit code for patch failure (because it is