Remove bogus master/builder/slave/force parameters from bot_update.

recipe_modules/bot_update/resources/bot_update.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # TODO(hinoka): Use logging.
 
 import cStringIO
 import codecs
 import collections
@@ skipping 149 matching lines @@
         'src/native_client/': 'got_nacl_revision',
         'src/tools/swarm_client/': 'got_swarm_client_revision',
         'src/tools/swarming_client/': 'got_swarming_client_revision',
         'src/third_party/WebKit/': 'got_webkit_revision',
         'src/third_party/webrtc/': 'got_webrtc_revision',
         'src/v8/': 'got_v8_revision',
     }
 }
 
 
-BOT_UPDATE_MESSAGE = """
-What is the "Bot Update" step?
-==============================
-
-This step ensures that the source checkout on the bot (e.g. Chromium's src/ and
-its dependencies) is checked out in a consistent state. This means that all of
-the necessary repositories are checked out, no extra repositories are checked
-out, and no locally modified files are present.
-
-These actions used to be taken care of by the "gclient revert" and "update"
-steps. However, those steps are known to be buggy and occasionally flaky. This
-step has two main advantages over them:
-  * it only operates in Git, so the logic can be clearer and cleaner; and
-  * it is a slave-side script, so its behavior can be modified without
-    restarting the master.
-
-Why Git, you ask? Because that is the direction that the Chromium project is
-heading. This step is an integral part of the transition from using the SVN repo
-at chrome/trunk/src to using the Git repo src.git. Please pardon the dust while
-we fully convert everything to Git. This message will get out of your way
-eventually, and the waterfall will be a happier place because of it.
-
-This step can be activated or deactivated independently on every builder on
-every master. When it is active, the "gclient revert" and "update" steps become
-no-ops. When it is inactive, it prints this message, cleans up after itself, and
-lets everything else continue as though nothing has changed. Eventually, when
-everything is stable enough, this step will replace them entirely.
-
-Debugging information:
-(master/builder/slave may be unspecified on recipes)
-master: %(master)s
-builder: %(builder)s
-slave: %(slave)s
-forced by recipes: %(recipe)s
-CURRENT_DIR: %(CURRENT_DIR)s
-BUILDER_DIR: %(BUILDER_DIR)s
-SLAVE_DIR: %(SLAVE_DIR)s
-THIS_DIR: %(THIS_DIR)s
-SCRIPTS_DIR: %(SCRIPTS_DIR)s
-BUILD_DIR: %(BUILD_DIR)s
-ROOT_DIR: %(ROOT_DIR)s
-DEPOT_TOOLS_DIR: %(DEPOT_TOOLS_DIR)s
-bot_update.py is:"""
-
-ACTIVATED_MESSAGE = """ACTIVE.
-The bot will perform a Git checkout in this step.
-The "gclient revert" and "update" steps are no-ops.
-
-"""
-
-NOT_ACTIVATED_MESSAGE = """INACTIVE.
-This step does nothing. You actually want to look at the "update" step.
-
-"""
-
-
 GCLIENT_TEMPLATE = """solutions = %(solutions)s
 
 cache_dir = r%(cache_dir)s
 %(target_os)s
 %(target_os_only)s
 """
 
 
 internal_data = {}
 if BUILD_INTERNAL_DIR:
@@ skipping 12 matching lines @@
     # If SVN path matches key, the entire URL is rewritten to the Git url.
     '/chrome/trunk/src':
         CHROMIUM_SRC_URL,
     '/chrome/trunk/src/tools/cros.DEPS':
         CHROMIUM_GIT_HOST + '/chromium/src/tools/cros.DEPS.git',
     '/chrome-internal/trunk/src-internal':
         'https://chrome-internal.googlesource.com/chrome/src-internal.git',
 }
 RECOGNIZED_PATHS.update(internal_data.get('RECOGNIZED_PATHS', {}))
 
-ENABLED_MASTERS = [
-    'bot_update.always_on',
-    'chromium.android',
-    'chromium.angle',
-    'chromium.chrome',
-    'chromium.chromedriver',
-    'chromium.chromiumos',
-    'chromium',
-    'chromium.fyi',
-    'chromium.goma',
-    'chromium.gpu',
-    'chromium.gpu.fyi',
-    'chromium.infra',
-    'chromium.infra.cron',
-    'chromium.linux',
-    'chromium.lkgr',
-    'chromium.mac',
-    'chromium.memory',
-    'chromium.memory.fyi',
-    'chromium.perf',
-    'chromium.perf.fyi',
-    'chromium.swarm',
-    'chromium.webkit',
-    'chromium.webrtc',
-    'chromium.webrtc.fyi',
-    'chromium.win',
-    'client.catapult',
-    'client.drmemory',
-    'client.mojo',
-    'client.nacl',
-    'client.nacl.ports',
-    'client.nacl.sdk',
-    'client.nacl.toolchain',
-    'client.pdfium',
-    'client.skia',
-    'client.skia.fyi',
-    'client.v8',
-    'client.v8.branches',
-    'client.v8.fyi',
-    'client.v8.ports',
-    'client.webrtc',
-    'client.webrtc.fyi',
-    'tryserver.blink',
-    'tryserver.client.catapult',
-    'tryserver.client.mojo',
-    'tryserver.chromium.android',
-    'tryserver.chromium.angle',
-    'tryserver.chromium.linux',
-    'tryserver.chromium.mac',
-    'tryserver.chromium.perf',
-    'tryserver.chromium.win',
-    'tryserver.infra',
-    'tryserver.nacl',
-    'tryserver.v8',
-    'tryserver.webrtc',
-]
-ENABLED_MASTERS += internal_data.get('ENABLED_MASTERS', [])
-
-ENABLED_BUILDERS = {
-    'client.dart.fyi': [
-        'v8-linux-release',
-        'v8-mac-release',
-        'v8-win-release',
-    ],
-    'client.dynamorio': [
-        'linux-v8-dr',
-    ],
-}
-ENABLED_BUILDERS.update(internal_data.get('ENABLED_BUILDERS', {}))
-
-ENABLED_SLAVES = {}
-ENABLED_SLAVES.update(internal_data.get('ENABLED_SLAVES', {}))
-
-# Disabled filters get run AFTER enabled filters, so for example if a builder
-# config is enabled, but a bot on that builder is disabled, that bot will
-# be disabled.
-DISABLED_BUILDERS = {}
-DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {}))
-
-DISABLED_SLAVES = {}
-DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {}))
-
-# These masters work only in Git, meaning for got_revision, always output
-# a git hash rather than a SVN rev.
-GIT_MASTERS = [
-    'client.v8',
-    'client.v8.branches',
-    'client.v8.ports',
-    'tryserver.v8',
-]
-GIT_MASTERS += internal_data.get('GIT_MASTERS', [])
-
 
 # How many times to try before giving up.
 ATTEMPTS = 5
 
 GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py')
 
 # Find the patch tool.
 if sys.platform.startswith('win'):
   if not BUILD_INTERNAL_DIR:
     print 'Warning: could not find patch tool because there is no '
@@ skipping 158 matching lines @@
 
 def get_gclient_spec(solutions, target_os, target_os_only, git_cache_dir):
   return GCLIENT_TEMPLATE % {
       'solutions': pprint.pformat(solutions, indent=4),
       'cache_dir': '"%s"' % git_cache_dir,
       'target_os': ('\ntarget_os=%s' % target_os) if target_os else '',
       'target_os_only': '\ntarget_os_only=%s' % target_os_only
   }
 
 
-def check_enabled(master, builder, slave):
-  if master in ENABLED_MASTERS:
-    return True
-  builder_list = ENABLED_BUILDERS.get(master)
-  if builder_list and builder in builder_list:
-    return True
-  slave_list = ENABLED_SLAVES.get(master)
-  if slave_list and slave in slave_list:
-    return True
-  return False
-
-
-def check_disabled(master, builder, slave):
-  """Returns True if disabled, False if not disabled."""
-  builder_list = DISABLED_BUILDERS.get(master)
-  if builder_list and builder in builder_list:
-    return True
-  slave_list = DISABLED_SLAVES.get(master)
-  if slave_list and slave in slave_list:
-    return True
-  return False
-
-
-def check_valid_host(master, builder, slave):
-  return (check_enabled(master, builder, slave)
-          and not check_disabled(master, builder, slave))
-
-
 def maybe_ignore_revision(revision, buildspec):
   """Handle builders that don't care what buildbot tells them to build.
 
   This is especially the case with branch builders that build from buildspecs
   and/or trigger off multiple repositories, where the --revision passed in has
   nothing to do with the solution being built. Clearing the revision in this
   case causes bot_update to use HEAD rather that trying to checkout an
   inappropriate version of the solution.
   """
   if buildspec and buildspec.container == 'branches':
@@ skipping 815 matching lines @@
                    default='codereview.chromium.org',
                    help='Rietveld server.')
   parse.add_option('--gerrit_repo',
                    help='Gerrit repository to pull the ref from.')
   parse.add_option('--gerrit_ref', help='Gerrit ref to apply.')
   parse.add_option('--gerrit_no_rebase_patch_ref', action='store_true',
                    help='Bypass rebase of Gerrit patch ref after checkout.')
   parse.add_option('--gerrit_no_reset', action='store_true',
                    help='Bypass calling reset after applying a gerrit ref.')
   parse.add_option('--specs', help='Gcilent spec.')
-  parse.add_option('--master',
-                   help='Master name. If specified and it is not in '
-                        'bot_update\'s whitelist, bot_update will be noop.')
   parse.add_option('-f', '--force', action='store_true',
                    help='Bypass check to see if we want to be run. '
                         'Should ONLY be used locally or by smart recipes.')
   parse.add_option('--revision_mapping',
                    help='{"path/to/repo/": "property_name"}')
   parse.add_option('--revision_mapping_file',
                    help=('Same as revision_mapping, except its a path to a json'
                          ' file containing that format.'))
   parse.add_option('--revision', action='append', default=[],
                    help='Revision to check out. Can be an SVN revision number, '
                         'git hash, or any form of git ref.  Can prepend '
                         'root@<rev> to specify which repository, where root '
                         'is either a filesystem path, git https url, or '
                         'svn url. To specify Tip of Tree, set rev to HEAD.'
                         'To specify a git branch and an SVN rev, <rev> can be '
                         'set to <branch>:<revision>.')
   parse.add_option('--output_manifest', action='store_true',
                    help=('Add manifest json to the json output.'))
   parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0],
                    help='Hostname of the current machine, '
                    'used for determining whether or not to activate.')
-  parse.add_option('--builder_name', help='Name of the builder, '
-                   'used for determining whether or not to activate.')
   parse.add_option('--build_dir', default=os.getcwd())
   parse.add_option('--flag_file', default=path.join(os.getcwd(),
                                                     'update.flag'))
   parse.add_option('--shallow', action='store_true',
                    help='Use shallow clones for cache repositories.')
   parse.add_option('--gyp_env', action='append', default=[],
                    help='Environment variables to pass into gclient runhooks.')
   parse.add_option('--clobber', action='store_true',
                    help='Delete checkout first, always')
   parse.add_option('--bot_update_clobber', action='store_true', dest='clobber',
@@ skipping 51 matching lines @@
 def prepare(options, git_slns, active):
   """Prepares the target folder before we checkout."""
   dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
   # If we're active now, but the flag file doesn't exist (we weren't active
   # last run) or vice versa, blow away all checkouts.
   if bool(active) != bool(check_flag(options.flag_file)):
     ensure_no_checkout(dir_names, '*')
   if options.output_json:
     # Make sure we tell recipes that we didn't run if the script exits here.
     emit_json(options.output_json, did_run=active)
   if active:

[Ryan Tseng, 2016/09/08 18:34:55]

always active

     if options.clobber:
       ensure_no_checkout(dir_names, '*')
     else:
       ensure_no_checkout(dir_names, '.svn')
     emit_flag(options.flag_file)
   else:
     delete_flag(options.flag_file)
     raise Inactive  # This is caught in main() and we exit cleanly.
 
   # Do a shallow checkout if the disk is less than 100GB.
@@ skipping 14 matching lines @@
   # The first solution is where the primary DEPS file resides.
   first_sln = dir_names[0]
 
   # Split all the revision specifications into a nice dict.
   print 'Revisions: %s' % options.revision
   revisions = parse_revisions(options.revision, first_sln)
   print 'Fetching Git checkout at %s@%s' % (first_sln, revisions[first_sln])
   return revisions, step_text
 
 
-def checkout(options, git_slns, specs, buildspec, master,
+def checkout(options, git_slns, specs, buildspec,
              svn_root, revisions, step_text):
   first_sln = git_slns[0]['name']
   dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
   try:
     # Outer try is for catching patch failures and exiting gracefully.
     # Inner try is for catching gclient failures and retrying gracefully.
     try:
       checkout_parameters = dict(
           # First, pass in the base of what we want to check out.
           solutions=git_slns,
@@ skipping 42 matching lines @@
                 patch_root=options.patch_root,
                 patch_failure=True,
                 step_text='%s PATCH FAILED' % step_text,
                 fixed_revisions=revisions)
     else:
       # If we're not on recipes, tell annotator about our got_revisions.
       emit_log_lines('patch error', e.output)
       print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text
     raise
 
-  # Revision is an svn revision, unless it's a git master.
-  use_svn_rev = master not in GIT_MASTERS
+  use_svn_rev = False
 
   # Take care of got_revisions outputs.
   revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {}))
   if options.revision_mapping:
     revision_mapping.update(options.revision_mapping)
 
   # If the repo is not in the default GOT_REVISION_MAPPINGS and no
   # revision_mapping were specified on the command line then
   # default to setting 'got_revision' based on the first solution.
   if not revision_mapping:
@@ skipping 18 matching lines @@
               patch_root=options.patch_root,
               step_text=step_text,
               fixed_revisions=revisions,
               properties=got_revisions,
               manifest=manifest)
   else:
     # If we're not on recipes, tell annotator about our got_revisions.
     emit_properties(got_revisions)
 
 
-def print_help_text(force, output_json, active, master, builder, slave):
-  """Print helpful messages to tell devs whats going on."""
-  if force and output_json:
-    recipe_force = 'Forced on by recipes'
-  elif active and output_json:
-    recipe_force = 'Off by recipes, but forced on by bot update'
-  elif not active and output_json:
-    recipe_force = 'Forced off by recipes'
-  else:
-    recipe_force = 'N/A. Was not called by recipes'
-
-  print BOT_UPDATE_MESSAGE % {
-    'master': master or 'Not specified',
-    'builder': builder or 'Not specified',
-    'slave': slave or 'Not specified',
-    'recipe': recipe_force,
+def print_debug_info():
+  print "Debugging info:"
+  debug_params = {
     'CURRENT_DIR': CURRENT_DIR,
     'BUILDER_DIR': BUILDER_DIR,
     'SLAVE_DIR': SLAVE_DIR,
     'THIS_DIR': THIS_DIR,
     'SCRIPTS_DIR': SCRIPTS_DIR,
     'BUILD_DIR': BUILD_DIR,
     'ROOT_DIR': ROOT_DIR,
     'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR,
   },
-  print ACTIVATED_MESSAGE if active else NOT_ACTIVATED_MESSAGE
+  for k, v in sorted(debug_params.iteritems()):
+    print "%s: %r" % (k, v)
 
 
 def main():
   # Get inputs.
   options, _ = parse_args()
-  builder = options.builder_name
-  slave = options.slave_name
-  master = options.master
 
   # Always run. This option will be removed in a later CL, but for now make sure
   # that bot_update is ALWAYS set to run, no matter what.
   options.force = True
 
   # Check if this script should activate or not.
-  active = options.force or check_valid_host(master, builder, slave)
+  active = True
 
   # Print a helpful message to tell developers whats going on with this step.
-  print_help_text(
-      options.force, options.output_json, active, master, builder, slave)
+  print_debug_info()
 
   # Parse, munipulate, and print the gclient solutions.
   specs = {}
   exec(options.specs, specs)
   svn_solutions = specs.get('solutions', [])
   git_slns, svn_root, buildspec = solutions_to_git(svn_solutions)
   options.revision = maybe_ignore_revision(options.revision, buildspec)
 
   solutions_printer(git_slns)
 
   try:
     # Dun dun dun, the main part of bot_update.
     revisions, step_text = prepare(options, git_slns, active)
-    checkout(options, git_slns, specs, buildspec, master, svn_root, revisions,
+    checkout(options, git_slns, specs, buildspec, svn_root, revisions,
              step_text)
 
   except Inactive:

[Ryan Tseng, 2016/09/08 18:34:54]

not used

[iannucci, 2016/09/08 18:39:05]

Yeah, I want to clean these things up in a future CL. Trying to keep this one
small.

     # Not active, should count as passing.
     pass
   except PatchFailed as e:
     emit_flag(options.flag_file)
     # Return a specific non-zero exit code for patch failure (because it is
     # a failure), but make it different than other failures to distinguish
     # between infra failures (independent from patch author), and patch
     # failures (that patch author can fix). However, PatchFailure due to
     # download patch failure is still an infra problem.
     if e.code == 3:
       # Patch download problem.
       return 87
     # Genuine patch problem.
     return 88
   except Exception:
     # Unexpected failure.
     emit_flag(options.flag_file)
     raise
   else:
     emit_flag(options.flag_file)
 
 
 if __name__ == '__main__':
   sys.exit(main())