Add concept of bot config and test spec database (BotConfigAndTestDB).

scripts/slave/recipe_modules/chromium_tests/api.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import ast
 import collections
 import contextlib
 import copy
 import itertools
 import json
 
 from recipe_engine.types import freeze
 from recipe_engine import recipe_api
 from recipe_engine import util as recipe_util
 
+from . import bot_config_and_test_db as bdb_module
 from . import builders
 from . import steps
 from . import trybots
 
 
 MB_CONFIG_FILENAME = ['tools', 'mb', 'mb_config.pyl']
 
 
 # Paths which affect recipe config and behavior in a way that survives
 # deapplying user's patch.
@@ skipping 35 matching lines @@
     return steps
 
   @property
   def trybots(self):
     return trybots.TRYBOTS
 
   def add_builders(self, builders):
     """Adds builders to our builder map"""
     self._builders.update(builders)
 
-  def get_bot_config(self, mastername, buildername):
+  def _get_bot_config(self, mastername, buildername):
     master_dict = self.builders.get(mastername, {})
     return freeze(master_dict.get('builders', {}).get(buildername))
 
   def configure_build(self, mastername, buildername, override_bot_type=None):
     master_dict = self.builders.get(mastername, {})
     bot_config = master_dict.get('builders', {}).get(buildername)
 
     # Get the buildspec version. It can be supplied as a build property or as
     # a recipe config value.
     buildspec_version = (self.m.properties.get('buildspec_version') or
@@ skipping 52 matching lines @@
       # builds to match also the chromium revision from the builder.
       component_rev = self.m.properties.get('revision') or 'HEAD'
       if bot_type == 'tester':
         component_rev = self.m.properties.get(
             'parent_got_revision', component_rev)
       dep = bot_config.get('set_component_rev')
       self.m.gclient.c.revisions[dep['name']] = dep['rev_str'] % component_rev
 
   def ensure_checkout(self, mastername, buildername,
                       root_solution_revision=None):
-    bot_config = self.get_bot_config(mastername, buildername)
+    bot_config = self._get_bot_config(mastername, buildername)
 
     if self.m.platform.is_win:
       self.m.chromium.taskkill()
 
     # Bot Update re-uses the gclient configs.
     update_step = self.m.bot_update.ensure_checkout(
         patch_root=bot_config.get('patch_root'),
         root_solution_revision=root_solution_revision,
         clobber=bot_config.get('clobber', False))
     assert update_step.json.output['did_run']
     # HACK(dnj): Remove after 'crbug.com/398105' has landed
     self.m.chromium.set_build_properties(update_step.json.output['properties'])
 
     return update_step
 
   def set_up_swarming(self, mastername, buildername):
-    bot_config = self.get_bot_config(mastername, buildername)
+    bot_config = self._get_bot_config(mastername, buildername)
 
     enable_swarming = bot_config.get('enable_swarming')
     if enable_swarming:
       self.m.isolate.set_isolate_environment(self.m.chromium.c)
       self.m.swarming.check_client_version()
       for key, value in bot_config.get('swarming_dimensions', {}).iteritems():
         self.m.swarming.set_default_dimension(key, value)
 
   def runhooks(self, update_step):
     if self.m.tryserver.is_tryserver:
       try:
         self.m.chromium.runhooks(name='runhooks (with patch)')
       except self.m.step.StepFailure:
         # As part of deapplying patch we call runhooks without the patch.
         self.deapply_patch(update_step)
         raise
     else:
       self.m.chromium.runhooks()
 
   def get_test_spec(self, mastername, buildername):
-    bot_config = self.get_bot_config(mastername, buildername)
+    bot_config = self._get_bot_config(mastername, buildername)
 
     test_spec_file = bot_config.get('testing', {}).get(
         'test_spec_file', '%s.json' % mastername)
 
     # TODO(phajdan.jr): Bots should have no generators instead.
     if bot_config.get('disable_tests'):
       return {}
     return self.read_test_spec(self.m, test_spec_file)
 
-  def get_master_dict_with_dynamic_tests(
+  def _get_master_dict_with_dynamic_tests(
       self, mastername, buildername, test_spec, scripts_compile_targets):
     # We manually thaw the path to the elements we are modifying, since the
     # builders are frozen.
     master_dict = dict(self.builders[mastername])
     builders = master_dict['builders'] = dict(master_dict['builders'])
     bot_config = builders[buildername]
     for loop_buildername in builders:
       builder_dict = builders[loop_buildername] = (
           dict(builders[loop_buildername]))
       builders[loop_buildername]['tests'] = (
           self.generate_tests_from_test_spec(
               self.m, test_spec, builder_dict, loop_buildername, mastername,
               # TODO(phajdan.jr): Get enable_swarming value from builder_dict.
               # Above should remove the need to get bot_config and buildername
               # in this method.
               bot_config.get('enable_swarming', False),
               scripts_compile_targets, builder_dict.get('test_generators', [])
           ))
 
     return freeze(master_dict)
 
+  def create_bot_db_from_master_dict(self, mastername, master_dict):
+    bot_db = bdb_module.BotConfigAndTestDB()
+    bot_db._add_master_dict_and_test_spec(mastername, master_dict, {})
+    return bot_db
+
   def prepare_checkout(self, mastername, buildername,
                        root_solution_revision=None):
-    bot_config = self.get_bot_config(mastername, buildername)
+    bot_config = self._get_bot_config(mastername, buildername)
 
     update_step = self.ensure_checkout(mastername, buildername,
                                        root_solution_revision)
     # TODO(robertocn): Remove this hack by the end of Q1/2016.
     if (mastername == 'tryserver.chromium.perf'
         and bot_config.get('bot_type') == 'builder'
         and buildername.endswith('builder')):
       force_legacy_compile = self.should_force_legacy_compiling(
           mastername, buildername)
       if force_legacy_compile:
         self.m.chromium.c.project_generator.tool = 'gyp'
 
     self.set_up_swarming(mastername, buildername)
     self.runhooks(update_step)
 
     test_spec = self.get_test_spec(mastername, buildername)
 
     # TODO(phajdan.jr): Bots should have no generators instead.
     if bot_config.get('disable_tests'):
       scripts_compile_targets = {}
     else:
       scripts_compile_targets = \
           self.get_compile_targets_for_scripts().json.output
 
-    master_dict = self.get_master_dict_with_dynamic_tests(
+    master_dict = self._get_master_dict_with_dynamic_tests(
         mastername, buildername, test_spec, scripts_compile_targets)
 
     if self.m.chromium.c.lto and \
         not self.m.chromium.c.env.LLVM_FORCE_HEAD_REVISION:
       self.m.chromium.download_lto_plugin()
 
-    return update_step, master_dict, test_spec
+    bot_db = bdb_module.BotConfigAndTestDB()
+    bot_db._add_master_dict_and_test_spec(mastername, master_dict, test_spec)
+
+    return update_step, bot_db
 
   def generate_tests_from_test_spec(self, api, test_spec, builder_dict,
       buildername, mastername, enable_swarming, scripts_compile_targets,
       generators):
     tests = builder_dict.get('tests', ())
     # TODO(phajdan.jr): Switch everything to scripts generators and simplify.
     for generator in generators:
       tests = (
           tuple(generator(api, mastername, buildername, test_spec,
                          enable_swarming=enable_swarming,
@@ skipping 57 matching lines @@
             raise
 
       if failed_tests:
         failed_tests_names = [t.name for t in failed_tests]
         raise self.m.step.StepFailure(
             '%d tests failed: %r' % (len(failed_tests), failed_tests_names))
 
     return test_runner
 
   def get_compile_targets_and_tests(
-      self, mastername, buildername, master_dict, test_spec,
+      self, mastername, buildername, bot_db,
       override_bot_type=None, override_tests=None):
     """Returns a tuple: list of compile targets and list of tests.
 
     The list of tests includes ones on the triggered testers."""
 
-    bot_config = master_dict.get('builders', {}).get(buildername)
+    assert isinstance(bot_db, bdb_module.BotConfigAndTestDB), \
+        "bot_db argument %r was not a BotConfigAndTestDB" % (bot_db)
+    bot_config = bot_db.get_bot_config(mastername, buildername)
     bot_type = override_bot_type or bot_config.get('bot_type', 'builder_tester')
 
     tests = [copy.deepcopy(t) for t in bot_config.get('tests', [])]
     if override_tests is not None:
       tests = override_tests
 
     if bot_type not in ['builder', 'builder_tester']:
       return [], []
 
     compile_targets = set(bot_config.get('compile_targets', []))
     tests_including_triggered = list(tests)
-    for _, builder_dict in master_dict.get('builders', {}).iteritems():
-      if builder_dict.get('parent_buildername') == buildername:
-        tests_including_triggered.extend(builder_dict.get('tests', []))
+    for _, test_bot in bot_db.bot_configs_matching_parent_buildername(
+        mastername, buildername):
+      tests_including_triggered.extend(test_bot.get('tests', []))
 
     if bot_config.get('add_tests_as_compile_targets', True):
       for t in tests_including_triggered:
         compile_targets.update(t.compile_targets(self.m))
 
     # Only add crash_service when we have explicit compile targets.
     if (self.m.platform.is_win and
         compile_targets and
         'all' not in compile_targets):
       compile_targets.add('crash_service')
 
     # Lastly, add any targets the checkout-side test spec told us to use.
-    compile_targets.update(test_spec.get(buildername, {}).get(
+    compile_targets.update(bot_db.get_test_spec(mastername, buildername).get(
         'additional_compile_targets', []))
 
     return sorted(compile_targets), tests_including_triggered
 
   def transient_check(self, update_step, command):
     """Runs command, checking for transience if this is a try job.
 
     * command is a function which takes an argument of type (str -> str),
       which is a test name transformation (it adds "with patch" or "without
       patch") and runs the command.
     * update_step is the bot_update step used for deapplying the patch.
     """
     if self.m.tryserver.is_tryserver:
       try:
         command(lambda name: '%s (with patch)' % name)
       except self.m.step.StepFailure:
         self.deapply_patch(update_step)
         command(lambda name: '%s (without patch)' % name)
         raise
     else:
       command(lambda name: name)
 
 
-  def compile(self, mastername, buildername, update_step, master_dict,
-              test_spec, mb_mastername=None, mb_buildername=None):
+  def compile(self, mastername, buildername, update_step, bot_db,
+              mb_mastername=None, mb_buildername=None):
     """Runs compile and related steps for given builder."""
+    assert isinstance(bot_db, bdb_module.BotConfigAndTestDB), \
+        "bot_db argument %r was not a BotConfigAndTestDB" % (bot_db)
     compile_targets, tests_including_triggered = \
         self.get_compile_targets_and_tests(
             mastername,
             buildername,
-            master_dict, test_spec)
+            bot_db)
     self.compile_specific_targets(
-        mastername, buildername, update_step, master_dict,
+        mastername, buildername, update_step, bot_db,
         compile_targets, tests_including_triggered,
         mb_mastername=mb_mastername, mb_buildername=mb_buildername)
 
   def compile_specific_targets(
-      self, mastername, buildername, update_step, master_dict,
+      self, mastername, buildername, update_step, bot_db,
       compile_targets, tests_including_triggered,
       mb_mastername=None, mb_buildername=None, override_bot_type=None):
     """Runs compile and related steps for given builder.
 
     Allows finer-grained control about exact compile targets used.
 
     We don't use the given `mastername` and `buildername` to run MB, because
     they may be the values of the continuous builder the trybot may be
     configured to match; instead we need to use the actual mastername and
     buildername we're running on (Default to the "mastername" and
     "buildername" in the build properties -- self.m.properties, but could be
     overridden by `mb_mastername` and `mb_buildername`), because it may be
     configured with different MB settings.
 
     However, recipes used by Findit for culprit finding may still set
     (mb_mastername, mb_buildername) = (mastername, buildername) to exactly match
     a given continuous builder."""
 
-    bot_config = master_dict.get('builders', {}).get(buildername)
-    master_config = master_dict.get('settings', {})
+    assert isinstance(bot_db, bdb_module.BotConfigAndTestDB), \
+        "bot_db argument %r was not a BotConfigAndTestDB" % (bot_db)
+    bot_config = bot_db.get_bot_config(mastername, buildername)
+    master_config = bot_db.get_master_settings(mastername)
     bot_type = override_bot_type or bot_config.get('bot_type', 'builder_tester')
 
     self.m.chromium.cleanup_temp()
     if self.m.chromium.c.TARGET_PLATFORM == 'android':
       self.m.chromium_android.clean_local_files()
       self.m.chromium_android.run_tree_truth()
 
     if bot_type in ['builder', 'builder_tester']:
       isolated_targets = [
           t.isolate_target(self.m)
@@ skipping 58 matching lines @@
             build_url=self._build_gs_archive_url(
                 mastername, master_config, buildername),
             build_revision=build_revision,
             cros_board=self.m.chromium.c.TARGET_CROS_BOARD,
             # TODO(machenbach): Make asan a configuration switch.
             package_dsym_files=(
                 self.m.chromium.c.gyp_env.GYP_DEFINES.get('asan') and
                 self.m.chromium.c.HOST_PLATFORM == 'mac'),
         )
 
-      for loop_buildername, builder_dict in sorted(master_dict.get(
-          'builders', {}).iteritems()):
-        if builder_dict.get('parent_buildername') == buildername:
-          trigger_spec = {
+      for loop_buildername, builder_dict in sorted(
+          bot_db.bot_configs_matching_parent_buildername(
+              mastername, buildername)):
+        trigger_spec = {
             'builder_name': loop_buildername,
             'properties': {},
-          }
-          for name, value in update_step.presentation.properties.iteritems():
-            if name.startswith('got_'):
-              trigger_spec['properties']['parent_' + name] = value
-          self.m.trigger(trigger_spec)
+        }
+        for name, value in update_step.presentation.properties.iteritems():
+          if name.startswith('got_'):
+            trigger_spec['properties']['parent_' + name] = value
+        self.m.trigger(trigger_spec)
 
     if bot_config.get('archive_build') and not self.m.tryserver.is_tryserver:
       self.m.chromium.archive_build(
           'archive_build',
           bot_config['gs_bucket'],
           bot_config.get('gs_acl'),
           mode='dev'
       )
 
   def run_mb_and_compile(self, compile_targets, isolated_targets, name_suffix,
                          mb_mastername=None, mb_buildername=None):
     if self.m.chromium.c.project_generator.tool == 'mb':
       mb_mastername = mb_mastername or self.m.properties['mastername']
       mb_buildername = mb_buildername or self.m.properties['buildername']
       self.m.chromium.run_mb(mb_mastername, mb_buildername,
                              isolated_targets=isolated_targets,
                              name='generate_build_files%s' % name_suffix)
 
     self.m.chromium.compile(compile_targets, name='compile%s' % name_suffix)
 
   def download_and_unzip_build(self, mastername, buildername, update_step,
-                               master_dict, build_archive_url=None,
+                               bot_db, build_archive_url=None,
                                build_revision=None, override_bot_type=None):
+    assert isinstance(bot_db, bdb_module.BotConfigAndTestDB), \
+        "bot_db argument %r was not a BotConfigAndTestDB" % (bot_db)
     # We only want to do this for tester bots (i.e. those which do not compile
     # locally).
-    bot_type = override_bot_type or master_dict.get('builders', {}).get(
-        buildername, {}).get('bot_type')
+    bot_type = override_bot_type or bot_db.get_bot_config(
+        mastername, buildername).get('bot_type')
     if bot_type != 'tester':
       return
 
     # Protect against hard to debug mismatches between directory names
     # used to run tests from and extract build to. We've had several cases
     # where a stale build directory was used on a tester, and the extracted
     # build was not used at all, leading to confusion why source code changes
     # are not taking effect.
     #
     # The best way to ensure the old build directory is not used is to
     # remove it.
     self.m.file.rmtree(
       'build directory',
       self.m.chromium.c.build_dir.join(self.m.chromium.c.build_config_fs))
 
     legacy_build_url = None
     got_revision = update_step.presentation.properties['got_revision']
     build_revision = build_revision or self.m.properties.get(
         'parent_got_revision') or got_revision
     build_archive_url = build_archive_url or self.m.properties.get(
         'parent_build_archive_url')
     if build_archive_url is None:
-      master_config = master_dict.get('settings', {})
+      master_config = bot_db.get_master_settings(mastername)
       legacy_build_url = self._make_legacy_build_url(master_config, mastername)
 
     self.m.archive.download_and_unzip_build(
       step_name='extract build',
       target=self.m.chromium.c.build_config_fs,
       build_url=legacy_build_url,
       build_revision=build_revision,
       build_archive_url=build_archive_url)
 
-  def tests_for_builder(self, mastername, buildername, update_step, master_dict,
+  def tests_for_builder(self, mastername, buildername, update_step, bot_db,
                         override_bot_type=None):
-
-    bot_config = master_dict.get('builders', {}).get(buildername)
+    assert isinstance(bot_db, bdb_module.BotConfigAndTestDB), \
+        "bot_db argument %r was not a BotConfigAndTestDB" % (bot_db)
+    bot_config = bot_db.get_bot_config(mastername, buildername)
     bot_type = override_bot_type or bot_config.get('bot_type', 'builder_tester')
     # TODO(shinyak): bot_config.get('tests', []) sometimes return tuple.
     tests = [copy.deepcopy(t) for t in bot_config.get('tests', [])]
 
     if bot_config.get('goma_canary') or bot_config.get('goma_staging'):
       tests.insert(0, steps.DiagnoseGomaTest())
 
     if bot_type == 'tester':
       if (self.m.chromium.c.TARGET_PLATFORM == 'android' and
           bot_config.get('root_devices')):
@@ skipping 295 matching lines @@
       result_text = 'MB is enabled for this builder at this revision.'
       log_name = 'Builder MB-ready'
       self.m.step.active_result.presentation.logs[log_name] = [result_text]
       return False
     except (self.m.step.StepFailure, KeyError):
       result_text = 'MB is not enabled for this builder at this revision.'
       log_name = 'Builder NOT MB-ready'
       self.m.step.active_result.presentation.logs[log_name] = [result_text]
       self.m.step.active_result.presentation.status = self.m.step.WARNING
       return True