Allow a single trybot to mirror multiple waterfall bots.

scripts/slave/recipes/chromium_trybot.py

Index: scripts/slave/recipes/chromium_trybot.py
diff --git a/scripts/slave/recipes/chromium_trybot.py b/scripts/slave/recipes/chromium_trybot.py
index 46bab948d91688e7115452ad1585b80ea3841a6c..af62e182076e44d1d878757784fa1fdddfa5e22c 100644
--- a/scripts/slave/recipes/chromium_trybot.py
+++ b/scripts/slave/recipes/chromium_trybot.py
@@ -199,13 +199,19 @@ def is_source_file(api, filepath):
   return ext in ['.c', '.cc', '.cpp', '.h', '.java', '.mm']
 
 def _RunStepsInternal(api):
-  def _get_bot_config(mastername, buildername):
+  # This returns either a single dictionary or list of dictionaries
+  # that contain the 'mastername' and 'buildername' properties. This
+  # is the 'bot_desc' concept rather than the 'bot_config' concept.
+  def _get_bot_desc(mastername, buildername):
     master_dict = api.chromium_tests.trybots.get(mastername, {})
     return master_dict.get('builders', {}).get(buildername)
 
   mastername = api.properties.get('mastername')
   buildername = api.properties.get('buildername')
-  bot_config = _get_bot_config(mastername, buildername)
+  bot_descs = _get_bot_desc(mastername, buildername)
+  # Support the case where a single trybot picks up compile targets
+  # and tests from multiple bots.
+  bot_descs = api.chromium_tests.normalize_bot_descs(bot_descs)
 
   # TODO(sergiyb): This is a temporary hack to run GPU tests on tryserver
   # only. This should be removed when we will convert chromium.gpu waterfall
@@ -215,16 +221,18 @@ def _RunStepsInternal(api):
   enable_gpu_tests = builder in CHROMIUM_GPU_DIMENSION_SETS.get(master, {})
 
   api.chromium_tests.configure_build(
-      bot_config['mastername'],
-      bot_config['buildername'],
+      bot_descs,
       override_bot_type='builder_tester')
 
   api.chromium_tests.configure_swarming('chromium', precommit=True)
 
   api.chromium.apply_config('trybot_flavor')
   if enable_gpu_tests:
-    api.chromium.apply_config('archive_gpu_tests')
-    api.chromium.apply_config('chrome_with_codecs')
+    # TODO(kbr): the "optional" flag is only being used during the
+    # transition from the GPU recipe to the Chromium recipe. This
+    # entire if-test will soon be removed.
+    api.chromium.apply_config('archive_gpu_tests', optional=True)
+    api.chromium.apply_config('chrome_with_codecs', optional=True)
 
   if api.properties.get('patch_project') == 'blink':  # pragma: no cover
     raise Exception('CLs which use blink project are not supported. '
@@ -232,27 +240,27 @@ def _RunStepsInternal(api):
                     'the blink merge.')
 
   bot_update_step, bot_db = \
-      api.chromium_tests.prepare_checkout(
-          bot_config['mastername'],
-          bot_config['buildername'])
-
-  tests = list(api.chromium_tests.tests_for_builder(
-      bot_config['mastername'],
-      bot_config['buildername'],
-      bot_update_step,
-      bot_db,
-      override_bot_type='builder_tester'))
-  tester = bot_config.get('tester', '')
-  if tester:
-    test_config = bot_db.get_bot_config(bot_config['mastername'], tester)
-    for key, value in test_config.get('swarming_dimensions', {}).iteritems():
-      api.swarming.set_default_dimension(key, value)
+      api.chromium_tests.prepare_checkout(bot_descs)
+
+  tests = []
+  for bot_desc in bot_descs:
     tests.extend(api.chromium_tests.tests_for_builder(
-        bot_config['mastername'],
-        tester,
+        bot_desc['mastername'],
+        bot_desc['buildername'],
         bot_update_step,
         bot_db,
         override_bot_type='builder_tester'))
+    tester = bot_desc.get('tester', '')
+    if tester:
+      test_config = bot_db.get_bot_config(bot_desc['mastername'], tester)
+      for key, value in test_config.get('swarming_dimensions', {}).iteritems():
+        api.swarming.set_default_dimension(key, value)
+      tests.extend(api.chromium_tests.tests_for_builder(
+          bot_desc['mastername'],
+          tester,
+          bot_update_step,
+          bot_db,
+          override_bot_type='builder_tester'))
 
   if enable_gpu_tests:
     tests.extend(api.gpu.create_tests(
@@ -295,8 +303,7 @@ def _RunStepsInternal(api):
 
   compile_targets, tests_including_triggered = \
       api.chromium_tests.get_compile_targets_and_tests(
-          bot_config['mastername'],
-          bot_config['buildername'],
+          bot_descs,
           bot_db,
           override_bot_type='builder_tester',
           override_tests=tests)
@@ -311,7 +318,7 @@ def _RunStepsInternal(api):
                                  additional_compile_targets,
                                  'trybot_analyze_config.json')
 
-  if bot_config.get('analyze_mode') == 'compile':
+  if bot_descs[0].get('analyze_mode') == 'compile':

[Sergey Berezin, 2016/01/12 01:44:23]

nit: this makes the [0]'th configuration "special", supplying extra parameters
for the trybot as a whole. Is this intentional?

[Ken Russell (switch to Gerrit), 2016/01/12 05:35:51]

I didn't understand this code nor the ramifications of making it iterate over
all the bot_descs so leaving it checking the 0'th one seemed safest. I probably
should have added an assert.

Looking more deeply, this zaps all of the tests, and is used only on the
compile-only tryservers. I guess it makes sense to take the union of the
settings in all of the bot_descs.

     tests = []
     tests_including_triggered = []
 
@@ -341,8 +348,8 @@ def _RunStepsInternal(api):
       compile_targets = sorted(set(compile_targets))
 
     api.chromium_tests.compile_specific_targets(
-        bot_config['mastername'],
-        bot_config['buildername'],
+        bot_descs[0]['mastername'],
+        bot_descs[0]['buildername'],
         bot_update_step,
         bot_db,
         compile_targets,
@@ -418,15 +425,19 @@ def GenTests(api):
   # While not strictly required for coverage, record expectations for each
   # of the configs so we can see when and how they change.
   for mastername, master_config in api.chromium_tests.trybots.iteritems():
-    for buildername, bot_config in master_config['builders'].iteritems():
+    for buildername, bot_desc in master_config['builders'].iteritems():
       for analyze in ['', '_analyze']:
         test_name = 'full_%s_%s%s' % (_sanitize_nonalpha(mastername),
                                       _sanitize_nonalpha(buildername),
                                       analyze)
+        bot_desc = api.chromium_tests.normalize_bot_descs(bot_desc)
         yield (
           api.test(test_name) +
+          # In the case where a trybot mirrors multiple waterfall
+          # bots, pick up the platform from the first one. (They
+          # should all be identical, anyway.)
           api.chromium_tests.platform(
-              bot_config['mastername'], bot_config['buildername']) +
+              bot_desc[0]['mastername'], bot_desc[0]['buildername']) +
           (api.empty_test_data() if analyze else suppress_analyze()) +
           props(mastername=mastername, buildername=buildername)
         )
@@ -1082,3 +1093,26 @@ def GenTests(api):
           patch_project='v8') +
     api.platform.name('mac')
   )
+
+  # This tests the scenario where one trybot mirrors multiple
+  # waterfall bots. Right now, linux_chromium_rel_ng has the GPU tests
+  # hardwired in. The presence of base_unittests in the output with
+  # the configuration below shows that the mirroring is working
+  # correctly. (Once the bot is switched from the GPU to the Chromium
+  # recipe, the test setup and expectations will be updated -- no
+  # reminder necessary, they'll have to be.)
+  yield (
+    api.test('trybot_mirrors_multiple_waterfall_bots') +
+    api.properties.tryserver(
+      mastername='tryserver.chromium.linux',
+      buildername='linux_chromium_rel_ng',
+      path_config='swarming',
+    ) +
+    api.platform.name('linux') +
+    suppress_analyze() +
+    api.override_step_data('read test spec (chromium.gpu)', api.json.output({
+        'Fake Linux Release (NVIDIA)': {
+            'gtest_tests': ['base_unittests'],
+        },
+    }))
+  )