[tools/perf] Make trybot bails out early if the benchmark is disabled on trybot platform

tools/perf/core/trybot_command.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import argparse
 import os
 import logging
 import platform
 import re
 import subprocess
@@ skipping 60 matching lines @@
     return '%s\nError running tryjob.' % self.args[0]
 
 
 def _GetTrybotList(builders):
   builders = ['%s' % bot.replace('_perf_bisect', '').replace('_', '-')
               for bot in builders]
   builders.extend(INCLUDE_BOTS)
   return sorted(builders)
 
 
+def _GetBotPlatformFromTrybotName(trybot_name):
+  os_names = ['linux', 'android', 'mac', 'win']
+  try:
+    return next(b for b in os_names if b in trybot_name)
+  except StopIteration:
+    raise TrybotError('Trybot "%s" unsupported for tryjobs.' % trybot_name)
+
+
 def _GetBuilderNames(trybot_name, builders):
   """ Return platform and its available bot name as dictionary."""
   os_names = ['linux', 'android', 'mac', 'win']
   if 'all' not in trybot_name:
     bot = ['%s_perf_bisect' % trybot_name.replace('-', '_')]
-    try:
-      bot_platform = next(b for b in os_names if b in trybot_name)
-    except StopIteration:
-      raise TrybotError('Trybot "%s" unsupported for tryjobs.' % trybot_name)
+    bot_platform = _GetBotPlatformFromTrybotName(trybot_name)
     if 'x64' in trybot_name:
       bot_platform += '-x64'
     return {bot_platform: bot}
 
   platform_and_bots = {}
   for os_name in os_names:
     platform_and_bots[os_name] = [bot for bot in builders if os_name in bot]
 
   # Special case for Windows x64, consider it as separate platform
   # config config should contain target_arch=x64 and --browser=release_x64.
@@ skipping 75 matching lines @@
         formatter_class=argparse.RawTextHelpFormatter)
     return parser
 
   @classmethod
   def ProcessCommandLineArgs(cls, parser, options, extra_args, environment):
     del environment  # unused
     for arg in extra_args:
       if arg == '--browser' or arg.startswith('--browser='):
         parser.error('--browser=... is not allowed when running trybot.')
     all_benchmarks = discover.DiscoverClasses(
-        start_dir=path_util.GetPerfBenchmarksDir(),
-        top_level_dir=path_util.GetPerfDir(),
-        base_class=benchmark.Benchmark).values()
+      start_dir=path_util.GetPerfBenchmarksDir(),
+      top_level_dir=path_util.GetPerfDir(),
+      base_class=benchmark.Benchmark).values()
     all_benchmark_names = [b.Name() for b in all_benchmarks]
-    if options.benchmark_name not in all_benchmark_names:
+    all_benchmarks_by_names = {b.Name(): b for b in all_benchmarks}
+    benchmark_class = all_benchmarks_by_names.get(options.benchmark_name, None)
+    if not benchmark_class:
       possible_benchmark_names = matching.GetMostLikelyMatchedObject(
           all_benchmark_names, options.benchmark_name)
       parser.error(
          'No benchmark named "%s". Do you mean any of those benchmarks '
          'below?\n%s' %
          (options.benchmark_name, '\n'.join(possible_benchmark_names)))
+    is_benchmark_disabled, reason = cls.IsBenchmarkDisabledOnTrybotPlatform(
+        benchmark_class, options.trybot)
+    also_run_disabled_option = '--also-run-disabled-tests'
+    if is_benchmark_disabled and also_run_disabled_option not in extra_args:
+      parser.error('%s To run the benchmark on trybot anyway, add '
+                   '%s option.' % (reason, also_run_disabled_option))
+
+  @classmethod
+  def IsBenchmarkDisabledOnTrybotPlatform(cls, benchmark_class, trybot_name):
+    """ Return whether benchmark will be disabled on trybot platform.
+
+    Note that we cannot tell with certainty whether the benchmark will be
+    disabled on the trybot platform since the disable logic in ShouldDisable()
+    can be very dynamic and can only be verified on the trybot server platform.
+
+    We are biased on the side of enabling the benchmark, and attempt to
+    early discover whether the benchmark will be disabled as our best.
+
+    It should never be the case that the benchmark will be enabled on the test
+    platform but this method returns True.
+
+    Returns:
+      A tuple (is_benchmark_disabled, reason) whereas |is_benchmark_disabled| is
+      a boolean that tells whether we are sure that the benchmark will be
+      disabled, and |reason| is a string that shows the reason why we think the
+      benchmark is disabled for sure.
+    """
+    benchmark_name = benchmark_class.Name()
+    benchmark_disabled_strings = set()
+    if hasattr(benchmark_class, '_disabled_strings'):
+      # pylint: disable=protected-access
+      benchmark_disabled_strings = benchmark_class._disabled_strings
+      # pylint: enable=protected-access
+    if 'all' in benchmark_disabled_strings:
+      return True, 'Benchmark %s is disabled on all platform.' % benchmark_name
+    if trybot_name == 'all':
+      return False, ''
+    trybot_platform = _GetBotPlatformFromTrybotName(trybot_name)
+    if trybot_platform in benchmark_disabled_strings:
+      return True, (
+          "Benchmark %s is disabled on %s, and trybot's platform is %s." %
+          (benchmark_name, ', '.join(benchmark_disabled_strings),
+           trybot_platform))
+    benchmark_enabled_strings = None
+    if hasattr(benchmark_class, '_enabled_strings'):
+      # pylint: disable=protected-access
+      benchmark_enabled_strings = benchmark_class._enabled_strings
+      # pylint: enable=protected-access
+    if (benchmark_enabled_strings and
+        trybot_platform not in benchmark_enabled_strings and
+        'all' not in benchmark_enabled_strings):
+      return True, (
+          "Benchmark %s is only enabled on %s, and trybot's platform is %s." %
+          (benchmark_name, ', '.join(benchmark_enabled_strings),
+           trybot_platform))
+    if benchmark_class.ShouldDisable != benchmark.Benchmark.ShouldDisable:
+      logging.warning(
+          'Benchmark %s has ShouldDisable() method defined. If your trybot run '
+          'does not produce any results, it is possible that the benchmark '
+          'is disabled on the target trybot platform.', benchmark_name)
+    return False, ''
 
   @classmethod
   def AddCommandLineArgs(cls, parser, environment):
     del environment  # unused
     available_bots = _GetTrybotList(cls._GetBuilderList())
     parser.add_argument(
         'trybot', choices=available_bots,
         help=('specify which bots to run telemetry benchmarks on. '
               ' Allowed values are:\n' + '\n'.join(available_bots)),
         metavar='<trybot name>')
@@ skipping 228 matching lines @@
         return ERROR  # pylint: disable=lost-exception
       logging.info('Checked out original branch: %s', original_branchname)
       returncode, out, err = _RunProcess(
           [_GIT_CMD, 'branch', '-D', 'telemetry-tryjob'])
       if returncode:
         logging.error('Could not delete telemetry-tryjob branch. '
                       'Please delete it manually: %s', err)
         return ERROR  # pylint: disable=lost-exception
       logging.info('Deleted temp branch: telemetry-tryjob')
     return SUCCESS