Add gpu and cc perf tests to new fyi tester

tools/perf/generate_perf_json.py

 #!/usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Script to generate chromium.perf.json and chromium.perf.fyi.json in
 the src/testing/buildbot directory. Maintaining these files by hand is
 too unwieldy.
 """
 
 import json
 import os
 import sys
 
 from chrome_telemetry_build import chromium_config
 
 sys.path.append(chromium_config.GetTelemetryDir())
 from telemetry import benchmark as benchmark_module
 from telemetry.core import discover
 from telemetry.util import bot_utils
 
 
 SCRIPT_TESTS = [
   {
     'args': [
       'gpu_perftests'
     ],
     'name': 'gpu_perftests',
     'script': 'gtest_perf_test.py',
-    'testers': [
-      {
-        'name': 'Android Galaxy S5 Perf',
-        'shards': [3]
-      },
-      {
-        'name': 'Android Nexus5 Perf',
-        'shards': [2]
-      },
-      {
-        'name': 'Android Nexus7v2 Perf',
-        'shards': [2]
-      },
-      {
-        'name': 'Android Nexus9 Perf',
-        'shards': [2]
-      },
-    ]
+    'testers': {
+      'chromium.perf': [
+        {
+          'name': 'Android Galaxy S5 Perf',
+          'shards': [3]
+        },
+        {
+          'name': 'Android Nexus5 Perf',
+          'shards': [2]
+        },
+        {
+          'name': 'Android Nexus7v2 Perf',
+          'shards': [2]
+        },
+        {
+          'name': 'Android Nexus9 Perf',
+          'shards': [2]
+        }
+      ],
+      'chromium.perf.fyi': [
+        {
+          'name': 'Android Galaxy S5 Perf',
+          'shards': [1]
+        },
+      ]
+    }
   },
   {
     'args': [
       'cc_perftests'
     ],
     'name': 'cc_perftests',
     'script': 'gtest_perf_test.py',
-    'testers': [
-      {
-        'name': 'Android Galaxy S5 Perf',
-        'shards': [3]
-      },
-      {
-        'name': 'Android Nexus5 Perf',
-        'shards': [2]
-      },
-      {
-        'name': 'Android Nexus6 Perf',
-        'shards': [2]
-      },
-      {
-        'name': 'Android Nexus7v2 Perf',
-        'shards': [2]
-      },
-      {
-        'name': 'Android Nexus9 Perf',
-        'shards': [2]
-      },
-    ]
+    'testers': {
+      'chromium.perf': [
+        {
+          'name': 'Android Galaxy S5 Perf',
+          'shards': [3]
+        },
+        {
+          'name': 'Android Nexus5 Perf',
+          'shards': [2]
+        },
+        {
+          'name': 'Android Nexus6 Perf',
+          'shards': [2]
+        },
+        {
+          'name': 'Android Nexus7v2 Perf',
+          'shards': [2]
+        },
+        {
+          'name': 'Android Nexus9 Perf',
+          'shards': [2]
+        },
+      ],
+      'chromium.perf.fyi': [
+        {
+          'name': 'Android Galaxy S5 Perf',
+          'shards': [1]
+        },
+      ]
+    }
   },
   {
     'args': [
       'cc_perftests',
       '--test-launcher-print-test-stdio=always'
     ],
     'name': 'cc_perftests',
     'script': 'gtest_perf_test.py',
-    'testers': [
-      {
-        'name': 'Linux Perf',
-        'shards': [3]
-      },
-    ]
+    'testers': {
+      'chromium.perf': [
+        {
+          'name': 'Linux Perf',
+          'shards': [3]
+        },
+      ]
+    }
   },
   {
     'args': [
       'load_library_perf_tests',
       '--test-launcher-print-test-stdio=always'
     ],
     'name': 'load_library_perf_tests',
     'script': 'gtest_perf_test.py',
-    'testers': [
-      {
-        'name': 'Linux Perf',
-        'shards': [3]
-      },
-      {
-        'name': 'Win 7 ATI GPU Perf',
-        'shards': [2]
-      },
-      {
-        'name': 'Win 7 Nvidia GPU Perf',
-        'shards': [2]
-      },
-      {
-        'name': 'Win 7 Perf',
-        'shards': [3]
-      },
-      {
-        'name': 'Win 7 x64 Perf',
-        'shards': [2]
-      },
-      {
-        'name': 'Win 8 Perf',
-        'shards': [2]
-      },
-    ]
+    'testers': {
+      'chromium.perf': [
+        {
+          'name': 'Linux Perf',
+          'shards': [3]
+        },
+        {
+          'name': 'Win 7 ATI GPU Perf',
+          'shards': [2]
+        },
+        {
+          'name': 'Win 7 Nvidia GPU Perf',
+          'shards': [2]
+        },
+        {
+          'name': 'Win 7 Perf',
+          'shards': [3]
+        },
+        {
+          'name': 'Win 7 x64 Perf',
+          'shards': [2]
+        },
+        {
+          'name': 'Win 8 Perf',
+          'shards': [2]
+        },
+      ]
+    }
   },
   {
     'args': [
       'performance_browser_tests',
       '--test-launcher-print-test-stdio=always',
       '--gtest_filter=TabCapturePerformanceTest.*:CastV2PerformanceTest.*',
       '--test-launcher-jobs=1',
       '--enable-gpu'
     ],
     'name': 'performance_browser_tests',
     'script': 'gtest_perf_test.py',
-    'testers': [
-      {
-        'name': 'Mac 10.8 Perf',
-        'shards': [3]
-      },
-      {
-        'name': 'Mac 10.9 Perf',
-        'shards': [3]
-      },
-      {
-        'name': 'Win 7 ATI GPU Perf',
-        'shards': [2]
-      },
-      {
-        'name': 'Win 7 Nvidia GPU Perf',
-        'shards': [2]
-      },
-      {
-        'name': 'Win 7 Perf',
-        'shards': [3]
-      },
-      {
-        'name': 'Win 7 x64 Perf',
-        'shards': [2]
-      },
-      {
-        'name': 'Win 8 Perf',
-        'shards': [2]
-      },
-    ]
+    'testers': {
+      'chromium.perf': [
+        {
+          'name': 'Mac 10.8 Perf',
+          'shards': [3]
+        },
+        {
+          'name': 'Mac 10.9 Perf',
+          'shards': [3]
+        },
+        {
+          'name': 'Win 7 ATI GPU Perf',
+          'shards': [2]
+        },
+        {
+          'name': 'Win 7 Nvidia GPU Perf',
+          'shards': [2]
+        },
+        {
+          'name': 'Win 7 Perf',
+          'shards': [3]
+        },
+        {
+          'name': 'Win 7 x64 Perf',
+          'shards': [2]
+        },
+        {
+          'name': 'Win 8 Perf',
+          'shards': [2]
+        },
+      ]
+    }
   },
   {
     'args': [
       'angle_perftests',
       '--test-launcher-print-test-stdio=always',
       '--test-launcher-jobs=1'
     ],
     'name': 'angle_perftests',
     'script': 'gtest_perf_test.py',
-    'testers': [
-      {
-        'name': 'Win 7 ATI GPU Perf',
-        'shards': [2]
-      },
-      {
-        'name': 'Win 7 Nvidia GPU Perf',
-        'shards': [2]
-      },
-    ]
+    'testers': {
+      'chromium.perf': [
+        {
+          'name': 'Win 7 ATI GPU Perf',
+          'shards': [2]
+        },
+        {
+          'name': 'Win 7 Nvidia GPU Perf',
+          'shards': [2]
+        },
+      ]
+    }
   },
 ]
 
 def add_tester(waterfall, name, perf_id, platform, target_bits=64,
               num_host_shards=1, num_device_shards=1, swarming=None):
   del perf_id # this will be needed
   waterfall['testers'][name] = {
     'platform': platform,
     'num_device_shards': num_device_shards,
     'num_host_shards': num_host_shards,
     'target_bits': target_bits,
   }
 
   if swarming:
     waterfall['testers'][name]['swarming_dimensions'] = swarming
     waterfall['testers'][name]['swarming'] = True
 
   return waterfall
 
 def get_fyi_waterfall_config():
   waterfall = {'builders':[], 'testers': {}}
   waterfall = add_tester(
+    waterfall, 'Android Galaxy S5 Perf',
+    'android-galaxy-s5-perf', 'android')
+  waterfall = add_tester(
     waterfall, 'Win 10 Low-End Perf Tests',
     'win-low-end-2-core', 'win',
     swarming=[
       {
        'device_ids': ['build187-b4']
       },
       {
        'device_ids': ['build171-b4', 'build186-b4']
       }
     ])
@@ skipping 79 matching lines @@
     '--output-format=chartjson',
     '--browser=%s' % browser
   ]
   # When this is enabled on more than just windows machines we will need
   # --device=android
 
   step_name = benchmark_name
   if browser == 'reference':
     test_args.append('--output-trace-tag=_ref')
     step_name += '.reference'
-  swarming = {
-    # Always say this is true regardless of whether the tester
-    # supports swarming. It doesn't hurt.
-    'can_use_on_swarming_builders': True,
-    'expiration': 14400,
-    'dimension_sets': swarming_dimensions
-  }
+
+  swarming = None
+  if swarming_dimensions:
+    swarming = {
+      # Always say this is true regardless of whether the tester
+      # supports swarming. It doesn't hurt.
+      'can_use_on_swarming_builders': True,
+      'expiration': 14400,
+      'dimension_sets': swarming_dimensions
+    }
 
   result = {
     'args': test_args,
     'isolate_name': 'telemetry_perf_tests',
     'name': step_name,
     'override_compile_targets': ['telemetry_perf_tests'],
-    'swarming': swarming,
   }
+  if swarming:
+    result['swarming'] = swarming
 
   return result
 
-def script_test_enabled_on_tester(test, tester_name, shard):
-  for enabled_tester in test['testers']:
+def script_test_enabled_on_tester(master, test, tester_name, shard):
+  for enabled_tester in test['testers'].get(master, []):
     if enabled_tester['name'] == tester_name:
       if shard in enabled_tester['shards']:
         return True
   return False
 
-def generate_script_tests(tester_name, shard):
+def generate_script_tests(master, tester_name, shard):
   script_tests = []
   for test in SCRIPT_TESTS:
-    if script_test_enabled_on_tester(test, tester_name, shard):
+    if script_test_enabled_on_tester(master, test, tester_name, shard):
       script = {
         'args': test['args'],
         'name': test['name'],
         'script': test['script']
       }
       script_tests.append(script)
   return script_tests
 
 def generate_telemetry_tests(tester_config, benchmarks):
   isolated_scripts = []
@@ skipping 25 matching lines @@
     test = generate_telemetry_test(
       swarming_dimensions, benchmark.Name(), browser_name)
     isolated_scripts.append(test)
     # Now create another executable for this benchmark on the reference browser
     reference_test = generate_telemetry_test(
       swarming_dimensions, benchmark.Name(),'reference')
     isolated_scripts.append(reference_test)
 
   return isolated_scripts
 
+
+BENCHMARK_NAME_WHITELIST = set([

[martiniss, 2016/10/11 22:10:33]

I added this because otherwise it added all the benchmarks in the
//tools/benchmark directory, which I figured we wouldn't want to do?

[eyaich1, 2016/10/11 23:05:02]

Right.  Thanks for the addition.

+    u'smoothness.top_25_smooth',
+    u'smoothness.top_25_smooth.reference',
+    u'sunspider',
+    u'sunspider.reference',
+    u'system_health.webview_startup',
+    u'system_health.webview_startup.reference',
+    u'page_cycler_v2.intl_hi_ru',
+    u'page_cycler_v2.intl_hi_ru.reference',
+    u'dromaeo.cssqueryjquery',
+    u'dromaeo.cssqueryjquery.reference'

[eyaich1, 2016/10/11 23:05:02]

You can remove all the *.reference from this list.  We call
generate_telemetry_test twice for each benchmark name and that method adds the
.reference.  IT doesn't hurt because DiscoverClasses will never return something
with a .reference so it is a noop, but it is dead code.

[martiniss, 2016/10/11 23:32:02]

Removed.

+])
+
+
 def current_benchmarks():
   current_dir = os.path.dirname(__file__)
   benchmarks_dir = os.path.join(current_dir, 'benchmarks')
   top_level_dir = os.path.dirname(benchmarks_dir)
 
-  return discover.DiscoverClasses(
+  all_benchmarks = discover.DiscoverClasses(
       benchmarks_dir, top_level_dir, benchmark_module.Benchmark,
       index_by_class_name=True).values()
+  return sorted((
+      bench for bench in all_benchmarks
+      if bench.Name() in BENCHMARK_NAME_WHITELIST), key=lambda b: b.Name())
 
-def generate_all_tests(waterfall, is_fyi):
+
+def generate_all_tests(waterfall):
   tests = {}
   for builder in waterfall['builders']:
     tests[builder] = {}
+  all_benchmarks = current_benchmarks()
+
   for name, config in waterfall['testers'].iteritems():
-    if is_fyi:
+    if config.get('swarming', False):
       # Right now we are only generating benchmarks for the fyi waterfall
-      all_benchmarks = current_benchmarks()
       isolated_scripts = generate_telemetry_tests(config, all_benchmarks)
       tests[name] = {
         'isolated_scripts': sorted(isolated_scripts, key=lambda x: x['name'])
       }
     else:
       # scripts are only currently run in addition to the main waterfall.  They
       # are currently the only thing generated in the perf json file.
       # TODO eyaich: will need to handle the sharding differently when we have
       # swarmed bots on the main waterfall.
       for shard in range(0, config['num_host_shards']):
         tester_name = '%s (%d)' % (name, shard + 1)
-        scripts = generate_script_tests(name, shard + 1)
+        scripts = generate_script_tests(waterfall['name'], name, shard + 1)
         if scripts:
           tests[tester_name] = {
             'scripts': sorted(scripts, key=lambda x: x['name'])
           }
 
   tests['AAAAA1 AUTOGENERATED FILE DO NOT EDIT'] = {}
-  tests['AAAAA2 See generate_perf_json.py to make changes'] = {}
-  filename = 'chromium.perf.fyi.json' if is_fyi else 'chromium.perf.json'
+  tests['AAAAA2 See //tools/perf/generate_perf_json.py to make changes'] = {}
+  filename = '%s.json' % waterfall['name']
+
   current_dir = os.path.dirname(os.path.abspath(__file__))
   src_dir = os.path.dirname(os.path.dirname(current_dir))
 
   with open(os.path.join(src_dir, 'testing', 'buildbot', filename), 'w') as fp:
     json.dump(tests, fp, indent=2, separators=(',', ': '), sort_keys=True)
     fp.write('\n')
 
+
 def main():
   waterfall = get_waterfall_config()
+  waterfall['name'] = 'chromium.perf'
   fyi_waterfall = get_fyi_waterfall_config()
-  generate_all_tests(fyi_waterfall, True)
-  generate_all_tests(waterfall, False)
+  fyi_waterfall['name'] = 'chromium.perf.fyi'
+
+  generate_all_tests(fyi_waterfall)
+  generate_all_tests(waterfall)
   return 0
 
 if __name__ == "__main__":
   sys.exit(main())