Generating benchmark.csv file

tools/perf/core/perf_json_generator.py

 #!/usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Script to generate chromium.perf.json and chromium.perf.fyi.json in
-the src/testing/buildbot directory. Maintaining these files by hand is
-too unwieldy.
+the src/testing/buildbot directory and benchmark.csv in the src/tools/perf
+directory. Maintaining these files by hand is too unwieldy.
 """
 import argparse
+import collections
+import csv
 import json
 import os
 import sys
 
 from chrome_telemetry_build import chromium_config
 
 sys.path.append(chromium_config.GetTelemetryDir())
 from telemetry import benchmark as benchmark_module
+from telemetry import decorators
 from telemetry.core import discover
 from telemetry.util import bot_utils
 
 
 SCRIPT_TESTS = [
   {
     'args': [
       'gpu_perftests',
       '--adb-path',
       'src/third_party/catapult/devil/bin/deps/linux2/x86_64/bin/adb',
@@ skipping 618 matching lines @@
 }
 
 # Certain swarming bots are not sharding correctly with the new device affinity
 # algorithm.  Reverting to legacy algorithm to try and get them to complete.
 # See crbug.com/670284
 LEGACY_DEVICE_AFFIINITY_ALGORITHM = [
   'Win Zenbook Perf',
   'Win 10 High-DPI Perf',
 ]
 
-def current_benchmarks(use_whitelist):
+def current_benchmarks(use_whitelist, use_blacklist = True):
   benchmarks_dir = os.path.join(src_dir(), 'tools', 'perf', 'benchmarks')
   top_level_dir = os.path.dirname(benchmarks_dir)
 
   all_benchmarks = discover.DiscoverClasses(
       benchmarks_dir, top_level_dir, benchmark_module.Benchmark,
       index_by_class_name=True).values()
-  # Remove all blacklisted benchmarks
-  for blacklisted in BENCHMARK_NAME_BLACKLIST:
-    for benchmark in all_benchmarks:
-      if benchmark.Name() == blacklisted:
-        all_benchmarks.remove(benchmark)
-        break
+
+  if use_blacklist:
+    # Remove all blacklisted benchmarks
+    for blacklisted in BENCHMARK_NAME_BLACKLIST:
+      for benchmark in all_benchmarks:
+        if benchmark.Name() == blacklisted:
+          all_benchmarks.remove(benchmark)
+          break
 
   if use_whitelist:
     all_benchmarks = (
         bench for bench in all_benchmarks
         if bench.Name() in BENCHMARK_NAME_WHITELIST)
   return sorted(all_benchmarks, key=lambda b: b.Name())
 
 
 # Returns a sorted list of (benchmark, avg_runtime) pairs for every
 # benchmark in the all_benchmarks list where avg_runtime is in seconds.  Also
@@ skipping 126 matching lines @@
     json.dump(tests, fp, indent=2, separators=(',', ': '), sort_keys=True)
     fp.write('\n')
 
 
 def src_dir():
   file_path = os.path.abspath(__file__)
   return os.path.dirname(os.path.dirname(
       os.path.dirname(os.path.dirname(file_path))))
 
 
+BenchmarkMetadata = collections.namedtuple(
+    'BenchmarkMetadata', 'emails component')
+NON_TELEMETRY_BENCHMARKS = {

[nednguyen, 2017/03/16 21:08:17]

My concern here is that there is no way to make sure that this list is in synced
with the manual list people put in the json config file. For example, if people
remove tracing_perftests at line 85 in this file, how do we make sure that they
don't forget to also remove tracing_perftests here?

[ashleymarie1, 2017/03/16 21:18:02]

I, too, am concerned about that. I'm having trouble finding any other way to
compile this information though since not all the non telemetry benchmarks are
in SCRIPT_TESTS or any other list as far as I've seen. Any ideas for how to make
this more robust?

[nednguyen, 2017/03/16 21:23:43]

One idea I have is make a method that return a map of all benchmark names by
theirs metadata (Telemetry + hard code one). Then somewhere, you can make an
assert statement that all the benchmark to be generated in the perf waterfall
json config must have name match one of benchmark_names_by_metadata's keys.

[ashleymarie1, 2017/03/17 15:30:19]

So this solves the problem of a user adding a test above but failing to add it
here. It doesn't solve the reverse problem of removing a test above and failing
to remove it here. So we also need to assert that all the
benchmark_names_by_metadata's keys are in the benchmark names generated in the
perf waterfall json config.
I feel like there should be a cleaner solution here, I just can't think of it :(
Maybe add all the benchmark names by metadata to a set and then add all the test
names from the json config file to a set and assert the sets are the same?

+    "angle_perftests": BenchmarkMetadata("jmadill@chromium.org", None),
+    "cc_perftest": BenchmarkMetadata("enne@chromium.org", None),
+    "gpu_perftests": BenchmarkMetadata("reveman@chromium.org", None),
+    "tracing_perftests": BenchmarkMetadata(
+        "kkraynov@chromium.org, primiano@chromium.org", None),
+    "indexeddb_perf": BenchmarkMetadata("cmumford@chromium.org", None),
+    "load_library_perf_tests": BenchmarkMetadata(None, None),
+    "media_perftests": BenchmarkMetadata("crouleau@chromium.org", None),
+    "performance_browser_tests": BenchmarkMetadata(
+        "hubbe@chromium.org, justinlin@chromium.org, miu@chromium.org", None)
+}
+
+
+def update_benchmark_csv():
+  """Updates go/chrome-benchmarks.
+
+  Updates telemetry/perf/benchmark.csv containing the current benchmark names,
+  owners, and components.
+  """
+  benchmark_list = current_benchmarks(
+      use_whitelist = False, use_blacklist = False)
+
+  header_data = [['AUTOGENERATED FILE DO NOT EDIT'],
+      ['See //tools/perf/generate_perf_data.py to make changes'],
+      ['Benchmark name', 'Individual owners', 'Component']
+  ]
+  data = []
+  for benchmark_name in NON_TELEMETRY_BENCHMARKS:
+    data.append([
+        benchmark_name,
+        NON_TELEMETRY_BENCHMARKS[benchmark_name].emails,
+        NON_TELEMETRY_BENCHMARKS[benchmark_name].component
+    ])
+  for benchmark in benchmark_list:
+    emails = decorators.GetEmails(benchmark)
+    if emails:
+      emails = ", ".join(emails)
+    data.append([
+        benchmark.Name(),
+        emails,
+        decorators.GetComponent(benchmark)
+    ])
+
+  data = sorted(data, key=lambda b: b[0])
+  data = header_data + data
+  perf_dir = os.path.join(src_dir(), 'tools', 'perf')
+  benchmark_file = os.path.join(perf_dir, 'benchmark.csv')
+  with open(benchmark_file, 'wb') as f:
+    writer = csv.writer(f)
+    writer.writerows(data)
+
+
 def main(args):
   parser = argparse.ArgumentParser(
-      description=('Generate perf test\' json config. This need to be done '
-                   'anytime you add/remove any existing benchmarks in '
-                   'tools/perf/benchmarks.'))
+      description=('Generate perf test\' json config and benchmark.csv. '
+                   'This needs to be done anytime you add/remove any existing'
+                   'benchmarks in tools/perf/benchmarks.'))
   parser.add_argument(
       '--validate-only', action='store_true', default=False,
       help=('Validate whether the perf json generated will be the same as the '
             'existing configs. This does not change the contain of existing '
             'configs'))
   options = parser.parse_args(args)
 
   waterfall = get_waterfall_config()
   waterfall['name'] = 'chromium.perf'
   fyi_waterfall = get_fyi_waterfall_config()
   fyi_waterfall['name'] = 'chromium.perf.fyi'
 
   if options.validate_only:
     if tests_are_up_to_date(fyi_waterfall) and tests_are_up_to_date(waterfall):
       print 'All the perf JSON config files are up-to-date. \\o/'
       return 0
     else:
       print ('The perf JSON config files are not up-to-date. Please run %s '
              'without --validate-only flag to update the perf JSON '
              'configs.') % sys.argv[0]
       return 1
   else:
     update_all_tests(fyi_waterfall)
     update_all_tests(waterfall)
+    update_benchmark_csv()
   return 0