Generating benchmark.csv file

tools/perf/core/perf_json_generator.py

Index: tools/perf/core/perf_json_generator.py
diff --git a/tools/perf/core/perf_json_generator.py b/tools/perf/core/perf_json_generator.py
index 5e908f6f53d8be4f045c53ba7daf1b58e89ebc84..2540089efa42f1145ecc295afe80d49a4dcf9716 100755
--- a/tools/perf/core/perf_json_generator.py
+++ b/tools/perf/core/perf_json_generator.py
@@ -4,10 +4,12 @@
 # found in the LICENSE file.
 
 """Script to generate chromium.perf.json and chromium.perf.fyi.json in
-the src/testing/buildbot directory. Maintaining these files by hand is
-too unwieldy.
+the src/testing/buildbot directory and benchmark.csv in the src/tools/perf
+directory. Maintaining these files by hand is too unwieldy.
 """
 import argparse
+import collections
+import csv
 import json
 import os
 import sys
@@ -16,6 +18,7 @@ from chrome_telemetry_build import chromium_config
 
 sys.path.append(chromium_config.GetTelemetryDir())
 from telemetry import benchmark as benchmark_module
+from telemetry import decorators
 from telemetry.core import discover
 from telemetry.util import bot_utils
 
@@ -654,19 +657,21 @@ LEGACY_DEVICE_AFFIINITY_ALGORITHM = [
   'Win 10 High-DPI Perf',
 ]
 
-def current_benchmarks(use_whitelist):
+def current_benchmarks(use_whitelist, use_blacklist = True):
   benchmarks_dir = os.path.join(src_dir(), 'tools', 'perf', 'benchmarks')
   top_level_dir = os.path.dirname(benchmarks_dir)
 
   all_benchmarks = discover.DiscoverClasses(
       benchmarks_dir, top_level_dir, benchmark_module.Benchmark,
       index_by_class_name=True).values()
-  # Remove all blacklisted benchmarks
-  for blacklisted in BENCHMARK_NAME_BLACKLIST:
-    for benchmark in all_benchmarks:
-      if benchmark.Name() == blacklisted:
-        all_benchmarks.remove(benchmark)
-        break
+
+  if use_blacklist:
+    # Remove all blacklisted benchmarks
+    for blacklisted in BENCHMARK_NAME_BLACKLIST:
+      for benchmark in all_benchmarks:
+        if benchmark.Name() == blacklisted:
+          all_benchmarks.remove(benchmark)
+          break
 
   if use_whitelist:
     all_benchmarks = (
@@ -813,11 +818,66 @@ def src_dir():
       os.path.dirname(os.path.dirname(file_path))))
 
 
+BenchmarkMetadata = collections.namedtuple(
+    'BenchmarkMetadata', 'emails component')
+NON_TELEMETRY_BENCHMARKS = {

[nednguyen, 2017/03/16 21:08:17]

My concern here is that there is no way to make sure that this list is in synced
with the manual list people put in the json config file. For example, if people
remove tracing_perftests at line 85 in this file, how do we make sure that they
don't forget to also remove tracing_perftests here?

[ashleymarie1, 2017/03/16 21:18:02]

I, too, am concerned about that. I'm having trouble finding any other way to
compile this information though since not all the non telemetry benchmarks are
in SCRIPT_TESTS or any other list as far as I've seen. Any ideas for how to make
this more robust?

[nednguyen, 2017/03/16 21:23:43]

One idea I have is make a method that return a map of all benchmark names by
theirs metadata (Telemetry + hard code one). Then somewhere, you can make an
assert statement that all the benchmark to be generated in the perf waterfall
json config must have name match one of benchmark_names_by_metadata's keys.

[ashleymarie1, 2017/03/17 15:30:19]

So this solves the problem of a user adding a test above but failing to add it
here. It doesn't solve the reverse problem of removing a test above and failing
to remove it here. So we also need to assert that all the
benchmark_names_by_metadata's keys are in the benchmark names generated in the
perf waterfall json config.
I feel like there should be a cleaner solution here, I just can't think of it :(
Maybe add all the benchmark names by metadata to a set and then add all the test
names from the json config file to a set and assert the sets are the same?

+    "angle_perftests": BenchmarkMetadata("jmadill@chromium.org", None),
+    "cc_perftest": BenchmarkMetadata("enne@chromium.org", None),
+    "gpu_perftests": BenchmarkMetadata("reveman@chromium.org", None),
+    "tracing_perftests": BenchmarkMetadata(
+        "kkraynov@chromium.org, primiano@chromium.org", None),
+    "indexeddb_perf": BenchmarkMetadata("cmumford@chromium.org", None),
+    "load_library_perf_tests": BenchmarkMetadata(None, None),
+    "media_perftests": BenchmarkMetadata("crouleau@chromium.org", None),
+    "performance_browser_tests": BenchmarkMetadata(
+        "hubbe@chromium.org, justinlin@chromium.org, miu@chromium.org", None)
+}
+
+
+def update_benchmark_csv():
+  """Updates go/chrome-benchmarks.
+
+  Updates telemetry/perf/benchmark.csv containing the current benchmark names,
+  owners, and components.
+  """
+  benchmark_list = current_benchmarks(
+      use_whitelist = False, use_blacklist = False)
+
+  header_data = [['AUTOGENERATED FILE DO NOT EDIT'],
+      ['See //tools/perf/generate_perf_data.py to make changes'],
+      ['Benchmark name', 'Individual owners', 'Component']
+  ]
+  data = []
+  for benchmark_name in NON_TELEMETRY_BENCHMARKS:
+    data.append([
+        benchmark_name,
+        NON_TELEMETRY_BENCHMARKS[benchmark_name].emails,
+        NON_TELEMETRY_BENCHMARKS[benchmark_name].component
+    ])
+  for benchmark in benchmark_list:
+    emails = decorators.GetEmails(benchmark)
+    if emails:
+      emails = ", ".join(emails)
+    data.append([
+        benchmark.Name(),
+        emails,
+        decorators.GetComponent(benchmark)
+    ])
+
+  data = sorted(data, key=lambda b: b[0])
+  data = header_data + data
+  perf_dir = os.path.join(src_dir(), 'tools', 'perf')
+  benchmark_file = os.path.join(perf_dir, 'benchmark.csv')
+  with open(benchmark_file, 'wb') as f:
+    writer = csv.writer(f)
+    writer.writerows(data)
+
+
 def main(args):
   parser = argparse.ArgumentParser(
-      description=('Generate perf test\' json config. This need to be done '
-                   'anytime you add/remove any existing benchmarks in '
-                   'tools/perf/benchmarks.'))
+      description=('Generate perf test\' json config and benchmark.csv. '
+                   'This needs to be done anytime you add/remove any existing'
+                   'benchmarks in tools/perf/benchmarks.'))
   parser.add_argument(
       '--validate-only', action='store_true', default=False,
       help=('Validate whether the perf json generated will be the same as the '
@@ -842,4 +902,5 @@ def main(args):
   else:
     update_all_tests(fyi_waterfall)
     update_all_tests(waterfall)
+    update_benchmark_csv()
   return 0