Generating benchmark.csv file

tools/perf/core/perf_data_generator.py

 #!/usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Script to generate chromium.perf.json and chromium.perf.fyi.json in
-the src/testing/buildbot directory. Maintaining these files by hand is
-too unwieldy.
+the src/testing/buildbot directory and benchmark.csv in the src/tools/perf
+directory. Maintaining these files by hand is too unwieldy.
 """
 import argparse
+import collections
+import csv
 import json
 import os
+import re
 import sys
+import sets
 
 from chrome_telemetry_build import chromium_config
 
 sys.path.append(chromium_config.GetTelemetryDir())
 from telemetry import benchmark as benchmark_module
+from telemetry import decorators
 from telemetry.core import discover
 from telemetry.util import bot_utils
 
 
 SCRIPT_TESTS = [
   {
     'args': [
       'gpu_perftests',
       '--adb-path',
       'src/third_party/catapult/devil/bin/deps/linux2/x86_64/bin/adb',
@@ skipping 763 matching lines @@
 def tests_are_up_to_date(waterfall):
   tests = generate_all_tests(waterfall)
   tests_data = json.dumps(tests, indent=2, separators=(',', ': '),
                           sort_keys=True)
   config_file = get_json_config_file_for_waterfall(waterfall)
   with open(config_file, 'r') as fp:
     config_data = fp.read().strip()
   return tests_data == config_data
 
 
-def update_all_tests(waterfall):
-  tests = generate_all_tests(waterfall)
-  config_file = get_json_config_file_for_waterfall(waterfall)
-  with open(config_file, 'w') as fp:
-    json.dump(tests, fp, indent=2, separators=(',', ': '), sort_keys=True)
-    fp.write('\n')
+def update_all_tests(waterfalls):
+  all_tests = {}
+  for w in waterfalls:
+    tests = generate_all_tests(w)
+    config_file = get_json_config_file_for_waterfall(w)
+    with open(config_file, 'w') as fp:
+      json.dump(tests, fp, indent=2, separators=(',', ': '), sort_keys=True)
+      fp.write('\n')
+    all_tests.update(tests)
+  verify_all_tests_in_benchmark_csv(all_tests, get_all_benchmarks_metadata())
 
 
 def src_dir():
   file_path = os.path.abspath(__file__)
   return os.path.dirname(os.path.dirname(
       os.path.dirname(os.path.dirname(file_path))))
 
 
+BenchmarkMetadata = collections.namedtuple(
+    'BenchmarkMetadata', 'emails component')
+NON_TELEMETRY_BENCHMARKS = {
+    "angle_perftests": BenchmarkMetadata("jmadill@chromium.org", None),

[nednguyen, 2017/03/21 21:56:03]

nits: string should always be singly quoted

[ashleymarie1, 2017/03/22 19:17:11]

Thanks! Updated all the double quotes I added to single quotes. Is there a
reason for this or is it just standard in Chromium to do single quoted strings?

[nednguyen, 2017/03/22 19:22:56]

It's just our convention to use singly quoted string everywhere:
https://google.github.io/styleguide/pyguide.html#Strings

+    "cc_perftests": BenchmarkMetadata("enne@chromium.org", None),
+    "gpu_perftests": BenchmarkMetadata("reveman@chromium.org", None),
+    "tracing_perftests": BenchmarkMetadata(
+        "kkraynov@chromium.org, primiano@chromium.org", None),
+    "load_library_perf_tests": BenchmarkMetadata(None, None),
+    "media_perftests": BenchmarkMetadata("crouleau@chromium.org", None),
+    "performance_browser_tests": BenchmarkMetadata(
+        "hubbe@chromium.org, justinlin@chromium.org, miu@chromium.org", None)
+}
+
+
+# Returns a dictionary mapping benchmark name to benchmark owner metadata
+def get_all_benchmarks_metadata():
+  metadata = NON_TELEMETRY_BENCHMARKS
+  benchmark_list = current_benchmarks(False)
+
+  for benchmark in benchmark_list:
+    emails = decorators.GetEmails(benchmark)
+    if emails:
+      emails = ", ".join(emails)
+    metadata[benchmark.Name()] = BenchmarkMetadata(
+        emails, decorators.GetComponent(benchmark))
+  return metadata
+
+
+def verify_all_tests_in_benchmark_csv(tests, benchmark_metadata):
+  benchmark_names = sets.Set(benchmark_metadata)
+  test_names = sets.Set()
+  for t in tests:
+    scripts = []
+    if 'isolated_scripts' in tests[t]:
+      scripts = tests[t]['isolated_scripts']
+    elif 'scripts' in tests[t]:
+      scripts = tests[t]['scripts']
+    else:
+      assert('Android Compile' == t
+        or 'Android arm64 Compile' == t
+        or t.startswith('AAAAA')), 'Unknown test data %s' % t
+    for s in scripts:
+      name = s['name']
+      name = re.sub('\\.reference$', '', name)
+      test_names.add(name)
+
+  error_messages = []
+  for test in benchmark_names - test_names:
+    error_messages.append('Remove ' + test + ' from NON_TELEMETRY_BENCHMARKS')
+  for test in test_names - benchmark_names:
+    error_messages.append('Add ' + test + ' to NON_TELEMETRY_BENCHMARKS')
+
+  assert benchmark_names == test_names, ('Please update '
+      'NON_TELEMETRY_BENCHMARKS as below:\n' + '\n'.join(error_messages))
+
+
+def update_benchmark_csv():
+  """Updates go/chrome-benchmarks.
+
+  Updates telemetry/perf/benchmark.csv containing the current benchmark names,
+  owners, and components.
+  """
+  header_data = [['AUTOGENERATED FILE DO NOT EDIT'],
+      ['See //tools/perf/generate_perf_data.py to make changes'],
+      ['Benchmark name', 'Individual owners', 'Component']
+  ]
+
+  csv_data = []
+  benchmark_metadata = get_all_benchmarks_metadata()
+  for benchmark_name in benchmark_metadata:
+    csv_data.append([
+        benchmark_name,
+        benchmark_metadata[benchmark_name].emails,
+        benchmark_metadata[benchmark_name].component
+    ])
+
+  csv_data = sorted(csv_data, key=lambda b: b[0])
+  csv_data = header_data + csv_data
+
+  perf_dir = os.path.join(src_dir(), 'tools', 'perf')
+  benchmark_file = os.path.join(perf_dir, 'benchmark.csv')
+  with open(benchmark_file, 'wb') as f:
+    writer = csv.writer(f)
+    writer.writerows(csv_data)
+
+
 def main(args):
   parser = argparse.ArgumentParser(
-      description=('Generate perf test\' json config. This need to be done '
-                   'anytime you add/remove any existing benchmarks in '
-                   'tools/perf/benchmarks.'))
+      description=('Generate perf test\' json config and benchmark.csv. '
+                   'This needs to be done anytime you add/remove any existing'
+                   'benchmarks in tools/perf/benchmarks.'))
   parser.add_argument(
       '--validate-only', action='store_true', default=False,
       help=('Validate whether the perf json generated will be the same as the '
             'existing configs. This does not change the contain of existing '
             'configs'))
   options = parser.parse_args(args)
 
   waterfall = get_waterfall_config()
   waterfall['name'] = 'chromium.perf'
   fyi_waterfall = get_fyi_waterfall_config()
   fyi_waterfall['name'] = 'chromium.perf.fyi'
 
   if options.validate_only:

[nednguyen, 2017/03/21 21:59:58]

actually I think this should also check for verify_all_tests_in_benchmark_csv()

The reason is someone can make a CL that only modify NON_TELEMETRY_BENCHMARKS
and PRESUBMIT would not validate whether NON_TELEMETRY_BENCHMARKS are in synced
because that doesn't change the the generated benchmarks.

[ashleymarie1, 2017/03/22 19:17:11]

Good point! I hadn't thought about that

     if tests_are_up_to_date(fyi_waterfall) and tests_are_up_to_date(waterfall):
       print 'All the perf JSON config files are up-to-date. \\o/'
       return 0
     else:
       print ('The perf JSON config files are not up-to-date. Please run %s '
              'without --validate-only flag to update the perf JSON '
-             'configs.') % sys.argv[0]
+             'configs and benchmark.csv.') % sys.argv[0]
       return 1
   else:
-    update_all_tests(fyi_waterfall)
-    update_all_tests(waterfall)
+    update_all_tests([fyi_waterfall, waterfall])
+    update_benchmark_csv()
   return 0