Generating benchmark.csv file

tools/perf/core/perf_json_generator.py

 #!/usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Script to generate chromium.perf.json and chromium.perf.fyi.json in
-the src/testing/buildbot directory. Maintaining these files by hand is
-too unwieldy.
+the src/testing/buildbot directory and benchmark.csv in the src/tools/perf
+directory. Maintaining these files by hand is too unwieldy.
 """
 import argparse
+import csv
 import json
 import os
 import sys
 
 from chrome_telemetry_build import chromium_config
 
 sys.path.append(chromium_config.GetTelemetryDir())
 from telemetry import benchmark as benchmark_module
+from telemetry import decorators
 from telemetry.core import discover
 from telemetry.util import bot_utils
 
 
 SCRIPT_TESTS = [
   {
     'args': [
       'gpu_perftests',
       '--adb-path',
       'src/third_party/catapult/devil/bin/deps/linux2/x86_64/bin/adb',
@@ skipping 618 matching lines @@
 }
 
 # Certain swarming bots are not sharding correctly with the new device affinity
 # algorithm.  Reverting to legacy algorithm to try and get them to complete.
 # See crbug.com/670284
 LEGACY_DEVICE_AFFIINITY_ALGORITHM = [
   'Win Zenbook Perf',
   'Win 10 High-DPI Perf',
 ]
 
-def current_benchmarks(use_whitelist):
+def current_benchmarks(use_whitelist, use_blacklist = True):
   benchmarks_dir = os.path.join(src_dir(), 'tools', 'perf', 'benchmarks')
   top_level_dir = os.path.dirname(benchmarks_dir)
 
   all_benchmarks = discover.DiscoverClasses(
       benchmarks_dir, top_level_dir, benchmark_module.Benchmark,
       index_by_class_name=True).values()
-  # Remove all blacklisted benchmarks
-  for blacklisted in BENCHMARK_NAME_BLACKLIST:
-    for benchmark in all_benchmarks:
-      if benchmark.Name() == blacklisted:
-        all_benchmarks.remove(benchmark)
-        break
+
+  if use_blacklist:
+    # Remove all blacklisted benchmarks
+    for blacklisted in BENCHMARK_NAME_BLACKLIST:
+      for benchmark in all_benchmarks:
+        if benchmark.Name() == blacklisted:
+          all_benchmarks.remove(benchmark)
+          break
 
   if use_whitelist:
     all_benchmarks = (
         bench for bench in all_benchmarks
         if bench.Name() in BENCHMARK_NAME_WHITELIST)
   return sorted(all_benchmarks, key=lambda b: b.Name())
 
 
 # Returns a sorted list of (benchmark, avg_runtime) pairs for every
 # benchmark in the all_benchmarks list where avg_runtime is in seconds.  Also
@@ skipping 126 matching lines @@
     json.dump(tests, fp, indent=2, separators=(',', ': '), sort_keys=True)
     fp.write('\n')
 
 
 def src_dir():
   file_path = os.path.abspath(__file__)
   return os.path.dirname(os.path.dirname(
       os.path.dirname(os.path.dirname(file_path))))
 
 
+MANUAL_BENCHMARKS = [

[nednguyen, 2017/03/16 19:11:18]

Let name this NON_TElEMETRY_BENCHMARKS. Also can you make it a dictionary with
keys are benchmark names and values are namedtuple with 2 fields: emails &
component?

[ashleymarie1, 2017/03/16 19:40:51]

Done. I had never heard of namedtuple before today! Thanks for teaching me
something new :)

+    ["angle_perftests", "jmadill@chromium.org", None],
+    ["cc_perftest", "enne@chromium.org", None],
+    ["gpu_perftests", "reveman@chromium.org", None],
+    ["tracing_perftests", "kkraynov@chromium.org, primiano@chromium.org",
+        None],
+    ["indexeddb_perf", "cmumford@chromium.org", None],
+    ["load_library_perf_tests", None, None],
+    ["media_perftests", "crouleau@chromium.org", None],
+    ["performance_browser_tests",
+        "hubbe@chromium.org, justinlin@chromium.org, miu@chromium.org", None]
+]
+
+
+def update_benchmark_csv():
+  """Updates go/chrome-benchmarks.
+
+  Updates telemetry/perf/benchmark.csv containing the current benchmark names,
+  owners, and components.
+  """
+  benchmark_list = current_benchmarks(
+      use_whitelist = False, use_blacklist = False)
+  data = MANUAL_BENCHMARKS
+  for benchmark in benchmark_list:
+    emails = decorators.GetEmails(benchmark)
+    if emails:
+      emails = ", ".join(emails)
+    data.append([
+        benchmark.Name(),
+        emails,
+        decorators.GetComponent(benchmark)
+    ])
+
+  data = sorted(data, key=lambda b: b[0])
+
+  perf_dir = os.path.join(src_dir(), 'tools', 'perf')
+  benchmark_file = os.path.join(perf_dir, 'benchmark.csv')
+  with open(benchmark_file, 'wb') as f:
+    writer = csv.writer(f)
+    writer.writerows(data)
+
+
 def main(args):
   parser = argparse.ArgumentParser(
       description=('Generate perf test\' json config. This need to be done '
                    'anytime you add/remove any existing benchmarks in '

[nednguyen, 2017/03/16 19:11:18]

The description needs update

[ashleymarie1, 2017/03/16 19:40:51]

Done.

                    'tools/perf/benchmarks.'))
   parser.add_argument(
       '--validate-only', action='store_true', default=False,
       help=('Validate whether the perf json generated will be the same as the '
             'existing configs. This does not change the contain of existing '
             'configs'))
   options = parser.parse_args(args)
 
+  update_benchmark_csv()

[nednguyen, 2017/03/16 19:11:18]

If --validate-only flag is enabled, we should not modify files

[ashleymarie1, 2017/03/16 19:40:51]

Done.

+
   waterfall = get_waterfall_config()
   waterfall['name'] = 'chromium.perf'
   fyi_waterfall = get_fyi_waterfall_config()
   fyi_waterfall['name'] = 'chromium.perf.fyi'
 
   if options.validate_only:
     if tests_are_up_to_date(fyi_waterfall) and tests_are_up_to_date(waterfall):
       print 'All the perf JSON config files are up-to-date. \\o/'
       return 0
     else:
       print ('The perf JSON config files are not up-to-date. Please run %s '
              'without --validate-only flag to update the perf JSON '
              'configs.') % sys.argv[0]
       return 1
   else:
     update_all_tests(fyi_waterfall)
     update_all_tests(waterfall)
   return 0