[Telemetry]Do not run power.idle_platform for reference build runs

tools/perf/core/perf_data_generator.py

 #!/usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Script to generate chromium.perf.json and chromium.perf.fyi.json in
 the src/testing/buildbot directory and benchmark.csv in the src/tools/perf
 directory. Maintaining these files by hand is too unwieldy.
 """
 import argparse
@@ skipping 563 matching lines @@
 def generate_cplusplus_isolate_script_test(dimension):
   return [
     generate_isolate_script_entry(
         [get_swarming_dimension(dimension, shard)], [], name, name,
         ignore_task_failure=False)
     for name, shard in dimension['perf_tests']
   ]
 
 
 def generate_telemetry_tests(
   tester_config, benchmarks, benchmark_sharding_map, use_whitelist):

[nednguyen, 2017/04/14 16:32:45]

can you make add a benchmark_ref_build_blacklist param here (and modify the call
site of this to pass
benchmark_ref_build_blacklist=BENCHMARK_REF_BUILD_BLACKLIST) so we can add
unittest for this?

[rnephew (Reviews Here), 2017/04/14 17:22:21]

I get "W:590, 0: Dangerous default value BENCHMARK_REF_BUILD_BLACKLIST
(__builtin__.list) as argument (dangerous-default-value)" when I do that. I have
to do it in a slightly more round about way to not get that linter warning.

   isolated_scripts = []
   # First determine the browser that you need based on the tester
   browser_name = ''
   if tester_config['platform'] == 'android':
     browser_name = 'android-chromium'
   elif (tester_config['platform'] == 'win'
     and tester_config['target_bits'] == 64):
     browser_name = 'release_x64'
   else:
     browser_name ='release'
@@ skipping 22 matching lines @@
       if device_affinity is None:
         raise Exception('Device affinity for benchmark %s not found'
           % benchmark.Name())
       swarming_dimensions.append(
           get_swarming_dimension(dimension, device_affinity))
 
     test = generate_telemetry_test(
       swarming_dimensions, benchmark.Name(), browser_name)
     isolated_scripts.append(test)
     # Now create another executable for this benchmark on the reference browser
-    reference_test = generate_telemetry_test(
-      swarming_dimensions, benchmark.Name(),'reference')
-    isolated_scripts.append(reference_test)
-    if current_shard == (num_shards - 1):
-      current_shard = 0
-    else:
-      current_shard += 1
+    # if it is not blacklisted from running on the reference browser.
+    if benchmark.Name() not in BENCHMARK_REF_BUILD_BLACKLIST:
+      reference_test = generate_telemetry_test(
+        swarming_dimensions, benchmark.Name(),'reference')
+      isolated_scripts.append(reference_test)
+      if current_shard == (num_shards - 1):
+        current_shard = 0
+      else:
+        current_shard += 1
 
   return isolated_scripts
 
 
 BENCHMARK_NAME_WHITELIST = set([
     u'smoothness.top_25_smooth',
     u'sunspider',
     u'system_health.webview_startup',
     u'page_cycler_v2.intl_hi_ru',
     u'dromaeo.cssqueryjquery',
@@ skipping 17 matching lines @@
 }
 
 # Certain swarming bots are not sharding correctly with the new device affinity
 # algorithm.  Reverting to legacy algorithm to try and get them to complete.
 # See crbug.com/670284
 LEGACY_DEVICE_AFFIINITY_ALGORITHM = [
   'Win Zenbook Perf',
   'Win 10 High-DPI Perf',
 ]
 
+# List of benchmarks that are to never be run with reference builds.
+BENCHMARK_REF_BUILD_BLACKLIST = [
+  'power.idle_platform',
+]
+
+
 def current_benchmarks(use_whitelist):
   benchmarks_dir = os.path.join(src_dir(), 'tools', 'perf', 'benchmarks')
   top_level_dir = os.path.dirname(benchmarks_dir)
 
   all_benchmarks = discover.DiscoverClasses(
       benchmarks_dir, top_level_dir, benchmark_module.Benchmark,
       index_by_class_name=True).values()
   # Remove all blacklisted benchmarks
   for blacklisted in BENCHMARK_NAME_BLACKLIST:
     for benchmark in all_benchmarks:
@@ skipping 285 matching lines @@
       return 0
     else:
       print ('The perf JSON config files are not up-to-date. Please run %s '
              'without --validate-only flag to update the perf JSON '
              'configs and benchmark.csv.') % sys.argv[0]
       return 1
   else:
     update_all_tests([fyi_waterfall, waterfall])
     update_benchmark_csv()
   return 0