| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright 2016 The Chromium Authors. All rights reserved. | 2 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 # pylint: disable=too-many-lines | 6 # pylint: disable=too-many-lines |
| 7 | 7 |
| 8 """Script to generate chromium.perf.json and chromium.perf.fyi.json in | 8 """Script to generate chromium.perf.json and chromium.perf.fyi.json in |
| 9 the src/testing/buildbot directory and benchmark.csv in the src/tools/perf | 9 the src/testing/buildbot directory and benchmark.csv in the src/tools/perf |
| 10 directory. Maintaining these files by hand is too unwieldy. | 10 directory. Maintaining these files by hand is too unwieldy. |
| (...skipping 828 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 839 } | 839 } |
| 840 | 840 |
| 841 for name, config in waterfall['builders'].iteritems(): | 841 for name, config in waterfall['builders'].iteritems(): |
| 842 tests[name] = config | 842 tests[name] = config |
| 843 | 843 |
| 844 tests['AAAAA1 AUTOGENERATED FILE DO NOT EDIT'] = {} | 844 tests['AAAAA1 AUTOGENERATED FILE DO NOT EDIT'] = {} |
| 845 tests['AAAAA2 See //tools/perf/generate_perf_data.py to make changes'] = {} | 845 tests['AAAAA2 See //tools/perf/generate_perf_data.py to make changes'] = {} |
| 846 return tests | 846 return tests |
| 847 | 847 |
| 848 | 848 |
| 849 def get_json_config_file_for_waterfall(waterfall): | 849 def get_json_config_file_for_waterfall(waterfall, filename_suffix=None): |
| 850 filename = '%s.json' % waterfall['name'] | 850 filename = '%s%s.json' % (waterfall['name'], |
| 851 filename_suffix if filename_suffix else "") |
| 851 buildbot_dir = os.path.join( | 852 buildbot_dir = os.path.join( |
| 852 path_util.GetChromiumSrcDir(), 'testing', 'buildbot') | 853 path_util.GetChromiumSrcDir(), 'testing', 'buildbot') |
| 853 return os.path.join(buildbot_dir, filename) | 854 return os.path.join(buildbot_dir, filename) |
| 854 | 855 |
| 855 | 856 |
| 857 def append_extra_tests(waterfall, tests): |
| 858 extra_config_file = get_json_config_file_for_waterfall(waterfall, '.extras') |
| 859 if os.path.isfile(extra_config_file): |
| 860 with open(extra_config_file) as extra_fp: |
| 861 extra_tests = json.load(extra_fp) |
| 862 for key, value in extra_tests.iteritems(): |
| 863 assert key not in tests |
| 864 tests[key] = value |
| 865 |
| 866 |
| 856 def tests_are_up_to_date(waterfalls): | 867 def tests_are_up_to_date(waterfalls): |
| 857 up_to_date = True | 868 up_to_date = True |
| 858 all_tests = {} | 869 all_tests = {} |
| 859 for w in waterfalls: | 870 for w in waterfalls: |
| 860 tests = generate_all_tests(w) | 871 tests = generate_all_tests(w) |
| 872 # Note: |all_tests| don't cover those manually-specified tests added by |
| 873 # append_extra_tests(). |
| 874 all_tests.update(tests) |
| 875 append_extra_tests(w, tests) |
| 861 tests_data = json.dumps(tests, indent=2, separators=(',', ': '), | 876 tests_data = json.dumps(tests, indent=2, separators=(',', ': '), |
| 862 sort_keys=True) | 877 sort_keys=True) |
| 863 config_file = get_json_config_file_for_waterfall(w) | 878 config_file = get_json_config_file_for_waterfall(w) |
| 864 with open(config_file, 'r') as fp: | 879 with open(config_file, 'r') as fp: |
| 865 config_data = fp.read().strip() | 880 config_data = fp.read().strip() |
| 866 all_tests.update(tests) | |
| 867 up_to_date &= tests_data == config_data | 881 up_to_date &= tests_data == config_data |
| 868 verify_all_tests_in_benchmark_csv(all_tests, | 882 verify_all_tests_in_benchmark_csv(all_tests, |
| 869 get_all_waterfall_benchmarks_metadata()) | 883 get_all_waterfall_benchmarks_metadata()) |
| 870 return up_to_date | 884 return up_to_date |
| 871 | 885 |
| 872 | 886 |
| 873 def update_all_tests(waterfalls): | 887 def update_all_tests(waterfalls): |
| 874 all_tests = {} | 888 all_tests = {} |
| 875 for w in waterfalls: | 889 for w in waterfalls: |
| 876 tests = generate_all_tests(w) | 890 tests = generate_all_tests(w) |
| 891 # Note: |all_tests| don't cover those manually-specified tests added by |
| 892 # append_extra_tests(). |
| 893 all_tests.update(tests) |
| 894 append_extra_tests(w, tests) |
| 877 config_file = get_json_config_file_for_waterfall(w) | 895 config_file = get_json_config_file_for_waterfall(w) |
| 878 with open(config_file, 'w') as fp: | 896 with open(config_file, 'w') as fp: |
| 879 json.dump(tests, fp, indent=2, separators=(',', ': '), sort_keys=True) | 897 json.dump(tests, fp, indent=2, separators=(',', ': '), sort_keys=True) |
| 880 fp.write('\n') | 898 fp.write('\n') |
| 881 all_tests.update(tests) | |
| 882 verify_all_tests_in_benchmark_csv(all_tests, | 899 verify_all_tests_in_benchmark_csv(all_tests, |
| 883 get_all_waterfall_benchmarks_metadata()) | 900 get_all_waterfall_benchmarks_metadata()) |
| 884 | 901 |
| 885 | 902 |
| 886 # not_scheduled means this test is not scheduled on any of the chromium.perf | 903 # not_scheduled means this test is not scheduled on any of the chromium.perf |
| 887 # waterfalls. Right now, all the below benchmarks are scheduled, but some other | 904 # waterfalls. Right now, all the below benchmarks are scheduled, but some other |
| 888 # benchmarks are not scheduled, because they're disabled on all platforms. | 905 # benchmarks are not scheduled, because they're disabled on all platforms. |
| 889 BenchmarkMetadata = collections.namedtuple( | 906 BenchmarkMetadata = collections.namedtuple( |
| 890 'BenchmarkMetadata', 'emails component not_scheduled') | 907 'BenchmarkMetadata', 'emails component not_scheduled') |
| 891 NON_TELEMETRY_BENCHMARKS = { | 908 NON_TELEMETRY_BENCHMARKS = { |
| (...skipping 160 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1052 return 0 | 1069 return 0 |
| 1053 else: | 1070 else: |
| 1054 print ('The perf JSON config files are not up-to-date. Please run %s ' | 1071 print ('The perf JSON config files are not up-to-date. Please run %s ' |
| 1055 'without --validate-only flag to update the perf JSON ' | 1072 'without --validate-only flag to update the perf JSON ' |
| 1056 'configs and benchmark.csv.') % sys.argv[0] | 1073 'configs and benchmark.csv.') % sys.argv[0] |
| 1057 return 1 | 1074 return 1 |
| 1058 else: | 1075 else: |
| 1059 update_all_tests([fyi_waterfall, waterfall]) | 1076 update_all_tests([fyi_waterfall, waterfall]) |
| 1060 update_benchmark_csv() | 1077 update_benchmark_csv() |
| 1061 return 0 | 1078 return 0 |
| OLD | NEW |