OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2015 The Chromium Authors. All rights reserved. | 2 # Copyright 2015 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Runs an isolate bundled Telemetry benchmark. | 6 """Runs an isolate bundled Telemetry benchmark. |
7 | 7 |
8 This script attempts to emulate the contract of gtest-style tests | 8 This script attempts to emulate the contract of gtest-style tests |
9 invoked via recipes. The main contract is that the caller passes the | 9 invoked via recipes. The main contract is that the caller passes the |
10 argument: | 10 argument: |
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
59 if args.xvfb and xvfb.should_start_xvfb(env): | 59 if args.xvfb and xvfb.should_start_xvfb(env): |
60 xvfb_proc, openbox_proc, xcompmgr_proc = xvfb.start_xvfb(env=env, | 60 xvfb_proc, openbox_proc, xcompmgr_proc = xvfb.start_xvfb(env=env, |
61 build_dir='.') | 61 build_dir='.') |
62 assert xvfb_proc and openbox_proc and xcompmgr_proc, 'Failed to start xvfb' | 62 assert xvfb_proc and openbox_proc and xcompmgr_proc, 'Failed to start xvfb' |
63 try: | 63 try: |
64 tempfile_dir = tempfile.mkdtemp('telemetry') | 64 tempfile_dir = tempfile.mkdtemp('telemetry') |
65 valid = True | 65 valid = True |
66 failures = [] | 66 failures = [] |
67 chartjson_results_present = '--output-format=chartjson' in rest_args | 67 chartjson_results_present = '--output-format=chartjson' in rest_args |
68 chartresults = None | 68 chartresults = None |
| 69 json_test_results_present = '--output-format=json-test-results' in rest_args |
| 70 json_test_results = None |
69 | 71 |
70 results = None | 72 results = None |
71 try: | 73 try: |
72 rc = common.run_command([sys.executable] + rest_args + [ | 74 rc = common.run_command([sys.executable] + rest_args + [ |
73 '--output-dir', tempfile_dir, | 75 '--output-dir', tempfile_dir, |
74 '--output-format=json' | 76 '--output-format=json' |
75 ], env=env) | 77 ], env=env) |
76 # If we have also output chartjson read it in and return it. | 78 # If we have also output chartjson read it in and return it. |
77 # results-chart.json is the file name output by telemetry when the | 79 # results-chart.json is the file name output by telemetry when the |
78 # chartjson output format is included | 80 # chartjson output format is included |
(...skipping 11 matching lines...) Expand all Loading... |
90 for value in results['per_page_values']: | 92 for value in results['per_page_values']: |
91 if value['type'] == 'failure': | 93 if value['type'] == 'failure': |
92 page_data = results['pages'][str(value['page_id'])] | 94 page_data = results['pages'][str(value['page_id'])] |
93 name = page_data.get('name') | 95 name = page_data.get('name') |
94 if not name: | 96 if not name: |
95 name = page_data['url'] | 97 name = page_data['url'] |
96 | 98 |
97 failures.append(name) | 99 failures.append(name) |
98 valid = bool(rc == 0 or failures) | 100 valid = bool(rc == 0 or failures) |
99 | 101 |
| 102 if json_test_results_present: |
| 103 tempfile_name = os.path.join(tempfile_dir, 'test-results.json') |
| 104 with open(tempfile_name) as f: |
| 105 json_test_results = json.load(f) |
| 106 |
100 except Exception: | 107 except Exception: |
101 traceback.print_exc() | 108 traceback.print_exc() |
102 if results: | 109 if results: |
103 print 'results, which possibly caused exception: %s' % json.dumps( | 110 print 'results, which possibly caused exception: %s' % json.dumps( |
104 results, indent=2) | 111 results, indent=2) |
105 valid = False | 112 valid = False |
106 finally: | 113 finally: |
107 shutil.rmtree(tempfile_dir) | 114 shutil.rmtree(tempfile_dir) |
108 | 115 |
109 if not valid and not failures: | 116 if not valid and not failures: |
110 failures = ['(entire test suite)'] | 117 failures = ['(entire test suite)'] |
111 if rc == 0: | 118 if rc == 0: |
112 rc = 1 # Signal an abnormal exit. | 119 rc = 1 # Signal an abnormal exit. |
113 | 120 |
114 if chartjson_results_present and args.isolated_script_test_chartjson_output: | 121 if chartjson_results_present and args.isolated_script_test_chartjson_output: |
115 chartjson_output_file = \ | 122 chartjson_output_file = \ |
116 open(args.isolated_script_test_chartjson_output, 'w') | 123 open(args.isolated_script_test_chartjson_output, 'w') |
117 json.dump(chartresults, chartjson_output_file) | 124 json.dump(chartresults, chartjson_output_file) |
118 | 125 |
119 json.dump({ | 126 if not json_test_results_present: |
120 'valid': valid, | 127 json_test_results = { |
121 'failures': failures | 128 'valid': valid, |
122 }, args.isolated_script_test_output) | 129 'failures': failures |
| 130 } |
| 131 |
| 132 json.dump(json_test_results, args.isolated_script_test_output) |
123 return rc | 133 return rc |
124 | 134 |
125 finally: | 135 finally: |
126 xvfb.kill(xvfb_proc) | 136 xvfb.kill(xvfb_proc) |
127 xvfb.kill(openbox_proc) | 137 xvfb.kill(openbox_proc) |
128 xvfb.kill(xcompmgr_proc) | 138 xvfb.kill(xcompmgr_proc) |
129 | 139 |
130 | 140 |
131 # This is not really a "script test" so does not need to manually add | 141 # This is not really a "script test" so does not need to manually add |
132 # any additional compile targets. | 142 # any additional compile targets. |
133 def main_compile_targets(args): | 143 def main_compile_targets(args): |
134 json.dump([], args.output) | 144 json.dump([], args.output) |
135 | 145 |
136 | 146 |
137 if __name__ == '__main__': | 147 if __name__ == '__main__': |
138 # Conform minimally to the protocol defined by ScriptTest. | 148 # Conform minimally to the protocol defined by ScriptTest. |
139 if 'compile_targets' in sys.argv: | 149 if 'compile_targets' in sys.argv: |
140 funcs = { | 150 funcs = { |
141 'run': None, | 151 'run': None, |
142 'compile_targets': main_compile_targets, | 152 'compile_targets': main_compile_targets, |
143 } | 153 } |
144 sys.exit(common.run_script(sys.argv[1:], funcs)) | 154 sys.exit(common.run_script(sys.argv[1:], funcs)) |
145 sys.exit(main()) | 155 sys.exit(main()) |
OLD | NEW |