OLD | NEW |
1 # Copyright 2015 The Chromium Authors. All rights reserved. | 1 # Copyright 2015 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """API for the perf try job recipe module. | 5 """API for the perf try job recipe module. |
6 | 6 |
7 This API is meant to enable the perf try job recipe on any chromium-supported | 7 This API is meant to enable the perf try job recipe on any chromium-supported |
8 platform for any test that can be run via buildbot, perf or otherwise. | 8 platform for any test that can be run via buildbot, perf or otherwise. |
9 """ | 9 """ |
10 | 10 |
11 import re | 11 import re |
12 import urllib | 12 import urllib |
13 import uuid | |
14 | 13 |
15 from recipe_engine import recipe_api | 14 from recipe_engine import recipe_api |
16 from . import build_state | |
17 | 15 |
18 PERF_CONFIG_FILE = 'tools/run-perf-test.cfg' | 16 PERF_CONFIG_FILE = 'tools/run-perf-test.cfg' |
19 WEBKIT_PERF_CONFIG_FILE = 'third_party/WebKit/Tools/run-perf-test.cfg' | 17 WEBKIT_PERF_CONFIG_FILE = 'third_party/WebKit/Tools/run-perf-test.cfg' |
20 PERF_BENCHMARKS_PATH = 'tools/perf/benchmarks' | 18 PERF_BENCHMARKS_PATH = 'tools/perf/benchmarks' |
21 PERF_MEASUREMENTS_PATH = 'tools/perf/measurements' | 19 PERF_MEASUREMENTS_PATH = 'tools/perf/measurements' |
22 BUILDBOT_BUILDERNAME = 'BUILDBOT_BUILDERNAME' | 20 BUILDBOT_BUILDERNAME = 'BUILDBOT_BUILDERNAME' |
23 BENCHMARKS_JSON_FILE = 'benchmarks.json' | 21 BENCHMARKS_JSON_FILE = 'benchmarks.json' |
24 | 22 |
25 CLOUD_RESULTS_LINK = (r'\s(?P<VALUES>http://storage.googleapis.com/' | 23 CLOUD_RESULTS_LINK = (r'\s(?P<VALUES>http://storage.googleapis.com/' |
26 'chromium-telemetry/html-results/results-[a-z0-9-_]+)\s') | 24 'chromium-telemetry/html-results/results-[a-z0-9-_]+)\s') |
27 PROFILER_RESULTS_LINK = (r'\s(?P<VALUES>https://console.developers.google.com/' | 25 PROFILER_RESULTS_LINK = (r'\s(?P<VALUES>https://console.developers.google.com/' |
28 'm/cloudstorage/b/[a-z-]+/o/profiler-[a-z0-9-_.]+)\s') | 26 'm/cloudstorage/b/[a-z-]+/o/profiler-[a-z0-9-_.]+)\s') |
29 RESULTS_BANNER = """ | 27 RESULTS_BANNER = """ |
30 ===== PERF TRY JOB RESULTS ===== | 28 ===== PERF TRY JOB RESULTS ===== |
31 | 29 |
32 Test Command: %(command)s | 30 Test Command: %(command)s |
33 Test Metric: %(metric)s | 31 Test Metric: %(metric)s |
34 Relative Change: %(relative_change).05f%% | 32 Relative Change: %(relative_change).05f%% |
35 Standard Error: +- %(std_err).05f delta | 33 Standard Error: +- %(std_err).05f delta |
36 | 34 |
37 %(results)s | 35 %(results)s |
38 """ | 36 """ |
39 SERVICE_ACCOUNT = 'chromium-bisect' | 37 |
40 | 38 |
41 class PerfTryJobApi(recipe_api.RecipeApi): | 39 class PerfTryJobApi(recipe_api.RecipeApi): |
42 | 40 |
43 def __init__(self, *args, **kwargs): | 41 def __init__(self, *args, **kwargs): |
44 super(PerfTryJobApi, self).__init__(*args, **kwargs) | 42 super(PerfTryJobApi, self).__init__(*args, **kwargs) |
45 | 43 |
46 def start_perf_try_job(self, affected_files, bot_update_step, bot_db): | 44 def start_perf_try_job(self, affected_files, bot_update_step, bot_db): |
47 """Entry point pert tryjob or CQ tryjob.""" | 45 """Entry point pert tryjob or CQ tryjob.""" |
48 perf_config = self._get_perf_config(affected_files) | 46 perf_config = self._get_perf_config(affected_files) |
49 if perf_config: | 47 if perf_config: |
(...skipping 139 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
189 # try job leading to unwanted cache and temp data. The best way to | 187 # try job leading to unwanted cache and temp data. The best way to |
190 # ensure the old build directory is removed before doing any | 188 # ensure the old build directory is removed before doing any |
191 # compilation. | 189 # compilation. |
192 self.m.file.rmtree( | 190 self.m.file.rmtree( |
193 'build directory', | 191 'build directory', |
194 self.m.chromium.c.build_dir.join(self.m.chromium.c.build_config_fs)) | 192 self.m.chromium.c.build_dir.join(self.m.chromium.c.build_config_fs)) |
195 self.m.chromium_tests.transient_check( | 193 self.m.chromium_tests.transient_check( |
196 update_step, | 194 update_step, |
197 lambda transform_name: self.m.chromium_tests.run_mb_and_compile( | 195 lambda transform_name: self.m.chromium_tests.run_mb_and_compile( |
198 compile_targets, None, name_suffix=transform_name(''))) | 196 compile_targets, None, name_suffix=transform_name(''))) |
199 else: # pragma: no cover | 197 else: |
200 self.m.chromium_tests.run_mb_and_compile( | 198 self.m.chromium_tests.run_mb_and_compile( |
201 compile_targets, None, name_suffix=' %s' % name) | 199 compile_targets, None, name_suffix=' %s' % name) |
202 | 200 |
203 def _run_test(self, cfg, **kwargs): | 201 def _run_test(self, cfg, **kwargs): |
204 """Runs test from config and return results.""" | 202 """Runs test from config and return results.""" |
205 values, overall_output, retcodes = self.m.bisect_tester.run_test( | 203 values, overall_output, retcodes = self.m.bisect_tester.run_test( |
206 cfg, **kwargs) | 204 cfg, **kwargs) |
207 all_values = self.m.bisect_tester.digest_run_results(values, retcodes, cfg) | 205 all_values = self.m.bisect_tester.digest_run_results(values, retcodes, cfg) |
208 overall_success = True | 206 overall_success = True |
209 if (not kwargs.get('allow_flakes', True) and | 207 if (not kwargs.get('allow_flakes', True) and |
210 cfg.get('test_type', 'perf') != 'return_code'): | 208 cfg.get('test_type', 'perf') != 'return_code'): |
211 overall_success = all(v == 0 for v in retcodes) | 209 overall_success = all(v == 0 for v in retcodes) |
212 return { | 210 return { |
213 'results': all_values, | 211 'results': all_values, |
214 'ret_code': overall_success, | 212 'ret_code': overall_success, |
215 'output': ''.join(overall_output) | 213 'output': ''.join(overall_output) |
216 } | 214 } |
217 | 215 |
218 def _build_and_run_tests(self, cfg, update_step, bot_db, revision_hash, | 216 def _build_and_run_tests(self, cfg, update_step, bot_db, revision, |
219 **kwargs): | 217 **kwargs): |
220 """Compiles binaries and runs tests for a given a revision.""" | 218 """Compiles binaries and runs tests for a given a revision.""" |
221 with_patch = kwargs.get('name') == 'With Patch' | 219 update_step = self._checkout_revision(update_step, bot_db, revision) |
222 update_step = self._checkout_revision(update_step, bot_db, revision_hash) | 220 self._compile(kwargs['name'], self.m.properties['mastername'], |
223 if update_step.presentation.properties: | 221 self.m.properties['buildername'], update_step, bot_db) |
224 revision_hash = update_step.presentation.properties['got_revision'] | 222 |
225 revision = build_state.BuildState(self, revision_hash, with_patch) | |
226 # request build and wait for it only when the build is nonexistent | |
227 if with_patch or not self._gsutil_file_exists(revision.build_file_path): | |
228 try: | |
229 self._request_build(revision, with_patch) | |
230 self._wait_for(revision) | |
231 except Exception, e: | |
232 raise self.m.step.StepFailure('Error occurs when building %s: %s' % ( | |
233 revision.build_id, str(e))) | |
234 self._download_build(update_step, bot_db, revision) | |
235 if self.m.chromium.c.TARGET_PLATFORM == 'android': | 223 if self.m.chromium.c.TARGET_PLATFORM == 'android': |
236 self.m.chromium_android.adb_install_apk('ChromePublic.apk') | 224 self.m.chromium_android.adb_install_apk('ChromePublic.apk') |
237 | 225 |
238 return self._run_test(cfg, **kwargs) | 226 return self._run_test(cfg, **kwargs) |
239 | 227 |
240 def _gsutil_file_exists(self, path): | |
241 """Returns True if a file exists at the given GS path.""" | |
242 try: | |
243 self.m.gsutil(['ls', path], name='exists') | |
244 except self.m.step.StepFailure: # pragma: no cover | |
245 return False | |
246 return True | |
247 | |
248 # Duplicate code from auto_bisect.revision_state._request_build | |
249 def _request_build(self, revision, with_patch): | |
250 if self.m.chromium.c.TARGET_PLATFORM == 'android': | |
251 self.m.chromium_android.clean_local_files() | |
252 else: | |
253 # Removes any chrome temporary files or build.dead directories. | |
254 self.m.chromium.cleanup_temp() | |
255 if with_patch: | |
256 properties = { | |
257 'parent_got_revision': revision.commit_hash, | |
258 'clobber': True, | |
259 'build_archive_url': revision.build_file_path, | |
260 'issue': self.m.properties['issue'], | |
261 'patch_storage': self.m.properties['patch_storage'], | |
262 'patchset': self.m.properties['patchset'], | |
263 'rietveld': self.m.properties['rietveld'] | |
264 } | |
265 else: | |
266 properties = { | |
267 'parent_got_revision': revision.commit_hash, | |
268 'clobber': True, | |
269 'build_archive_url': revision.build_file_path, | |
270 } | |
271 bot_name = self.get_builder_bot_for_this_platform() | |
272 if self.m.properties.get('is_test'): | |
273 client_operation_id = '123456' | |
274 else: | |
275 client_operation_id = uuid.uuid4().hex # pragma: no cover | |
276 build_details = { | |
277 'bucket': 'master.' + self.m.properties['mastername'], | |
278 'parameters': { | |
279 'buildername': bot_name, | |
280 'properties': properties | |
281 }, | |
282 'client_operation_id': client_operation_id, | |
283 'tags':{} | |
284 } | |
285 result = self.m.buildbucket.put( | |
286 [build_details], | |
287 self.m.service_account.get_json_path(SERVICE_ACCOUNT)) | |
288 revision.build_id = result.stdout['results'][0]['build']['id'] | |
289 | |
290 | |
291 def _wait_for(self, revision): # pragma: no cover | |
292 while True: | |
293 if revision.is_completed(): | |
294 if revision.is_build_archived: | |
295 break | |
296 raise self.m.step.StepFailure('Build %s fails' % revision.build_id) | |
297 else: | |
298 self.m.python.inline( | |
299 'sleeping', | |
300 """ | |
301 import sys | |
302 import time | |
303 time.sleep(20*60) | |
304 sys.exit(0) | |
305 """) | |
306 | |
307 # Duplicate code from auto_bisect.api.start_test_run_for_bisect | |
308 def _download_build(self, update_step, bot_db, | |
309 revision, run_locally=False, | |
310 skip_download=False): | |
311 mastername = self.m.properties.get('mastername') | |
312 buildername = self.m.properties.get('buildername') | |
313 bot_config = bot_db.get_bot_config(mastername, buildername) | |
314 build_archive_url = revision.build_file_path | |
315 if not skip_download: | |
316 if self.m.chromium.c.TARGET_PLATFORM == 'android': | |
317 # The best way to ensure the old build directory is not used is to | |
318 # remove it. | |
319 build_dir = self.m.chromium.c.build_dir.join( | |
320 self.m.chromium.c.build_config_fs) | |
321 self.m.file.rmtree('build directory', build_dir) | |
322 | |
323 # The way android builders on tryserver.chromium.perf are archived is | |
324 # different from builders on chromium.perf. In order to support both | |
325 # forms of archives, we added this temporary hack until builders are | |
326 # fixed. See http://crbug.com/535218. | |
327 zip_dir = self.m.path.join(self.m.path['checkout'], 'full-build-linux') | |
328 if self.m.path.exists(zip_dir): # pragma: no cover | |
329 self.m.file.rmtree('full-build-linux directory', zip_dir) | |
330 gs_bucket = 'gs://%s/' % revision.bucket | |
331 archive_path = build_archive_url[len(gs_bucket):] | |
332 self.m.chromium_android.download_build( | |
333 bucket=bot_config['bucket'], | |
334 path=archive_path) | |
335 | |
336 # The way android builders on tryserver.chromium.perf are archived is | |
337 # different from builders on chromium.perf. In order to support both | |
338 # forms of archives, we added this temporary hack until builders are | |
339 # fixed. See http://crbug.com/535218. | |
340 if self.m.path.exists(zip_dir): # pragma: no cover | |
341 self.m.python.inline( | |
342 'moving full-build-linux to out/Release', | |
343 """ | |
344 import shutil | |
345 import sys | |
346 shutil.move(sys.argv[1], sys.argv[2]) | |
347 """, | |
348 args=[zip_dir, build_dir]) | |
349 else: | |
350 self.m.chromium_tests.download_and_unzip_build( | |
351 mastername, buildername, update_step, bot_db, | |
352 build_archive_url=build_archive_url, | |
353 build_revision=revision.commit_hash, | |
354 override_bot_type='tester') | |
355 | |
356 # Duplicate code from auto_bisect.bisector.get_builder_bot_for_this_platform | |
357 def get_builder_bot_for_this_platform(self): # pragma: no cover | |
358 bot_name = self.m.properties.get('buildername', '') | |
359 if 'win' in bot_name: | |
360 if any(b in bot_name for b in ['x64', 'gpu']): | |
361 return 'winx64_bisect_builder' | |
362 return 'win_perf_bisect_builder' | |
363 | |
364 if 'android' in bot_name: | |
365 if 'nexus9' in bot_name: | |
366 return 'android_arm64_perf_bisect_builder' | |
367 return 'android_perf_bisect_builder' | |
368 | |
369 if 'mac' in bot_name: | |
370 return 'mac_perf_bisect_builder' | |
371 | |
372 return 'linux_perf_bisect_builder' | |
373 | |
374 def _load_config_file(self, name, src_path, **kwargs): | 228 def _load_config_file(self, name, src_path, **kwargs): |
375 """Attempts to load the specified config file and grab config dict.""" | 229 """Attempts to load the specified config file and grab config dict.""" |
376 step_result = self.m.python( | 230 step_result = self.m.python( |
377 name, | 231 name, |
378 self.resource('load_config_to_json.py'), | 232 self.resource('load_config_to_json.py'), |
379 ['--source', src_path, '--output_json', self.m.json.output()], | 233 ['--source', src_path, '--output_json', self.m.json.output()], |
380 **kwargs) | 234 **kwargs) |
381 if not step_result.json.output: # pragma: no cover | 235 if not step_result.json.output: # pragma: no cover |
382 raise self.m.step.StepFailure('Loading config file failed. [%s]' % | 236 raise self.m.step.StepFailure('Loading config file failed. [%s]' % |
383 src_path) | 237 src_path) |
(...skipping 273 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
657 | 511 |
658 def _prepend_src_to_path_in_command(test_cfg): | 512 def _prepend_src_to_path_in_command(test_cfg): |
659 command_to_run = [] | 513 command_to_run = [] |
660 for v in test_cfg.get('command').split(): | 514 for v in test_cfg.get('command').split(): |
661 if v in ['./tools/perf/run_benchmark', | 515 if v in ['./tools/perf/run_benchmark', |
662 'tools/perf/run_benchmark', | 516 'tools/perf/run_benchmark', |
663 'tools\\perf\\run_benchmark']: | 517 'tools\\perf\\run_benchmark']: |
664 v = 'src/tools/perf/run_benchmark' | 518 v = 'src/tools/perf/run_benchmark' |
665 command_to_run.append(v) | 519 command_to_run.append(v) |
666 test_cfg.update({'command': ' '.join(command_to_run)}) | 520 test_cfg.update({'command': ' '.join(command_to_run)}) |
OLD | NEW |