OLD | NEW |
1 # Copyright 2015 The Chromium Authors. All rights reserved. | 1 # Copyright 2015 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 import argparse | 4 import argparse |
5 import logging | 5 import logging |
6 import json | 6 import json |
7 import StringIO | 7 import StringIO |
8 import unittest | 8 import unittest |
9 | 9 |
10 import mock | 10 import mock |
(...skipping 315 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
326 | 326 |
327 command = trybot_command.Trybot() | 327 command = trybot_command.Trybot() |
328 options = argparse.Namespace(trybot='android-nexus4', benchmark_name='foo') | 328 options = argparse.Namespace(trybot='android-nexus4', benchmark_name='foo') |
329 command.Run(options, []) | 329 command.Run(options, []) |
330 self.assertEquals( | 330 self.assertEquals( |
331 ('Error creating branch telemetry-tryjob. ' | 331 ('Error creating branch telemetry-tryjob. ' |
332 'Please delete it if it exists.\n' | 332 'Please delete it if it exists.\n' |
333 'fatal: A branch named \'telemetry-try\' already exists.\n'), | 333 'fatal: A branch named \'telemetry-try\' already exists.\n'), |
334 self.log_output.getvalue()) | 334 self.log_output.getvalue()) |
335 | 335 |
336 def _GetConfigForTrybot(self, name, platform, branch, cfg_filename, | 336 def _SetupTrybotCommand(self, try_json_dict, trybot): |
337 is_blink=False, extra_benchmark_args=None): | 337 self._MockTryserverJson(try_json_dict) |
| 338 command = trybot_command.Trybot() |
| 339 command._InitializeBuilderNames(trybot) |
| 340 return command |
| 341 |
| 342 def _GetConfigForTrybot(self, name, platform, extra_benchmark_args=None, |
| 343 repo_name='chromium'): |
338 bot = '%s_perf_bisect' % name.replace('', '').replace('-', '_') | 344 bot = '%s_perf_bisect' % name.replace('', '').replace('-', '_') |
339 self._MockTryserverJson({bot: 'stuff'}) | 345 command = self._SetupTrybotCommand({bot: 'stuff'}, name) |
340 first_processes = () | 346 options = argparse.Namespace( |
341 if is_blink: | 347 trybot=name, benchmark_name='sunspider', repo_name=repo_name) |
342 first_processes = ( | 348 extra_benchmark_args = extra_benchmark_args or [] |
343 (['git', 'rev-parse', '--abbrev-ref', 'HEAD'], (0, 'br', None)), | 349 arguments = [options.benchmark_name] + extra_benchmark_args |
344 (['git', 'update-index', '--refresh', '-q'], (0, None, None,)), | 350 cfg = command._GetPerfConfig(platform, arguments) |
345 (['git', 'diff-index', 'HEAD'], (0, '', None)), | |
346 (['git', 'log', 'origin/master..HEAD'], (0, '', None)) | |
347 ) | |
348 self._ExpectProcesses(first_processes + ( | |
349 (['git', 'rev-parse', '--abbrev-ref', 'HEAD'], (0, branch, None)), | |
350 (['git', 'update-index', '--refresh', '-q'], (0, None, None,)), | |
351 (['git', 'diff-index', 'HEAD'], (0, '', None)), | |
352 (['git', 'log', 'origin/master..HEAD'], (0, 'logs here', None)), | |
353 (['git', 'checkout', '-b', 'telemetry-tryjob'], (0, None, None)), | |
354 (['git', 'branch', '--set-upstream-to', 'origin/master'], | |
355 (0, None, None)), | |
356 (['git', 'commit', '-a', '-m', 'bisect config: %s' % platform], | |
357 (0, None, None)), | |
358 (['git', 'cl', 'upload', '-f', '--bypass-hooks', '-m', | |
359 'CL for perf tryjob on %s' % platform], | |
360 (0, 'stuff https://codereview.chromium.org/12345 stuff', None)), | |
361 (['git', 'cl', 'try', '-m', 'tryserver.chromium.perf', '-b', bot], | |
362 (0, None, None)), | |
363 (['git', 'checkout', branch], (0, None, None)), | |
364 (['git', 'branch', '-D', 'telemetry-tryjob'], (0, None, None)) | |
365 )) | |
366 cfg = StringIO.StringIO() | |
367 self._stubs.open.files = {cfg_filename: cfg} | |
368 | 351 |
369 options = argparse.Namespace(trybot=name, benchmark_name='sunspider') | 352 return cfg |
370 command = trybot_command.Trybot() | 353 |
371 extra_benchmark_args = extra_benchmark_args or [] | 354 def _GetTestConfigForGitTryParams(self, name, platform, repo_name): |
372 command.Run(options, extra_benchmark_args) | 355 config = self._GetConfigForTrybot(name, platform, repo_name=repo_name) |
373 return cfg.getvalue() | 356 return 'perf_try_config=%s' % json.dumps( |
| 357 config, sort_keys=True, indent=2, separators=(',', ': ')) |
374 | 358 |
375 def testConfigAndroid(self): | 359 def testConfigAndroid(self): |
376 config = self._GetConfigForTrybot( | 360 config = self._GetConfigForTrybot('android-nexus4', 'android') |
377 'android-nexus4', 'android', 'somebranch', | |
378 'tools/run-perf-test.cfg') | |
379 self.assertEquals( | 361 self.assertEquals( |
380 ('config = {\n' | 362 {'command': ('src/tools/perf/run_benchmark ' |
381 ' "command": "./tools/perf/run_benchmark ' | 363 '--browser=android-chromium sunspider --verbose'), |
382 '--browser=android-chromium sunspider --verbose",\n' | 364 'max_time_minutes': '120', |
383 ' "max_time_minutes": "120",\n' | 365 'repeat_count': '1', |
384 ' "repeat_count": "1",\n' | 366 'target_arch': 'ia32', |
385 ' "target_arch": "ia32",\n' | 367 'truncate_percent': '0' |
386 ' "truncate_percent": "0"\n' | 368 }, config) |
387 '}'), config) | |
388 | 369 |
389 def testConfigMac(self): | 370 def testConfigMac(self): |
390 config = self._GetConfigForTrybot( | 371 config = self._GetConfigForTrybot('mac-10-9', 'mac') |
391 'mac-10-9', 'mac', 'currentwork', 'tools/run-perf-test.cfg') | |
392 self.assertEquals( | 372 self.assertEquals( |
393 ('config = {\n' | 373 {'command': ('src/tools/perf/run_benchmark ' |
394 ' "command": "./tools/perf/run_benchmark ' | 374 '--browser=release sunspider --verbose'), |
395 '--browser=release sunspider --verbose",\n' | 375 'max_time_minutes': '120', |
396 ' "max_time_minutes": "120",\n' | 376 'repeat_count': '1', |
397 ' "repeat_count": "1",\n' | 377 'target_arch': 'ia32', |
398 ' "target_arch": "ia32",\n' | 378 'truncate_percent': '0' |
399 ' "truncate_percent": "0"\n' | 379 }, config) |
400 '}'), config) | |
401 | 380 |
402 def testConfigWinX64(self): | 381 def testConfigWinX64(self): |
403 config = self._GetConfigForTrybot( | 382 config = self._GetConfigForTrybot('win-x64', 'win-x64') |
404 'win-x64', 'win-x64', 'currentwork', 'tools/run-perf-test.cfg') | 383 |
405 self.assertEquals( | 384 self.assertEquals( |
406 ('config = {\n' | 385 {'command': ('src/tools/perf/run_benchmark ' |
407 ' "command": "python tools\\\\perf\\\\run_benchmark ' | 386 '--browser=release_x64 sunspider --verbose'), |
408 '--browser=release_x64 sunspider --verbose",\n' | 387 'max_time_minutes': '120', |
409 ' "max_time_minutes": "120",\n' | 388 'repeat_count': '1', |
410 ' "repeat_count": "1",\n' | 389 'target_arch': 'x64', |
411 ' "target_arch": "x64",\n' | 390 'truncate_percent': '0' |
412 ' "truncate_percent": "0"\n' | 391 }, config) |
413 '}'), config) | 392 |
414 | |
415 def testVerboseOptionIsNotAddedTwice(self): | 393 def testVerboseOptionIsNotAddedTwice(self): |
416 config = self._GetConfigForTrybot( | 394 config = self._GetConfigForTrybot('win-x64', 'win-x64', |
417 'win-x64', 'win-x64', 'currentwork', 'tools/run-perf-test.cfg', | 395 extra_benchmark_args=['-v']) |
418 extra_benchmark_args=['-v']) | |
419 self.assertEquals( | 396 self.assertEquals( |
420 ('config = {\n' | 397 {'command': ('src/tools/perf/run_benchmark ' |
421 ' "command": "python tools\\\\perf\\\\run_benchmark ' | 398 '--browser=release_x64 sunspider -v'), |
422 '--browser=release_x64 sunspider -v",\n' | 399 'max_time_minutes': '120', |
423 ' "max_time_minutes": "120",\n' | 400 'repeat_count': '1', |
424 ' "repeat_count": "1",\n' | 401 'target_arch': 'x64', |
425 ' "target_arch": "x64",\n' | 402 'truncate_percent': '0' |
426 ' "truncate_percent": "0"\n' | 403 }, config) |
427 '}'), config) | |
428 | 404 |
429 def testConfigWinX64WithNoHyphen(self): | 405 def testConfigWinX64WithNoHyphen(self): |
430 config = self._GetConfigForTrybot( | 406 config = self._GetConfigForTrybot('winx64nvidia', 'win-x64') |
431 'winx64nvidia', 'win-x64', 'currentwork', 'tools/run-perf-test.cfg') | |
432 self.assertEquals( | 407 self.assertEquals( |
433 ('config = {\n' | 408 {'command': ('src/tools/perf/run_benchmark ' |
434 ' "command": "python tools\\\\perf\\\\run_benchmark ' | 409 '--browser=release_x64 sunspider --verbose'), |
435 '--browser=release_x64 sunspider --verbose",\n' | 410 'max_time_minutes': '120', |
436 ' "max_time_minutes": "120",\n' | 411 'repeat_count': '1', |
437 ' "repeat_count": "1",\n' | 412 'target_arch': 'x64', |
438 ' "target_arch": "x64",\n' | 413 'truncate_percent': '0' |
439 ' "truncate_percent": "0"\n' | 414 }, config) |
440 '}'), config) | |
441 | 415 |
442 def testUnsupportedTrybot(self): | 416 def testUnsupportedTrybot(self): |
443 self.assertRaises( | 417 self.assertRaises( |
444 trybot_command.TrybotError, | 418 trybot_command.TrybotError, |
445 trybot_command._GetBuilderNames, | 419 trybot_command._GetBuilderNames, |
446 'arms-nvidia', | 420 'arms-nvidia', |
447 {'win_perf_bisect': 'stuff'} | 421 {'win_perf_bisect': 'stuff'} |
448 ) | 422 ) |
449 | 423 |
450 def testConfigBlink(self): | 424 def testUpdateConfigGitCommitTrybotError(self): |
451 config = self._GetConfigForTrybot( | |
452 'mac-10-9', 'mac', 'blinkbranch', | |
453 'Tools/run-perf-test.cfg', True) | |
454 self.assertEquals( | |
455 ('config = {\n' | |
456 ' "command": "./tools/perf/run_benchmark ' | |
457 '--browser=release sunspider --verbose",\n' | |
458 ' "max_time_minutes": "120",\n' | |
459 ' "repeat_count": "1",\n' | |
460 ' "target_arch": "ia32",\n' | |
461 ' "truncate_percent": "0"\n' | |
462 '}'), config) | |
463 | 425 |
464 def testUpdateConfigGitCommitTrybotError(self): | 426 command = self._SetupTrybotCommand( |
465 self._MockTryserverJson({'android_nexus4_perf_bisect': 'stuff'}) | 427 ('android_nexus4_perf_bisect': 'stuff'}, 'android-nexus4') |
466 command = trybot_command.Trybot() | |
467 command._InitializeBuilderNames('android-nexus4') | |
468 self._ExpectProcesses(( | 428 self._ExpectProcesses(( |
469 (['git', 'commit', '-a', '-m', 'bisect config: android'], | 429 (['git', 'commit', '-a', '-m', 'bisect config: android'], |
470 (128, 'None', 'commit failed')), | 430 (128, 'None', 'commit failed')), |
471 (['git', 'cl', 'upload', '-f', '--bypass-hooks', '-m', | 431 (['git', 'cl', 'upload', '-f', '--bypass-hooks', '-m', |
472 'CL for perf tryjob on android'], | 432 'CL for perf tryjob on android'], |
473 (0, 'stuff https://codereview.chromium.org/12345 stuff', None)), | 433 (0, 'stuff https://codereview.chromium.org/12345 stuff', None)), |
474 (['git', 'cl', 'try', '-m', 'tryserver.chromium.perf', '-b', | 434 (['git', 'cl', 'try', '-m', 'tryserver.chromium.perf', '-b', |
475 'android_nexus4_perf_bisect'], (0, None, None)))) | 435 'android_nexus4_perf_bisect', |
| 436 '=p', ], (0, None, None)))) |
476 cfg_filename = 'tools/run-perf-test.cfg' | 437 cfg_filename = 'tools/run-perf-test.cfg' |
477 cfg = StringIO.StringIO() | 438 cfg = StringIO.StringIO() |
478 self._stubs.open.files = {cfg_filename: cfg} | 439 self._stubs.open.files = {cfg_filename: cfg} |
479 self.assertRaises( | 440 self.assertRaises( |
480 trybot_command.TrybotError, command._UpdateConfigAndRunTryjob, | 441 trybot_command.TrybotError, command._UpdateConfigAndRunTryjob, |
481 'android', cfg_filename, []) | 442 'android', cfg_filename, []) |
482 | 443 |
483 def testUpdateConfigGitUploadTrybotError(self): | 444 def testUpdateConfigGitUploadTrybotError(self): |
484 self._MockTryserverJson({'android_nexus4_perf_bisect': 'stuff'}) | 445 self._MockTryserverJson({'android_nexus4_perf_bisect': 'stuff'}) |
485 command = trybot_command.Trybot() | 446 command = trybot_command.Trybot() |
(...skipping 186 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
672 @benchmark.Enabled('win', 'mac') | 633 @benchmark.Enabled('win', 'mac') |
673 class FooBenchmark(benchmark.Benchmark): | 634 class FooBenchmark(benchmark.Benchmark): |
674 pass | 635 pass |
675 self.assertFalse(self.IsBenchmarkDisabled(FooBenchmark, 'all')) | 636 self.assertFalse(self.IsBenchmarkDisabled(FooBenchmark, 'all')) |
676 self.assertFalse(self.IsBenchmarkDisabled(FooBenchmark, 'all-mac')) | 637 self.assertFalse(self.IsBenchmarkDisabled(FooBenchmark, 'all-mac')) |
677 self.assertFalse(self.IsBenchmarkDisabled(FooBenchmark, 'winx64ati')) | 638 self.assertFalse(self.IsBenchmarkDisabled(FooBenchmark, 'winx64ati')) |
678 | 639 |
679 self.assertTrue(self.IsBenchmarkDisabled(FooBenchmark, 'android-s5')) | 640 self.assertTrue(self.IsBenchmarkDisabled(FooBenchmark, 'android-s5')) |
680 self.assertTrue(self.IsBenchmarkDisabled(FooBenchmark, 'linux')) | 641 self.assertTrue(self.IsBenchmarkDisabled(FooBenchmark, 'linux')) |
681 self.assertTrue(self.IsBenchmarkDisabled(FooBenchmark, 'all-linux')) | 642 self.assertTrue(self.IsBenchmarkDisabled(FooBenchmark, 'all-linux')) |
OLD | NEW |