| OLD | NEW |
| 1 # Copyright (C) 2012 Google Inc. All rights reserved. | 1 # Copyright (C) 2012 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 96 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 107 | 107 |
| 108 def test_collect_tests_with_ignored_skipped_list(self): | 108 def test_collect_tests_with_ignored_skipped_list(self): |
| 109 runner, port = self.create_runner(args=['--force']) | 109 runner, port = self.create_runner(args=['--force']) |
| 110 | 110 |
| 111 self._add_file(runner, 'inspector', 'test1.html') | 111 self._add_file(runner, 'inspector', 'test1.html') |
| 112 self._add_file(runner, 'inspector', 'unsupported_test1.html') | 112 self._add_file(runner, 'inspector', 'unsupported_test1.html') |
| 113 self._add_file(runner, 'inspector', 'test2.html') | 113 self._add_file(runner, 'inspector', 'test2.html') |
| 114 self._add_file(runner, 'inspector/resources', 'resource_file.html') | 114 self._add_file(runner, 'inspector/resources', 'resource_file.html') |
| 115 self._add_file(runner, 'unsupported', 'unsupported_test2.html') | 115 self._add_file(runner, 'unsupported', 'unsupported_test2.html') |
| 116 port.skipped_perf_tests = lambda: ['inspector/unsupported_test1.html', '
unsupported'] | 116 port.skipped_perf_tests = lambda: ['inspector/unsupported_test1.html', '
unsupported'] |
| 117 self.assertItemsEqual(self._collect_tests_and_sort_test_name(runner), [ | 117 self.assertItemsEqual( |
| 118 'inspector/test1.html', 'inspector/test2.html', 'i
nspector/unsupported_test1.html', 'unsupported/unsupported_test2.html']) | 118 self._collect_tests_and_sort_test_name(runner), |
| 119 [ |
| 120 'inspector/test1.html', |
| 121 'inspector/test2.html', |
| 122 'inspector/unsupported_test1.html', |
| 123 'unsupported/unsupported_test2.html' |
| 124 ]) |
| 119 | 125 |
| 120 def test_default_args(self): | 126 def test_default_args(self): |
| 121 options, _ = PerfTestsRunner._parse_args([]) | 127 options, _ = PerfTestsRunner._parse_args([]) |
| 122 self.assertTrue(options.build) | 128 self.assertTrue(options.build) |
| 123 self.assertEqual(options.time_out_ms, 600 * 1000) | 129 self.assertEqual(options.time_out_ms, 600 * 1000) |
| 124 self.assertTrue(options.generate_results) | 130 self.assertTrue(options.generate_results) |
| 125 self.assertTrue(options.show_results) | 131 self.assertTrue(options.show_results) |
| 126 self.assertTrue(options.use_skipped_list) | 132 self.assertTrue(options.use_skipped_list) |
| 127 self.assertEqual(options.repeat, 1) | 133 self.assertEqual(options.repeat, 1) |
| 128 self.assertEqual(options.test_runner_count, DEFAULT_TEST_RUNNER_COUNT) | 134 self.assertEqual(options.test_runner_count, DEFAULT_TEST_RUNNER_COUNT) |
| 129 | 135 |
| 130 def test_parse_args(self): | 136 def test_parse_args(self): |
| 131 options, _ = PerfTestsRunner._parse_args([ | 137 options, _ = PerfTestsRunner._parse_args([ |
| 132 '--build-directory=folder42', | 138 '--build-directory=folder42', |
| 133 '--platform=platform42', | 139 '--platform=platform42', |
| 134 '--builder-name', 'webkit-mac-1', | 140 '--builder-name', 'webkit-mac-1', |
| 135 '--build-number=56', | 141 '--build-number=56', |
| 136 '--time-out-ms=42', | 142 '--time-out-ms=42', |
| 137 '--no-show-results', | 143 '--no-show-results', |
| 138 '--reset-results', | 144 '--reset-results', |
| 139 '--output-json-path=a/output.json', | 145 '--output-json-path=a/output.json', |
| 140 '--slave-config-json-path=a/source.json', | 146 '--slave-config-json-path=a/source.json', |
| 141 '--test-results-server=somehost', | 147 '--test-results-server=somehost', |
| 142 '--additional-driver-flag=--enable-threaded-parser', | 148 '--additional-driver-flag=--enable-threaded-parser', |
| 143 '--additional-driver-flag=--awesomesauce', | 149 '--additional-driver-flag=--awesomesauce', |
| 144 '--repeat=5', | 150 '--repeat=5', |
| 145 '--test-runner-count=5', | 151 '--test-runner-count=5', |
| 146 '--debug']) | 152 '--debug' |
| 153 ]) |
| 147 self.assertTrue(options.build) | 154 self.assertTrue(options.build) |
| 148 self.assertEqual(options.build_directory, 'folder42') | 155 self.assertEqual(options.build_directory, 'folder42') |
| 149 self.assertEqual(options.platform, 'platform42') | 156 self.assertEqual(options.platform, 'platform42') |
| 150 self.assertEqual(options.builder_name, 'webkit-mac-1') | 157 self.assertEqual(options.builder_name, 'webkit-mac-1') |
| 151 self.assertEqual(options.build_number, '56') | 158 self.assertEqual(options.build_number, '56') |
| 152 self.assertEqual(options.time_out_ms, '42') | 159 self.assertEqual(options.time_out_ms, '42') |
| 153 self.assertEqual(options.configuration, 'Debug') | 160 self.assertEqual(options.configuration, 'Debug') |
| 154 self.assertFalse(options.show_results) | 161 self.assertFalse(options.show_results) |
| 155 self.assertTrue(options.reset_results) | 162 self.assertTrue(options.reset_results) |
| 156 self.assertEqual(options.output_json_path, 'a/output.json') | 163 self.assertEqual(options.output_json_path, 'a/output.json') |
| (...skipping 384 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 541 self._test_run_with_json_output(runner, port.host.filesystem, upload_suc
ceeds=True) | 548 self._test_run_with_json_output(runner, port.host.filesystem, upload_suc
ceeds=True) |
| 542 self.assertEqual(self._load_output_json(runner), [{ | 549 self.assertEqual(self._load_output_json(runner), [{ |
| 543 "buildTime": "2013-02-08T15:19:37.460000", "tests": self._event_targ
et_wrapper_and_inspector_results, | 550 "buildTime": "2013-02-08T15:19:37.460000", "tests": self._event_targ
et_wrapper_and_inspector_results, |
| 544 "revisions": {"chromium": {"timestamp": "2013-02-01 08:48:05 +0000",
"revision": "5678"}}}]) | 551 "revisions": {"chromium": {"timestamp": "2013-02-01 08:48:05 +0000",
"revision": "5678"}}}]) |
| 545 | 552 |
| 546 filesystem = port.host.filesystem | 553 filesystem = port.host.filesystem |
| 547 self.assertTrue(filesystem.isfile(runner._output_json_path())) | 554 self.assertTrue(filesystem.isfile(runner._output_json_path())) |
| 548 self.assertTrue(filesystem.isfile(filesystem.splitext(runner._output_jso
n_path())[0] + '.html')) | 555 self.assertTrue(filesystem.isfile(filesystem.splitext(runner._output_jso
n_path())[0] + '.html')) |
| 549 | 556 |
| 550 def test_run_with_description(self): | 557 def test_run_with_description(self): |
| 551 runner, port = self.create_runner_and_setup_results_template(args=['--ou
tput-json-path=/mock-checkout/output.json', | 558 runner, port = self.create_runner_and_setup_results_template( |
| 552 '--te
st-results-server=some.host', '--description', 'some description']) | 559 args=['--output-json-path=/mock-checkout/output.json', |
| 560 '--test-results-server=some.host', '--description', 'some desc
ription']) |
| 553 self._test_run_with_json_output(runner, port.host.filesystem, upload_suc
ceeds=True) | 561 self._test_run_with_json_output(runner, port.host.filesystem, upload_suc
ceeds=True) |
| 554 self.assertEqual(self._load_output_json(runner), [{ | 562 self.assertEqual(self._load_output_json(runner), [{ |
| 555 "buildTime": "2013-02-08T15:19:37.460000", "description": "some desc
ription", | 563 "buildTime": "2013-02-08T15:19:37.460000", "description": "some desc
ription", |
| 556 "tests": self._event_target_wrapper_and_inspector_results, | 564 "tests": self._event_target_wrapper_and_inspector_results, |
| 557 "revisions": {"chromium": {"timestamp": "2013-02-01 08:48:05 +0000",
"revision": "5678"}}}]) | 565 "revisions": {"chromium": {"timestamp": "2013-02-01 08:48:05 +0000",
"revision": "5678"}}}]) |
| 558 | 566 |
| 559 def create_runner_and_setup_results_template(self, args=[]): | 567 def create_runner_and_setup_results_template(self, args=[]): |
| 560 runner, port = self.create_runner(args) | 568 runner, port = self.create_runner(args) |
| 561 filesystem = port.host.filesystem | 569 filesystem = port.host.filesystem |
| 562 filesystem.write_text_file(runner._base_path + '/resources/results-templ
ate.html', | 570 filesystem.write_text_file( |
| 563 'BEGIN<script src="%AbsolutePathToWebKitTrunk
%/some.js"></script>' | 571 runner._base_path + '/resources/results-template.html', # pylint: d
isable=protected-access |
| 564 '<script src="%AbsolutePathToWebKitTrunk%/oth
er.js"></script><script>%PeformanceTestsResultsJSON%</script>END') | 572 ('BEGIN<script src="%AbsolutePathToWebKitTrunk%/some.js"></script>' |
| 573 '<script src="%AbsolutePathToWebKitTrunk%/other.js"></script><scrip
t>%PeformanceTestsResultsJSON%</script>END')) |
| 565 filesystem.write_text_file(runner._base_path + '/Dromaeo/resources/droma
eo/web/lib/jquery-1.6.4.js', 'jquery content') | 574 filesystem.write_text_file(runner._base_path + '/Dromaeo/resources/droma
eo/web/lib/jquery-1.6.4.js', 'jquery content') |
| 566 return runner, port | 575 return runner, port |
| 567 | 576 |
| 568 def test_run_respects_no_results(self): | 577 def test_run_respects_no_results(self): |
| 569 runner, port = self.create_runner(args=['--output-json-path=/mock-checko
ut/output.json', | 578 runner, port = self.create_runner(args=['--output-json-path=/mock-checko
ut/output.json', |
| 570 '--test-results-server=some.host
', '--no-results']) | 579 '--test-results-server=some.host
', '--no-results']) |
| 571 self._test_run_with_json_output(runner, port.host.filesystem, upload_suc
ceeds=False, results_shown=False) | 580 self._test_run_with_json_output(runner, port.host.filesystem, upload_suc
ceeds=False, results_shown=False) |
| 572 self.assertFalse(port.host.filesystem.isfile('/mock-checkout/output.json
')) | 581 self.assertFalse(port.host.filesystem.isfile('/mock-checkout/output.json
')) |
| 573 | 582 |
| 574 def test_run_generates_json_by_default(self): | 583 def test_run_generates_json_by_default(self): |
| 575 runner, port = self.create_runner_and_setup_results_template() | 584 runner, port = self.create_runner_and_setup_results_template() |
| 576 filesystem = port.host.filesystem | 585 filesystem = port.host.filesystem |
| 577 output_json_path = runner._output_json_path() | 586 output_json_path = runner._output_json_path() |
| 578 results_page_path = filesystem.splitext(output_json_path)[0] + '.html' | 587 results_page_path = filesystem.splitext(output_json_path)[0] + '.html' |
| 579 | 588 |
| 580 self.assertFalse(filesystem.isfile(output_json_path)) | 589 self.assertFalse(filesystem.isfile(output_json_path)) |
| 581 self.assertFalse(filesystem.isfile(results_page_path)) | 590 self.assertFalse(filesystem.isfile(results_page_path)) |
| 582 | 591 |
| 583 self._test_run_with_json_output(runner, port.host.filesystem) | 592 self._test_run_with_json_output(runner, port.host.filesystem) |
| 584 | 593 |
| 585 self.assertEqual(self._load_output_json(runner), [{ | 594 self.assertEqual( |
| 586 "buildTime": "2013-02-08T15:19:37.460000", "tests": self._event_targ
et_wrapper_and_inspector_results, | 595 self._load_output_json(runner), |
| 587 "revisions": {"chromium": {"timestamp": "2013-02-01 08:48:05 +0000",
"revision": "5678"}}}]) | 596 [ |
| 597 { |
| 598 "buildTime": "2013-02-08T15:19:37.460000", |
| 599 "tests": self._event_target_wrapper_and_inspector_results, |
| 600 "revisions": {"chromium": {"timestamp": "2013-02-01 08:48:05
+0000", "revision": "5678"}} |
| 601 } |
| 602 ]) |
| 588 | 603 |
| 589 self.assertTrue(filesystem.isfile(output_json_path)) | 604 self.assertTrue(filesystem.isfile(output_json_path)) |
| 590 self.assertTrue(filesystem.isfile(results_page_path)) | 605 self.assertTrue(filesystem.isfile(results_page_path)) |
| 591 | 606 |
| 592 def test_run_merges_output_by_default(self): | 607 def test_run_merges_output_by_default(self): |
| 593 runner, port = self.create_runner_and_setup_results_template() | 608 runner, port = self.create_runner_and_setup_results_template() |
| 594 filesystem = port.host.filesystem | 609 filesystem = port.host.filesystem |
| 595 output_json_path = runner._output_json_path() | 610 output_json_path = runner._output_json_path() |
| 596 | 611 |
| 597 filesystem.write_text_file(output_json_path, '[{"previous": "results"}]'
) | 612 filesystem.write_text_file(output_json_path, '[{"previous": "results"}]'
) |
| (...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 660 self.assertEqual(page_shown, []) | 675 self.assertEqual(page_shown, []) |
| 661 | 676 |
| 662 def test_run_with_bad_output_json(self): | 677 def test_run_with_bad_output_json(self): |
| 663 runner, port = self.create_runner_and_setup_results_template(args=['--ou
tput-json-path=/mock-checkout/output.json']) | 678 runner, port = self.create_runner_and_setup_results_template(args=['--ou
tput-json-path=/mock-checkout/output.json']) |
| 664 port.host.filesystem.write_text_file('/mock-checkout/output.json', 'bad
json') | 679 port.host.filesystem.write_text_file('/mock-checkout/output.json', 'bad
json') |
| 665 self._test_run_with_json_output(runner, port.host.filesystem, expected_e
xit_code=PerfTestsRunner.EXIT_CODE_BAD_MERGE) | 680 self._test_run_with_json_output(runner, port.host.filesystem, expected_e
xit_code=PerfTestsRunner.EXIT_CODE_BAD_MERGE) |
| 666 port.host.filesystem.write_text_file('/mock-checkout/output.json', '{"an
other bad json": "1"}') | 681 port.host.filesystem.write_text_file('/mock-checkout/output.json', '{"an
other bad json": "1"}') |
| 667 self._test_run_with_json_output(runner, port.host.filesystem, expected_e
xit_code=PerfTestsRunner.EXIT_CODE_BAD_MERGE) | 682 self._test_run_with_json_output(runner, port.host.filesystem, expected_e
xit_code=PerfTestsRunner.EXIT_CODE_BAD_MERGE) |
| 668 | 683 |
| 669 def test_run_with_slave_config_json(self): | 684 def test_run_with_slave_config_json(self): |
| 670 runner, port = self.create_runner_and_setup_results_template(args=['--ou
tput-json-path=/mock-checkout/output.json', | 685 runner, port = self.create_runner_and_setup_results_template( |
| 671 '--sl
ave-config-json-path=/mock-checkout/slave-config.json', '--test-results-server=s
ome.host']) | 686 args=['--output-json-path=/mock-checkout/output.json', |
| 687 '--slave-config-json-path=/mock-checkout/slave-config.json', |
| 688 '--test-results-server=some.host']) |
| 672 port.host.filesystem.write_text_file('/mock-checkout/slave-config.json',
'{"key": "value"}') | 689 port.host.filesystem.write_text_file('/mock-checkout/slave-config.json',
'{"key": "value"}') |
| 673 self._test_run_with_json_output(runner, port.host.filesystem, upload_suc
ceeds=True) | 690 self._test_run_with_json_output(runner, port.host.filesystem, upload_suc
ceeds=True) |
| 674 self.assertEqual(self._load_output_json(runner), [{ | 691 self.assertEqual(self._load_output_json(runner), [{ |
| 675 "buildTime": "2013-02-08T15:19:37.460000", "tests": self._event_targ
et_wrapper_and_inspector_results, | 692 "buildTime": "2013-02-08T15:19:37.460000", "tests": self._event_targ
et_wrapper_and_inspector_results, |
| 676 "revisions": {"chromium": {"timestamp": "2013-02-01 08:48:05 +0000",
"revision": "5678"}}, "builderKey": "value"}]) | 693 "revisions": {"chromium": {"timestamp": "2013-02-01 08:48:05 +0000",
"revision": "5678"}}, "builderKey": "value"}]) |
| 677 | 694 |
| 678 def test_run_with_bad_slave_config_json(self): | 695 def test_run_with_bad_slave_config_json(self): |
| 679 runner, port = self.create_runner_and_setup_results_template(args=['--ou
tput-json-path=/mock-checkout/output.json', | 696 runner, port = self.create_runner_and_setup_results_template( |
| 680 '--sl
ave-config-json-path=/mock-checkout/slave-config.json', '--test-results-server=s
ome.host']) | 697 args=['--output-json-path=/mock-checkout/output.json', |
| 698 '--slave-config-json-path=/mock-checkout/slave-config.json', |
| 699 '--test-results-server=some.host']) |
| 681 logs = self._test_run_with_json_output(runner, port.host.filesystem, | 700 logs = self._test_run_with_json_output(runner, port.host.filesystem, |
| 682 expected_exit_code=PerfTestsRunne
r.EXIT_CODE_BAD_SOURCE_JSON) | 701 expected_exit_code=PerfTestsRunne
r.EXIT_CODE_BAD_SOURCE_JSON) |
| 683 self.assertTrue('Missing slave configuration JSON file: /mock-checkout/s
lave-config.json' in logs) | 702 self.assertTrue('Missing slave configuration JSON file: /mock-checkout/s
lave-config.json' in logs) |
| 684 port.host.filesystem.write_text_file('/mock-checkout/slave-config.json',
'bad json') | 703 port.host.filesystem.write_text_file('/mock-checkout/slave-config.json',
'bad json') |
| 685 self._test_run_with_json_output(runner, port.host.filesystem, expected_e
xit_code=PerfTestsRunner.EXIT_CODE_BAD_SOURCE_JSON) | 704 self._test_run_with_json_output(runner, port.host.filesystem, expected_e
xit_code=PerfTestsRunner.EXIT_CODE_BAD_SOURCE_JSON) |
| 686 port.host.filesystem.write_text_file('/mock-checkout/slave-config.json',
'["another bad json"]') | 705 port.host.filesystem.write_text_file('/mock-checkout/slave-config.json',
'["another bad json"]') |
| 687 self._test_run_with_json_output(runner, port.host.filesystem, expected_e
xit_code=PerfTestsRunner.EXIT_CODE_BAD_SOURCE_JSON) | 706 self._test_run_with_json_output(runner, port.host.filesystem, expected_e
xit_code=PerfTestsRunner.EXIT_CODE_BAD_SOURCE_JSON) |
| 688 | 707 |
| 689 def test_run_with_multiple_repositories(self): | 708 def test_run_with_multiple_repositories(self): |
| 690 runner, port = self.create_runner_and_setup_results_template(args=['--ou
tput-json-path=/mock-checkout/output.json', | 709 runner, port = self.create_runner_and_setup_results_template(args=['--ou
tput-json-path=/mock-checkout/output.json', |
| (...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 776 self._test_run_with_json_output(runner, port.host.filesystem, compare_lo
gs=False) | 795 self._test_run_with_json_output(runner, port.host.filesystem, compare_lo
gs=False) |
| 777 generated_json = json.loads(port.host.filesystem.files['/mock-checkout/o
utput.json']) | 796 generated_json = json.loads(port.host.filesystem.files['/mock-checkout/o
utput.json']) |
| 778 self.assertTrue(isinstance(generated_json, list)) | 797 self.assertTrue(isinstance(generated_json, list)) |
| 779 self.assertEqual(len(generated_json), 1) | 798 self.assertEqual(len(generated_json), 1) |
| 780 | 799 |
| 781 output = generated_json[0]['tests']['Bindings']['tests']['event-target-w
rapper']['metrics']['Time']['current'] | 800 output = generated_json[0]['tests']['Bindings']['tests']['event-target-w
rapper']['metrics']['Time']['current'] |
| 782 self.assertEqual(len(output), 3) | 801 self.assertEqual(len(output), 3) |
| 783 expectedMetrics = EventTargetWrapperTestData.results['metrics']['Time'][
'current'][0] | 802 expectedMetrics = EventTargetWrapperTestData.results['metrics']['Time'][
'current'][0] |
| 784 for metrics in output: | 803 for metrics in output: |
| 785 self.assertEqual(metrics, expectedMetrics) | 804 self.assertEqual(metrics, expectedMetrics) |
| OLD | NEW |