| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2006-2009 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2006-2009 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Run layout tests using the test_shell. | 6 """Run layout tests using the test_shell. |
| 7 | 7 |
| 8 This is a port of the existing webkit test script run-webkit-tests. | 8 This is a port of the existing webkit test script run-webkit-tests. |
| 9 | 9 |
| 10 The TestRunner class runs a series of tests (TestType interface) against a set | 10 The TestRunner class runs a series of tests (TestType interface) against a set |
| (...skipping 20 matching lines...) Expand all Loading... |
| 31 import re | 31 import re |
| 32 import shutil | 32 import shutil |
| 33 import subprocess | 33 import subprocess |
| 34 import sys | 34 import sys |
| 35 import time | 35 import time |
| 36 import traceback | 36 import traceback |
| 37 | 37 |
| 38 from layout_package import apache_http_server | 38 from layout_package import apache_http_server |
| 39 from layout_package import test_expectations | 39 from layout_package import test_expectations |
| 40 from layout_package import http_server | 40 from layout_package import http_server |
| 41 from layout_package import json_results_generator | 41 from layout_package import json_layout_results_generator |
| 42 from layout_package import metered_stream | 42 from layout_package import metered_stream |
| 43 from layout_package import path_utils | 43 from layout_package import path_utils |
| 44 from layout_package import platform_utils | 44 from layout_package import platform_utils |
| 45 from layout_package import test_failures | 45 from layout_package import test_failures |
| 46 from layout_package import test_shell_thread | 46 from layout_package import test_shell_thread |
| 47 from layout_package import test_files | 47 from layout_package import test_files |
| 48 from layout_package import websocket_server | 48 from layout_package import websocket_server |
| 49 from test_types import fuzzy_image_diff | 49 from test_types import fuzzy_image_diff |
| 50 from test_types import image_diff | 50 from test_types import image_diff |
| 51 from test_types import test_type_base | 51 from test_types import test_type_base |
| 52 from test_types import text_diff | 52 from test_types import text_diff |
| 53 | 53 |
| 54 sys.path.append(path_utils.PathFromBase('third_party')) | 54 sys.path.append(path_utils.PathFromBase('third_party')) |
| 55 import simplejson | 55 import simplejson |
| 56 | 56 |
| 57 # Indicates that we want detailed progress updates in the output (prints | 57 # Indicates that we want detailed progress updates in the output (prints |
| 58 # directory-by-directory feedback). | 58 # directory-by-directory feedback). |
| 59 LOG_DETAILED_PROGRESS = 'detailed-progress' | 59 LOG_DETAILED_PROGRESS = 'detailed-progress' |
| 60 | 60 |
| 61 # Log any unexpected results while running (instead of just at the end). | 61 # Log any unexpected results while running (instead of just at the end). |
| 62 LOG_UNEXPECTED = 'unexpected' | 62 LOG_UNEXPECTED = 'unexpected' |
| 63 | 63 |
| 64 # Builder base URL where we have the archived test results. |
| 65 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/" |
| 66 |
| 64 TestExpectationsFile = test_expectations.TestExpectationsFile | 67 TestExpectationsFile = test_expectations.TestExpectationsFile |
| 65 | 68 |
| 66 class TestInfo: | 69 class TestInfo: |
| 67 """Groups information about a test for easy passing of data.""" | 70 """Groups information about a test for easy passing of data.""" |
| 68 def __init__(self, filename, timeout): | 71 def __init__(self, filename, timeout): |
| 69 """Generates the URI and stores the filename and timeout for this test. | 72 """Generates the URI and stores the filename and timeout for this test. |
| 70 Args: | 73 Args: |
| 71 filename: Full path to the test. | 74 filename: Full path to the test. |
| 72 timeout: Timeout for running the test in TestShell. | 75 timeout: Timeout for running the test in TestShell. |
| 73 """ | 76 """ |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 122 if len(failures): | 125 if len(failures): |
| 123 self.failures[test] = failures | 126 self.failures[test] = failures |
| 124 if expected: | 127 if expected: |
| 125 self.expected += 1 | 128 self.expected += 1 |
| 126 else: | 129 else: |
| 127 self.unexpected_results[test] = result | 130 self.unexpected_results[test] = result |
| 128 self.unexpected += 1 | 131 self.unexpected += 1 |
| 129 | 132 |
| 130 | 133 |
| 131 class TestRunner: | 134 class TestRunner: |
| 132 """A class for managing running a series of tests on a series of test | 135 """A class for managing running a series of tests on a series of layout test |
| 133 files.""" | 136 files.""" |
| 134 | 137 |
| 135 HTTP_SUBDIR = os.sep.join(['', 'http', '']) | 138 HTTP_SUBDIR = os.sep.join(['', 'http', '']) |
| 136 WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', '']) | 139 WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', '']) |
| 137 | 140 |
| 138 # The per-test timeout in milliseconds, if no --time-out-ms option was given | 141 # The per-test timeout in milliseconds, if no --time-out-ms option was given |
| 139 # to run_webkit_tests. This should correspond to the default timeout in | 142 # to run_webkit_tests. This should correspond to the default timeout in |
| 140 # test_shell.exe. | 143 # test_shell.exe. |
| 141 DEFAULT_TEST_TIMEOUT_MS = 6 * 1000 | 144 DEFAULT_TEST_TIMEOUT_MS = 6 * 1000 |
| 142 | 145 |
| (...skipping 734 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 877 unexpected_file.close() | 880 unexpected_file.close() |
| 878 | 881 |
| 879 # Write a json file of the test_expectations.txt file for the layout tests | 882 # Write a json file of the test_expectations.txt file for the layout tests |
| 880 # dashboard. | 883 # dashboard. |
| 881 expectations_file = open(os.path.join(self._options.results_directory, | 884 expectations_file = open(os.path.join(self._options.results_directory, |
| 882 "expectations.json"), "w") | 885 "expectations.json"), "w") |
| 883 expectations_json = self._expectations.GetExpectationsJsonForAllPlatforms() | 886 expectations_json = self._expectations.GetExpectationsJsonForAllPlatforms() |
| 884 expectations_file.write(("ADD_EXPECTATIONS(" + expectations_json + ");")) | 887 expectations_file.write(("ADD_EXPECTATIONS(" + expectations_json + ");")) |
| 885 expectations_file.close() | 888 expectations_file.close() |
| 886 | 889 |
| 887 json_results_generator.JSONResultsGenerator(self._options, | 890 json_layout_results_generator.JSONLayoutResultsGenerator( |
| 888 self._expectations, result_summary, individual_test_timings, | 891 self._options.builder_name, self._options.build_name, |
| 889 self._options.results_directory, self._test_files_list) | 892 self._options.build_number, self._options.results_directory, |
| 893 BUILDER_BASE_URL, individual_test_timings, |
| 894 self._expectations, result_summary, self._test_files_list) |
| 890 | 895 |
| 891 logging.debug("Finished writing JSON files.") | 896 logging.debug("Finished writing JSON files.") |
| 892 | 897 |
| 893 def _PrintExpectedResultsOfType(self, write, result_summary, result_type, | 898 def _PrintExpectedResultsOfType(self, write, result_summary, result_type, |
| 894 result_type_str): | 899 result_type_str): |
| 895 """Print the number of the tests in a given result class. | 900 """Print the number of the tests in a given result class. |
| 896 | 901 |
| 897 Args: | 902 Args: |
| 898 write: A callback to write info to (e.g., a LoggingWriter) or | 903 write: A callback to write info to (e.g., a LoggingWriter) or |
| 899 sys.stdout.write. | 904 sys.stdout.write. |
| (...skipping 693 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1593 option_parser.add_option("", "--build-number", | 1598 option_parser.add_option("", "--build-number", |
| 1594 default="DUMMY_BUILD_NUMBER", | 1599 default="DUMMY_BUILD_NUMBER", |
| 1595 help=("The build number of the builder running" | 1600 help=("The build number of the builder running" |
| 1596 "this script.")) | 1601 "this script.")) |
| 1597 option_parser.add_option("", "--experimental-fully-parallel", | 1602 option_parser.add_option("", "--experimental-fully-parallel", |
| 1598 action="store_true", default=False, | 1603 action="store_true", default=False, |
| 1599 help="run all tests in parallel") | 1604 help="run all tests in parallel") |
| 1600 | 1605 |
| 1601 options, args = option_parser.parse_args() | 1606 options, args = option_parser.parse_args() |
| 1602 main(options, args) | 1607 main(options, args) |
| OLD | NEW |