OLD | NEW |
1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
11 # in the documentation and/or other materials provided with the | 11 # in the documentation and/or other materials provided with the |
12 # distribution. | 12 # distribution. |
13 # * Neither the Google name nor the names of its | 13 # * Neither the Google name nor the names of its |
14 # contributors may be used to endorse or promote products derived from | 14 # contributors may be used to endorse or promote products derived from |
15 # this software without specific prior written permission. | 15 # this software without specific prior written permission. |
16 # | 16 # |
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
28 | 28 |
29 """Abstract base class of Port-specific entry points for the layout tests | 29 """Abstract base class of Port-specific entry points for the layout tests |
30 test infrastructure (the Port and Driver classes).""" | 30 test infrastructure (the Port and Driver classes). |
| 31 """ |
31 | 32 |
32 import collections | 33 import collections |
33 import cgi | 34 import cgi |
34 import difflib | 35 import difflib |
35 import errno | 36 import errno |
36 import itertools | 37 import itertools |
37 import json | 38 import json |
38 import logging | 39 import logging |
39 import os | 40 import os |
40 import operator | 41 import operator |
(...skipping 259 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
300 return [self._filesystem.join(path, suite.name) for path in self.default
_baseline_search_path()] | 301 return [self._filesystem.join(path, suite.name) for path in self.default
_baseline_search_path()] |
301 | 302 |
302 def baseline_search_path(self): | 303 def baseline_search_path(self): |
303 return (self.get_option('additional_platform_directory', []) + | 304 return (self.get_option('additional_platform_directory', []) + |
304 self._flag_specific_baseline_search_path() + | 305 self._flag_specific_baseline_search_path() + |
305 self._compare_baseline() + | 306 self._compare_baseline() + |
306 self.default_baseline_search_path()) | 307 self.default_baseline_search_path()) |
307 | 308 |
308 def default_baseline_search_path(self): | 309 def default_baseline_search_path(self): |
309 """Return a list of absolute paths to directories to search under for | 310 """Return a list of absolute paths to directories to search under for |
310 baselines. The directories are searched in order.""" | 311 baselines. The directories are searched in order. |
| 312 """ |
311 return map(self._webkit_baseline_path, self.FALLBACK_PATHS[self.version(
)]) | 313 return map(self._webkit_baseline_path, self.FALLBACK_PATHS[self.version(
)]) |
312 | 314 |
313 @memoized | 315 @memoized |
314 def _compare_baseline(self): | 316 def _compare_baseline(self): |
315 factory = PortFactory(self.host) | 317 factory = PortFactory(self.host) |
316 target_port = self.get_option('compare_port') | 318 target_port = self.get_option('compare_port') |
317 if target_port: | 319 if target_port: |
318 return factory.get(target_port).default_baseline_search_path() | 320 return factory.get(target_port).default_baseline_search_path() |
319 return [] | 321 return [] |
320 | 322 |
(...skipping 206 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
527 err_str = "Image diff returned an exit code of %s. See http://cr
bug.com/278596" % exit_code | 529 err_str = "Image diff returned an exit code of %s. See http://cr
bug.com/278596" % exit_code |
528 except OSError as e: | 530 except OSError as e: |
529 err_str = 'error running image diff: %s' % str(e) | 531 err_str = 'error running image diff: %s' % str(e) |
530 finally: | 532 finally: |
531 self._filesystem.rmtree(str(tempdir)) | 533 self._filesystem.rmtree(str(tempdir)) |
532 | 534 |
533 return (result, err_str or None) | 535 return (result, err_str or None) |
534 | 536 |
535 def diff_text(self, expected_text, actual_text, expected_filename, actual_fi
lename): | 537 def diff_text(self, expected_text, actual_text, expected_filename, actual_fi
lename): |
536 """Returns a string containing the diff of the two text strings | 538 """Returns a string containing the diff of the two text strings |
537 in 'unified diff' format.""" | 539 in 'unified diff' format. |
| 540 """ |
538 | 541 |
539 # The filenames show up in the diff output, make sure they're | 542 # The filenames show up in the diff output, make sure they're |
540 # raw bytes and not unicode, so that they don't trigger join() | 543 # raw bytes and not unicode, so that they don't trigger join() |
541 # trying to decode the input. | 544 # trying to decode the input. |
542 def to_raw_bytes(string_value): | 545 def to_raw_bytes(string_value): |
543 if isinstance(string_value, unicode): | 546 if isinstance(string_value, unicode): |
544 return string_value.encode('utf-8') | 547 return string_value.encode('utf-8') |
545 return string_value | 548 return string_value |
546 expected_filename = to_raw_bytes(expected_filename) | 549 expected_filename = to_raw_bytes(expected_filename) |
547 actual_filename = to_raw_bytes(actual_filename) | 550 actual_filename = to_raw_bytes(actual_filename) |
(...skipping 13 matching lines...) Expand all Loading... |
561 | 564 |
562 return ''.join(diff_fixup(diff)) | 565 return ''.join(diff_fixup(diff)) |
563 | 566 |
564 def driver_name(self): | 567 def driver_name(self): |
565 if self.get_option('driver_name'): | 568 if self.get_option('driver_name'): |
566 return self.get_option('driver_name') | 569 return self.get_option('driver_name') |
567 return self.CONTENT_SHELL_NAME | 570 return self.CONTENT_SHELL_NAME |
568 | 571 |
569 def expected_baselines_by_extension(self, test_name): | 572 def expected_baselines_by_extension(self, test_name): |
570 """Returns a dict mapping baseline suffix to relative path for each base
line in | 573 """Returns a dict mapping baseline suffix to relative path for each base
line in |
571 a test. For reftests, it returns ".==" or ".!=" instead of the suffix.""
" | 574 a test. For reftests, it returns ".==" or ".!=" instead of the suffix. |
| 575 """ |
572 # FIXME: The name similarity between this and expected_baselines() below
, is unfortunate. | 576 # FIXME: The name similarity between this and expected_baselines() below
, is unfortunate. |
573 # We should probably rename them both. | 577 # We should probably rename them both. |
574 baseline_dict = {} | 578 baseline_dict = {} |
575 reference_files = self.reference_files(test_name) | 579 reference_files = self.reference_files(test_name) |
576 if reference_files: | 580 if reference_files: |
577 # FIXME: How should this handle more than one type of reftest? | 581 # FIXME: How should this handle more than one type of reftest? |
578 baseline_dict['.' + reference_files[0][0]] = self.relative_test_file
name(reference_files[0][1]) | 582 baseline_dict['.' + reference_files[0][0]] = self.relative_test_file
name(reference_files[0][1]) |
579 | 583 |
580 for extension in self.baseline_extensions(): | 584 for extension in self.baseline_extensions(): |
581 path = self.expected_filename(test_name, extension, return_default=F
alse) | 585 path = self.expected_filename(test_name, extension, return_default=F
alse) |
(...skipping 107 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
689 | 693 |
690 def expected_audio(self, test_name): | 694 def expected_audio(self, test_name): |
691 baseline_path = self.expected_filename(test_name, '.wav') | 695 baseline_path = self.expected_filename(test_name, '.wav') |
692 if not self._filesystem.exists(baseline_path): | 696 if not self._filesystem.exists(baseline_path): |
693 return None | 697 return None |
694 return self._filesystem.read_binary_file(baseline_path) | 698 return self._filesystem.read_binary_file(baseline_path) |
695 | 699 |
696 def expected_text(self, test_name): | 700 def expected_text(self, test_name): |
697 """Returns the text output we expect the test to produce, or None | 701 """Returns the text output we expect the test to produce, or None |
698 if we don't expect there to be any text output. | 702 if we don't expect there to be any text output. |
699 End-of-line characters are normalized to '\n'.""" | 703 End-of-line characters are normalized to '\n'. |
| 704 """ |
700 # FIXME: DRT output is actually utf-8, but since we don't decode the | 705 # FIXME: DRT output is actually utf-8, but since we don't decode the |
701 # output from DRT (instead treating it as a binary string), we read the | 706 # output from DRT (instead treating it as a binary string), we read the |
702 # baselines as a binary string, too. | 707 # baselines as a binary string, too. |
703 baseline_path = self.expected_filename(test_name, '.txt') | 708 baseline_path = self.expected_filename(test_name, '.txt') |
704 if not self._filesystem.exists(baseline_path): | 709 if not self._filesystem.exists(baseline_path): |
705 return None | 710 return None |
706 text = self._filesystem.read_binary_file(baseline_path) | 711 text = self._filesystem.read_binary_file(baseline_path) |
707 return text.replace("\r\n", "\n") | 712 return text.replace("\r\n", "\n") |
708 | 713 |
709 def _get_reftest_list(self, test_name): | 714 def _get_reftest_list(self, test_name): |
(...skipping 250 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
960 if self._filesystem.isdir(category) and test_name.startswith(test_or
_category): | 965 if self._filesystem.isdir(category) and test_name.startswith(test_or
_category): |
961 return True | 966 return True |
962 return False | 967 return False |
963 | 968 |
964 def is_chromium(self): | 969 def is_chromium(self): |
965 return True | 970 return True |
966 | 971 |
967 def name(self): | 972 def name(self): |
968 """Returns a name that uniquely identifies this particular type of port | 973 """Returns a name that uniquely identifies this particular type of port |
969 (e.g., "mac-snowleopard" or "linux-trusty" and can be passed | 974 (e.g., "mac-snowleopard" or "linux-trusty" and can be passed |
970 to factory.get() to instantiate the port.""" | 975 to factory.get() to instantiate the port. |
| 976 """ |
971 return self._name | 977 return self._name |
972 | 978 |
973 def operating_system(self): | 979 def operating_system(self): |
974 # Subclasses should override this default implementation. | 980 # Subclasses should override this default implementation. |
975 return 'mac' | 981 return 'mac' |
976 | 982 |
977 def version(self): | 983 def version(self): |
978 """Returns a string indicating the version of a given platform, e.g. | 984 """Returns a string indicating the version of a given platform, e.g. |
979 'leopard' or 'win7'. | 985 'leopard' or 'win7'. |
980 | 986 |
981 This is used to help identify the exact port when parsing test | 987 This is used to help identify the exact port when parsing test |
982 expectations, determining search paths, and logging information.""" | 988 expectations, determining search paths, and logging information. |
| 989 """ |
983 return self._version | 990 return self._version |
984 | 991 |
985 def architecture(self): | 992 def architecture(self): |
986 return self._architecture | 993 return self._architecture |
987 | 994 |
988 def get_option(self, name, default_value=None): | 995 def get_option(self, name, default_value=None): |
989 return getattr(self._options, name, default_value) | 996 return getattr(self._options, name, default_value) |
990 | 997 |
991 def set_option_default(self, name, default_value): | 998 def set_option_default(self, name, default_value): |
992 return self._options.ensure_value(name, default_value) | 999 return self._options.ensure_value(name, default_value) |
993 | 1000 |
994 @memoized | 1001 @memoized |
995 def path_to_generic_test_expectations_file(self): | 1002 def path_to_generic_test_expectations_file(self): |
996 return self._filesystem.join(self.layout_tests_dir(), 'TestExpectations'
) | 1003 return self._filesystem.join(self.layout_tests_dir(), 'TestExpectations'
) |
997 | 1004 |
998 def relative_test_filename(self, filename): | 1005 def relative_test_filename(self, filename): |
999 """Returns a test_name a relative unix-style path for a filename under t
he LayoutTests | 1006 """Returns a test_name a relative unix-style path for a filename under t
he LayoutTests |
1000 directory. Ports may legitimately return abspaths here if no relpath mak
es sense.""" | 1007 directory. Ports may legitimately return abspaths here if no relpath mak
es sense. |
| 1008 """ |
1001 # Ports that run on windows need to override this method to deal with | 1009 # Ports that run on windows need to override this method to deal with |
1002 # filenames with backslashes in them. | 1010 # filenames with backslashes in them. |
1003 if filename.startswith(self.layout_tests_dir()): | 1011 if filename.startswith(self.layout_tests_dir()): |
1004 return self.host.filesystem.relpath(filename, self.layout_tests_dir(
)) | 1012 return self.host.filesystem.relpath(filename, self.layout_tests_dir(
)) |
1005 else: | 1013 else: |
1006 return self.host.filesystem.abspath(filename) | 1014 return self.host.filesystem.abspath(filename) |
1007 | 1015 |
1008 @memoized | 1016 @memoized |
1009 def abspath_for_test(self, test_name): | 1017 def abspath_for_test(self, test_name): |
1010 """Returns the full path to the file for a given test name. This is the | 1018 """Returns the full path to the file for a given test name. This is the |
1011 inverse of relative_test_filename().""" | 1019 inverse of relative_test_filename(). |
| 1020 """ |
1012 return self._filesystem.join(self.layout_tests_dir(), test_name) | 1021 return self._filesystem.join(self.layout_tests_dir(), test_name) |
1013 | 1022 |
1014 def results_directory(self): | 1023 def results_directory(self): |
1015 """Absolute path to the place to store the test results (uses --results-
directory).""" | 1024 """Absolute path to the place to store the test results (uses --results-
directory).""" |
1016 if not self._results_directory: | 1025 if not self._results_directory: |
1017 option_val = self.get_option('results_directory') or self.default_re
sults_directory() | 1026 option_val = self.get_option('results_directory') or self.default_re
sults_directory() |
1018 self._results_directory = self._filesystem.abspath(option_val) | 1027 self._results_directory = self._filesystem.abspath(option_val) |
1019 return self._results_directory | 1028 return self._results_directory |
1020 | 1029 |
1021 def bot_test_times_path(self): | 1030 def bot_test_times_path(self): |
(...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1104 clean_env[variable] = self.host.environ[variable] | 1113 clean_env[variable] = self.host.environ[variable] |
1105 | 1114 |
1106 for string_variable in self.get_option('additional_env_var', []): | 1115 for string_variable in self.get_option('additional_env_var', []): |
1107 [name, value] = string_variable.split('=', 1) | 1116 [name, value] = string_variable.split('=', 1) |
1108 clean_env[name] = value | 1117 clean_env[name] = value |
1109 | 1118 |
1110 return clean_env | 1119 return clean_env |
1111 | 1120 |
1112 def show_results_html_file(self, results_filename): | 1121 def show_results_html_file(self, results_filename): |
1113 """This routine should display the HTML file pointed at by | 1122 """This routine should display the HTML file pointed at by |
1114 results_filename in a users' browser.""" | 1123 results_filename in a users' browser. |
| 1124 """ |
1115 return self.host.user.open_url(abspath_to_uri(self.host.platform, result
s_filename)) | 1125 return self.host.user.open_url(abspath_to_uri(self.host.platform, result
s_filename)) |
1116 | 1126 |
1117 def create_driver(self, worker_number, no_timeout=False): | 1127 def create_driver(self, worker_number, no_timeout=False): |
1118 """Return a newly created Driver subclass for starting/stopping the test
driver.""" | 1128 """Return a newly created Driver subclass for starting/stopping the test
driver.""" |
1119 return self._driver_class()(self, worker_number, pixel_tests=self.get_op
tion('pixel_tests'), no_timeout=no_timeout) | 1129 return self._driver_class()(self, worker_number, pixel_tests=self.get_op
tion('pixel_tests'), no_timeout=no_timeout) |
1120 | 1130 |
1121 def start_helper(self): | 1131 def start_helper(self): |
1122 """If a port needs to reconfigure graphics settings or do other | 1132 """If a port needs to reconfigure graphics settings or do other |
1123 things to ensure a known test configuration, it should override this | 1133 things to ensure a known test configuration, it should override this |
1124 method.""" | 1134 method. |
| 1135 """ |
1125 helper_path = self._path_to_helper() | 1136 helper_path = self._path_to_helper() |
1126 if helper_path: | 1137 if helper_path: |
1127 _log.debug("Starting layout helper %s", helper_path) | 1138 _log.debug("Starting layout helper %s", helper_path) |
1128 # Note: Not thread safe: http://bugs.python.org/issue2320 | 1139 # Note: Not thread safe: http://bugs.python.org/issue2320 |
1129 self._helper = self._executive.popen([helper_path], | 1140 self._helper = self._executive.popen([helper_path], |
1130 stdin=self._executive.PIPE, std
out=self._executive.PIPE, stderr=None) | 1141 stdin=self._executive.PIPE, std
out=self._executive.PIPE, stderr=None) |
1131 is_ready = self._helper.stdout.readline() | 1142 is_ready = self._helper.stdout.readline() |
1132 if not is_ready.startswith('ready'): | 1143 if not is_ready.startswith('ready'): |
1133 _log.error("layout_test_helper failed to be ready") | 1144 _log.error("layout_test_helper failed to be ready") |
1134 | 1145 |
1135 def requires_http_server(self): | 1146 def requires_http_server(self): |
1136 """Does the port require an HTTP server for running tests? This could | 1147 """Does the port require an HTTP server for running tests? This could |
1137 be the case when the tests aren't run on the host platform.""" | 1148 be the case when the tests aren't run on the host platform. |
| 1149 """ |
1138 return False | 1150 return False |
1139 | 1151 |
1140 def start_http_server(self, additional_dirs, number_of_drivers): | 1152 def start_http_server(self, additional_dirs, number_of_drivers): |
1141 """Start a web server. Raise an error if it can't start or is already ru
nning. | 1153 """Start a web server. Raise an error if it can't start or is already ru
nning. |
1142 | 1154 |
1143 Ports can stub this out if they don't need a web server to be running.""
" | 1155 Ports can stub this out if they don't need a web server to be running. |
| 1156 """ |
1144 assert not self._http_server, 'Already running an http server.' | 1157 assert not self._http_server, 'Already running an http server.' |
1145 | 1158 |
1146 server = apache_http.ApacheHTTP(self, self.results_directory(), | 1159 server = apache_http.ApacheHTTP(self, self.results_directory(), |
1147 additional_dirs=additional_dirs, | 1160 additional_dirs=additional_dirs, |
1148 number_of_servers=(number_of_drivers * 4
)) | 1161 number_of_servers=(number_of_drivers * 4
)) |
1149 server.start() | 1162 server.start() |
1150 self._http_server = server | 1163 self._http_server = server |
1151 | 1164 |
1152 def start_websocket_server(self): | 1165 def start_websocket_server(self): |
1153 """Start a web server. Raise an error if it can't start or is already ru
nning. | 1166 """Start a web server. Raise an error if it can't start or is already ru
nning. |
1154 | 1167 |
1155 Ports can stub this out if they don't need a websocket server to be runn
ing.""" | 1168 Ports can stub this out if they don't need a websocket server to be runn
ing. |
| 1169 """ |
1156 assert not self._websocket_server, 'Already running a websocket server.' | 1170 assert not self._websocket_server, 'Already running a websocket server.' |
1157 | 1171 |
1158 server = pywebsocket.PyWebSocket(self, self.results_directory()) | 1172 server = pywebsocket.PyWebSocket(self, self.results_directory()) |
1159 server.start() | 1173 server.start() |
1160 self._websocket_server = server | 1174 self._websocket_server = server |
1161 | 1175 |
1162 def is_wptserve_enabled(self): | 1176 def is_wptserve_enabled(self): |
1163 """Used as feature flag for WPT Serve feature.""" | 1177 """Used as feature flag for WPT Serve feature.""" |
1164 return self._is_wptserve_enabled | 1178 return self._is_wptserve_enabled |
1165 | 1179 |
1166 @staticmethod | 1180 @staticmethod |
1167 def is_wptserve_test(test): | 1181 def is_wptserve_test(test): |
1168 """Whether wptserve should be used for a given test if enabled.""" | 1182 """Whether wptserve should be used for a given test if enabled.""" |
1169 return test.startswith("imported/wpt/") | 1183 return test.startswith("imported/wpt/") |
1170 | 1184 |
1171 def should_use_wptserve(self, test): | 1185 def should_use_wptserve(self, test): |
1172 return self.is_wptserve_enabled() and self.is_wptserve_test(test) | 1186 return self.is_wptserve_enabled() and self.is_wptserve_test(test) |
1173 | 1187 |
1174 def start_wptserve(self): | 1188 def start_wptserve(self): |
1175 """Start a WPT web server. Raise an error if it can't start or is alread
y running. | 1189 """Start a WPT web server. Raise an error if it can't start or is alread
y running. |
1176 | 1190 |
1177 Ports can stub this out if they don't need a WPT web server to be runnin
g.""" | 1191 Ports can stub this out if they don't need a WPT web server to be runnin
g. |
| 1192 """ |
1178 assert not self._wpt_server, 'Already running an http server.' | 1193 assert not self._wpt_server, 'Already running an http server.' |
1179 assert self.is_wptserve_enabled(), 'Cannot start server if WPT is not en
abled.' | 1194 assert self.is_wptserve_enabled(), 'Cannot start server if WPT is not en
abled.' |
1180 | 1195 |
1181 # We currently don't support any output mechanism for the WPT server. | 1196 # We currently don't support any output mechanism for the WPT server. |
1182 server = wptserve.WPTServe(self, self.results_directory()) | 1197 server = wptserve.WPTServe(self, self.results_directory()) |
1183 server.start() | 1198 server.start() |
1184 self._wpt_server = server | 1199 self._wpt_server = server |
1185 | 1200 |
1186 def stop_wptserve(self): | 1201 def stop_wptserve(self): |
1187 """Shut down the WPT server if it is running. Do nothing if it isn't.""" | 1202 """Shut down the WPT server if it is running. Do nothing if it isn't.""" |
1188 if self._wpt_server: | 1203 if self._wpt_server: |
1189 self._wpt_server.stop() | 1204 self._wpt_server.stop() |
1190 self._wpt_server = None | 1205 self._wpt_server = None |
1191 | 1206 |
1192 def http_server_supports_ipv6(self): | 1207 def http_server_supports_ipv6(self): |
1193 # Apache < 2.4 on win32 does not support IPv6, nor does cygwin apache. | 1208 # Apache < 2.4 on win32 does not support IPv6, nor does cygwin apache. |
1194 if self.host.platform.is_cygwin() or self.host.platform.is_win(): | 1209 if self.host.platform.is_cygwin() or self.host.platform.is_win(): |
1195 return False | 1210 return False |
1196 return True | 1211 return True |
1197 | 1212 |
1198 def stop_helper(self): | 1213 def stop_helper(self): |
1199 """Shut down the test helper if it is running. Do nothing if | 1214 """Shut down the test helper if it is running. Do nothing if |
1200 it isn't, or it isn't available. If a port overrides start_helper() | 1215 it isn't, or it isn't available. If a port overrides start_helper() |
1201 it must override this routine as well.""" | 1216 it must override this routine as well. |
| 1217 """ |
1202 if self._helper: | 1218 if self._helper: |
1203 _log.debug("Stopping layout test helper") | 1219 _log.debug("Stopping layout test helper") |
1204 try: | 1220 try: |
1205 self._helper.stdin.write("x\n") | 1221 self._helper.stdin.write("x\n") |
1206 self._helper.stdin.close() | 1222 self._helper.stdin.close() |
1207 self._helper.wait() | 1223 self._helper.wait() |
1208 except IOError: | 1224 except IOError: |
1209 pass | 1225 pass |
1210 finally: | 1226 finally: |
1211 self._helper = None | 1227 self._helper = None |
(...skipping 17 matching lines...) Expand all Loading... |
1229 def test_configuration(self): | 1245 def test_configuration(self): |
1230 """Returns the current TestConfiguration for the port.""" | 1246 """Returns the current TestConfiguration for the port.""" |
1231 if not self._test_configuration: | 1247 if not self._test_configuration: |
1232 self._test_configuration = TestConfiguration(self._version, self._ar
chitecture, self._options.configuration.lower()) | 1248 self._test_configuration = TestConfiguration(self._version, self._ar
chitecture, self._options.configuration.lower()) |
1233 return self._test_configuration | 1249 return self._test_configuration |
1234 | 1250 |
1235 # FIXME: Belongs on a Platform object. | 1251 # FIXME: Belongs on a Platform object. |
1236 @memoized | 1252 @memoized |
1237 def all_test_configurations(self): | 1253 def all_test_configurations(self): |
1238 """Returns a list of TestConfiguration instances, representing all avail
able | 1254 """Returns a list of TestConfiguration instances, representing all avail
able |
1239 test configurations for this port.""" | 1255 test configurations for this port. |
| 1256 """ |
1240 return self._generate_all_test_configurations() | 1257 return self._generate_all_test_configurations() |
1241 | 1258 |
1242 # FIXME: Belongs on a Platform object. | 1259 # FIXME: Belongs on a Platform object. |
1243 def configuration_specifier_macros(self): | 1260 def configuration_specifier_macros(self): |
1244 """Ports may provide a way to abbreviate configuration specifiers to con
veniently | 1261 """Ports may provide a way to abbreviate configuration specifiers to con
veniently |
1245 refer to them as one term or alias specific values to more generic ones.
For example: | 1262 refer to them as one term or alias specific values to more generic ones.
For example: |
1246 | 1263 |
1247 (vista, win7) -> win # Abbreviate all Windows versions into one namesake
. | 1264 (vista, win7) -> win # Abbreviate all Windows versions into one namesake
. |
1248 (precise, trusty) -> linux # Change specific name of Linux distro to a
more generic term. | 1265 (precise, trusty) -> linux # Change specific name of Linux distro to a
more generic term. |
1249 | 1266 |
1250 Returns a dictionary, each key representing a macro term ('win', for exa
mple), | 1267 Returns a dictionary, each key representing a macro term ('win', for exa
mple), |
1251 and value being a list of valid configuration specifiers (such as ['vist
a', 'win7']).""" | 1268 and value being a list of valid configuration specifiers (such as ['vist
a', 'win7']). |
| 1269 """ |
1252 return self.CONFIGURATION_SPECIFIER_MACROS | 1270 return self.CONFIGURATION_SPECIFIER_MACROS |
1253 | 1271 |
1254 def _generate_all_test_configurations(self): | 1272 def _generate_all_test_configurations(self): |
1255 """Returns a sequence of the TestConfigurations the port supports.""" | 1273 """Returns a sequence of the TestConfigurations the port supports.""" |
1256 # By default, we assume we want to test every graphics type in | 1274 # By default, we assume we want to test every graphics type in |
1257 # every configuration on every system. | 1275 # every configuration on every system. |
1258 test_configurations = [] | 1276 test_configurations = [] |
1259 for version, architecture in self.ALL_SYSTEMS: | 1277 for version, architecture in self.ALL_SYSTEMS: |
1260 for build_type in self.ALL_BUILD_TYPES: | 1278 for build_type in self.ALL_BUILD_TYPES: |
1261 test_configurations.append(TestConfiguration(version, architectu
re, build_type)) | 1279 test_configurations.append(TestConfiguration(version, architectu
re, build_type)) |
(...skipping 11 matching lines...) Expand all Loading... |
1273 return [self._filesystem.join(self.layout_tests_dir(), 'flag-specific',
flag.lstrip('-')) | 1291 return [self._filesystem.join(self.layout_tests_dir(), 'flag-specific',
flag.lstrip('-')) |
1274 for flag in self.get_option('additional_driver_flag', [])] | 1292 for flag in self.get_option('additional_driver_flag', [])] |
1275 | 1293 |
1276 def expectations_dict(self): | 1294 def expectations_dict(self): |
1277 """Returns an OrderedDict of name -> expectations strings. | 1295 """Returns an OrderedDict of name -> expectations strings. |
1278 The names are expected to be (but not required to be) paths in the files
ystem. | 1296 The names are expected to be (but not required to be) paths in the files
ystem. |
1279 If the name is a path, the file can be considered updatable for things l
ike rebaselining, | 1297 If the name is a path, the file can be considered updatable for things l
ike rebaselining, |
1280 so don't use names that are paths if they're not paths. | 1298 so don't use names that are paths if they're not paths. |
1281 Generally speaking the ordering should be files in the filesystem in cas
cade order | 1299 Generally speaking the ordering should be files in the filesystem in cas
cade order |
1282 (TestExpectations followed by Skipped, if the port honors both formats), | 1300 (TestExpectations followed by Skipped, if the port honors both formats), |
1283 then any built-in expectations (e.g., from compile-time exclusions), the
n --additional-expectations options.""" | 1301 then any built-in expectations (e.g., from compile-time exclusions), the
n --additional-expectations options. |
| 1302 """ |
1284 # FIXME: rename this to test_expectations() once all the callers are upd
ated to know about the ordered dict. | 1303 # FIXME: rename this to test_expectations() once all the callers are upd
ated to know about the ordered dict. |
1285 expectations = collections.OrderedDict() | 1304 expectations = collections.OrderedDict() |
1286 | 1305 |
1287 for path in self.expectations_files(): | 1306 for path in self.expectations_files(): |
1288 if self._filesystem.exists(path): | 1307 if self._filesystem.exists(path): |
1289 expectations[path] = self._filesystem.read_text_file(path) | 1308 expectations[path] = self._filesystem.read_text_file(path) |
1290 | 1309 |
1291 for path in self.get_option('additional_expectations', []): | 1310 for path in self.get_option('additional_expectations', []): |
1292 expanded_path = self._filesystem.expanduser(path) | 1311 expanded_path = self._filesystem.expanduser(path) |
1293 if self._filesystem.exists(expanded_path): | 1312 if self._filesystem.exists(expanded_path): |
(...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1360 expected_filename] | 1379 expected_filename] |
1361 | 1380 |
1362 @staticmethod | 1381 @staticmethod |
1363 def _handle_wdiff_error(script_error): | 1382 def _handle_wdiff_error(script_error): |
1364 # Exit 1 means the files differed, any other exit code is an error. | 1383 # Exit 1 means the files differed, any other exit code is an error. |
1365 if script_error.exit_code != 1: | 1384 if script_error.exit_code != 1: |
1366 raise script_error | 1385 raise script_error |
1367 | 1386 |
1368 def _run_wdiff(self, actual_filename, expected_filename): | 1387 def _run_wdiff(self, actual_filename, expected_filename): |
1369 """Runs wdiff and may throw exceptions. | 1388 """Runs wdiff and may throw exceptions. |
1370 This is mostly a hook for unit testing.""" | 1389 This is mostly a hook for unit testing. |
| 1390 """ |
1371 # Diffs are treated as binary as they may include multiple files | 1391 # Diffs are treated as binary as they may include multiple files |
1372 # with conflicting encodings. Thus we do not decode the output. | 1392 # with conflicting encodings. Thus we do not decode the output. |
1373 command = self._wdiff_command(actual_filename, expected_filename) | 1393 command = self._wdiff_command(actual_filename, expected_filename) |
1374 wdiff = self._executive.run_command(command, decode_output=False, | 1394 wdiff = self._executive.run_command(command, decode_output=False, |
1375 error_handler=self._handle_wdiff_err
or) | 1395 error_handler=self._handle_wdiff_err
or) |
1376 return self._format_wdiff_output_as_html(wdiff) | 1396 return self._format_wdiff_output_as_html(wdiff) |
1377 | 1397 |
1378 _wdiff_error_html = "Failed to run wdiff, see error log." | 1398 _wdiff_error_html = "Failed to run wdiff, see error log." |
1379 | 1399 |
1380 def wdiff_text(self, actual_filename, expected_filename): | 1400 def wdiff_text(self, actual_filename, expected_filename): |
1381 """Returns a string of HTML indicating the word-level diff of the | 1401 """Returns a string of HTML indicating the word-level diff of the |
1382 contents of the two filenames. Returns an empty string if word-level | 1402 contents of the two filenames. Returns an empty string if word-level |
1383 diffing isn't available.""" | 1403 diffing isn't available. |
| 1404 """ |
1384 if not self.wdiff_available(): | 1405 if not self.wdiff_available(): |
1385 return "" | 1406 return "" |
1386 try: | 1407 try: |
1387 # It's possible to raise a ScriptError we pass wdiff invalid paths. | 1408 # It's possible to raise a ScriptError we pass wdiff invalid paths. |
1388 return self._run_wdiff(actual_filename, expected_filename) | 1409 return self._run_wdiff(actual_filename, expected_filename) |
1389 except OSError as e: | 1410 except OSError as e: |
1390 if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]: | 1411 if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]: |
1391 # Silently ignore cases where wdiff is missing. | 1412 # Silently ignore cases where wdiff is missing. |
1392 self._wdiff_available = False | 1413 self._wdiff_available = False |
1393 return "" | 1414 return "" |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1437 This is needed only by ports that use the apache_http_server module. | 1458 This is needed only by ports that use the apache_http_server module. |
1438 """ | 1459 """ |
1439 raise NotImplementedError('Port.path_to_apache') | 1460 raise NotImplementedError('Port.path_to_apache') |
1440 | 1461 |
1441 def path_to_apache_config_file(self): | 1462 def path_to_apache_config_file(self): |
1442 """Returns the full path to the apache configuration file. | 1463 """Returns the full path to the apache configuration file. |
1443 | 1464 |
1444 If the WEBKIT_HTTP_SERVER_CONF_PATH environment variable is set, its | 1465 If the WEBKIT_HTTP_SERVER_CONF_PATH environment variable is set, its |
1445 contents will be used instead. | 1466 contents will be used instead. |
1446 | 1467 |
1447 This is needed only by ports that use the apache_http_server module.""" | 1468 This is needed only by ports that use the apache_http_server module. |
| 1469 """ |
1448 config_file_from_env = self.host.environ.get('WEBKIT_HTTP_SERVER_CONF_PA
TH') | 1470 config_file_from_env = self.host.environ.get('WEBKIT_HTTP_SERVER_CONF_PA
TH') |
1449 if config_file_from_env: | 1471 if config_file_from_env: |
1450 if not self._filesystem.exists(config_file_from_env): | 1472 if not self._filesystem.exists(config_file_from_env): |
1451 raise IOError('%s was not found on the system' % config_file_fro
m_env) | 1473 raise IOError('%s was not found on the system' % config_file_fro
m_env) |
1452 return config_file_from_env | 1474 return config_file_from_env |
1453 | 1475 |
1454 config_file_name = self._apache_config_file_name_for_platform() | 1476 config_file_name = self._apache_config_file_name_for_platform() |
1455 return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', co
nfig_file_name) | 1477 return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', co
nfig_file_name) |
1456 | 1478 |
1457 # | 1479 # |
(...skipping 25 matching lines...) Expand all Loading... |
1483 | 1505 |
1484 def _path_to_webcore_library(self): | 1506 def _path_to_webcore_library(self): |
1485 """Returns the full path to a built copy of WebCore.""" | 1507 """Returns the full path to a built copy of WebCore.""" |
1486 return None | 1508 return None |
1487 | 1509 |
1488 def _path_to_helper(self): | 1510 def _path_to_helper(self): |
1489 """Returns the full path to the layout_test_helper binary, which | 1511 """Returns the full path to the layout_test_helper binary, which |
1490 is used to help configure the system for the test run, or None | 1512 is used to help configure the system for the test run, or None |
1491 if no helper is needed. | 1513 if no helper is needed. |
1492 | 1514 |
1493 This is likely only used by start/stop_helper().""" | 1515 This is likely only used by start/stop_helper(). |
| 1516 """ |
1494 return None | 1517 return None |
1495 | 1518 |
1496 def _path_to_image_diff(self): | 1519 def _path_to_image_diff(self): |
1497 """Returns the full path to the image_diff binary, or None if it is not
available. | 1520 """Returns the full path to the image_diff binary, or None if it is not
available. |
1498 | 1521 |
1499 This is likely used only by diff_image()""" | 1522 This is likely used only by diff_image() |
| 1523 """ |
1500 return self._build_path('image_diff') | 1524 return self._build_path('image_diff') |
1501 | 1525 |
1502 @memoized | 1526 @memoized |
1503 def _path_to_wdiff(self): | 1527 def _path_to_wdiff(self): |
1504 """Returns the full path to the wdiff binary, or None if it is not avail
able. | 1528 """Returns the full path to the wdiff binary, or None if it is not avail
able. |
1505 | 1529 |
1506 This is likely used only by wdiff_text()""" | 1530 This is likely used only by wdiff_text() |
| 1531 """ |
1507 for path in ("/usr/bin/wdiff", "/usr/bin/dwdiff"): | 1532 for path in ("/usr/bin/wdiff", "/usr/bin/dwdiff"): |
1508 if self._filesystem.exists(path): | 1533 if self._filesystem.exists(path): |
1509 return path | 1534 return path |
1510 return None | 1535 return None |
1511 | 1536 |
1512 def _webkit_baseline_path(self, platform): | 1537 def _webkit_baseline_path(self, platform): |
1513 """Return the full path to the top of the baseline tree for a | 1538 """Return the full path to the top of the baseline tree for a |
1514 given platform.""" | 1539 given platform. |
| 1540 """ |
1515 return self._filesystem.join(self.layout_tests_dir(), 'platform', platfo
rm) | 1541 return self._filesystem.join(self.layout_tests_dir(), 'platform', platfo
rm) |
1516 | 1542 |
1517 def _driver_class(self): | 1543 def _driver_class(self): |
1518 """Returns the port's driver implementation.""" | 1544 """Returns the port's driver implementation.""" |
1519 return driver.Driver | 1545 return driver.Driver |
1520 | 1546 |
1521 def output_contains_sanitizer_messages(self, output): | 1547 def output_contains_sanitizer_messages(self, output): |
1522 if not output: | 1548 if not output: |
1523 return None | 1549 return None |
1524 if 'AddressSanitizer' in output: | 1550 if 'AddressSanitizer' in output: |
(...skipping 262 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1787 | 1813 |
1788 def __init__(self, base, args, reference_args=None): | 1814 def __init__(self, base, args, reference_args=None): |
1789 self.name = base | 1815 self.name = base |
1790 self.base = base | 1816 self.base = base |
1791 self.args = args | 1817 self.args = args |
1792 self.reference_args = args if reference_args is None else reference_args | 1818 self.reference_args = args if reference_args is None else reference_args |
1793 self.tests = set() | 1819 self.tests = set() |
1794 | 1820 |
1795 def __repr__(self): | 1821 def __repr__(self): |
1796 return "PhysicalTestSuite('%s', '%s', %s, %s)" % (self.name, self.base,
self.args, self.reference_args) | 1822 return "PhysicalTestSuite('%s', '%s', %s, %s)" % (self.name, self.base,
self.args, self.reference_args) |
OLD | NEW |