OLD | NEW |
1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
101 ('mac10.9', 'x86'), | 101 ('mac10.9', 'x86'), |
102 ('mac10.10', 'x86'), | 102 ('mac10.10', 'x86'), |
103 ('mac10.11', 'x86'), | 103 ('mac10.11', 'x86'), |
104 ('win7', 'x86'), | 104 ('win7', 'x86'), |
105 ('win10', 'x86'), | 105 ('win10', 'x86'), |
106 ('precise', 'x86_64'), | 106 ('precise', 'x86_64'), |
107 ('trusty', 'x86_64'), | 107 ('trusty', 'x86_64'), |
108 # FIXME: Technically this should be 'arm', but adding a third architectu
re type breaks TestConfigurationConverter. | 108 # FIXME: Technically this should be 'arm', but adding a third architectu
re type breaks TestConfigurationConverter. |
109 # If we need this to be 'arm' in the future, then we first have to fix T
estConfigurationConverter. | 109 # If we need this to be 'arm' in the future, then we first have to fix T
estConfigurationConverter. |
110 ('icecreamsandwich', 'x86'), | 110 ('icecreamsandwich', 'x86'), |
111 ) | 111 ) |
112 | 112 |
113 CONFIGURATION_SPECIFIER_MACROS = { | 113 CONFIGURATION_SPECIFIER_MACROS = { |
114 'mac': ['retina', 'mac10.9', 'mac10.10', 'mac10.11'], | 114 'mac': ['retina', 'mac10.9', 'mac10.10', 'mac10.11'], |
115 'win': ['win7', 'win10'], | 115 'win': ['win7', 'win10'], |
116 'linux': ['precise', 'trusty'], | 116 'linux': ['precise', 'trusty'], |
117 'android': ['icecreamsandwich'], | 117 'android': ['icecreamsandwich'], |
118 } | 118 } |
119 | 119 |
120 DEFAULT_BUILD_DIRECTORIES = ('out',) | 120 DEFAULT_BUILD_DIRECTORIES = ('out',) |
121 | 121 |
(...skipping 597 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
719 for line in reftest_list_file.split('\n'): | 719 for line in reftest_list_file.split('\n'): |
720 line = re.sub('#.+$', '', line) | 720 line = re.sub('#.+$', '', line) |
721 split_line = line.split() | 721 split_line = line.split() |
722 if len(split_line) == 4: | 722 if len(split_line) == 4: |
723 # FIXME: Probably one of mozilla's extensions in the reftest.lis
t format. Do we need to support this? | 723 # FIXME: Probably one of mozilla's extensions in the reftest.lis
t format. Do we need to support this? |
724 _log.warning("unsupported reftest.list line '%s' in %s" % (line,
reftest_list_path)) | 724 _log.warning("unsupported reftest.list line '%s' in %s" % (line,
reftest_list_path)) |
725 continue | 725 continue |
726 if len(split_line) < 3: | 726 if len(split_line) < 3: |
727 continue | 727 continue |
728 expectation_type, test_file, ref_file = split_line | 728 expectation_type, test_file, ref_file = split_line |
729 parsed_list.setdefault(filesystem.join(test_dirpath, test_file), [])
.append((expectation_type, filesystem.join(test_dirpath, ref_file))) | 729 parsed_list.setdefault(filesystem.join(test_dirpath, test_file), [])
.append( |
| 730 (expectation_type, filesystem.join(test_dirpath, ref_file))) |
730 return parsed_list | 731 return parsed_list |
731 | 732 |
732 def reference_files(self, test_name): | 733 def reference_files(self, test_name): |
733 """Return a list of expectation (== or !=) and filename pairs""" | 734 """Return a list of expectation (== or !=) and filename pairs""" |
734 | 735 |
735 reftest_list = self._get_reftest_list(test_name) | 736 reftest_list = self._get_reftest_list(test_name) |
736 if not reftest_list: | 737 if not reftest_list: |
737 reftest_list = [] | 738 reftest_list = [] |
738 for expectation, prefix in (('==', ''), ('!=', '-mismatch')): | 739 for expectation, prefix in (('==', ''), ('!=', '-mismatch')): |
739 for extension in Port._supported_file_extensions: | 740 for extension in Port._supported_file_extensions: |
(...skipping 11 matching lines...) Expand all Loading... |
751 suites = self.virtual_test_suites() | 752 suites = self.virtual_test_suites() |
752 if paths: | 753 if paths: |
753 tests.extend(self._virtual_tests_matching_paths(paths, suites)) | 754 tests.extend(self._virtual_tests_matching_paths(paths, suites)) |
754 else: | 755 else: |
755 tests.extend(self._all_virtual_tests(suites)) | 756 tests.extend(self._all_virtual_tests(suites)) |
756 return tests | 757 return tests |
757 | 758 |
758 def _real_tests(self, paths): | 759 def _real_tests(self, paths): |
759 # When collecting test cases, skip these directories | 760 # When collecting test cases, skip these directories |
760 skipped_directories = set(['.svn', '_svn', 'platform', 'resources', 'sup
port', 'script-tests', 'reference', 'reftest']) | 761 skipped_directories = set(['.svn', '_svn', 'platform', 'resources', 'sup
port', 'script-tests', 'reference', 'reftest']) |
761 files = find_files.find(self._filesystem, self.layout_tests_dir(), paths
, skipped_directories, Port.is_test_file, self.test_key) | 762 files = find_files.find(self._filesystem, self.layout_tests_dir(), paths
, |
| 763 skipped_directories, Port.is_test_file, self.tes
t_key) |
762 return [self.relative_test_filename(f) for f in files] | 764 return [self.relative_test_filename(f) for f in files] |
763 | 765 |
764 # When collecting test cases, we include any file with these extensions. | 766 # When collecting test cases, we include any file with these extensions. |
765 _supported_file_extensions = set(['.html', '.xml', '.xhtml', '.xht', '.pl', | 767 _supported_file_extensions = set(['.html', '.xml', '.xhtml', '.xht', '.pl', |
766 '.htm', '.php', '.svg', '.mht', '.pdf']) | 768 '.htm', '.php', '.svg', '.mht', '.pdf']) |
767 | 769 |
768 @staticmethod | 770 @staticmethod |
769 # If any changes are made here be sure to update the isUsedInReftest method
in old-run-webkit-tests as well. | 771 # If any changes are made here be sure to update the isUsedInReftest method
in old-run-webkit-tests as well. |
770 def is_reference_html_file(filesystem, dirname, filename): | 772 def is_reference_html_file(filesystem, dirname, filename): |
771 if filename.startswith('ref-') or filename.startswith('notref-'): | 773 if filename.startswith('ref-') or filename.startswith('notref-'): |
(...skipping 350 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1122 | 1124 |
1123 def start_helper(self): | 1125 def start_helper(self): |
1124 """If a port needs to reconfigure graphics settings or do other | 1126 """If a port needs to reconfigure graphics settings or do other |
1125 things to ensure a known test configuration, it should override this | 1127 things to ensure a known test configuration, it should override this |
1126 method.""" | 1128 method.""" |
1127 helper_path = self._path_to_helper() | 1129 helper_path = self._path_to_helper() |
1128 if helper_path: | 1130 if helper_path: |
1129 _log.debug("Starting layout helper %s" % helper_path) | 1131 _log.debug("Starting layout helper %s" % helper_path) |
1130 # Note: Not thread safe: http://bugs.python.org/issue2320 | 1132 # Note: Not thread safe: http://bugs.python.org/issue2320 |
1131 self._helper = self._executive.popen([helper_path], | 1133 self._helper = self._executive.popen([helper_path], |
1132 stdin=self._executive.PIPE, stdout=self._executive.PIPE, stderr=
None) | 1134 stdin=self._executive.PIPE, std
out=self._executive.PIPE, stderr=None) |
1133 is_ready = self._helper.stdout.readline() | 1135 is_ready = self._helper.stdout.readline() |
1134 if not is_ready.startswith('ready'): | 1136 if not is_ready.startswith('ready'): |
1135 _log.error("layout_test_helper failed to be ready") | 1137 _log.error("layout_test_helper failed to be ready") |
1136 | 1138 |
1137 def requires_http_server(self): | 1139 def requires_http_server(self): |
1138 """Does the port require an HTTP server for running tests? This could | 1140 """Does the port require an HTTP server for running tests? This could |
1139 be the case when the tests aren't run on the host platform.""" | 1141 be the case when the tests aren't run on the host platform.""" |
1140 return False | 1142 return False |
1141 | 1143 |
1142 def start_http_server(self, additional_dirs, number_of_drivers): | 1144 def start_http_server(self, additional_dirs, number_of_drivers): |
(...skipping 218 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1361 if script_error.exit_code != 1: | 1363 if script_error.exit_code != 1: |
1362 raise script_error | 1364 raise script_error |
1363 | 1365 |
1364 def _run_wdiff(self, actual_filename, expected_filename): | 1366 def _run_wdiff(self, actual_filename, expected_filename): |
1365 """Runs wdiff and may throw exceptions. | 1367 """Runs wdiff and may throw exceptions. |
1366 This is mostly a hook for unit testing.""" | 1368 This is mostly a hook for unit testing.""" |
1367 # Diffs are treated as binary as they may include multiple files | 1369 # Diffs are treated as binary as they may include multiple files |
1368 # with conflicting encodings. Thus we do not decode the output. | 1370 # with conflicting encodings. Thus we do not decode the output. |
1369 command = self._wdiff_command(actual_filename, expected_filename) | 1371 command = self._wdiff_command(actual_filename, expected_filename) |
1370 wdiff = self._executive.run_command(command, decode_output=False, | 1372 wdiff = self._executive.run_command(command, decode_output=False, |
1371 error_handler=self._handle_wdiff_error) | 1373 error_handler=self._handle_wdiff_err
or) |
1372 return self._format_wdiff_output_as_html(wdiff) | 1374 return self._format_wdiff_output_as_html(wdiff) |
1373 | 1375 |
1374 _wdiff_error_html = "Failed to run wdiff, see error log." | 1376 _wdiff_error_html = "Failed to run wdiff, see error log." |
1375 | 1377 |
1376 def wdiff_text(self, actual_filename, expected_filename): | 1378 def wdiff_text(self, actual_filename, expected_filename): |
1377 """Returns a string of HTML indicating the word-level diff of the | 1379 """Returns a string of HTML indicating the word-level diff of the |
1378 contents of the two filenames. Returns an empty string if word-level | 1380 contents of the two filenames. Returns an empty string if word-level |
1379 diffing isn't available.""" | 1381 diffing isn't available.""" |
1380 if not self.wdiff_available(): | 1382 if not self.wdiff_available(): |
1381 return "" | 1383 return "" |
(...skipping 138 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1520 return 'AddressSanitizer' | 1522 return 'AddressSanitizer' |
1521 if 'MemorySanitizer' in output: | 1523 if 'MemorySanitizer' in output: |
1522 return 'MemorySanitizer' | 1524 return 'MemorySanitizer' |
1523 return None | 1525 return None |
1524 | 1526 |
1525 def _get_crash_log(self, name, pid, stdout, stderr, newer_than): | 1527 def _get_crash_log(self, name, pid, stdout, stderr, newer_than): |
1526 if self.output_contains_sanitizer_messages(stderr): | 1528 if self.output_contains_sanitizer_messages(stderr): |
1527 # Running the symbolizer script can take a lot of memory, so we need
to | 1529 # Running the symbolizer script can take a lot of memory, so we need
to |
1528 # serialize access to it across all the concurrently running drivers
. | 1530 # serialize access to it across all the concurrently running drivers
. |
1529 | 1531 |
1530 llvm_symbolizer_path = self.path_from_chromium_base('third_party', '
llvm-build', 'Release+Asserts', 'bin', 'llvm-symbolizer') | 1532 llvm_symbolizer_path = self.path_from_chromium_base( |
| 1533 'third_party', 'llvm-build', 'Release+Asserts', 'bin', 'llvm-sym
bolizer') |
1531 if self._filesystem.exists(llvm_symbolizer_path): | 1534 if self._filesystem.exists(llvm_symbolizer_path): |
1532 env = os.environ.copy() | 1535 env = os.environ.copy() |
1533 env['LLVM_SYMBOLIZER_PATH'] = llvm_symbolizer_path | 1536 env['LLVM_SYMBOLIZER_PATH'] = llvm_symbolizer_path |
1534 else: | 1537 else: |
1535 env = None | 1538 env = None |
1536 sanitizer_filter_path = self.path_from_chromium_base('tools', 'valgr
ind', 'asan', 'asan_symbolize.py') | 1539 sanitizer_filter_path = self.path_from_chromium_base('tools', 'valgr
ind', 'asan', 'asan_symbolize.py') |
1537 sanitizer_strip_path_prefix = 'Release/../../' | 1540 sanitizer_strip_path_prefix = 'Release/../../' |
1538 if self._filesystem.exists(sanitizer_filter_path): | 1541 if self._filesystem.exists(sanitizer_filter_path): |
1539 stderr = self._executive.run_command(['flock', sys.executable, s
anitizer_filter_path, sanitizer_strip_path_prefix], input=stderr, decode_output=
False, env=env) | 1542 stderr = self._executive.run_command( |
| 1543 ['flock', sys.executable, sanitizer_filter_path, sanitizer_s
trip_path_prefix], input=stderr, decode_output=False, env=env) |
1540 | 1544 |
1541 name_str = name or '<unknown process name>' | 1545 name_str = name or '<unknown process name>' |
1542 pid_str = str(pid or '<unknown>') | 1546 pid_str = str(pid or '<unknown>') |
1543 | 1547 |
1544 # We require stdout and stderr to be bytestrings, not character strings. | 1548 # We require stdout and stderr to be bytestrings, not character strings. |
1545 if stdout: | 1549 if stdout: |
1546 assert isinstance(stdout, str) | 1550 assert isinstance(stdout, str) |
1547 stdout_lines = stdout.decode('utf8', 'replace').splitlines() | 1551 stdout_lines = stdout.decode('utf8', 'replace').splitlines() |
1548 else: | 1552 else: |
1549 stdout_lines = [u'<empty>'] | 1553 stdout_lines = [u'<empty>'] |
1550 if stderr: | 1554 if stderr: |
1551 assert isinstance(stderr, str) | 1555 assert isinstance(stderr, str) |
1552 stderr_lines = stderr.decode('utf8', 'replace').splitlines() | 1556 stderr_lines = stderr.decode('utf8', 'replace').splitlines() |
1553 else: | 1557 else: |
1554 stderr_lines = [u'<empty>'] | 1558 stderr_lines = [u'<empty>'] |
1555 | 1559 |
1556 return (stderr, 'crash log for %s (pid %s):\n%s\n%s\n' % (name_str, pid_
str, | 1560 return (stderr, 'crash log for %s (pid %s):\n%s\n%s\n' % (name_str, pid_
str, |
1557 '\n'.join(('STDOUT: ' + l) for l in stdout_lines), | 1561 '\n'.join(('ST
DOUT: ' + l) for l in stdout_lines), |
1558 '\n'.join(('STDERR: ' + l) for l in stderr_lines))) | 1562 '\n'.join(('ST
DERR: ' + l) for l in stderr_lines))) |
1559 | 1563 |
1560 def look_for_new_crash_logs(self, crashed_processes, start_time): | 1564 def look_for_new_crash_logs(self, crashed_processes, start_time): |
1561 pass | 1565 pass |
1562 | 1566 |
1563 def look_for_new_samples(self, unresponsive_processes, start_time): | 1567 def look_for_new_samples(self, unresponsive_processes, start_time): |
1564 pass | 1568 pass |
1565 | 1569 |
1566 def sample_process(self, name, pid): | 1570 def sample_process(self, name, pid): |
1567 pass | 1571 pass |
1568 | 1572 |
(...skipping 184 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1753 pass | 1757 pass |
1754 return True | 1758 return True |
1755 | 1759 |
1756 def _chromium_baseline_path(self, platform): | 1760 def _chromium_baseline_path(self, platform): |
1757 if platform is None: | 1761 if platform is None: |
1758 platform = self.name() | 1762 platform = self.name() |
1759 return self.path_from_webkit_base('LayoutTests', 'platform', platform) | 1763 return self.path_from_webkit_base('LayoutTests', 'platform', platform) |
1760 | 1764 |
1761 | 1765 |
1762 class VirtualTestSuite(object): | 1766 class VirtualTestSuite(object): |
| 1767 |
1763 def __init__(self, prefix=None, base=None, args=None, references_use_default
_args=False): | 1768 def __init__(self, prefix=None, base=None, args=None, references_use_default
_args=False): |
1764 assert base | 1769 assert base |
1765 assert args | 1770 assert args |
1766 assert prefix.find('/') == -1, "Virtual test suites prefixes cannot cont
ain /'s: %s" % prefix | 1771 assert prefix.find('/') == -1, "Virtual test suites prefixes cannot cont
ain /'s: %s" % prefix |
1767 self.name = 'virtual/' + prefix + '/' + base | 1772 self.name = 'virtual/' + prefix + '/' + base |
1768 self.base = base | 1773 self.base = base |
1769 self.args = args | 1774 self.args = args |
1770 self.reference_args = [] if references_use_default_args else args | 1775 self.reference_args = [] if references_use_default_args else args |
1771 self.tests = {} | 1776 self.tests = {} |
1772 | 1777 |
1773 def __repr__(self): | 1778 def __repr__(self): |
1774 return "VirtualTestSuite('%s', '%s', %s, %s)" % (self.name, self.base, s
elf.args, self.reference_args) | 1779 return "VirtualTestSuite('%s', '%s', %s, %s)" % (self.name, self.base, s
elf.args, self.reference_args) |
1775 | 1780 |
1776 | 1781 |
1777 class PhysicalTestSuite(object): | 1782 class PhysicalTestSuite(object): |
| 1783 |
1778 def __init__(self, base, args, reference_args=None): | 1784 def __init__(self, base, args, reference_args=None): |
1779 self.name = base | 1785 self.name = base |
1780 self.base = base | 1786 self.base = base |
1781 self.args = args | 1787 self.args = args |
1782 self.reference_args = args if reference_args is None else reference_args | 1788 self.reference_args = args if reference_args is None else reference_args |
1783 self.tests = set() | 1789 self.tests = set() |
1784 | 1790 |
1785 def __repr__(self): | 1791 def __repr__(self): |
1786 return "PhysicalTestSuite('%s', '%s', %s, %s)" % (self.name, self.base,
self.args, self.reference_args) | 1792 return "PhysicalTestSuite('%s', '%s', %s, %s)" % (self.name, self.base,
self.args, self.reference_args) |
OLD | NEW |