| OLD | NEW |
| 1 # Copyright (C) 2013 Adobe Systems Incorporated. All rights reserved. | 1 # Copyright (C) 2013 Adobe Systems Incorporated. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions | 4 # modification, are permitted provided that the following conditions |
| 5 # are met: | 5 # are met: |
| 6 # | 6 # |
| 7 # 1. Redistributions of source code must retain the above | 7 # 1. Redistributions of source code must retain the above |
| 8 # copyright notice, this list of conditions and the following | 8 # copyright notice, this list of conditions and the following |
| 9 # disclaimer. | 9 # disclaimer. |
| 10 # 2. Redistributions in binary form must reproduce the above | 10 # 2. Redistributions in binary form must reproduce the above |
| (...skipping 15 matching lines...) Expand all Loading... |
| 26 # SUCH DAMAGE. | 26 # SUCH DAMAGE. |
| 27 | 27 |
| 28 """Logic for converting and copying files from a W3C repo. | 28 """Logic for converting and copying files from a W3C repo. |
| 29 | 29 |
| 30 This module is responsible for modifying and copying a subset of the tests from | 30 This module is responsible for modifying and copying a subset of the tests from |
| 31 a local W3C repository source directory into a destination directory. | 31 a local W3C repository source directory into a destination directory. |
| 32 """ | 32 """ |
| 33 | 33 |
| 34 import logging | 34 import logging |
| 35 import mimetypes | 35 import mimetypes |
| 36 import os | |
| 37 import re | 36 import re |
| 38 | 37 |
| 39 from webkitpy.common.webkit_finder import WebKitFinder | 38 from webkitpy.common.webkit_finder import WebKitFinder |
| 40 from webkitpy.layout_tests.models.test_expectations import TestExpectationParser | 39 from webkitpy.layout_tests.models.test_expectations import TestExpectationParser |
| 41 from webkitpy.w3c.test_parser import TestParser | 40 from webkitpy.w3c.test_parser import TestParser |
| 42 from webkitpy.w3c.test_converter import convert_for_webkit | 41 from webkitpy.w3c.test_converter import convert_for_webkit |
| 43 | 42 |
| 44 # Maximum length of import path relative to the upstream repository base. | |
| 45 # This limit is here because the Windows builders cannot create paths that are | |
| 46 # longer than the Windows max path length (260). If the absolute path to the | |
| 47 # destination directory is up to 100 characters on Windows, then that would | |
| 48 # give us 160 characters from the base of the upstream repo. | |
| 49 # See: http://crbug.com/609871. | |
| 50 MAX_PATH_LENGTH = 160 | |
| 51 | |
| 52 _log = logging.getLogger(__name__) | 43 _log = logging.getLogger(__name__) |
| 53 | 44 |
| 54 | 45 |
| 55 class TestCopier(object): | 46 class TestCopier(object): |
| 56 | 47 |
| 57 def __init__(self, host, source_repo_path, dest_dir_name='external'): | 48 def __init__(self, host, source_repo_path, dest_dir_name='external'): |
| 58 """Initializes variables to prepare for copying and converting files. | 49 """Initializes variables to prepare for copying and converting files. |
| 59 | 50 |
| 60 Args: | 51 Args: |
| 61 host: An instance of Host. | 52 host: An instance of Host. |
| 62 source_repo_path: Path to the local checkout of a | 53 source_repo_path: Path to the local checkout of a |
| 63 web-platform-tests or csswg-test repository. | 54 web-platform-tests or csswg-test repository. |
| 64 dest_dir_name: The name of the directory under the layout tests | 55 dest_dir_name: The name of the directory under the layout tests |
| 65 directory where imported tests should be copied to. | 56 directory where imported tests should be copied to. |
| 66 TODO(qyearsley): This can be made into a constant. | 57 TODO(qyearsley): This can be made into a constant. |
| 67 """ | 58 """ |
| 68 self.host = host | 59 self.host = host |
| 69 | 60 |
| 70 assert self.host.filesystem.exists(source_repo_path) | 61 assert self.host.filesystem.exists(source_repo_path) |
| 71 self.source_repo_path = source_repo_path | 62 self.source_repo_path = source_repo_path |
| 72 self.dest_dir_name = dest_dir_name | 63 self.dest_dir_name = dest_dir_name |
| 73 | 64 |
| (...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 150 | 141 |
| 151 if filename.startswith('.') or filename.endswith('.pl'): | 142 if filename.startswith('.') or filename.endswith('.pl'): |
| 152 _log.debug('Skipping: %s', path_full) | 143 _log.debug('Skipping: %s', path_full) |
| 153 _log.debug(' Reason: Hidden files and perl scripts are not
necessary.') | 144 _log.debug(' Reason: Hidden files and perl scripts are not
necessary.') |
| 154 continue | 145 continue |
| 155 if filename == 'OWNERS' or filename == 'reftest.list': | 146 if filename == 'OWNERS' or filename == 'reftest.list': |
| 156 # See http://crbug.com/584660 and http://crbug.com/582838. | 147 # See http://crbug.com/584660 and http://crbug.com/582838. |
| 157 _log.debug('Skipping: %s', path_full) | 148 _log.debug('Skipping: %s', path_full) |
| 158 _log.debug(' Reason: This file may cause Chromium presubmit
to fail.') | 149 _log.debug(' Reason: This file may cause Chromium presubmit
to fail.') |
| 159 continue | 150 continue |
| 160 if self.path_too_long(path_full): | |
| 161 _log.warning('Skipping: %s', path_full) | |
| 162 _log.warning(' Reason: Long path. Max length %d; see http:/
/crbug.com/609871.', MAX_PATH_LENGTH) | |
| 163 continue | |
| 164 | 151 |
| 165 mimetype = mimetypes.guess_type(path_full) | 152 mimetype = mimetypes.guess_type(path_full) |
| 166 if ('html' not in str(mimetype[0]) and | 153 if ('html' not in str(mimetype[0]) and |
| 167 'application/xhtml+xml' not in str(mimetype[0]) and | 154 'application/xhtml+xml' not in str(mimetype[0]) and |
| 168 'application/xml' not in str(mimetype[0])): | 155 'application/xml' not in str(mimetype[0])): |
| 169 copy_list.append({'src': path_full, 'dest': filename}) | 156 copy_list.append({'src': path_full, 'dest': filename}) |
| 170 continue | 157 continue |
| 171 | 158 |
| 172 if self.filesystem.basename(root) in dirs_to_include: | 159 if self.filesystem.basename(root) in dirs_to_include: |
| 173 copy_list.append({'src': path_full, 'dest': filename}) | 160 copy_list.append({'src': path_full, 'dest': filename}) |
| 174 continue | 161 continue |
| 175 | 162 |
| 176 test_parser = TestParser(path_full, self.host) | 163 test_parser = TestParser(path_full, self.host) |
| 177 test_info = test_parser.analyze_test() | 164 test_info = test_parser.analyze_test() |
| 178 if test_info is None: | 165 if test_info is None: |
| 179 copy_list.append({'src': path_full, 'dest': filename}) | 166 copy_list.append({'src': path_full, 'dest': filename}) |
| 180 continue | 167 continue |
| 181 | 168 |
| 182 if 'reference' in test_info.keys(): | 169 if 'reference' in test_info.keys(): |
| 183 ref_path_full = test_info['reference'] | 170 ref_path_full = test_info['reference'] |
| 184 if not self.filesystem.exists(ref_path_full): | 171 if not self.filesystem.exists(ref_path_full): |
| 185 _log.warning('Skipping: %s', path_full) | 172 _log.warning('Skipping: %s', path_full) |
| 186 _log.warning(' Reason: Ref file "%s" was not found.', r
ef_path_full) | 173 _log.warning(' Reason: Ref file "%s" was not found.', r
ef_path_full) |
| 187 continue | 174 continue |
| 188 | 175 |
| 189 if self.is_wpt: | 176 if not self.is_wpt: |
| 190 if self.path_too_long(ref_path_full): | 177 # For csswg-test, we still need to add a ref file |
| 191 _log.warning('Skipping: %s', path_full) | 178 # using WebKit naming conventions. See crbug.com/268729. |
| 192 _log.warning(' Reason: Ref file path length would b
e too long: %s.', ref_path_full) | 179 # FIXME: Remove this when csswg-test is merged into wpt. |
| 193 _log.warning(' Max length %d; see http://crbug.com/
609871.', MAX_PATH_LENGTH) | |
| 194 continue | |
| 195 # For WPT, we don't need to rename the reference file to | |
| 196 # WebKit style name. | |
| 197 # We don't ask to copy ref_path_full here because this | |
| 198 # filesystem walk will find the reference file, and copy | |
| 199 # it as a non-test file. | |
| 200 else: | |
| 201 test_basename = self.filesystem.basename(test_info['test
']) | 180 test_basename = self.filesystem.basename(test_info['test
']) |
| 202 # Add the ref file, following WebKit style. | |
| 203 # FIXME: Ideally we'd support reading the metadata | |
| 204 # directly rather than relying on a naming convention. | |
| 205 # Using a naming convention creates duplicate copies of
the | |
| 206 # reference files (http://crrev.com/268729). | |
| 207 ref_file = self.filesystem.splitext(test_basename)[0] +
'-expected' | 181 ref_file = self.filesystem.splitext(test_basename)[0] +
'-expected' |
| 208 # Make sure to use the extension from the *reference*, n
ot | |
| 209 # from the test, because at least flexbox tests use XHTM
L | |
| 210 # references but HTML tests. | |
| 211 ref_file += self.filesystem.splitext(ref_path_full)[1] | 182 ref_file += self.filesystem.splitext(ref_path_full)[1] |
| 212 | 183 copy_list.append({ |
| 213 if self.path_too_long(path_full.replace(filename, ref_fi
le)): | 184 'src': test_info['reference'], |
| 214 _log.warning('Skipping: %s', path_full) | 185 'dest': ref_file, |
| 215 _log.warning(' Reason: Ref file path length would b
e too long: %s.', ref_file) | 186 'reference_support_info': test_info['reference_suppo
rt_info'], |
| 216 _log.warning(' Max length %d; see http://crbug.com/
609871.', MAX_PATH_LENGTH) | 187 }) |
| 217 continue | |
| 218 copy_list.append({'src': test_info['reference'], 'dest':
ref_file, | |
| 219 'reference_support_info': test_info['r
eference_support_info']}) | |
| 220 | 188 |
| 221 reftests += 1 | 189 reftests += 1 |
| 222 total_tests += 1 | 190 total_tests += 1 |
| 223 copy_list.append({'src': test_info['test'], 'dest': filename
}) | 191 copy_list.append({'src': test_info['test'], 'dest': filename
}) |
| 224 | 192 |
| 225 elif 'jstest' in test_info.keys(): | 193 elif 'jstest' in test_info.keys(): |
| 226 jstests += 1 | 194 jstests += 1 |
| 227 total_tests += 1 | 195 total_tests += 1 |
| 228 copy_list.append({'src': path_full, 'dest': filename, 'is_js
test': True}) | 196 copy_list.append({'src': path_full, 'dest': filename, 'is_js
test': True}) |
| 229 | 197 |
| (...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 353 return False | 321 return False |
| 354 | 322 |
| 355 # Conversion is not necessary for any tests in wpt now; see http://crbug
.com/654081. | 323 # Conversion is not necessary for any tests in wpt now; see http://crbug
.com/654081. |
| 356 # Note, we want to move away from converting files, see http://crbug.com
/663773. | 324 # Note, we want to move away from converting files, see http://crbug.com
/663773. |
| 357 if re.search(r'[/\\]external[/\\]wpt[/\\]', dest_dir): | 325 if re.search(r'[/\\]external[/\\]wpt[/\\]', dest_dir): |
| 358 return False | 326 return False |
| 359 | 327 |
| 360 # Only HTML, XHTML and CSS files should be converted. | 328 # Only HTML, XHTML and CSS files should be converted. |
| 361 mimetype, _ = mimetypes.guess_type(source_path) | 329 mimetype, _ = mimetypes.guess_type(source_path) |
| 362 return mimetype in ('text/html', 'application/xhtml+xml', 'text/css') | 330 return mimetype in ('text/html', 'application/xhtml+xml', 'text/css') |
| 363 | |
| 364 def path_too_long(self, source_path): | |
| 365 """Checks whether a source path is too long to import. | |
| 366 | |
| 367 Args: | |
| 368 Absolute path of file to be imported. | |
| 369 | |
| 370 Returns: | |
| 371 True if the path is too long to import, False if it's OK. | |
| 372 """ | |
| 373 path_from_repo_base = os.path.relpath(source_path, self.source_repo_path
) | |
| 374 return len(path_from_repo_base) > MAX_PATH_LENGTH | |
| OLD | NEW |