| OLD | NEW |
| 1 # Copyright (C) 2013 Adobe Systems Incorporated. All rights reserved. | 1 # Copyright (C) 2013 Adobe Systems Incorporated. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions | 4 # modification, are permitted provided that the following conditions |
| 5 # are met: | 5 # are met: |
| 6 # | 6 # |
| 7 # 1. Redistributions of source code must retain the above | 7 # 1. Redistributions of source code must retain the above |
| 8 # copyright notice, this list of conditions and the following | 8 # copyright notice, this list of conditions and the following |
| 9 # disclaimer. | 9 # disclaimer. |
| 10 # 2. Redistributions in binary form must reproduce the above | 10 # 2. Redistributions in binary form must reproduce the above |
| (...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 66 """ | 66 """ |
| 67 | 67 |
| 68 import logging | 68 import logging |
| 69 import mimetypes | 69 import mimetypes |
| 70 import optparse | 70 import optparse |
| 71 import os | 71 import os |
| 72 import sys | 72 import sys |
| 73 | 73 |
| 74 from webkitpy.common.host import Host | 74 from webkitpy.common.host import Host |
| 75 from webkitpy.common.webkit_finder import WebKitFinder | 75 from webkitpy.common.webkit_finder import WebKitFinder |
| 76 from webkitpy.common.system.executive import ScriptError | |
| 77 from webkitpy.layout_tests.models.test_expectations import TestExpectationParser | 76 from webkitpy.layout_tests.models.test_expectations import TestExpectationParser |
| 78 from webkitpy.w3c.test_parser import TestParser | 77 from webkitpy.w3c.test_parser import TestParser |
| 79 from webkitpy.w3c.test_converter import convert_for_webkit | 78 from webkitpy.w3c.test_converter import convert_for_webkit |
| 80 | 79 |
| 81 | 80 |
| 82 # Maximum length of import path starting from top of source repository. | 81 # Maximum length of import path starting from top of source repository. |
| 83 # This limit is here because the Windows builders cannot create paths that are | 82 # This limit is here because the Windows builders cannot create paths that are |
| 84 # longer than the Windows max path length (260). See http://crbug.com/609871. | 83 # longer than the Windows max path length (260). See http://crbug.com/609871. |
| 85 MAX_PATH_LENGTH = 125 | 84 MAX_PATH_LENGTH = 125 |
| 86 | 85 |
| 87 | 86 |
| 88 _log = logging.getLogger(__name__) | 87 _log = logging.getLogger(__name__) |
| 89 | 88 |
| 90 | 89 |
| 91 def main(_argv, _stdout, _stderr): | 90 def main(_argv, _stdout, _stderr): |
| 92 options, args = parse_args() | 91 options, args = parse_args() |
| 93 host = Host() | 92 host = Host() |
| 94 dir_to_import = host.filesystem.normpath(os.path.abspath(args[0])) | 93 source_repo_path = host.filesystem.normpath(os.path.abspath(args[0])) |
| 95 if len(args) == 1: | |
| 96 top_of_repo = dir_to_import | |
| 97 else: | |
| 98 top_of_repo = host.filesystem.normpath(os.path.abspath(args[1])) | |
| 99 | 94 |
| 100 if not host.filesystem.exists(dir_to_import): | 95 if not host.filesystem.exists(source_repo_path): |
| 101 sys.exit('Directory %s not found!' % dir_to_import) | 96 sys.exit('Repository directory %s not found!' % source_repo_path) |
| 102 if not host.filesystem.exists(top_of_repo): | |
| 103 sys.exit('Repository directory %s not found!' % top_of_repo) | |
| 104 if top_of_repo not in dir_to_import: | |
| 105 sys.exit('Repository directory %s must be a parent of %s' % (top_of_repo
, dir_to_import)) | |
| 106 | 97 |
| 107 configure_logging() | 98 configure_logging() |
| 108 test_importer = TestImporter(host, dir_to_import, top_of_repo, options) | 99 test_importer = TestImporter(host, source_repo_path, options) |
| 109 test_importer.do_import() | 100 test_importer.do_import() |
| 110 | 101 |
| 111 | 102 |
| 112 def configure_logging(): | 103 def configure_logging(): |
| 113 class LogHandler(logging.StreamHandler): | 104 class LogHandler(logging.StreamHandler): |
| 114 | 105 |
| 115 def format(self, record): | 106 def format(self, record): |
| 116 if record.levelno > logging.INFO: | 107 if record.levelno > logging.INFO: |
| 117 return "%s: %s" % (record.levelname, record.getMessage()) | 108 return "%s: %s" % (record.levelname, record.getMessage()) |
| 118 return record.getMessage() | 109 return record.getMessage() |
| 119 | 110 |
| 120 logger = logging.getLogger() | 111 logger = logging.getLogger() |
| 121 logger.setLevel(logging.INFO) | 112 logger.setLevel(logging.INFO) |
| 122 handler = LogHandler() | 113 handler = LogHandler() |
| 123 handler.setLevel(logging.INFO) | 114 handler.setLevel(logging.INFO) |
| 124 logger.addHandler(handler) | 115 logger.addHandler(handler) |
| 125 return handler | 116 return handler |
| 126 | 117 |
| 127 | 118 |
| 128 def parse_args(): | 119 def parse_args(): |
| 129 parser = optparse.OptionParser(usage='usage: %prog [options] [dir_to_import]
[top_of_repo]') | 120 parser = optparse.OptionParser(usage='usage: %prog [options] source_repo_pat
h') |
| 130 parser.add_option('-n', '--no-overwrite', dest='overwrite', action='store_fa
lse', default=True, | 121 parser.add_option('-n', '--no-overwrite', dest='overwrite', action='store_fa
lse', default=True, |
| 131 help='Flag to prevent duplicate test files from overwritin
g existing tests. By default, they will be overwritten.') | 122 help='Flag to prevent duplicate test files from overwritin
g existing tests. By default, they will be overwritten.') |
| 132 parser.add_option('-a', '--all', action='store_true', default=False, | 123 parser.add_option('-a', '--all', action='store_true', default=False, |
| 133 help='Import all tests including reftests, JS tests, and m
anual/pixel tests. By default, only reftests and JS tests are imported.') | 124 help='Import all tests including reftests, JS tests, and m
anual/pixel tests. By default, only reftests and JS tests are imported.') |
| 134 parser.add_option('-d', '--dest-dir', dest='destination', default='w3c', | 125 parser.add_option('-d', '--dest-dir', dest='destination', default='w3c', |
| 135 help='Import into a specified directory relative to the La
youtTests root. By default, files are imported under LayoutTests/w3c.') | 126 help='Import into a specified directory relative to the La
youtTests root. By default, files are imported under LayoutTests/w3c.') |
| 136 parser.add_option('--ignore-expectations', action='store_true', default=Fals
e, | 127 parser.add_option('--ignore-expectations', action='store_true', default=Fals
e, |
| 137 help='Ignore the W3CImportExpectations file and import eve
rything.') | 128 help='Ignore the W3CImportExpectations file and import eve
rything.') |
| 138 parser.add_option('--dry-run', action='store_true', default=False, | 129 parser.add_option('--dry-run', action='store_true', default=False, |
| 139 help='Dryrun only (don\'t actually write any results).') | 130 help='Dryrun only (don\'t actually write any results).') |
| 140 | 131 |
| 141 options, args = parser.parse_args() | 132 options, args = parser.parse_args() |
| 142 if len(args) > 2: | 133 if len(args) != 1: |
| 143 parser.error('Incorrect number of arguments') | 134 parser.error('Incorrect number of arguments; source repo path is require
d.') |
| 144 elif len(args) == 0: | |
| 145 # If no top-of-repo path was given, then assume that the user means to | |
| 146 # use the current working directory as the top-of-repo path. | |
| 147 # TODO(qyearsley): Remove this behavior, since it's a little confusing. | |
| 148 args = (os.getcwd(),) | |
| 149 return options, args | 135 return options, args |
| 150 | 136 |
| 151 | 137 |
| 152 class TestImporter(object): | 138 class TestImporter(object): |
| 153 | 139 |
| 154 def __init__(self, host, dir_to_import, top_of_repo, options): | 140 def __init__(self, host, source_repo_path, options): |
| 155 self.host = host | 141 self.host = host |
| 156 self.dir_to_import = dir_to_import | 142 self.source_repo_path = source_repo_path |
| 157 self.top_of_repo = top_of_repo | |
| 158 self.options = options | 143 self.options = options |
| 159 | 144 |
| 160 self.filesystem = self.host.filesystem | 145 self.filesystem = self.host.filesystem |
| 161 self.webkit_finder = WebKitFinder(self.filesystem) | 146 self.webkit_finder = WebKitFinder(self.filesystem) |
| 162 self._webkit_root = self.webkit_finder.webkit_base() | 147 self._webkit_root = self.webkit_finder.webkit_base() |
| 163 self.layout_tests_dir = self.webkit_finder.path_from_webkit_base('Layout
Tests') | 148 self.layout_tests_dir = self.webkit_finder.path_from_webkit_base('Layout
Tests') |
| 164 self.destination_directory = self.filesystem.normpath(self.filesystem.jo
in(self.layout_tests_dir, options.destination, | 149 self.destination_directory = self.filesystem.normpath(self.filesystem.jo
in(self.layout_tests_dir, options.destination, |
| 165
self.filesystem.basename(self.top_of_repo))) | 150
self.filesystem.basename(self.source_repo_path))) |
| 166 self.import_in_place = (self.dir_to_import == self.destination_directory
) | 151 self.import_in_place = (self.source_repo_path == self.destination_direct
ory) |
| 167 self.dir_above_repo = self.filesystem.dirname(self.top_of_repo) | 152 self.dir_above_repo = self.filesystem.dirname(self.source_repo_path) |
| 168 | 153 |
| 169 self.import_list = [] | 154 self.import_list = [] |
| 170 | 155 |
| 171 def do_import(self): | 156 def do_import(self): |
| 172 _log.info("Importing %s into %s", self.dir_to_import, self.destination_d
irectory) | 157 _log.info("Importing %s into %s", self.source_repo_path, self.destinatio
n_directory) |
| 173 self.find_importable_tests(self.dir_to_import) | 158 self.find_importable_tests() |
| 174 self.import_tests() | 159 self.import_tests() |
| 175 | 160 |
| 176 def find_importable_tests(self, directory): | 161 def find_importable_tests(self): |
| 177 """Walks through the source directory to find what tests should be impor
ted. | 162 """Walks through the source directory to find what tests should be impor
ted. |
| 178 | 163 |
| 179 This function sets self.import_list, which contains information about ho
w many | 164 This function sets self.import_list, which contains information about ho
w many |
| 180 tests are being imported, and their source and destination paths. | 165 tests are being imported, and their source and destination paths. |
| 181 """ | 166 """ |
| 182 paths_to_skip = self.find_paths_to_skip() | 167 paths_to_skip = self.find_paths_to_skip() |
| 183 | 168 |
| 184 for root, dirs, files in self.filesystem.walk(directory): | 169 for root, dirs, files in self.filesystem.walk(self.source_repo_path): |
| 185 cur_dir = root.replace(self.dir_above_repo + '/', '') + '/' | 170 cur_dir = root.replace(self.dir_above_repo + '/', '') + '/' |
| 186 _log.info(' scanning ' + cur_dir + '...') | 171 _log.info(' scanning ' + cur_dir + '...') |
| 187 total_tests = 0 | 172 total_tests = 0 |
| 188 reftests = 0 | 173 reftests = 0 |
| 189 jstests = 0 | 174 jstests = 0 |
| 190 | 175 |
| 191 # Files in 'tools' are not for browser testing (e.g., a script for g
enerating test files). | 176 # Files in 'tools' are not for browser testing (e.g., a script for g
enerating test files). |
| 192 # http://testthewebforward.org/docs/test-format-guidelines.html#tool
s | 177 # http://testthewebforward.org/docs/test-format-guidelines.html#tool
s |
| 193 DIRS_TO_SKIP = ('.git', 'test-plan', 'tools') | 178 DIRS_TO_SKIP = ('.git', 'test-plan', 'tools') |
| 194 | 179 |
| (...skipping 15 matching lines...) Expand all Loading... |
| 210 if not self.options.dry_run and self.import_in_place: | 195 if not self.options.dry_run and self.import_in_place: |
| 211 _log.info(" pruning %s" % path_base) | 196 _log.info(" pruning %s" % path_base) |
| 212 self.filesystem.rmtree(path_full) | 197 self.filesystem.rmtree(path_full) |
| 213 else: | 198 else: |
| 214 _log.info(" skipping %s" % path_base) | 199 _log.info(" skipping %s" % path_base) |
| 215 | 200 |
| 216 copy_list = [] | 201 copy_list = [] |
| 217 | 202 |
| 218 for filename in files: | 203 for filename in files: |
| 219 path_full = self.filesystem.join(root, filename) | 204 path_full = self.filesystem.join(root, filename) |
| 220 path_base = path_full.replace(directory + '/', '') | 205 path_base = path_full.replace(self.source_repo_path + '/', '') |
| 221 path_base = self.destination_directory.replace(self.layout_tests
_dir + '/', '') + '/' + path_base | 206 path_base = self.destination_directory.replace(self.layout_tests
_dir + '/', '') + '/' + path_base |
| 222 if path_base in paths_to_skip: | 207 if path_base in paths_to_skip: |
| 223 if not self.options.dry_run and self.import_in_place: | 208 if not self.options.dry_run and self.import_in_place: |
| 224 _log.info(" pruning %s" % path_base) | 209 _log.info(" pruning %s" % path_base) |
| 225 self.filesystem.remove(path_full) | 210 self.filesystem.remove(path_full) |
| 226 continue | 211 continue |
| 227 else: | 212 else: |
| 228 continue | 213 continue |
| 229 # FIXME: This block should really be a separate function, but th
e early-continues make that difficult. | 214 # FIXME: This block should really be a separate function, but th
e early-continues make that difficult. |
| 230 | 215 |
| (...skipping 95 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 326 total_imported_reftests += dir_to_copy['reftests'] | 311 total_imported_reftests += dir_to_copy['reftests'] |
| 327 total_imported_jstests += dir_to_copy['jstests'] | 312 total_imported_jstests += dir_to_copy['jstests'] |
| 328 | 313 |
| 329 prefixed_properties = [] | 314 prefixed_properties = [] |
| 330 | 315 |
| 331 if not dir_to_copy['copy_list']: | 316 if not dir_to_copy['copy_list']: |
| 332 continue | 317 continue |
| 333 | 318 |
| 334 orig_path = dir_to_copy['dirname'] | 319 orig_path = dir_to_copy['dirname'] |
| 335 | 320 |
| 336 subpath = self.filesystem.relpath(orig_path, self.top_of_repo) | 321 subpath = self.filesystem.relpath(orig_path, self.source_repo_path) |
| 337 new_path = self.filesystem.join(self.destination_directory, subpath) | 322 new_path = self.filesystem.join(self.destination_directory, subpath) |
| 338 | 323 |
| 339 if not self.filesystem.exists(new_path): | 324 if not self.filesystem.exists(new_path): |
| 340 self.filesystem.maybe_make_directory(new_path) | 325 self.filesystem.maybe_make_directory(new_path) |
| 341 | 326 |
| 342 copied_files = [] | 327 copied_files = [] |
| 343 | 328 |
| 344 for file_to_copy in dir_to_copy['copy_list']: | 329 for file_to_copy in dir_to_copy['copy_list']: |
| 345 # FIXME: Split this block into a separate function. | 330 # FIXME: Split this block into a separate function. |
| 346 orig_filepath = self.filesystem.normpath(file_to_copy['src']) | 331 orig_filepath = self.filesystem.normpath(file_to_copy['src']) |
| (...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 413 _log.info('Properties needing prefixes (by count):') | 398 _log.info('Properties needing prefixes (by count):') |
| 414 for prefixed_property in sorted(total_prefixed_properties, key=lambd
a p: total_prefixed_properties[p]): | 399 for prefixed_property in sorted(total_prefixed_properties, key=lambd
a p: total_prefixed_properties[p]): |
| 415 _log.info(' %s: %s', prefixed_property, total_prefixed_properti
es[prefixed_property]) | 400 _log.info(' %s: %s', prefixed_property, total_prefixed_properti
es[prefixed_property]) |
| 416 | 401 |
| 417 def path_too_long(self, source_path): | 402 def path_too_long(self, source_path): |
| 418 """Checks whether a source path is too long to import. | 403 """Checks whether a source path is too long to import. |
| 419 | 404 |
| 420 Args: | 405 Args: |
| 421 Absolute path of file to be imported. | 406 Absolute path of file to be imported. |
| 422 """ | 407 """ |
| 423 path_from_repo_base = os.path.relpath(source_path, self.top_of_repo) | 408 path_from_repo_base = os.path.relpath(source_path, self.source_repo_path
) |
| 424 return len(path_from_repo_base) > MAX_PATH_LENGTH | 409 return len(path_from_repo_base) > MAX_PATH_LENGTH |
| OLD | NEW |