| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Fetches a copy of the latest state of a W3C test repository and commits. | 5 """Fetches a copy of the latest state of a W3C test repository and commits. |
| 6 | 6 |
| 7 If this script is given the argument --auto-update, it will also attempt to | 7 If this script is given the argument --auto-update, it will also: |
| 8 upload a CL, triggery try jobs, and make any changes that are required for | 8 1. Upload a CL. |
| 9 new failing tests before committing. | 9 2. Trigger try jobs and wait for them to complete. |
| 10 3. Make any changes that are required for new failing tests. |
| 11 4. Commit the CL. |
| 10 """ | 12 """ |
| 11 | 13 |
| 12 import logging | 14 import logging |
| 13 import argparse | 15 import argparse |
| 14 import json | 16 import json |
| 15 | 17 |
| 16 from webkitpy.common.net.git_cl import GitCL | 18 from webkitpy.common.net.git_cl import GitCL |
| 17 from webkitpy.common.webkit_finder import WebKitFinder | 19 from webkitpy.common.webkit_finder import WebKitFinder |
| 18 from webkitpy.layout_tests.models.test_expectations import TestExpectations, Tes
tExpectationParser | 20 from webkitpy.layout_tests.models.test_expectations import TestExpectations, Tes
tExpectationParser |
| 21 from webkitpy.w3c.test_importer import TestImporter |
| 19 | 22 |
| 20 # Import destination directories (under LayoutTests/external/). | 23 # Import destination directories (under LayoutTests/external/). |
| 21 WPT_DEST_NAME = 'wpt' | 24 WPT_DEST_NAME = 'wpt' |
| 22 CSS_DEST_NAME = 'csswg-test' | 25 CSS_DEST_NAME = 'csswg-test' |
| 23 | 26 |
| 24 # Our mirrors of the official w3c repos, which we pull from. | 27 # Our mirrors of the official w3c repos, which we pull from. |
| 25 WPT_REPO_URL = 'https://chromium.googlesource.com/external/w3c/web-platform-test
s.git' | 28 WPT_REPO_URL = 'https://chromium.googlesource.com/external/w3c/web-platform-test
s.git' |
| 26 CSS_REPO_URL = 'https://chromium.googlesource.com/external/w3c/csswg-test.git' | 29 CSS_REPO_URL = 'https://chromium.googlesource.com/external/w3c/csswg-test.git' |
| 27 | 30 |
| 28 | 31 |
| (...skipping 112 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 141 source = self.path_from_webkit_base('LayoutTests', 'resources', file
name) | 144 source = self.path_from_webkit_base('LayoutTests', 'resources', file
name) |
| 142 destination = self.path_from_webkit_base('LayoutTests', 'external',
WPT_DEST_NAME, wpt_subdir, filename) | 145 destination = self.path_from_webkit_base('LayoutTests', 'external',
WPT_DEST_NAME, wpt_subdir, filename) |
| 143 self.copyfile(source, destination) | 146 self.copyfile(source, destination) |
| 144 self.run(['git', 'add', destination]) | 147 self.run(['git', 'add', destination]) |
| 145 for filename, wpt_subdir in resources_to_copy_from_wpt: | 148 for filename, wpt_subdir in resources_to_copy_from_wpt: |
| 146 source = self.path_from_webkit_base('LayoutTests', 'external', WPT_D
EST_NAME, wpt_subdir, filename) | 149 source = self.path_from_webkit_base('LayoutTests', 'external', WPT_D
EST_NAME, wpt_subdir, filename) |
| 147 destination = self.path_from_webkit_base('LayoutTests', 'resources',
filename) | 150 destination = self.path_from_webkit_base('LayoutTests', 'resources',
filename) |
| 148 self.copyfile(source, destination) | 151 self.copyfile(source, destination) |
| 149 self.run(['git', 'add', destination]) | 152 self.run(['git', 'add', destination]) |
| 150 | 153 |
| 151 def _generate_manifest(self, original_repo_path, dest_path): | 154 def _generate_manifest(self, dest_path): |
| 152 """Generates MANIFEST.json for imported tests. | 155 """Generates MANIFEST.json for imported tests. |
| 153 | 156 |
| 154 Args: | 157 Args: |
| 155 original_repo_path: Path to the temporary source WPT repo directory. | 158 original_repo_path: Path to the temporary source WPT repo directory. |
| 156 dest_path: Path to the destination WPT directory. | 159 dest_path: Path to the destination WPT directory. |
| 157 | 160 |
| 158 Runs the (newly-updated) manifest command if it's found, and then | 161 Runs the (newly-updated) manifest command if it's found, and then |
| 159 stages the generated MANIFEST.json in the git index, ready to commit. | 162 stages the generated MANIFEST.json in the git index, ready to commit. |
| 160 """ | 163 """ |
| 161 manifest_command = self.finder.path_from_webkit_base('Tools', 'Scripts',
'webkitpy', 'thirdparty', 'wpt', 'wpt', 'manifest') | 164 manifest_command = self.finder.path_from_webkit_base('Tools', 'Scripts',
'webkitpy', 'thirdparty', 'wpt', 'wpt', 'manifest') |
| (...skipping 29 matching lines...) Expand all Loading... |
| 191 master_commitish = show_ref_output.split()[0] | 194 master_commitish = show_ref_output.split()[0] |
| 192 | 195 |
| 193 _log.info('Cleaning out tests from LayoutTests/external/%s.', dest_dir_n
ame) | 196 _log.info('Cleaning out tests from LayoutTests/external/%s.', dest_dir_n
ame) |
| 194 dest_path = self.path_from_webkit_base('LayoutTests', 'external', dest_d
ir_name) | 197 dest_path = self.path_from_webkit_base('LayoutTests', 'external', dest_d
ir_name) |
| 195 is_not_baseline_filter = lambda fs, dirname, basename: not self.is_basel
ine(basename) | 198 is_not_baseline_filter = lambda fs, dirname, basename: not self.is_basel
ine(basename) |
| 196 files_to_delete = self.fs.files_under(dest_path, file_filter=is_not_base
line_filter) | 199 files_to_delete = self.fs.files_under(dest_path, file_filter=is_not_base
line_filter) |
| 197 for subpath in files_to_delete: | 200 for subpath in files_to_delete: |
| 198 self.remove('LayoutTests', 'external', subpath) | 201 self.remove('LayoutTests', 'external', subpath) |
| 199 | 202 |
| 200 _log.info('Importing the tests.') | 203 _log.info('Importing the tests.') |
| 201 src_repo = self.path_from_webkit_base(dest_dir_name) | 204 test_importer = TestImporter(self.host, temp_repo_path) |
| 202 import_path = self.path_from_webkit_base('Tools', 'Scripts', 'import-w3c
-tests') | 205 test_importer.do_import() |
| 203 self.run([self.host.executable, import_path, '-d', 'external', src_repo]
) | |
| 204 | 206 |
| 205 self.run(['git', 'add', '--all', 'LayoutTests/external/%s' % dest_dir_na
me]) | 207 self.run(['git', 'add', '--all', 'LayoutTests/external/%s' % dest_dir_na
me]) |
| 206 | 208 |
| 207 _log.info('Deleting any orphaned baselines.') | 209 _log.info('Deleting any orphaned baselines.') |
| 208 | 210 |
| 209 is_baseline_filter = lambda fs, dirname, basename: self.is_baseline(base
name) | 211 is_baseline_filter = lambda fs, dirname, basename: self.is_baseline(base
name) |
| 210 previous_baselines = self.fs.files_under(dest_path, file_filter=is_basel
ine_filter) | 212 previous_baselines = self.fs.files_under(dest_path, file_filter=is_basel
ine_filter) |
| 211 | 213 |
| 212 for subpath in previous_baselines: | 214 for subpath in previous_baselines: |
| 213 full_path = self.fs.join(dest_path, subpath) | 215 full_path = self.fs.join(dest_path, subpath) |
| 214 if self.fs.glob(full_path.replace('-expected.txt', '*')) == [full_pa
th]: | 216 if self.fs.glob(full_path.replace('-expected.txt', '*')) == [full_pa
th]: |
| 215 self.fs.remove(full_path) | 217 self.fs.remove(full_path) |
| 216 | 218 |
| 217 self._generate_manifest(temp_repo_path, dest_path) | 219 self._generate_manifest(dest_path) |
| 218 | 220 |
| 219 if not keep_w3c_repos_around: | 221 if not keep_w3c_repos_around: |
| 220 _log.info('Deleting temp repo directory %s.', temp_repo_path) | 222 _log.info('Deleting temp repo directory %s.', temp_repo_path) |
| 221 self.rmtree(temp_repo_path) | 223 self.rmtree(temp_repo_path) |
| 222 | 224 |
| 223 _log.info('Updating TestExpectations for any removed or renamed tests.') | 225 _log.info('Updating TestExpectations for any removed or renamed tests.') |
| 224 self.update_all_test_expectations_files(self._list_deleted_tests(), self
._list_renamed_tests()) | 226 self.update_all_test_expectations_files(self._list_deleted_tests(), self
._list_renamed_tests()) |
| 225 | 227 |
| 226 return '%s@%s' % (dest_dir_name, master_commitish) | 228 return '%s@%s' % (dest_dir_name, master_commitish) |
| 227 | 229 |
| (...skipping 224 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 452 """Returns a dict mapping source to dest name for layout tests that have
been renamed.""" | 454 """Returns a dict mapping source to dest name for layout tests that have
been renamed.""" |
| 453 out = self.check_run(['git', 'diff', 'origin/master', '-M100%', '--diff-
filter=R', '--name-status']) | 455 out = self.check_run(['git', 'diff', 'origin/master', '-M100%', '--diff-
filter=R', '--name-status']) |
| 454 renamed_tests = {} | 456 renamed_tests = {} |
| 455 for line in out.splitlines(): | 457 for line in out.splitlines(): |
| 456 _, source_path, dest_path = line.split() | 458 _, source_path, dest_path = line.split() |
| 457 source_test = self.finder.layout_test_name(source_path) | 459 source_test = self.finder.layout_test_name(source_path) |
| 458 dest_test = self.finder.layout_test_name(dest_path) | 460 dest_test = self.finder.layout_test_name(dest_path) |
| 459 if source_test and dest_test: | 461 if source_test and dest_test: |
| 460 renamed_tests[source_test] = dest_test | 462 renamed_tests[source_test] = dest_test |
| 461 return renamed_tests | 463 return renamed_tests |
| OLD | NEW |