| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Fetches a copy of the latest state of a W3C test repository and commits. | 5 """Fetches a copy of the latest state of a W3C test repository and commits. |
| 6 | 6 |
| 7 If this script is given the argument --auto-update, it will also: | 7 If this script is given the argument --auto-update, it will also: |
| 8 1. Upload a CL. | 8 1. Upload a CL. |
| 9 2. Trigger try jobs and wait for them to complete. | 9 2. Trigger try jobs and wait for them to complete. |
| 10 3. Make any changes that are required for new failing tests. | 10 3. Make any changes that are required for new failing tests. |
| (...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 60 | 60 |
| 61 _log.debug('Noting the current Chromium commit.') | 61 _log.debug('Noting the current Chromium commit.') |
| 62 _, show_ref_output = self.run(['git', 'show-ref', 'HEAD']) | 62 _, show_ref_output = self.run(['git', 'show-ref', 'HEAD']) |
| 63 chromium_commit = show_ref_output.split()[0] | 63 chromium_commit = show_ref_output.split()[0] |
| 64 | 64 |
| 65 dest_dir_name = WPT_DEST_NAME | 65 dest_dir_name = WPT_DEST_NAME |
| 66 repo_url = WPT_REPO_URL | 66 repo_url = WPT_REPO_URL |
| 67 | 67 |
| 68 # TODO(qyearsley): Simplify this to use LocalWPT.fetch when csswg-test | 68 # TODO(qyearsley): Simplify this to use LocalWPT.fetch when csswg-test |
| 69 # is merged into web-platform-tests (crbug.com/706118). | 69 # is merged into web-platform-tests (crbug.com/706118). |
| 70 temp_repo_path = self.finder.path_from_webkit_base(dest_dir_name) | 70 temp_repo_path = self.finder.path_from_layout_tests(dest_dir_name) |
| 71 _log.info('Cloning repo: %s', repo_url) | 71 _log.info('Cloning repo: %s', repo_url) |
| 72 _log.info('Local path: %s', temp_repo_path) | 72 _log.info('Local path: %s', temp_repo_path) |
| 73 self.run(['git', 'clone', repo_url, temp_repo_path]) | 73 self.run(['git', 'clone', repo_url, temp_repo_path]) |
| 74 | 74 |
| 75 if not options.ignore_exportable_commits: | 75 if not options.ignore_exportable_commits: |
| 76 commits = self.exportable_but_not_exported_commits(temp_repo_path) | 76 commits = self.exportable_but_not_exported_commits(temp_repo_path) |
| 77 if commits: | 77 if commits: |
| 78 # If there are exportable commits, then there's no more work | 78 # If there are exportable commits, then there's no more work |
| 79 # to do for now. This isn't really an error case; we expect | 79 # to do for now. This isn't really an error case; we expect |
| 80 # to hit this case some of the time. | 80 # to hit this case some of the time. |
| (...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 132 git_diff_retcode, _ = self.run(['git', 'diff', '--quiet', 'HEAD'], exit_
on_failure=False) | 132 git_diff_retcode, _ = self.run(['git', 'diff', '--quiet', 'HEAD'], exit_
on_failure=False) |
| 133 if git_diff_retcode: | 133 if git_diff_retcode: |
| 134 _log.warning('Checkout is dirty; aborting.') | 134 _log.warning('Checkout is dirty; aborting.') |
| 135 return False | 135 return False |
| 136 | 136 |
| 137 local_commits = self.run(['git', 'log', '--oneline', 'origin/master..HEA
D'])[1] | 137 local_commits = self.run(['git', 'log', '--oneline', 'origin/master..HEA
D'])[1] |
| 138 if local_commits and not allow_local_commits: | 138 if local_commits and not allow_local_commits: |
| 139 _log.warning('Checkout has local commits; aborting. Use --allow-loca
l-commits to allow this.') | 139 _log.warning('Checkout has local commits; aborting. Use --allow-loca
l-commits to allow this.') |
| 140 return False | 140 return False |
| 141 | 141 |
| 142 if self.fs.exists(self.finder.path_from_webkit_base(WPT_DEST_NAME)): | 142 temp_repo_path = self.finder.path_from_layout_tests(WPT_DEST_NAME) |
| 143 _log.warning('WebKit/%s exists; aborting.', WPT_DEST_NAME) | 143 if self.fs.exists(temp_repo_path): |
| 144 _log.warning('%s exists; aborting.', temp_repo_path) |
| 144 return False | 145 return False |
| 145 | 146 |
| 146 return True | 147 return True |
| 147 | 148 |
| 148 def exportable_but_not_exported_commits(self, wpt_path): | 149 def exportable_but_not_exported_commits(self, wpt_path): |
| 149 """Checks for commits that might be overwritten by importing. | 150 """Checks for commits that might be overwritten by importing. |
| 150 | 151 |
| 151 Args: | 152 Args: |
| 152 wpt_path: The path to a local checkout of web-platform-tests. | 153 wpt_path: The path to a local checkout of web-platform-tests. |
| 153 | 154 |
| 154 Returns: | 155 Returns: |
| 155 A list of commits in the Chromium repo that are exportable | 156 A list of commits in the Chromium repo that are exportable |
| 156 but not yet exported to the web-platform-tests repo. | 157 but not yet exported to the web-platform-tests repo. |
| 157 """ | 158 """ |
| 158 local_wpt = LocalWPT(self.host, path=wpt_path) | 159 local_wpt = LocalWPT(self.host, path=wpt_path) |
| 159 assert self.host.filesystem.exists(wpt_path) | 160 assert self.host.filesystem.exists(wpt_path) |
| 160 _, chromium_commit = local_wpt.most_recent_chromium_commit() | 161 _, chromium_commit = local_wpt.most_recent_chromium_commit() |
| 161 return exportable_commits_since(chromium_commit.sha, self.host, local_wp
t) | 162 return exportable_commits_since(chromium_commit.sha, self.host, local_wp
t) |
| 162 | 163 |
| 163 def clean_up_temp_repo(self, temp_repo_path): | 164 def clean_up_temp_repo(self, temp_repo_path): |
| 164 _log.info('Deleting temp repo directory %s.', temp_repo_path) | 165 _log.info('Deleting temp repo directory %s.', temp_repo_path) |
| 165 self.rmtree(temp_repo_path) | 166 self.fs.rmtree(temp_repo_path) |
| 166 | 167 |
| 167 def _copy_resources(self): | 168 def _copy_resources(self): |
| 168 """Copies resources from wpt to LayoutTests/resources. | 169 """Copies resources from wpt to LayoutTests/resources. |
| 169 | 170 |
| 170 We copy idlharness.js and testharness.js in wpt to LayoutTests/resources | 171 We copy idlharness.js and testharness.js in wpt to LayoutTests/resources |
| 171 in order to use them in non-imported tests. | 172 in order to use them in non-imported tests. |
| 172 | 173 |
| 173 If this method is changed, the lists of files expected to be identical | 174 If this method is changed, the lists of files expected to be identical |
| 174 in LayoutTests/PRESUBMIT.py should also be changed. | 175 in LayoutTests/PRESUBMIT.py should also be changed. |
| 175 """ | 176 """ |
| (...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 241 return '%s@%s' % (dest_dir_name, master_commitish) | 242 return '%s@%s' % (dest_dir_name, master_commitish) |
| 242 | 243 |
| 243 def _clear_out_dest_path(self, dest_path): | 244 def _clear_out_dest_path(self, dest_path): |
| 244 _log.info('Cleaning out tests from %s.', dest_path) | 245 _log.info('Cleaning out tests from %s.', dest_path) |
| 245 should_remove = lambda fs, dirname, basename: ( | 246 should_remove = lambda fs, dirname, basename: ( |
| 246 not self.is_baseline(basename) and | 247 not self.is_baseline(basename) and |
| 247 # See http://crbug.com/702283 for context. | 248 # See http://crbug.com/702283 for context. |
| 248 basename != 'OWNERS') | 249 basename != 'OWNERS') |
| 249 files_to_delete = self.fs.files_under(dest_path, file_filter=should_remo
ve) | 250 files_to_delete = self.fs.files_under(dest_path, file_filter=should_remo
ve) |
| 250 for subpath in files_to_delete: | 251 for subpath in files_to_delete: |
| 251 self.remove('LayoutTests', 'external', subpath) | 252 self.remove(self.finder.path_from_layout_tests('external', subpath)) |
| 252 | 253 |
| 253 def _commit_changes(self, commit_message): | 254 def _commit_changes(self, commit_message): |
| 254 _log.info('Committing changes.') | 255 _log.info('Committing changes.') |
| 255 self.run(['git', 'commit', '--all', '-F', '-'], stdin=commit_message) | 256 self.run(['git', 'commit', '--all', '-F', '-'], stdin=commit_message) |
| 256 | 257 |
| 257 def _has_changes(self): | 258 def _has_changes(self): |
| 258 return_code, _ = self.run(['git', 'diff', '--quiet', 'HEAD'], exit_on_fa
ilure=False) | 259 return_code, _ = self.run(['git', 'diff', '--quiet', 'HEAD'], exit_on_fa
ilure=False) |
| 259 return return_code == 1 | 260 return return_code == 1 |
| 260 | 261 |
| 261 def _commit_message(self, chromium_commit, import_commit): | 262 def _commit_message(self, chromium_commit, import_commit): |
| (...skipping 17 matching lines...) Expand all Loading... |
| 279 | 280 |
| 280 @staticmethod | 281 @staticmethod |
| 281 def is_baseline(basename): | 282 def is_baseline(basename): |
| 282 # TODO(qyearsley): Find a better, centralized place for this. | 283 # TODO(qyearsley): Find a better, centralized place for this. |
| 283 # Also, the name for this method should be is_text_baseline. | 284 # Also, the name for this method should be is_text_baseline. |
| 284 return basename.endswith('-expected.txt') | 285 return basename.endswith('-expected.txt') |
| 285 | 286 |
| 286 def run(self, cmd, exit_on_failure=True, cwd=None, stdin=''): | 287 def run(self, cmd, exit_on_failure=True, cwd=None, stdin=''): |
| 287 _log.debug('Running command: %s', ' '.join(cmd)) | 288 _log.debug('Running command: %s', ' '.join(cmd)) |
| 288 | 289 |
| 289 cwd = cwd or self.finder.webkit_base() | 290 cwd = cwd or self.finder.path_from_layout_tests() |
| 290 proc = self.executive.popen(cmd, stdout=self.executive.PIPE, stderr=self
.executive.PIPE, stdin=self.executive.PIPE, cwd=cwd) | 291 proc = self.executive.popen(cmd, stdout=self.executive.PIPE, stderr=self
.executive.PIPE, stdin=self.executive.PIPE, cwd=cwd) |
| 291 out, err = proc.communicate(stdin) | 292 out, err = proc.communicate(stdin) |
| 292 if proc.returncode or self.verbose: | 293 if proc.returncode or self.verbose: |
| 293 _log.info('# ret> %d', proc.returncode) | 294 _log.info('# ret> %d', proc.returncode) |
| 294 if out: | 295 if out: |
| 295 for line in out.splitlines(): | 296 for line in out.splitlines(): |
| 296 _log.info('# out> %s', line) | 297 _log.info('# out> %s', line) |
| 297 if err: | 298 if err: |
| 298 for line in err.splitlines(): | 299 for line in err.splitlines(): |
| 299 _log.info('# err> %s', line) | 300 _log.info('# err> %s', line) |
| 300 if exit_on_failure and proc.returncode: | 301 if exit_on_failure and proc.returncode: |
| 301 self.host.exit(proc.returncode) | 302 self.host.exit(proc.returncode) |
| 302 return proc.returncode, out | 303 return proc.returncode, out |
| 303 | 304 |
| 304 def check_run(self, command): | 305 def check_run(self, command): |
| 305 return_code, out = self.run(command) | 306 return_code, out = self.run(command) |
| 306 if return_code: | 307 if return_code: |
| 307 raise Exception('%s failed with exit code %d.' % ' '.join(command),
return_code) | 308 raise Exception('%s failed with exit code %d.' % ' '.join(command),
return_code) |
| 308 return out | 309 return out |
| 309 | 310 |
| 310 def copyfile(self, source, destination): | 311 def copyfile(self, source, destination): |
| 311 _log.debug('cp %s %s', source, destination) | 312 _log.debug('cp %s %s', source, destination) |
| 312 self.fs.copyfile(source, destination) | 313 self.fs.copyfile(source, destination) |
| 313 | 314 |
| 314 def remove(self, *comps): | 315 def remove(self, dest): |
| 315 dest = self.finder.path_from_webkit_base(*comps) | |
| 316 _log.debug('rm %s', dest) | 316 _log.debug('rm %s', dest) |
| 317 self.fs.remove(dest) | 317 self.fs.remove(dest) |
| 318 | 318 |
| 319 def rmtree(self, *comps): | |
| 320 dest = self.finder.path_from_webkit_base(*comps) | |
| 321 _log.debug('rm -fr %s', dest) | |
| 322 self.fs.rmtree(dest) | |
| 323 | |
| 324 def do_auto_update(self): | 319 def do_auto_update(self): |
| 325 """Attempts to upload a CL, make any required adjustments, and commit. | 320 """Attempts to upload a CL, make any required adjustments, and commit. |
| 326 | 321 |
| 327 This function assumes that the imported repo has already been updated, | 322 This function assumes that the imported repo has already been updated, |
| 328 and that change has been committed. There may be newly-failing tests, | 323 and that change has been committed. There may be newly-failing tests, |
| 329 so before being able to commit these new changes, we may need to update | 324 so before being able to commit these new changes, we may need to update |
| 330 TestExpectations or download new baselines. | 325 TestExpectations or download new baselines. |
| 331 | 326 |
| 332 Returns: | 327 Returns: |
| 333 True if successfully committed, False otherwise. | 328 True if successfully committed, False otherwise. |
| (...skipping 145 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 479 """Returns a dict mapping source to dest name for layout tests that have
been renamed.""" | 474 """Returns a dict mapping source to dest name for layout tests that have
been renamed.""" |
| 480 out = self.check_run(['git', 'diff', 'origin/master', '-M100%', '--diff-
filter=R', '--name-status']) | 475 out = self.check_run(['git', 'diff', 'origin/master', '-M100%', '--diff-
filter=R', '--name-status']) |
| 481 renamed_tests = {} | 476 renamed_tests = {} |
| 482 for line in out.splitlines(): | 477 for line in out.splitlines(): |
| 483 _, source_path, dest_path = line.split() | 478 _, source_path, dest_path = line.split() |
| 484 source_test = self.finder.layout_test_name(source_path) | 479 source_test = self.finder.layout_test_name(source_path) |
| 485 dest_test = self.finder.layout_test_name(dest_path) | 480 dest_test = self.finder.layout_test_name(dest_path) |
| 486 if source_test and dest_test: | 481 if source_test and dest_test: |
| 487 renamed_tests[source_test] = dest_test | 482 renamed_tests[source_test] = dest_test |
| 488 return renamed_tests | 483 return renamed_tests |
| OLD | NEW |