OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Fetches a copy of the latest state of a W3C test repository and commits. | 5 """Fetches a copy of the latest state of a W3C test repository and commits. |
6 | 6 |
7 If this script is given the argument --auto-update, it will also: | 7 If this script is given the argument --auto-update, it will also: |
8 1. Upload a CL. | 8 1. Upload a CL. |
9 2. Trigger try jobs and wait for them to complete. | 9 2. Trigger try jobs and wait for them to complete. |
10 3. Make any changes that are required for new failing tests. | 10 3. Make any changes that are required for new failing tests. |
(...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
60 | 60 |
61 _log.debug('Noting the current Chromium commit.') | 61 _log.debug('Noting the current Chromium commit.') |
62 _, show_ref_output = self.run(['git', 'show-ref', 'HEAD']) | 62 _, show_ref_output = self.run(['git', 'show-ref', 'HEAD']) |
63 chromium_commit = show_ref_output.split()[0] | 63 chromium_commit = show_ref_output.split()[0] |
64 | 64 |
65 dest_dir_name = WPT_DEST_NAME | 65 dest_dir_name = WPT_DEST_NAME |
66 repo_url = WPT_REPO_URL | 66 repo_url = WPT_REPO_URL |
67 | 67 |
68 # TODO(qyearsley): Simplify this to use LocalWPT.fetch when csswg-test | 68 # TODO(qyearsley): Simplify this to use LocalWPT.fetch when csswg-test |
69 # is merged into web-platform-tests (crbug.com/706118). | 69 # is merged into web-platform-tests (crbug.com/706118). |
70 temp_repo_path = self.finder.path_from_webkit_base(dest_dir_name) | 70 temp_repo_path = self.finder.path_from_layout_tests(dest_dir_name) |
71 _log.info('Cloning repo: %s', repo_url) | 71 _log.info('Cloning repo: %s', repo_url) |
72 _log.info('Local path: %s', temp_repo_path) | 72 _log.info('Local path: %s', temp_repo_path) |
73 self.run(['git', 'clone', repo_url, temp_repo_path]) | 73 self.run(['git', 'clone', repo_url, temp_repo_path]) |
74 | 74 |
75 if not options.ignore_exportable_commits: | 75 if not options.ignore_exportable_commits: |
76 commits = self.exportable_but_not_exported_commits(temp_repo_path) | 76 commits = self.exportable_but_not_exported_commits(temp_repo_path) |
77 if commits: | 77 if commits: |
78 # If there are exportable commits, then there's no more work | 78 # If there are exportable commits, then there's no more work |
79 # to do for now. This isn't really an error case; we expect | 79 # to do for now. This isn't really an error case; we expect |
80 # to hit this case some of the time. | 80 # to hit this case some of the time. |
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
132 git_diff_retcode, _ = self.run(['git', 'diff', '--quiet', 'HEAD'], exit_
on_failure=False) | 132 git_diff_retcode, _ = self.run(['git', 'diff', '--quiet', 'HEAD'], exit_
on_failure=False) |
133 if git_diff_retcode: | 133 if git_diff_retcode: |
134 _log.warning('Checkout is dirty; aborting.') | 134 _log.warning('Checkout is dirty; aborting.') |
135 return False | 135 return False |
136 | 136 |
137 local_commits = self.run(['git', 'log', '--oneline', 'origin/master..HEA
D'])[1] | 137 local_commits = self.run(['git', 'log', '--oneline', 'origin/master..HEA
D'])[1] |
138 if local_commits and not allow_local_commits: | 138 if local_commits and not allow_local_commits: |
139 _log.warning('Checkout has local commits; aborting. Use --allow-loca
l-commits to allow this.') | 139 _log.warning('Checkout has local commits; aborting. Use --allow-loca
l-commits to allow this.') |
140 return False | 140 return False |
141 | 141 |
142 if self.fs.exists(self.finder.path_from_webkit_base(WPT_DEST_NAME)): | 142 temp_repo_path = self.finder.path_from_layout_tests(WPT_DEST_NAME) |
143 _log.warning('WebKit/%s exists; aborting.', WPT_DEST_NAME) | 143 if self.fs.exists(temp_repo_path): |
| 144 _log.warning('%s exists; aborting.', temp_repo_path) |
144 return False | 145 return False |
145 | 146 |
146 return True | 147 return True |
147 | 148 |
148 def exportable_but_not_exported_commits(self, wpt_path): | 149 def exportable_but_not_exported_commits(self, wpt_path): |
149 """Checks for commits that might be overwritten by importing. | 150 """Checks for commits that might be overwritten by importing. |
150 | 151 |
151 Args: | 152 Args: |
152 wpt_path: The path to a local checkout of web-platform-tests. | 153 wpt_path: The path to a local checkout of web-platform-tests. |
153 | 154 |
154 Returns: | 155 Returns: |
155 A list of commits in the Chromium repo that are exportable | 156 A list of commits in the Chromium repo that are exportable |
156 but not yet exported to the web-platform-tests repo. | 157 but not yet exported to the web-platform-tests repo. |
157 """ | 158 """ |
158 local_wpt = LocalWPT(self.host, path=wpt_path) | 159 local_wpt = LocalWPT(self.host, path=wpt_path) |
159 assert self.host.filesystem.exists(wpt_path) | 160 assert self.host.filesystem.exists(wpt_path) |
160 _, chromium_commit = local_wpt.most_recent_chromium_commit() | 161 _, chromium_commit = local_wpt.most_recent_chromium_commit() |
161 return exportable_commits_since(chromium_commit.sha, self.host, local_wp
t) | 162 return exportable_commits_since(chromium_commit.sha, self.host, local_wp
t) |
162 | 163 |
163 def clean_up_temp_repo(self, temp_repo_path): | 164 def clean_up_temp_repo(self, temp_repo_path): |
164 _log.info('Deleting temp repo directory %s.', temp_repo_path) | 165 _log.info('Deleting temp repo directory %s.', temp_repo_path) |
165 self.rmtree(temp_repo_path) | 166 self.fs.rmtree(temp_repo_path) |
166 | 167 |
167 def _copy_resources(self): | 168 def _copy_resources(self): |
168 """Copies resources from wpt to LayoutTests/resources. | 169 """Copies resources from wpt to LayoutTests/resources. |
169 | 170 |
170 We copy idlharness.js and testharness.js in wpt to LayoutTests/resources | 171 We copy idlharness.js and testharness.js in wpt to LayoutTests/resources |
171 in order to use them in non-imported tests. | 172 in order to use them in non-imported tests. |
172 | 173 |
173 If this method is changed, the lists of files expected to be identical | 174 If this method is changed, the lists of files expected to be identical |
174 in LayoutTests/PRESUBMIT.py should also be changed. | 175 in LayoutTests/PRESUBMIT.py should also be changed. |
175 """ | 176 """ |
(...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
241 return '%s@%s' % (dest_dir_name, master_commitish) | 242 return '%s@%s' % (dest_dir_name, master_commitish) |
242 | 243 |
243 def _clear_out_dest_path(self, dest_path): | 244 def _clear_out_dest_path(self, dest_path): |
244 _log.info('Cleaning out tests from %s.', dest_path) | 245 _log.info('Cleaning out tests from %s.', dest_path) |
245 should_remove = lambda fs, dirname, basename: ( | 246 should_remove = lambda fs, dirname, basename: ( |
246 not self.is_baseline(basename) and | 247 not self.is_baseline(basename) and |
247 # See http://crbug.com/702283 for context. | 248 # See http://crbug.com/702283 for context. |
248 basename != 'OWNERS') | 249 basename != 'OWNERS') |
249 files_to_delete = self.fs.files_under(dest_path, file_filter=should_remo
ve) | 250 files_to_delete = self.fs.files_under(dest_path, file_filter=should_remo
ve) |
250 for subpath in files_to_delete: | 251 for subpath in files_to_delete: |
251 self.remove('LayoutTests', 'external', subpath) | 252 self.remove(self.finder.path_from_layout_tests('external', subpath)) |
252 | 253 |
253 def _commit_changes(self, commit_message): | 254 def _commit_changes(self, commit_message): |
254 _log.info('Committing changes.') | 255 _log.info('Committing changes.') |
255 self.run(['git', 'commit', '--all', '-F', '-'], stdin=commit_message) | 256 self.run(['git', 'commit', '--all', '-F', '-'], stdin=commit_message) |
256 | 257 |
257 def _has_changes(self): | 258 def _has_changes(self): |
258 return_code, _ = self.run(['git', 'diff', '--quiet', 'HEAD'], exit_on_fa
ilure=False) | 259 return_code, _ = self.run(['git', 'diff', '--quiet', 'HEAD'], exit_on_fa
ilure=False) |
259 return return_code == 1 | 260 return return_code == 1 |
260 | 261 |
261 def _commit_message(self, chromium_commit, import_commit): | 262 def _commit_message(self, chromium_commit, import_commit): |
(...skipping 17 matching lines...) Expand all Loading... |
279 | 280 |
280 @staticmethod | 281 @staticmethod |
281 def is_baseline(basename): | 282 def is_baseline(basename): |
282 # TODO(qyearsley): Find a better, centralized place for this. | 283 # TODO(qyearsley): Find a better, centralized place for this. |
283 # Also, the name for this method should be is_text_baseline. | 284 # Also, the name for this method should be is_text_baseline. |
284 return basename.endswith('-expected.txt') | 285 return basename.endswith('-expected.txt') |
285 | 286 |
286 def run(self, cmd, exit_on_failure=True, cwd=None, stdin=''): | 287 def run(self, cmd, exit_on_failure=True, cwd=None, stdin=''): |
287 _log.debug('Running command: %s', ' '.join(cmd)) | 288 _log.debug('Running command: %s', ' '.join(cmd)) |
288 | 289 |
289 cwd = cwd or self.finder.webkit_base() | 290 cwd = cwd or self.finder.path_from_layout_tests() |
290 proc = self.executive.popen(cmd, stdout=self.executive.PIPE, stderr=self
.executive.PIPE, stdin=self.executive.PIPE, cwd=cwd) | 291 proc = self.executive.popen(cmd, stdout=self.executive.PIPE, stderr=self
.executive.PIPE, stdin=self.executive.PIPE, cwd=cwd) |
291 out, err = proc.communicate(stdin) | 292 out, err = proc.communicate(stdin) |
292 if proc.returncode or self.verbose: | 293 if proc.returncode or self.verbose: |
293 _log.info('# ret> %d', proc.returncode) | 294 _log.info('# ret> %d', proc.returncode) |
294 if out: | 295 if out: |
295 for line in out.splitlines(): | 296 for line in out.splitlines(): |
296 _log.info('# out> %s', line) | 297 _log.info('# out> %s', line) |
297 if err: | 298 if err: |
298 for line in err.splitlines(): | 299 for line in err.splitlines(): |
299 _log.info('# err> %s', line) | 300 _log.info('# err> %s', line) |
300 if exit_on_failure and proc.returncode: | 301 if exit_on_failure and proc.returncode: |
301 self.host.exit(proc.returncode) | 302 self.host.exit(proc.returncode) |
302 return proc.returncode, out | 303 return proc.returncode, out |
303 | 304 |
304 def check_run(self, command): | 305 def check_run(self, command): |
305 return_code, out = self.run(command) | 306 return_code, out = self.run(command) |
306 if return_code: | 307 if return_code: |
307 raise Exception('%s failed with exit code %d.' % ' '.join(command),
return_code) | 308 raise Exception('%s failed with exit code %d.' % ' '.join(command),
return_code) |
308 return out | 309 return out |
309 | 310 |
310 def copyfile(self, source, destination): | 311 def copyfile(self, source, destination): |
311 _log.debug('cp %s %s', source, destination) | 312 _log.debug('cp %s %s', source, destination) |
312 self.fs.copyfile(source, destination) | 313 self.fs.copyfile(source, destination) |
313 | 314 |
314 def remove(self, *comps): | 315 def remove(self, dest): |
315 dest = self.finder.path_from_webkit_base(*comps) | |
316 _log.debug('rm %s', dest) | 316 _log.debug('rm %s', dest) |
317 self.fs.remove(dest) | 317 self.fs.remove(dest) |
318 | 318 |
319 def rmtree(self, *comps): | |
320 dest = self.finder.path_from_webkit_base(*comps) | |
321 _log.debug('rm -fr %s', dest) | |
322 self.fs.rmtree(dest) | |
323 | |
324 def do_auto_update(self): | 319 def do_auto_update(self): |
325 """Attempts to upload a CL, make any required adjustments, and commit. | 320 """Attempts to upload a CL, make any required adjustments, and commit. |
326 | 321 |
327 This function assumes that the imported repo has already been updated, | 322 This function assumes that the imported repo has already been updated, |
328 and that change has been committed. There may be newly-failing tests, | 323 and that change has been committed. There may be newly-failing tests, |
329 so before being able to commit these new changes, we may need to update | 324 so before being able to commit these new changes, we may need to update |
330 TestExpectations or download new baselines. | 325 TestExpectations or download new baselines. |
331 | 326 |
332 Returns: | 327 Returns: |
333 True if successfully committed, False otherwise. | 328 True if successfully committed, False otherwise. |
(...skipping 145 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
479 """Returns a dict mapping source to dest name for layout tests that have
been renamed.""" | 474 """Returns a dict mapping source to dest name for layout tests that have
been renamed.""" |
480 out = self.check_run(['git', 'diff', 'origin/master', '-M100%', '--diff-
filter=R', '--name-status']) | 475 out = self.check_run(['git', 'diff', 'origin/master', '-M100%', '--diff-
filter=R', '--name-status']) |
481 renamed_tests = {} | 476 renamed_tests = {} |
482 for line in out.splitlines(): | 477 for line in out.splitlines(): |
483 _, source_path, dest_path = line.split() | 478 _, source_path, dest_path = line.split() |
484 source_test = self.finder.layout_test_name(source_path) | 479 source_test = self.finder.layout_test_name(source_path) |
485 dest_test = self.finder.layout_test_name(dest_path) | 480 dest_test = self.finder.layout_test_name(dest_path) |
486 if source_test and dest_test: | 481 if source_test and dest_test: |
487 renamed_tests[source_test] = dest_test | 482 renamed_tests[source_test] = dest_test |
488 return renamed_tests | 483 return renamed_tests |
OLD | NEW |