| OLD | NEW |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import json | 5 import json |
| 6 import optparse | 6 import optparse |
| 7 | 7 |
| 8 from webkitpy.common.net.buildbot import Build | 8 from webkitpy.common.net.buildbot import Build |
| 9 from webkitpy.common.net.git_cl import GitCL | 9 from webkitpy.common.net.git_cl import GitCL |
| 10 from webkitpy.common.checkout.git_mock import MockGit |
| 10 from webkitpy.common.net.layout_test_results import LayoutTestResults | 11 from webkitpy.common.net.layout_test_results import LayoutTestResults |
| 11 from webkitpy.common.net.rietveld import Rietveld | |
| 12 from webkitpy.common.net.web_mock import MockWeb | |
| 13 from webkitpy.common.system.log_testing import LoggingTestCase | 12 from webkitpy.common.system.log_testing import LoggingTestCase |
| 14 from webkitpy.layout_tests.builder_list import BuilderList | 13 from webkitpy.layout_tests.builder_list import BuilderList |
| 15 from webkitpy.tool.commands.rebaseline_cl import RebaselineCL | 14 from webkitpy.tool.commands.rebaseline_cl import RebaselineCL |
| 16 from webkitpy.tool.commands.rebaseline_unittest import BaseTestCase | 15 from webkitpy.tool.commands.rebaseline_unittest import BaseTestCase |
| 17 | 16 |
| 18 | 17 |
| 19 class RebaselineCLTest(BaseTestCase, LoggingTestCase): | 18 class RebaselineCLTest(BaseTestCase, LoggingTestCase): |
| 20 command_constructor = RebaselineCL | 19 command_constructor = RebaselineCL |
| 21 | 20 |
| 22 def setUp(self): | 21 def setUp(self): |
| 23 BaseTestCase.setUp(self) | 22 BaseTestCase.setUp(self) |
| 24 LoggingTestCase.setUp(self) | 23 LoggingTestCase.setUp(self) |
| 25 web = MockWeb(urls={ | 24 |
| 26 'https://codereview.chromium.org/api/11112222': json.dumps({ | 25 git_cl = GitCL(self.tool) |
| 27 'patchsets': [1, 2], | 26 git_cl.get_issue_number = lambda: '11112222' |
| 28 }), | 27 git_cl.latest_try_jobs = lambda _: [Build('MOCK Try Win', 5000)] |
| 29 'https://codereview.chromium.org/api/11112222/2': json.dumps({ | 28 self.command.git_cl = lambda: git_cl |
| 30 'try_job_results': [ | 29 |
| 31 { | 30 git = MockGit(filesystem=self.tool.filesystem, executive=self.tool.execu
tive) |
| 32 'builder': 'MOCK Try Win', | 31 git.changed_files = lambda **_: [ |
| 33 'buildnumber': 5000, | 32 'third_party/WebKit/LayoutTests/fast/dom/prototype-inheritance.html'
, |
| 34 'result': 0, | 33 'third_party/WebKit/LayoutTests/fast/dom/prototype-taco.html', |
| 35 }, | 34 ] |
| 36 { | 35 self.tool.git = lambda: git |
| 37 'builder': 'MOCK Try Mac', | 36 |
| 38 'buildnumber': 4000, | |
| 39 'result': 0, | |
| 40 }, | |
| 41 ], | |
| 42 'files': { | |
| 43 'third_party/WebKit/LayoutTests/fast/dom/prototype-inheritan
ce.html': {'status': 'M'}, | |
| 44 'third_party/WebKit/LayoutTests/fast/dom/prototype-taco.html
': {'status': 'M'}, | |
| 45 }, | |
| 46 }), | |
| 47 }) | |
| 48 self.tool.builders = BuilderList({ | 37 self.tool.builders = BuilderList({ |
| 49 "MOCK Try Win": { | 38 "MOCK Try Win": { |
| 50 "port_name": "test-win-win7", | 39 "port_name": "test-win-win7", |
| 51 "specifiers": ["Win7", "Release"], | 40 "specifiers": ["Win7", "Release"], |
| 52 "is_try_builder": True, | 41 "is_try_builder": True, |
| 53 }, | 42 }, |
| 54 "MOCK Try Linux": { | 43 "MOCK Try Linux": { |
| 55 "port_name": "test-linux-trusty", | 44 "port_name": "test-linux-trusty", |
| 56 "specifiers": ["Trusty", "Release"], | 45 "specifiers": ["Trusty", "Release"], |
| 57 "is_try_builder": True, | 46 "is_try_builder": True, |
| 58 }, | 47 }, |
| 59 }) | 48 }) |
| 60 self.command.rietveld = Rietveld(web) | |
| 61 | |
| 62 layout_test_results = LayoutTestResults({ | 49 layout_test_results = LayoutTestResults({ |
| 63 'tests': { | 50 'tests': { |
| 64 'fast': { | 51 'fast': { |
| 65 'dom': { | 52 'dom': { |
| 66 'prototype-inheritance.html': { | 53 'prototype-inheritance.html': { |
| 67 'expected': 'PASS', | 54 'expected': 'PASS', |
| 68 'actual': 'TEXT', | 55 'actual': 'TEXT', |
| 69 'is_unexpected': True, | 56 'is_unexpected': True, |
| 70 }, | 57 }, |
| 71 'prototype-banana.html': { | 58 'prototype-banana.html': { |
| (...skipping 67 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 139 | 126 |
| 140 def tearDown(self): | 127 def tearDown(self): |
| 141 BaseTestCase.tearDown(self) | 128 BaseTestCase.tearDown(self) |
| 142 LoggingTestCase.tearDown(self) | 129 LoggingTestCase.tearDown(self) |
| 143 | 130 |
| 144 @staticmethod | 131 @staticmethod |
| 145 def command_options(**kwargs): | 132 def command_options(**kwargs): |
| 146 options = { | 133 options = { |
| 147 'only_changed_tests': False, | 134 'only_changed_tests': False, |
| 148 'dry_run': False, | 135 'dry_run': False, |
| 149 'issue': None, | |
| 150 'optimize': True, | 136 'optimize': True, |
| 151 'results_directory': None, | 137 'results_directory': None, |
| 152 'verbose': False, | 138 'verbose': False, |
| 153 'trigger_jobs': False, | 139 'trigger_jobs': False, |
| 154 } | 140 } |
| 155 options.update(kwargs) | 141 options.update(kwargs) |
| 156 return optparse.Values(dict(**options)) | 142 return optparse.Values(dict(**options)) |
| 157 | 143 |
| 158 def test_execute_with_issue_number_given(self): | 144 def test_execute_with_issue_number_given(self): |
| 159 return_code = self.command.execute(self.command_options(issue=11112222),
[], self.tool) | 145 return_code = self.command.execute(self.command_options(), [], self.tool
) |
| 160 self.assertEqual(return_code, 0) | 146 self.assertEqual(return_code, 0) |
| 161 self.assertLog([ | 147 self.assertLog([ |
| 162 'INFO: Rebaselining fast/dom/prototype-inheritance.html\n', | 148 'INFO: Rebaselining fast/dom/prototype-inheritance.html\n', |
| 163 'INFO: Rebaselining fast/dom/prototype-newtest.html\n', | 149 'INFO: Rebaselining fast/dom/prototype-newtest.html\n', |
| 164 'INFO: Rebaselining fast/dom/prototype-slowtest.html\n', | 150 'INFO: Rebaselining fast/dom/prototype-slowtest.html\n', |
| 165 'INFO: Rebaselining fast/dom/prototype-taco.html\n', | 151 'INFO: Rebaselining fast/dom/prototype-taco.html\n', |
| 166 'INFO: Rebaselining svg/dynamic-updates/SVGFEDropShadowElement-dom-s
tdDeviation-attr.html\n', | 152 'INFO: Rebaselining svg/dynamic-updates/SVGFEDropShadowElement-dom-s
tdDeviation-attr.html\n', |
| 167 ]) | 153 ]) |
| 168 | 154 |
| 169 def test_execute_with_no_issue_number(self): | 155 def test_execute_with_no_issue_number(self): |
| 156 git_cl = GitCL(self.tool) |
| 157 git_cl.get_issue_number = lambda: 'None' |
| 158 self.command.git_cl = lambda: git_cl |
| 170 return_code = self.command.execute(self.command_options(), [], self.tool
) | 159 return_code = self.command.execute(self.command_options(), [], self.tool
) |
| 171 self.assertEqual(return_code, 1) | 160 self.assertEqual(return_code, 1) |
| 172 self.assertLog(['ERROR: No issue number given and no issue for current b
ranch. This tool requires a CL\n' | 161 self.assertLog(['ERROR: No CL number for current branch.\n']) |
| 173 'to operate on; please run `git cl upload` on this branc
h first, or use the --issue\n' | |
| 174 'option to download baselines for another existing CL.\n
']) | |
| 175 | 162 |
| 176 def test_execute_with_issue_number_from_branch(self): | 163 def test_execute_with_issue_number_from_branch(self): |
| 177 git_cl = GitCL(self.tool) | |
| 178 git_cl.get_issue_number = lambda: '11112222' | |
| 179 git_cl.latest_try_jobs = lambda _: [Build('MOCK Try Win', 5000)] | |
| 180 self.command.git_cl = lambda: git_cl | |
| 181 return_code = self.command.execute(self.command_options(), [], self.tool
) | 164 return_code = self.command.execute(self.command_options(), [], self.tool
) |
| 182 self.assertEqual(return_code, 0) | 165 self.assertEqual(return_code, 0) |
| 183 self.assertLog([ | 166 self.assertLog([ |
| 184 'INFO: Rebaselining fast/dom/prototype-inheritance.html\n', | 167 'INFO: Rebaselining fast/dom/prototype-inheritance.html\n', |
| 185 'INFO: Rebaselining fast/dom/prototype-newtest.html\n', | 168 'INFO: Rebaselining fast/dom/prototype-newtest.html\n', |
| 186 'INFO: Rebaselining fast/dom/prototype-slowtest.html\n', | 169 'INFO: Rebaselining fast/dom/prototype-slowtest.html\n', |
| 187 'INFO: Rebaselining fast/dom/prototype-taco.html\n', | 170 'INFO: Rebaselining fast/dom/prototype-taco.html\n', |
| 188 'INFO: Rebaselining svg/dynamic-updates/SVGFEDropShadowElement-dom-s
tdDeviation-attr.html\n', | 171 'INFO: Rebaselining svg/dynamic-updates/SVGFEDropShadowElement-dom-s
tdDeviation-attr.html\n', |
| 189 ]) | 172 ]) |
| 190 | 173 |
| 191 def test_execute_with_only_changed_tests_option(self): | 174 def test_execute_with_only_changed_tests_option(self): |
| 192 return_code = self.command.execute(self.command_options(issue=11112222,
only_changed_tests=True), [], self.tool) | 175 return_code = self.command.execute(self.command_options(only_changed_tes
ts=True), [], self.tool) |
| 193 self.assertEqual(return_code, 0) | 176 self.assertEqual(return_code, 0) |
| 194 # svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html | 177 # svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html |
| 195 # is in the list of failed tests, but not in the list of files modified | 178 # is in the list of failed tests, but not in the list of files modified |
| 196 # in the given CL; it should be included because all_tests is set to Tru
e. | 179 # in the given CL; it should be included because all_tests is set to Tru
e. |
| 197 self.assertLog([ | 180 self.assertLog([ |
| 198 'INFO: Rebaselining fast/dom/prototype-inheritance.html\n', | 181 'INFO: Rebaselining fast/dom/prototype-inheritance.html\n', |
| 199 'INFO: Rebaselining fast/dom/prototype-taco.html\n', | 182 'INFO: Rebaselining fast/dom/prototype-taco.html\n', |
| 200 ]) | 183 ]) |
| 201 | 184 |
| 202 def test_execute_with_flaky_test_that_fails_on_retry(self): | 185 def test_execute_with_flaky_test_that_fails_on_retry(self): |
| 203 # In this example, the --only-changed-tests flag is not given, but | 186 # In this example, the --only-changed-tests flag is not given, but |
| 204 # svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html | 187 # svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html |
| 205 # failed both with and without the patch in the try job, so it is not | 188 # failed both with and without the patch in the try job, so it is not |
| 206 # rebaselined. | 189 # rebaselined. |
| 207 self.tool.buildbot.set_retry_sumary_json(Build('MOCK Try Win', 5000), js
on.dumps({ | 190 self.tool.buildbot.set_retry_sumary_json(Build('MOCK Try Win', 5000), js
on.dumps({ |
| 208 'failures': [ | 191 'failures': [ |
| 209 'fast/dom/prototype-taco.html', | 192 'fast/dom/prototype-taco.html', |
| 210 'fast/dom/prototype-inheritance.html', | 193 'fast/dom/prototype-inheritance.html', |
| 211 ], | 194 ], |
| 212 'ignored': ['svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDevia
tion-attr.html'], | 195 'ignored': ['svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDevia
tion-attr.html'], |
| 213 })) | 196 })) |
| 214 return_code = self.command.execute(self.command_options(issue=11112222),
[], self.tool) | 197 return_code = self.command.execute(self.command_options(), [], self.tool
) |
| 215 self.assertEqual(return_code, 0) | 198 self.assertEqual(return_code, 0) |
| 216 self.assertLog([ | 199 self.assertLog([ |
| 217 'INFO: Rebaselining fast/dom/prototype-inheritance.html\n', | 200 'INFO: Rebaselining fast/dom/prototype-inheritance.html\n', |
| 218 'INFO: Rebaselining fast/dom/prototype-taco.html\n', | 201 'INFO: Rebaselining fast/dom/prototype-taco.html\n', |
| 219 ]) | 202 ]) |
| 220 | 203 |
| 221 def test_execute_with_no_retry_summary_downloaded(self): | 204 def test_execute_with_no_retry_summary_downloaded(self): |
| 222 # In this example, the --only-changed-tests flag is not given, but | 205 # In this example, the --only-changed-tests flag is not given, but |
| 223 # svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html | 206 # svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html |
| 224 # failed both with and without the patch in the try job, so it is not | 207 # failed both with and without the patch in the try job, so it is not |
| 225 # rebaselined. | 208 # rebaselined. |
| 226 self.tool.buildbot.set_retry_sumary_json( | 209 self.tool.buildbot.set_retry_sumary_json(Build('MOCK Try Win', 5000), No
ne) |
| 227 Build('MOCK Try Win', 5000), None) | 210 return_code = self.command.execute(self.command_options(), [], self.tool
) |
| 228 return_code = self.command.execute(self.command_options(issue=11112222),
[], self.tool) | |
| 229 self.assertEqual(return_code, 0) | 211 self.assertEqual(return_code, 0) |
| 230 self.assertLog([ | 212 self.assertLog([ |
| 231 'WARNING: No retry summary available for build Build(builder_name=u\
'MOCK Try Win\', build_number=5000).\n', | 213 'WARNING: No retry summary available for build Build(builder_name=\'
MOCK Try Win\', build_number=5000).\n', |
| 232 'INFO: Rebaselining fast/dom/prototype-inheritance.html\n', | 214 'INFO: Rebaselining fast/dom/prototype-inheritance.html\n', |
| 233 'INFO: Rebaselining fast/dom/prototype-newtest.html\n', | 215 'INFO: Rebaselining fast/dom/prototype-newtest.html\n', |
| 234 'INFO: Rebaselining fast/dom/prototype-slowtest.html\n', | 216 'INFO: Rebaselining fast/dom/prototype-slowtest.html\n', |
| 235 'INFO: Rebaselining fast/dom/prototype-taco.html\n', | 217 'INFO: Rebaselining fast/dom/prototype-taco.html\n', |
| 236 'INFO: Rebaselining svg/dynamic-updates/SVGFEDropShadowElement-dom-s
tdDeviation-attr.html\n', | 218 'INFO: Rebaselining svg/dynamic-updates/SVGFEDropShadowElement-dom-s
tdDeviation-attr.html\n', |
| 237 ]) | 219 ]) |
| 238 | 220 |
| 239 def test_execute_with_trigger_jobs_option(self): | 221 def test_execute_with_trigger_jobs_option(self): |
| 240 return_code = self.command.execute(self.command_options(issue=11112222,
trigger_jobs=True), [], self.tool) | 222 return_code = self.command.execute(self.command_options(trigger_jobs=Tru
e), [], self.tool) |
| 241 self.assertEqual(return_code, 1) | 223 self.assertEqual(return_code, 1) |
| 242 self.assertLog([ | 224 self.assertLog([ |
| 243 'INFO: Triggering try jobs for:\n', | 225 'INFO: Triggering try jobs for:\n', |
| 244 'INFO: MOCK Try Linux\n', | 226 'INFO: MOCK Try Linux\n', |
| 245 'INFO: Please re-run webkit-patch rebaseline-cl once all pending try
jobs have finished.\n', | 227 'INFO: Please re-run webkit-patch rebaseline-cl once all pending try
jobs have finished.\n', |
| 246 ]) | 228 ]) |
| 247 self.assertEqual( | 229 self.assertEqual( |
| 248 self.tool.executive.calls, | 230 self.tool.executive.calls, |
| 249 [['git', 'cl', 'try', '-b', 'MOCK Try Linux']]) | 231 [['git', 'cl', 'try', '-b', 'MOCK Try Linux']]) |
| 250 | 232 |
| 251 def test_rebaseline_calls(self): | 233 def test_rebaseline_calls(self): |
| 252 """Tests the list of commands that are invoked when rebaseline is called
.""" | 234 """Tests the list of commands that are invoked when rebaseline is called
.""" |
| 253 # First write test contents to the mock filesystem so that | 235 # First write test contents to the mock filesystem so that |
| 254 # fast/dom/prototype-taco.html is considered a real test to rebaseline. | 236 # fast/dom/prototype-taco.html is considered a real test to rebaseline. |
| 255 port = self.tool.port_factory.get('test-win-win7') | 237 port = self.tool.port_factory.get('test-win-win7') |
| 256 self._write( | 238 self._write( |
| 257 port.host.filesystem.join(port.layout_tests_dir(), 'fast/dom/prototy
pe-taco.html'), | 239 port.host.filesystem.join(port.layout_tests_dir(), 'fast/dom/prototy
pe-taco.html'), |
| 258 'test contents') | 240 'test contents') |
| 259 | 241 |
| 260 self.command.rebaseline( | 242 self.command.rebaseline( |
| 261 self.command_options(issue=11112222), | 243 self.command_options(), |
| 262 {"fast/dom/prototype-taco.html": {Build("MOCK Try Win", 5000): ["txt
", "png"]}}) | 244 {"fast/dom/prototype-taco.html": {Build("MOCK Try Win", 5000): ["txt
", "png"]}}) |
| 263 | 245 |
| 264 self.assertEqual( | 246 self.assertEqual( |
| 265 self.tool.executive.calls, | 247 self.tool.executive.calls, |
| 266 [ | 248 [ |
| 267 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt', | 249 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt', |
| 268 '--builder', 'MOCK Try Win', '--test', 'fast/dom/prototype-tac
o.html']], | 250 '--builder', 'MOCK Try Win', '--test', 'fast/dom/prototype-tac
o.html']], |
| 269 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt', | 251 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt', |
| 270 '--builder', 'MOCK Try Win', '--test', 'fast/dom/prototype-tac
o.html', '--build-number', '5000']], | 252 '--builder', 'MOCK Try Win', '--test', 'fast/dom/prototype-tac
o.html', '--build-number', '5000']], |
| 271 [['python', 'echo', 'optimize-baselines', '--suffixes', 'txt', '
fast/dom/prototype-taco.html']] | 253 [['python', 'echo', 'optimize-baselines', '--suffixes', 'txt', '
fast/dom/prototype-taco.html']] |
| (...skipping 30 matching lines...) Expand all Loading... |
| 302 self.assertEqual(self.tool.executive.calls, [['git', 'cl', 'try', '-b',
'MOCK Try Win']]) | 284 self.assertEqual(self.tool.executive.calls, [['git', 'cl', 'try', '-b',
'MOCK Try Win']]) |
| 303 self.assertLog([ | 285 self.assertLog([ |
| 304 'INFO: There are existing pending builds for:\n', | 286 'INFO: There are existing pending builds for:\n', |
| 305 'INFO: MOCK Try Linux\n', | 287 'INFO: MOCK Try Linux\n', |
| 306 'INFO: Triggering try jobs for:\n', | 288 'INFO: Triggering try jobs for:\n', |
| 307 'INFO: MOCK Try Win\n', | 289 'INFO: MOCK Try Win\n', |
| 308 ]) | 290 ]) |
| 309 | 291 |
| 310 def test_bails_when_one_build_is_missing_results(self): | 292 def test_bails_when_one_build_is_missing_results(self): |
| 311 self.tool.buildbot.set_results(Build("MOCK Try Win", 5000), None) | 293 self.tool.buildbot.set_results(Build("MOCK Try Win", 5000), None) |
| 312 return_code = self.command.execute(self.command_options(issue=11112222),
[], self.tool) | 294 return_code = self.command.execute(self.command_options(), [], self.tool
) |
| 313 self.assertEqual(return_code, 1) | 295 self.assertEqual(return_code, 1) |
| 314 self.assertLog([ | 296 self.assertLog([ |
| 315 'ERROR: Failed to fetch results from ' | 297 'ERROR: Failed to fetch results from ' |
| 316 '"https://storage.googleapis.com/chromium-layout-test-archives/MOCK_
Try_Win/5000/layout-test-results".\n' | 298 '"https://storage.googleapis.com/chromium-layout-test-archives/MOCK_
Try_Win/5000/layout-test-results".\n' |
| 317 'Try starting a new job for MOCK Try Win by running :\n' | 299 'Try starting a new job for MOCK Try Win by running :\n' |
| 318 ' git cl try -b MOCK Try Win\n' | 300 ' git cl try -b MOCK Try Win\n' |
| 319 ]) | 301 ]) |
| 320 | 302 |
| 321 def test_bails_when_there_are_unstaged_baselines(self): | 303 def test_bails_when_there_are_unstaged_baselines(self): |
| 322 git = self.tool.git() | 304 git = self.tool.git() |
| 323 git.unstaged_changes = lambda: {'third_party/WebKit/LayoutTests/my-test-
expected.txt': '?'} | 305 git.unstaged_changes = lambda: {'third_party/WebKit/LayoutTests/my-test-
expected.txt': '?'} |
| 324 return_code = self.command.execute(self.command_options(issue=11112222),
[], self.tool) | 306 return_code = self.command.execute(self.command_options(), [], self.tool
) |
| 325 self.assertEqual(return_code, 1) | 307 self.assertEqual(return_code, 1) |
| 326 self.assertLog([ | 308 self.assertLog([ |
| 327 'ERROR: Aborting: there are unstaged baselines:\n', | 309 'ERROR: Aborting: there are unstaged baselines:\n', |
| 328 'ERROR: /mock-checkout/third_party/WebKit/LayoutTests/my-test-expe
cted.txt\n', | 310 'ERROR: /mock-checkout/third_party/WebKit/LayoutTests/my-test-expe
cted.txt\n', |
| 329 ]) | 311 ]) |
| OLD | NEW |