| OLD | NEW |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import optparse |
| 5 import unittest | 6 import unittest |
| 6 | 7 |
| 7 from webkitpy.common.net.buildbot import Build | 8 from webkitpy.common.net.buildbot import Build |
| 8 from webkitpy.common.net.layouttestresults import LayoutTestResults | 9 from webkitpy.common.net.layouttestresults import LayoutTestResults |
| 9 from webkitpy.common.net.rietveld import Build | 10 from webkitpy.common.net.rietveld import Build |
| 10 from webkitpy.common.system.executive_mock import MockExecutive | 11 from webkitpy.common.system.executive_mock import MockExecutive |
| 11 from webkitpy.common.system.executive_mock import MockExecutive2 | 12 from webkitpy.common.system.executive_mock import MockExecutive2 |
| 12 from webkitpy.common.system.outputcapture import OutputCapture | 13 from webkitpy.common.system.outputcapture import OutputCapture |
| 13 from webkitpy.layout_tests.builder_list import BuilderList | 14 from webkitpy.layout_tests.builder_list import BuilderList |
| 15 from webkitpy.tool.commands.rebaseline import AbstractParallelRebaselineCommand |
| 14 from webkitpy.tool.commands.rebaseline import CopyExistingBaselinesInternal | 16 from webkitpy.tool.commands.rebaseline import CopyExistingBaselinesInternal |
| 17 from webkitpy.tool.commands.rebaseline import Rebaseline |
| 15 from webkitpy.tool.commands.rebaseline import RebaselineExpectations | 18 from webkitpy.tool.commands.rebaseline import RebaselineExpectations |
| 16 from webkitpy.tool.commands.rebaseline import Rebaseline | |
| 17 from webkitpy.tool.commands.rebaseline import RebaselineJson | 19 from webkitpy.tool.commands.rebaseline import RebaselineJson |
| 18 from webkitpy.tool.commands.rebaseline import RebaselineTest | 20 from webkitpy.tool.commands.rebaseline import RebaselineTest |
| 19 from webkitpy.tool.commands.rebaseline import AbstractParallelRebaselineCommand | |
| 20 from webkitpy.tool.mock_tool import MockOptions | |
| 21 from webkitpy.tool.mock_tool import MockWebKitPatch | 21 from webkitpy.tool.mock_tool import MockWebKitPatch |
| 22 | 22 |
| 23 | 23 |
| 24 # pylint: disable=protected-access | 24 # pylint: disable=protected-access |
| 25 class BaseTestCase(unittest.TestCase): | 25 class BaseTestCase(unittest.TestCase): |
| 26 MOCK_WEB_RESULT = 'MOCK Web result, convert 404 to None=True' | 26 MOCK_WEB_RESULT = 'MOCK Web result, convert 404 to None=True' |
| 27 WEB_PREFIX = 'https://storage.googleapis.com/chromium-layout-test-archives/M
OCK_Mac10_11/results/layout-test-results' | 27 WEB_PREFIX = 'https://storage.googleapis.com/chromium-layout-test-archives/M
OCK_Mac10_11/results/layout-test-results' |
| 28 | 28 |
| 29 command_constructor = None | 29 command_constructor = None |
| 30 | 30 |
| (...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 117 | 117 |
| 118 port = self.tool.port_factory.get('test-mac-mac10.10') | 118 port = self.tool.port_factory.get('test-mac-mac10.10') |
| 119 self._write( | 119 self._write( |
| 120 port._filesystem.join( | 120 port._filesystem.join( |
| 121 port.layout_tests_dir(), | 121 port.layout_tests_dir(), |
| 122 'platform/test-mac-mac10.10/failures/expected/image-expected.txt
'), | 122 'platform/test-mac-mac10.10/failures/expected/image-expected.txt
'), |
| 123 'original mac10.11 result') | 123 'original mac10.11 result') |
| 124 | 124 |
| 125 oc = OutputCapture() | 125 oc = OutputCapture() |
| 126 try: | 126 try: |
| 127 options = MockOptions(builder="MOCK Mac10.11", suffixes="txt", verbo
se=True, | 127 options = optparse.Values({ |
| 128 test="failures/expected/image.html", results_d
irectory=None) | 128 'builder': "MOCK Mac10.11", |
| 129 | 129 'suffixes': 'txt', |
| 130 'verbose': True, |
| 131 'test': "failures/expected/image.html", |
| 132 'results_directory': None |
| 133 }) |
| 130 oc.capture_output() | 134 oc.capture_output() |
| 131 self.command.execute(options, [], self.tool) | 135 self.command.execute(options, [], self.tool) |
| 132 finally: | 136 finally: |
| 133 out, _, _ = oc.restore_output() | 137 out, _, _ = oc.restore_output() |
| 134 | 138 |
| 135 self.assertMultiLineEqual( | 139 self.assertMultiLineEqual( |
| 136 self._read(self.tool.filesystem.join( | 140 self._read(self.tool.filesystem.join( |
| 137 port.layout_tests_dir(), | 141 port.layout_tests_dir(), |
| 138 'platform/test-mac-mac10.10/failures/expected/image-expected.txt
')), | 142 'platform/test-mac-mac10.10/failures/expected/image-expected.txt
')), |
| 139 'original mac10.11 result') | 143 'original mac10.11 result') |
| 140 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete"
: []}\n') | 144 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete"
: []}\n') |
| 141 | 145 |
| 142 def test_copying_overwritten_baseline_to_multiple_locations(self): | 146 def test_copying_overwritten_baseline_to_multiple_locations(self): |
| 143 self.tool.executive = MockExecutive2() | 147 self.tool.executive = MockExecutive2() |
| 144 | 148 |
| 145 port = self.tool.port_factory.get('test-win-win7') | 149 port = self.tool.port_factory.get('test-win-win7') |
| 146 self._write( | 150 self._write( |
| 147 port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-wi
n7/failures/expected/image-expected.txt'), | 151 port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-wi
n7/failures/expected/image-expected.txt'), |
| 148 'original win7 result') | 152 'original win7 result') |
| 149 | 153 |
| 150 oc = OutputCapture() | 154 oc = OutputCapture() |
| 151 try: | 155 try: |
| 152 options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=T
rue, | 156 options = optparse.Values({ |
| 153 test="failures/expected/image.html", results_d
irectory=None) | 157 'builder': "MOCK Win7", |
| 154 | 158 'suffixes': "txt", |
| 159 'verbose': True, |
| 160 'test': "failures/expected/image.html", |
| 161 'results_directory': None |
| 162 }) |
| 155 oc.capture_output() | 163 oc.capture_output() |
| 156 self.command.execute(options, [], self.tool) | 164 self.command.execute(options, [], self.tool) |
| 157 finally: | 165 finally: |
| 158 out, _, _ = oc.restore_output() | 166 out, _, _ = oc.restore_output() |
| 159 | 167 |
| 160 self.assertMultiLineEqual( | 168 self.assertMultiLineEqual( |
| 161 self._read(self.tool.filesystem.join( | 169 self._read(self.tool.filesystem.join( |
| 162 port.layout_tests_dir(), | 170 port.layout_tests_dir(), |
| 163 'platform/test-linux-trusty/failures/expected/image-expected.txt
')), | 171 'platform/test-linux-trusty/failures/expected/image-expected.txt
')), |
| 164 'original win7 result') | 172 'original win7 result') |
| 165 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( | 173 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( |
| 166 port.layout_tests_dir(), 'platform/test-linux-precise/userscripts/an
other-test-expected.txt'))) | 174 port.layout_tests_dir(), 'platform/test-linux-precise/userscripts/an
other-test-expected.txt'))) |
| 167 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( | 175 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( |
| 168 port.layout_tests_dir(), 'platform/test-mac-mac10.10/userscripts/ano
ther-test-expected.txt'))) | 176 port.layout_tests_dir(), 'platform/test-mac-mac10.10/userscripts/ano
ther-test-expected.txt'))) |
| 169 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete"
: []}\n') | 177 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete"
: []}\n') |
| 170 | 178 |
| 171 def test_no_copy_existing_baseline(self): | 179 def test_no_copy_existing_baseline(self): |
| 172 self.tool.executive = MockExecutive2() | 180 self.tool.executive = MockExecutive2() |
| 173 | 181 |
| 174 port = self.tool.port_factory.get('test-win-win7') | 182 port = self.tool.port_factory.get('test-win-win7') |
| 175 self._write(port._filesystem.join(port.layout_tests_dir(), | 183 self._write(port._filesystem.join(port.layout_tests_dir(), |
| 176 'platform/test-win-win7/failures/expec
ted/image-expected.txt'), 'original win7 result') | 184 'platform/test-win-win7/failures/expec
ted/image-expected.txt'), 'original win7 result') |
| 177 | 185 |
| 178 oc = OutputCapture() | 186 oc = OutputCapture() |
| 179 try: | 187 try: |
| 180 options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=T
rue, | 188 options = optparse.Values({ |
| 181 test="failures/expected/image.html", results_d
irectory=None) | 189 'builder': "MOCK Win7", |
| 182 | 190 'suffixes': "txt", |
| 191 'verbose': True, |
| 192 'test': "failures/expected/image.html", |
| 193 'results_directory': None |
| 194 }) |
| 183 oc.capture_output() | 195 oc.capture_output() |
| 184 self.command.execute(options, [], self.tool) | 196 self.command.execute(options, [], self.tool) |
| 185 finally: | 197 finally: |
| 186 out, _, _ = oc.restore_output() | 198 out, _, _ = oc.restore_output() |
| 187 | 199 |
| 188 self.assertMultiLineEqual( | 200 self.assertMultiLineEqual( |
| 189 self._read(self.tool.filesystem.join( | 201 self._read(self.tool.filesystem.join( |
| 190 port.layout_tests_dir(), | 202 port.layout_tests_dir(), |
| 191 'platform/test-linux-trusty/failures/expected/image-expected.txt
')), | 203 'platform/test-linux-trusty/failures/expected/image-expected.txt
')), |
| 192 'original win7 result') | 204 'original win7 result') |
| (...skipping 15 matching lines...) Expand all Loading... |
| 208 port.layout_tests_dir(), | 220 port.layout_tests_dir(), |
| 209 'platform/test-win-win7/failures/expected/image-expected.txt'), | 221 'platform/test-win-win7/failures/expected/image-expected.txt'), |
| 210 'original win7 result') | 222 'original win7 result') |
| 211 expectations_path = fs.join(port.path_to_generic_test_expectations_file(
)) | 223 expectations_path = fs.join(port.path_to_generic_test_expectations_file(
)) |
| 212 self._write( | 224 self._write( |
| 213 expectations_path, | 225 expectations_path, |
| 214 ("[ Win ] failures/expected/image.html [ Failure ]\n" | 226 ("[ Win ] failures/expected/image.html [ Failure ]\n" |
| 215 "[ Linux ] failures/expected/image.html [ Skip ]\n")) | 227 "[ Linux ] failures/expected/image.html [ Skip ]\n")) |
| 216 oc = OutputCapture() | 228 oc = OutputCapture() |
| 217 try: | 229 try: |
| 218 options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=T
rue, | 230 options = optparse.Values({ |
| 219 test="failures/expected/image.html", results_d
irectory=None) | 231 'builder': "MOCK Win7", |
| 220 | 232 'suffixes': "txt", 'verbose': True, |
| 233 'test': "failures/expected/image.html", |
| 234 'results_directory': None |
| 235 }) |
| 221 oc.capture_output() | 236 oc.capture_output() |
| 222 self.command.execute(options, [], self.tool) | 237 self.command.execute(options, [], self.tool) |
| 223 finally: | 238 finally: |
| 224 oc.restore_output() | 239 oc.restore_output() |
| 225 | 240 |
| 226 self.assertFalse( | 241 self.assertFalse( |
| 227 fs.exists(fs.join( | 242 fs.exists(fs.join( |
| 228 port.layout_tests_dir(), | 243 port.layout_tests_dir(), |
| 229 'platform/test-mac-mac10.10/failures/expected/image-expected.txt
'))) | 244 'platform/test-mac-mac10.10/failures/expected/image-expected.txt
'))) |
| 230 self.assertFalse( | 245 self.assertFalse( |
| 231 fs.exists(fs.join( | 246 fs.exists(fs.join( |
| 232 port.layout_tests_dir(), | 247 port.layout_tests_dir(), |
| 233 'platform/test-linux-trusty/failures/expected/image-expected.txt
'))) | 248 'platform/test-linux-trusty/failures/expected/image-expected.txt
'))) |
| 234 self.assertFalse( | 249 self.assertFalse( |
| 235 fs.exists(fs.join( | 250 fs.exists(fs.join( |
| 236 port.layout_tests_dir(), | 251 port.layout_tests_dir(), |
| 237 'platform/test-linux-precise/failures/expected/image-expected.tx
t'))) | 252 'platform/test-linux-precise/failures/expected/image-expected.tx
t'))) |
| 238 self.assertEqual( | 253 self.assertEqual( |
| 239 self._read(fs.join(port.layout_tests_dir(), 'platform/test-win-win7/
failures/expected/image-expected.txt')), | 254 self._read(fs.join(port.layout_tests_dir(), 'platform/test-win-win7/
failures/expected/image-expected.txt')), |
| 240 'original win7 result') | 255 'original win7 result') |
| 241 | 256 |
| 242 | 257 |
| 243 class TestRebaselineTest(BaseTestCase): | 258 class TestRebaselineTest(BaseTestCase): |
| 244 command_constructor = RebaselineTest # AKA webkit-patch rebaseline-test-int
ernal | 259 command_constructor = RebaselineTest # AKA webkit-patch rebaseline-test-int
ernal |
| 245 | 260 |
| 246 def setUp(self): | 261 def setUp(self): |
| 247 super(TestRebaselineTest, self).setUp() | 262 super(TestRebaselineTest, self).setUp() |
| 248 self.options = MockOptions( | 263 |
| 249 builder="MOCK Mac10.11", test="userscripts/another-test.html", suffi
xes="txt", results_directory=None, | 264 @staticmethod |
| 250 build_number=None) | 265 def options(**kwargs): |
| 266 return optparse.Values(dict({ |
| 267 'builder': "MOCK Mac10.11", |
| 268 'test': "userscripts/another-test.html", |
| 269 'suffixes': "txt", |
| 270 'results_directory': None, |
| 271 'build_number': None |
| 272 }, **kwargs)) |
| 251 | 273 |
| 252 def test_baseline_directory(self): | 274 def test_baseline_directory(self): |
| 253 command = self.command | 275 command = self.command |
| 254 self.assertMultiLineEqual(command._baseline_directory("MOCK Mac10.11"), | 276 self.assertMultiLineEqual(command._baseline_directory("MOCK Mac10.11"), |
| 255 "/test.checkout/LayoutTests/platform/test-mac-
mac10.11") | 277 "/test.checkout/LayoutTests/platform/test-mac-
mac10.11") |
| 256 self.assertMultiLineEqual(command._baseline_directory("MOCK Mac10.10"), | 278 self.assertMultiLineEqual(command._baseline_directory("MOCK Mac10.10"), |
| 257 "/test.checkout/LayoutTests/platform/test-mac-
mac10.10") | 279 "/test.checkout/LayoutTests/platform/test-mac-
mac10.10") |
| 258 self.assertMultiLineEqual(command._baseline_directory("MOCK Trusty"), | 280 self.assertMultiLineEqual(command._baseline_directory("MOCK Trusty"), |
| 259 "/test.checkout/LayoutTests/platform/test-linu
x-trusty") | 281 "/test.checkout/LayoutTests/platform/test-linu
x-trusty") |
| 260 self.assertMultiLineEqual(command._baseline_directory("MOCK Precise"), | 282 self.assertMultiLineEqual(command._baseline_directory("MOCK Precise"), |
| 261 "/test.checkout/LayoutTests/platform/test-linu
x-precise") | 283 "/test.checkout/LayoutTests/platform/test-linu
x-precise") |
| 262 | 284 |
| 263 def test_rebaseline_updates_expectations_file_noop(self): | 285 def test_rebaseline_updates_expectations_file_noop(self): |
| 264 self._zero_out_test_expectations() | 286 self._zero_out_test_expectations() |
| 265 self._write( | 287 self._write( |
| 266 self.mac_expectations_path, | 288 self.mac_expectations_path, |
| 267 ("Bug(B) [ Mac Linux Win7 Debug ] fast/dom/Window/window-postmessage
-clone-really-deep-array.html [ Pass ]\n" | 289 ("Bug(B) [ Mac Linux Win7 Debug ] fast/dom/Window/window-postmessage
-clone-really-deep-array.html [ Pass ]\n" |
| 268 "Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failu
re ]\n")) | 290 "Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failu
re ]\n")) |
| 269 self._write("fast/dom/Window/window-postmessage-clone-really-deep-array.
html", "Dummy test contents") | 291 self._write("fast/dom/Window/window-postmessage-clone-really-deep-array.
html", "Dummy test contents") |
| 270 self._write("fast/css/large-list-of-rules-crash.html", "Dummy test conte
nts") | 292 self._write("fast/css/large-list-of-rules-crash.html", "Dummy test conte
nts") |
| 271 self._write("userscripts/another-test.html", "Dummy test contents") | 293 self._write("userscripts/another-test.html", "Dummy test contents") |
| 272 | 294 |
| 273 self.options.suffixes = "png,wav,txt" | 295 self.command._rebaseline_test_and_update_expectations(self.options(suffi
xes="png,wav,txt")) |
| 274 self.command._rebaseline_test_and_update_expectations(self.options) | |
| 275 | 296 |
| 276 self.assertItemsEqual(self.tool.web.urls_fetched, | 297 self.assertItemsEqual(self.tool.web.urls_fetched, |
| 277 [self.WEB_PREFIX + '/userscripts/another-test-actu
al.png', | 298 [self.WEB_PREFIX + '/userscripts/another-test-actu
al.png', |
| 278 self.WEB_PREFIX + '/userscripts/another-test-actu
al.wav', | 299 self.WEB_PREFIX + '/userscripts/another-test-actu
al.wav', |
| 279 self.WEB_PREFIX + '/userscripts/another-test-actu
al.txt']) | 300 self.WEB_PREFIX + '/userscripts/another-test-actu
al.txt']) |
| 280 new_expectations = self._read(self.mac_expectations_path) | 301 new_expectations = self._read(self.mac_expectations_path) |
| 281 self.assertMultiLineEqual( | 302 self.assertMultiLineEqual( |
| 282 new_expectations, | 303 new_expectations, |
| 283 ("Bug(B) [ Mac Linux Win7 Debug ] fast/dom/Window/window-postmessage
-clone-really-deep-array.html [ Pass ]\n" | 304 ("Bug(B) [ Mac Linux Win7 Debug ] fast/dom/Window/window-postmessage
-clone-really-deep-array.html [ Pass ]\n" |
| 284 "Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failu
re ]\n")) | 305 "Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failu
re ]\n")) |
| 285 | 306 |
| 286 def test_rebaseline_test(self): | 307 def test_rebaseline_test(self): |
| 287 self.command._rebaseline_test("MOCK Trusty", "userscripts/another-test.h
tml", "txt", self.WEB_PREFIX) | 308 self.command._rebaseline_test("MOCK Trusty", "userscripts/another-test.h
tml", "txt", self.WEB_PREFIX) |
| 288 self.assertItemsEqual(self.tool.web.urls_fetched, [self.WEB_PREFIX + '/u
serscripts/another-test-actual.txt']) | 309 self.assertItemsEqual(self.tool.web.urls_fetched, [self.WEB_PREFIX + '/u
serscripts/another-test-actual.txt']) |
| 289 | 310 |
| 290 def test_rebaseline_test_with_results_directory(self): | 311 def test_rebaseline_test_with_results_directory(self): |
| 291 self._write("userscripts/another-test.html", "test data") | 312 self._write("userscripts/another-test.html", "test data") |
| 292 self._write( | 313 self._write( |
| 293 self.mac_expectations_path, | 314 self.mac_expectations_path, |
| 294 ("Bug(x) [ Mac ] userscripts/another-test.html [ Failure ]\n" | 315 ("Bug(x) [ Mac ] userscripts/another-test.html [ Failure ]\n" |
| 295 "bug(z) [ Linux ] userscripts/another-test.html [ Failure ]\n")) | 316 "bug(z) [ Linux ] userscripts/another-test.html [ Failure ]\n")) |
| 296 self.options.results_directory = '/tmp' | 317 self.command._rebaseline_test_and_update_expectations(self.options(resul
ts_directory='/tmp')) |
| 297 self.command._rebaseline_test_and_update_expectations(self.options) | |
| 298 self.assertItemsEqual(self.tool.web.urls_fetched, ['file:///tmp/userscri
pts/another-test-actual.txt']) | 318 self.assertItemsEqual(self.tool.web.urls_fetched, ['file:///tmp/userscri
pts/another-test-actual.txt']) |
| 299 | 319 |
| 300 def test_rebaseline_reftest(self): | 320 def test_rebaseline_reftest(self): |
| 301 self._write("userscripts/another-test.html", "test data") | 321 self._write("userscripts/another-test.html", "test data") |
| 302 self._write("userscripts/another-test-expected.html", "generic result") | 322 self._write("userscripts/another-test-expected.html", "generic result") |
| 303 self.options.suffixes = 'png' | 323 OutputCapture().assert_outputs(self, self.command._rebaseline_test_and_u
pdate_expectations, args=[self.options(suffixes='png')], |
| 304 OutputCapture().assert_outputs(self, self.command._rebaseline_test_and_u
pdate_expectations, args=[self.options], | |
| 305 expected_logs="Cannot rebaseline image re
sult for reftest: userscripts/another-test.html\n") | 324 expected_logs="Cannot rebaseline image re
sult for reftest: userscripts/another-test.html\n") |
| 306 self.assertDictEqual(self.command._scm_changes, {'add': [], 'remove-line
s': [], "delete": []}) | 325 self.assertDictEqual(self.command._scm_changes, {'add': [], 'remove-line
s': [], "delete": []}) |
| 307 | 326 |
| 308 def test_rebaseline_test_and_print_scm_changes(self): | 327 def test_rebaseline_test_and_print_scm_changes(self): |
| 309 self.command._print_scm_changes = True | 328 self.command._print_scm_changes = True |
| 310 self.command._scm_changes = {'add': [], 'delete': []} | 329 self.command._scm_changes = {'add': [], 'delete': []} |
| 311 self.tool._scm.exists = lambda x: False | 330 self.tool._scm.exists = lambda x: False |
| 312 | 331 |
| 313 self.command._rebaseline_test("MOCK Trusty", "userscripts/another-test.h
tml", "txt", None) | 332 self.command._rebaseline_test("MOCK Trusty", "userscripts/another-test.h
tml", "txt", None) |
| 314 | 333 |
| 315 self.assertDictEqual(self.command._scm_changes, { | 334 self.assertDictEqual(self.command._scm_changes, { |
| 316 'add': ['/test.checkout/LayoutTests/platform/test-linux-trusty/users
cripts/another-test-expected.txt'], | 335 'add': ['/test.checkout/LayoutTests/platform/test-linux-trusty/users
cripts/another-test-expected.txt'], |
| 317 'delete': [] | 336 'delete': [] |
| 318 }) | 337 }) |
| 319 | 338 |
| 320 def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self): | 339 def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self): |
| 321 self.tool.executive = MockExecutive2() | 340 self.tool.executive = MockExecutive2() |
| 322 | 341 |
| 323 port = self.tool.port_factory.get('test-win-win7') | 342 port = self.tool.port_factory.get('test-win-win7') |
| 324 self._write(port._filesystem.join(port.layout_tests_dir(), | 343 self._write(port._filesystem.join(port.layout_tests_dir(), |
| 325 'platform/test-win-win10/failures/expe
cted/image-expected.txt'), 'original win10 result') | 344 'platform/test-win-win10/failures/expe
cted/image-expected.txt'), 'original win10 result') |
| 326 | 345 |
| 327 oc = OutputCapture() | 346 oc = OutputCapture() |
| 328 try: | 347 try: |
| 329 options = MockOptions( | 348 options = optparse.Values({ |
| 330 optimize=True, builder="MOCK Win10", suffixes="txt", verbose=Tru
e, test="failures/expected/image.html", | 349 'optimize': True, |
| 331 results_directory=None, build_number=None) | 350 'builder': "MOCK Win10", |
| 332 | 351 'suffixes': "txt", |
| 352 'verbose': True, |
| 353 'test': "failures/expected/image.html", |
| 354 'results_directory': None, |
| 355 'build_number': None |
| 356 }) |
| 333 oc.capture_output() | 357 oc.capture_output() |
| 334 self.command.execute(options, [], self.tool) | 358 self.command.execute(options, [], self.tool) |
| 335 finally: | 359 finally: |
| 336 out, _, _ = oc.restore_output() | 360 out, _, _ = oc.restore_output() |
| 337 | 361 |
| 338 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layo
ut_tests_dir( | 362 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layo
ut_tests_dir( |
| 339 ), 'platform/test-win-win10/failures/expected/image-expected.txt')), 'MO
CK Web result, convert 404 to None=True') | 363 ), 'platform/test-win-win10/failures/expected/image-expected.txt')), 'MO
CK Web result, convert 404 to None=True') |
| 340 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( | 364 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( |
| 341 port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/i
mage-expected.txt'))) | 365 port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/i
mage-expected.txt'))) |
| 342 self.assertMultiLineEqual( | 366 self.assertMultiLineEqual( |
| (...skipping 12 matching lines...) Expand all Loading... |
| 355 class TestRebaselineJson(BaseTestCase): | 379 class TestRebaselineJson(BaseTestCase): |
| 356 command_constructor = RebaselineJson | 380 command_constructor = RebaselineJson |
| 357 | 381 |
| 358 def setUp(self): | 382 def setUp(self): |
| 359 super(TestRebaselineJson, self).setUp() | 383 super(TestRebaselineJson, self).setUp() |
| 360 self.tool.executive = MockExecutive2() | 384 self.tool.executive = MockExecutive2() |
| 361 | 385 |
| 362 def tearDown(self): | 386 def tearDown(self): |
| 363 super(TestRebaselineJson, self).tearDown() | 387 super(TestRebaselineJson, self).tearDown() |
| 364 | 388 |
| 389 @staticmethod |
| 390 def options(**kwargs): |
| 391 return optparse.Values(dict({ |
| 392 'optimize': True, |
| 393 'verbose': True, |
| 394 'results_directory': None |
| 395 }, **kwargs)) |
| 396 |
| 365 def test_rebaseline_test_passes_on_all_builders(self): | 397 def test_rebaseline_test_passes_on_all_builders(self): |
| 366 self._setup_mock_build_data() | 398 self._setup_mock_build_data() |
| 367 | 399 |
| 368 self.tool.buildbot.set_results(Build('MOCK Win7'), LayoutTestResults({ | 400 self.tool.buildbot.set_results(Build('MOCK Win7'), LayoutTestResults({ |
| 369 "tests": { | 401 "tests": { |
| 370 "userscripts": { | 402 "userscripts": { |
| 371 "first-test.html": { | 403 "first-test.html": { |
| 372 "expected": "NEEDSREBASELINE", | 404 "expected": "NEEDSREBASELINE", |
| 373 "actual": "PASS" | 405 "actual": "PASS" |
| 374 } | 406 } |
| 375 } | 407 } |
| 376 } | 408 } |
| 377 })) | 409 })) |
| 378 | 410 |
| 379 options = MockOptions(optimize=True, verbose=True, results_directory=Non
e) | |
| 380 | |
| 381 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") | 411 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") |
| 382 self._write("userscripts/first-test.html", "Dummy test contents") | 412 self._write("userscripts/first-test.html", "Dummy test contents") |
| 383 | 413 |
| 384 self.command._rebaseline(options, {"userscripts/first-test.html": {Build
("MOCK Win7"): ["txt", "png"]}}) | 414 self.command._rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) |
| 385 | 415 |
| 386 self.assertEqual(self.tool.executive.calls, []) | 416 self.assertEqual(self.tool.executive.calls, []) |
| 387 | 417 |
| 388 def test_rebaseline_all(self): | 418 def test_rebaseline_all(self): |
| 389 self._setup_mock_build_data() | 419 self._setup_mock_build_data() |
| 390 | 420 |
| 391 options = MockOptions(optimize=True, verbose=True, results_directory=Non
e) | |
| 392 self._write("userscripts/first-test.html", "Dummy test contents") | 421 self._write("userscripts/first-test.html", "Dummy test contents") |
| 393 self.command._rebaseline(options, {"userscripts/first-test.html": {Build
("MOCK Win7"): ["txt", "png"]}}) | 422 self.command._rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) |
| 394 | 423 |
| 395 # Note that we have one run_in_parallel() call followed by a run_command
() | 424 # Note that we have one run_in_parallel() call followed by a run_command
() |
| 396 self.assertEqual( | 425 self.assertEqual( |
| 397 self.tool.executive.calls, | 426 self.tool.executive.calls, |
| 398 [ | 427 [ |
| 399 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 428 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 400 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], | 429 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], |
| 401 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', | 430 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', |
| 402 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], | 431 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], |
| 403 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', | 432 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', |
| 404 'userscripts/first-test.html', '--verbose']] | 433 'userscripts/first-test.html', '--verbose']] |
| 405 ]) | 434 ]) |
| 406 | 435 |
| 407 def test_rebaseline_debug(self): | 436 def test_rebaseline_debug(self): |
| 408 self._setup_mock_build_data() | 437 self._setup_mock_build_data() |
| 409 | 438 |
| 410 options = MockOptions(optimize=True, verbose=True, results_directory=Non
e) | |
| 411 self._write("userscripts/first-test.html", "Dummy test contents") | 439 self._write("userscripts/first-test.html", "Dummy test contents") |
| 412 self.command._rebaseline(options, {"userscripts/first-test.html": {Build
("MOCK Win7 (dbg)"): ["txt", "png"]}}) | 440 self.command._rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7 (dbg)"): ["txt", "png"]}}) |
| 413 | 441 |
| 414 # Note that we have one run_in_parallel() call followed by a run_command
() | 442 # Note that we have one run_in_parallel() call followed by a run_command
() |
| 415 self.assertEqual( | 443 self.assertEqual( |
| 416 self.tool.executive.calls, | 444 self.tool.executive.calls, |
| 417 [ | 445 [ |
| 418 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 446 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 419 '--builder', 'MOCK Win7 (dbg)', '--test', 'userscripts/first-t
est.html', '--verbose']], | 447 '--builder', 'MOCK Win7 (dbg)', '--test', 'userscripts/first-t
est.html', '--verbose']], |
| 420 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', '--builder', | 448 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', '--builder', |
| 421 'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '-
-verbose']], | 449 'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '-
-verbose']], |
| 422 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', | 450 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', |
| 423 'userscripts/first-test.html', '--verbose']] | 451 'userscripts/first-test.html', '--verbose']] |
| 424 ]) | 452 ]) |
| 425 | 453 |
| 426 def test_no_optimize(self): | 454 def test_no_optimize(self): |
| 427 self._setup_mock_build_data() | 455 self._setup_mock_build_data() |
| 428 print self.tool.buildbot._canned_results | 456 print self.tool.buildbot._canned_results |
| 429 | 457 |
| 430 options = MockOptions(optimize=False, verbose=True, results_directory=No
ne) | |
| 431 self._write("userscripts/first-test.html", "Dummy test contents") | 458 self._write("userscripts/first-test.html", "Dummy test contents") |
| 432 self.command._rebaseline(options, {"userscripts/first-test.html": {Build
("MOCK Win7"): ["txt", "png"]}}) | 459 self.command._rebaseline( |
| 460 self.options(optimize=False), |
| 461 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) |
| 433 | 462 |
| 434 # Note that we have only one run_in_parallel() call | 463 # Note that we have only one run_in_parallel() call |
| 435 self.assertEqual( | 464 self.assertEqual( |
| 436 self.tool.executive.calls, | 465 self.tool.executive.calls, |
| 437 [ | 466 [ |
| 438 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 467 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 439 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], | 468 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], |
| 440 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', | 469 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', |
| 441 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']] | 470 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']] |
| 442 ]) | 471 ]) |
| 443 | 472 |
| 444 def test_results_directory(self): | 473 def test_results_directory(self): |
| 445 self._setup_mock_build_data() | 474 self._setup_mock_build_data() |
| 446 | 475 |
| 447 options = MockOptions(optimize=False, verbose=True, results_directory='/
tmp') | |
| 448 self._write("userscripts/first-test.html", "Dummy test contents") | 476 self._write("userscripts/first-test.html", "Dummy test contents") |
| 449 self.command._rebaseline(options, {"userscripts/first-test.html": {Build
("MOCK Win7"): ["txt", "png"]}}) | 477 self.command._rebaseline( |
| 478 self.options(optimize=False, results_directory='/tmp'), |
| 479 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) |
| 450 | 480 |
| 451 # Note that we have only one run_in_parallel() call | 481 # Note that we have only one run_in_parallel() call |
| 452 self.assertEqual( | 482 self.assertEqual( |
| 453 self.tool.executive.calls, | 483 self.tool.executive.calls, |
| 454 [ | 484 [ |
| 455 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 485 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 456 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--results-directory', '/tmp', '--verbose']], | 486 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--results-directory', '/tmp', '--verbose']], |
| 457 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', | 487 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', |
| 458 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--results-directory', '/tmp', '--verbose']] | 488 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--results-directory', '/tmp', '--verbose']] |
| 459 ]) | 489 ]) |
| 460 | 490 |
| 461 | 491 |
| 462 class TestRebaselineJsonUpdatesExpectationsFiles(BaseTestCase): | 492 class TestRebaselineJsonUpdatesExpectationsFiles(BaseTestCase): |
| 463 command_constructor = RebaselineJson | 493 command_constructor = RebaselineJson |
| 464 | 494 |
| 465 def setUp(self): | 495 def setUp(self): |
| 466 super(TestRebaselineJsonUpdatesExpectationsFiles, self).setUp() | 496 super(TestRebaselineJsonUpdatesExpectationsFiles, self).setUp() |
| 467 self.tool.executive = MockExecutive2() | 497 self.tool.executive = MockExecutive2() |
| 468 | 498 |
| 469 def mock_run_command(*args, **kwargs): # pylint: disable=unused-argumen
t | 499 def mock_run_command(*args, **kwargs): # pylint: disable=unused-argumen
t |
| 470 return '{"add": [], "remove-lines": [{"test": "userscripts/first-tes
t.html", "builder": "MOCK Mac10.11"}]}\n' | 500 return '{"add": [], "remove-lines": [{"test": "userscripts/first-tes
t.html", "builder": "MOCK Mac10.11"}]}\n' |
| 471 self.tool.executive.run_command = mock_run_command | 501 self.tool.executive.run_command = mock_run_command |
| 472 | 502 |
| 503 @staticmethod |
| 504 def options(): |
| 505 return optparse.Values({ |
| 506 'optimize': False, |
| 507 'verbose': True, |
| 508 'results_directory': None |
| 509 }) |
| 510 |
| 473 def test_rebaseline_updates_expectations_file(self): | 511 def test_rebaseline_updates_expectations_file(self): |
| 474 options = MockOptions(optimize=False, verbose=True, results_directory=No
ne) | |
| 475 | |
| 476 self._write( | 512 self._write( |
| 477 self.mac_expectations_path, | 513 self.mac_expectations_path, |
| 478 ("Bug(x) [ Mac ] userscripts/first-test.html [ Failure ]\n" | 514 ("Bug(x) [ Mac ] userscripts/first-test.html [ Failure ]\n" |
| 479 "bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n")) | 515 "bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n")) |
| 480 self._write("userscripts/first-test.html", "Dummy test contents") | 516 self._write("userscripts/first-test.html", "Dummy test contents") |
| 481 self._setup_mock_build_data() | 517 self._setup_mock_build_data() |
| 482 | 518 |
| 483 self.command._rebaseline(options, {"userscripts/first-test.html": {Build
("MOCK Mac10.11"): ["txt", "png"]}}) | 519 self.command._rebaseline( |
| 520 self.options(), |
| 521 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
| 484 | 522 |
| 485 new_expectations = self._read(self.mac_expectations_path) | 523 new_expectations = self._read(self.mac_expectations_path) |
| 486 self.assertMultiLineEqual( | 524 self.assertMultiLineEqual( |
| 487 new_expectations, | 525 new_expectations, |
| 488 ("Bug(x) [ Mac10.10 ] userscripts/first-test.html [ Failure ]\n" | 526 ("Bug(x) [ Mac10.10 ] userscripts/first-test.html [ Failure ]\n" |
| 489 "bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n")) | 527 "bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n")) |
| 490 | 528 |
| 491 def test_rebaseline_updates_expectations_file_all_platforms(self): | 529 def test_rebaseline_updates_expectations_file_all_platforms(self): |
| 492 options = MockOptions(optimize=False, verbose=True, results_directory=No
ne) | |
| 493 | |
| 494 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") | 530 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") |
| 495 self._write("userscripts/first-test.html", "Dummy test contents") | 531 self._write("userscripts/first-test.html", "Dummy test contents") |
| 496 self._setup_mock_build_data() | 532 self._setup_mock_build_data() |
| 497 | 533 |
| 498 self.command._rebaseline(options, {"userscripts/first-test.html": {Build
("MOCK Mac10.11"): ["txt", "png"]}}) | 534 self.command._rebaseline( |
| 535 self.options(), |
| 536 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
| 499 | 537 |
| 500 new_expectations = self._read(self.mac_expectations_path) | 538 new_expectations = self._read(self.mac_expectations_path) |
| 501 self.assertMultiLineEqual( | 539 self.assertMultiLineEqual( |
| 502 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") | 540 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") |
| 503 | 541 |
| 504 def test_rebaseline_handles_platform_skips(self): | 542 def test_rebaseline_handles_platform_skips(self): |
| 505 # This test is just like test_rebaseline_updates_expectations_file_all_p
latforms(), | 543 # This test is just like test_rebaseline_updates_expectations_file_all_p
latforms(), |
| 506 # except that if a particular port happens to SKIP a test in an override
s file, | 544 # except that if a particular port happens to SKIP a test in an override
s file, |
| 507 # we count that as passing, and do not think that we still need to rebas
eline it. | 545 # we count that as passing, and do not think that we still need to rebas
eline it. |
| 508 options = MockOptions(optimize=False, verbose=True, results_directory=No
ne) | |
| 509 | |
| 510 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") | 546 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") |
| 511 self._write("NeverFixTests", "Bug(y) [ Android ] userscripts [ WontFix ]
\n") | 547 self._write("NeverFixTests", "Bug(y) [ Android ] userscripts [ WontFix ]
\n") |
| 512 self._write("userscripts/first-test.html", "Dummy test contents") | 548 self._write("userscripts/first-test.html", "Dummy test contents") |
| 513 self._setup_mock_build_data() | 549 self._setup_mock_build_data() |
| 514 | 550 |
| 515 self.command._rebaseline(options, {"userscripts/first-test.html": {Build
("MOCK Mac10.11"): ["txt", "png"]}}) | 551 self.command._rebaseline( |
| 552 self.options(), |
| 553 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
| 516 | 554 |
| 517 new_expectations = self._read(self.mac_expectations_path) | 555 new_expectations = self._read(self.mac_expectations_path) |
| 518 self.assertMultiLineEqual( | 556 self.assertMultiLineEqual( |
| 519 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") | 557 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") |
| 520 | 558 |
| 521 def test_rebaseline_handles_skips_in_file(self): | 559 def test_rebaseline_handles_skips_in_file(self): |
| 522 # This test is like test_Rebaseline_handles_platform_skips, except that
the | 560 # This test is like test_Rebaseline_handles_platform_skips, except that
the |
| 523 # Skip is in the same (generic) file rather than a platform file. In thi
s case, | 561 # Skip is in the same (generic) file rather than a platform file. In thi
s case, |
| 524 # the Skip line should be left unmodified. Note that the first line is n
ow | 562 # the Skip line should be left unmodified. Note that the first line is n
ow |
| 525 # qualified as "[Linux Mac Win]"; if it was unqualified, it would confli
ct with | 563 # qualified as "[Linux Mac Win]"; if it was unqualified, it would confli
ct with |
| 526 # the second line. | 564 # the second line. |
| 527 options = MockOptions(optimize=False, verbose=True, results_directory=No
ne) | |
| 528 | |
| 529 self._write(self.mac_expectations_path, | 565 self._write(self.mac_expectations_path, |
| 530 ("Bug(x) [ Linux Mac Win ] userscripts/first-test.html [ Fai
lure ]\n" | 566 ("Bug(x) [ Linux Mac Win ] userscripts/first-test.html [ Fai
lure ]\n" |
| 531 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n"
)) | 567 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n"
)) |
| 532 self._write("userscripts/first-test.html", "Dummy test contents") | 568 self._write("userscripts/first-test.html", "Dummy test contents") |
| 533 self._setup_mock_build_data() | 569 self._setup_mock_build_data() |
| 534 | 570 |
| 535 self.command._rebaseline(options, {"userscripts/first-test.html": {Build
("MOCK Mac10.11"): ["txt", "png"]}}) | 571 self.command._rebaseline( |
| 572 self.options(), |
| 573 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
| 536 | 574 |
| 537 new_expectations = self._read(self.mac_expectations_path) | 575 new_expectations = self._read(self.mac_expectations_path) |
| 538 self.assertMultiLineEqual( | 576 self.assertMultiLineEqual( |
| 539 new_expectations, | 577 new_expectations, |
| 540 ("Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failur
e ]\n" | 578 ("Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failur
e ]\n" |
| 541 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n")) | 579 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n")) |
| 542 | 580 |
| 543 def test_rebaseline_handles_smoke_tests(self): | 581 def test_rebaseline_handles_smoke_tests(self): |
| 544 # This test is just like test_rebaseline_handles_platform_skips, except
that we check for | 582 # This test is just like test_rebaseline_handles_platform_skips, except
that we check for |
| 545 # a test not being in the SmokeTests file, instead of using overrides fi
les. | 583 # a test not being in the SmokeTests file, instead of using overrides fi
les. |
| 546 # If a test is not part of the smoke tests, we count that as passing on
ports that only | 584 # If a test is not part of the smoke tests, we count that as passing on
ports that only |
| 547 # run smoke tests, and do not think that we still need to rebaseline it. | 585 # run smoke tests, and do not think that we still need to rebaseline it. |
| 548 options = MockOptions(optimize=False, verbose=True, results_directory=No
ne) | |
| 549 | |
| 550 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") | 586 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") |
| 551 self._write("SmokeTests", "fast/html/article-element.html") | 587 self._write("SmokeTests", "fast/html/article-element.html") |
| 552 self._write("userscripts/first-test.html", "Dummy test contents") | 588 self._write("userscripts/first-test.html", "Dummy test contents") |
| 553 self._setup_mock_build_data() | 589 self._setup_mock_build_data() |
| 554 | 590 |
| 555 self.command._rebaseline(options, {"userscripts/first-test.html": {Build
("MOCK Mac10.11"): ["txt", "png"]}}) | 591 self.command._rebaseline( |
| 592 self.options(), |
| 593 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
| 556 | 594 |
| 557 new_expectations = self._read(self.mac_expectations_path) | 595 new_expectations = self._read(self.mac_expectations_path) |
| 558 self.assertMultiLineEqual( | 596 self.assertMultiLineEqual( |
| 559 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") | 597 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") |
| 560 | 598 |
| 561 | 599 |
| 562 class TestRebaseline(BaseTestCase): | 600 class TestRebaseline(BaseTestCase): |
| 563 # This command shares most of its logic with RebaselineJson, so these tests
just test what is different. | 601 # This command shares most of its logic with RebaselineJson, so these tests
just test what is different. |
| 564 | 602 |
| 565 command_constructor = Rebaseline # AKA webkit-patch rebaseline | 603 command_constructor = Rebaseline # AKA webkit-patch rebaseline |
| 566 | 604 |
| 567 def test_rebaseline(self): | 605 def test_rebaseline(self): |
| 568 self.command._builders_to_pull_from = lambda: ['MOCK Win7'] | 606 self.command._builders_to_pull_from = lambda: ['MOCK Win7'] |
| 569 | 607 |
| 570 self._write("userscripts/first-test.html", "test data") | 608 self._write("userscripts/first-test.html", "test data") |
| 571 | 609 |
| 572 self._zero_out_test_expectations() | 610 self._zero_out_test_expectations() |
| 573 self._setup_mock_build_data() | 611 self._setup_mock_build_data() |
| 574 | 612 options = optparse.Values({ |
| 575 self.command.execute(MockOptions(results_directory=False, optimize=False
, builders=None, | 613 'results_directory': False, |
| 576 suffixes="txt,png", verbose=True), ['us
erscripts/first-test.html'], self.tool) | 614 'optimize': False, |
| 615 'builders': None, |
| 616 'suffixes': "txt,png", |
| 617 'verbose': True |
| 618 }) |
| 619 self.command.execute(options, ['userscripts/first-test.html'], self.tool
) |
| 577 | 620 |
| 578 self.assertEqual( | 621 self.assertEqual( |
| 579 self.tool.executive.calls, | 622 self.tool.executive.calls, |
| 580 [ | 623 [ |
| 581 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 624 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 582 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], | 625 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], |
| 583 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', | 626 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', |
| 584 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']] | 627 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']] |
| 585 ]) | 628 ]) |
| 586 | 629 |
| 587 def test_rebaseline_directory(self): | 630 def test_rebaseline_directory(self): |
| 588 self.command._builders_to_pull_from = lambda: ['MOCK Win7'] | 631 self.command._builders_to_pull_from = lambda: ['MOCK Win7'] |
| 589 | 632 |
| 590 self._write("userscripts/first-test.html", "test data") | 633 self._write("userscripts/first-test.html", "test data") |
| 591 self._write("userscripts/second-test.html", "test data") | 634 self._write("userscripts/second-test.html", "test data") |
| 592 | 635 |
| 593 self._setup_mock_build_data() | 636 self._setup_mock_build_data() |
| 594 | 637 options = optparse.Values({ |
| 595 self.command.execute(MockOptions(results_directory=False, optimize=False
, builders=None, | 638 'results_directory': False, |
| 596 suffixes="txt,png", verbose=True), ['us
erscripts'], self.tool) | 639 'optimize': False, |
| 640 'builders': None, |
| 641 'suffixes': "txt,png", |
| 642 'verbose': True |
| 643 }) |
| 644 self.command.execute(options, ['userscripts'], self.tool) |
| 597 | 645 |
| 598 self.assertEqual( | 646 self.assertEqual( |
| 599 self.tool.executive.calls, | 647 self.tool.executive.calls, |
| 600 [ | 648 [ |
| 601 [ | 649 [ |
| 602 ['python', 'echo', 'copy-existing-baselines-internal', '--su
ffixes', 'txt,png', | 650 ['python', 'echo', 'copy-existing-baselines-internal', '--su
ffixes', 'txt,png', |
| 603 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test
.html', '--verbose'], | 651 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test
.html', '--verbose'], |
| 604 ['python', 'echo', 'copy-existing-baselines-internal', '--su
ffixes', 'txt,png', | 652 ['python', 'echo', 'copy-existing-baselines-internal', '--su
ffixes', 'txt,png', |
| 605 '--builder', 'MOCK Win7', '--test', 'userscripts/second-tes
t.html', '--verbose'] | 653 '--builder', 'MOCK Win7', '--test', 'userscripts/second-tes
t.html', '--verbose'] |
| 606 ], | 654 ], |
| 607 [ | 655 [ |
| 608 ['python', 'echo', 'rebaseline-test-internal', '--suffixes',
'txt,png', | 656 ['python', 'echo', 'rebaseline-test-internal', '--suffixes',
'txt,png', |
| 609 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test
.html', '--verbose'], | 657 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test
.html', '--verbose'], |
| 610 ['python', 'echo', 'rebaseline-test-internal', '--suffixes',
'txt,png', | 658 ['python', 'echo', 'rebaseline-test-internal', '--suffixes',
'txt,png', |
| 611 '--builder', 'MOCK Win7', '--test', 'userscripts/second-tes
t.html', '--verbose'] | 659 '--builder', 'MOCK Win7', '--test', 'userscripts/second-tes
t.html', '--verbose'] |
| 612 ] | 660 ] |
| 613 ]) | 661 ]) |
| 614 | 662 |
| 615 | 663 |
| 616 | 664 |
| 617 class TestRebaselineExpectations(BaseTestCase): | 665 class TestRebaselineExpectations(BaseTestCase): |
| 618 command_constructor = RebaselineExpectations | 666 command_constructor = RebaselineExpectations |
| 619 | 667 |
| 620 def setUp(self): | 668 def setUp(self): |
| 621 super(TestRebaselineExpectations, self).setUp() | 669 super(TestRebaselineExpectations, self).setUp() |
| 622 self.options = MockOptions(optimize=False, builders=None, suffixes=['txt
'], | 670 |
| 623 verbose=False, platform=None, results_directo
ry=None) | 671 @staticmethod |
| 672 def options(): |
| 673 return optparse.Values({ |
| 674 'optimize': False, |
| 675 'builders': None, |
| 676 'suffixes': ['txt'], |
| 677 'verbose': False, |
| 678 'platform': None, |
| 679 'results_directory': None |
| 680 }) |
| 624 | 681 |
| 625 def _write_test_file(self, port, path, contents): | 682 def _write_test_file(self, port, path, contents): |
| 626 abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path) | 683 abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path) |
| 627 self.tool.filesystem.write_text_file(abs_path, contents) | 684 self.tool.filesystem.write_text_file(abs_path, contents) |
| 628 | 685 |
| 629 def test_rebaseline_expectations(self): | 686 def test_rebaseline_expectations(self): |
| 630 self._zero_out_test_expectations() | 687 self._zero_out_test_expectations() |
| 631 | 688 |
| 632 self.tool.executive = MockExecutive2() | 689 self.tool.executive = MockExecutive2() |
| 633 | 690 |
| (...skipping 14 matching lines...) Expand all Loading... |
| 648 })) | 705 })) |
| 649 | 706 |
| 650 self._write("userscripts/another-test.html", "Dummy test contents") | 707 self._write("userscripts/another-test.html", "Dummy test contents") |
| 651 self._write("userscripts/images.svg", "Dummy test contents") | 708 self._write("userscripts/images.svg", "Dummy test contents") |
| 652 self.command._tests_to_rebaseline = lambda port: { | 709 self.command._tests_to_rebaseline = lambda port: { |
| 653 'userscripts/another-test.html': set(['txt']), | 710 'userscripts/another-test.html': set(['txt']), |
| 654 'userscripts/images.svg': set(['png']), | 711 'userscripts/images.svg': set(['png']), |
| 655 'userscripts/not-actually-failing.html': set(['txt', 'png', 'wav']), | 712 'userscripts/not-actually-failing.html': set(['txt', 'png', 'wav']), |
| 656 } | 713 } |
| 657 | 714 |
| 658 self.command.execute(self.options, [], self.tool) | 715 self.command.execute(self.options(), [], self.tool) |
| 659 | 716 |
| 660 self.assertEqual(self.tool.executive.calls, [ | 717 self.assertEqual(self.tool.executive.calls, [ |
| 661 [ | 718 [ |
| 662 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'txt', | 719 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'txt', |
| 663 '--builder', 'MOCK Mac10.10', '--test', 'userscripts/another-te
st.html'], | 720 '--builder', 'MOCK Mac10.10', '--test', 'userscripts/another-te
st.html'], |
| 664 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'txt', | 721 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'txt', |
| 665 '--builder', 'MOCK Mac10.11', '--test', 'userscripts/another-te
st.html'], | 722 '--builder', 'MOCK Mac10.11', '--test', 'userscripts/another-te
st.html'], |
| 666 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'png', | 723 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'png', |
| 667 '--builder', 'MOCK Mac10.10', '--test', 'userscripts/images.svg
'], | 724 '--builder', 'MOCK Mac10.10', '--test', 'userscripts/images.svg
'], |
| 668 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'png', | 725 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'png', |
| (...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 707 | 764 |
| 708 self._write("userscripts/reftest-text.html", "Dummy test contents") | 765 self._write("userscripts/reftest-text.html", "Dummy test contents") |
| 709 self._write("userscripts/reftest-text-expected.html", "Dummy test conten
ts") | 766 self._write("userscripts/reftest-text-expected.html", "Dummy test conten
ts") |
| 710 self._write("userscripts/reftest-text-expected.html", "Dummy test conten
ts") | 767 self._write("userscripts/reftest-text-expected.html", "Dummy test conten
ts") |
| 711 self.command._tests_to_rebaseline = lambda port: { | 768 self.command._tests_to_rebaseline = lambda port: { |
| 712 'userscripts/reftest-text.html': set(['txt']), | 769 'userscripts/reftest-text.html': set(['txt']), |
| 713 'userscripts/reftest-image.html': set(['png']), | 770 'userscripts/reftest-image.html': set(['png']), |
| 714 'userscripts/reftest-image-text.html': set(['png', 'txt']), | 771 'userscripts/reftest-image-text.html': set(['png', 'txt']), |
| 715 } | 772 } |
| 716 | 773 |
| 717 self.command.execute(self.options, [], self.tool) | 774 self.command.execute(self.options(), [], self.tool) |
| 718 | 775 |
| 719 self.assertEqual(self.tool.executive.calls, [ | 776 self.assertEqual(self.tool.executive.calls, [ |
| 720 [ | 777 [ |
| 721 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'txt', | 778 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'txt', |
| 722 '--builder', 'MOCK Mac10.10', '--test', 'userscripts/reftest-te
xt.html'], | 779 '--builder', 'MOCK Mac10.10', '--test', 'userscripts/reftest-te
xt.html'], |
| 723 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'txt', | 780 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'txt', |
| 724 '--builder', 'MOCK Mac10.11', '--test', 'userscripts/reftest-te
xt.html'], | 781 '--builder', 'MOCK Mac10.11', '--test', 'userscripts/reftest-te
xt.html'], |
| 725 ], | 782 ], |
| 726 [ | 783 [ |
| 727 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'tx
t', | 784 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'tx
t', |
| 728 '--builder', 'MOCK Mac10.10', '--test', 'userscripts/reftest-te
xt.html'], | 785 '--builder', 'MOCK Mac10.10', '--test', 'userscripts/reftest-te
xt.html'], |
| 729 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'tx
t', | 786 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'tx
t', |
| 730 '--builder', 'MOCK Mac10.11', '--test', 'userscripts/reftest-te
xt.html'], | 787 '--builder', 'MOCK Mac10.11', '--test', 'userscripts/reftest-te
xt.html'], |
| 731 ], | 788 ], |
| 732 ]) | 789 ]) |
| 733 | 790 |
| 734 def test_rebaseline_expectations_noop(self): | 791 def test_rebaseline_expectations_noop(self): |
| 735 self._zero_out_test_expectations() | 792 self._zero_out_test_expectations() |
| 736 | 793 |
| 737 oc = OutputCapture() | 794 oc = OutputCapture() |
| 738 try: | 795 try: |
| 739 oc.capture_output() | 796 oc.capture_output() |
| 740 self.command.execute(self.options, [], self.tool) | 797 self.command.execute(self.options(), [], self.tool) |
| 741 finally: | 798 finally: |
| 742 _, _, logs = oc.restore_output() | 799 _, _, logs = oc.restore_output() |
| 743 self.assertEqual(self.tool.filesystem.written_files, {}) | 800 self.assertEqual(self.tool.filesystem.written_files, {}) |
| 744 self.assertEqual(logs, 'Did not find any tests marked Rebaseline.\n'
) | 801 self.assertEqual(logs, 'Did not find any tests marked Rebaseline.\n'
) |
| 745 | 802 |
| 746 def disabled_test_overrides_are_included_correctly(self): | 803 def disabled_test_overrides_are_included_correctly(self): |
| 747 # TODO(qyearsley): Fix or remove this test method. | 804 # TODO(qyearsley): Fix or remove this test method. |
| 748 # This tests that any tests marked as REBASELINE in the overrides are fo
und, but | 805 # This tests that any tests marked as REBASELINE in the overrides are fo
und, but |
| 749 # that the overrides do not get written into the main file. | 806 # that the overrides do not get written into the main file. |
| 750 self._zero_out_test_expectations() | 807 self._zero_out_test_expectations() |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 790 | 847 |
| 791 self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy
test contents") | 848 self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy
test contents") |
| 792 | 849 |
| 793 self.tool.executive = MockLineRemovingExecutive() | 850 self.tool.executive = MockLineRemovingExecutive() |
| 794 | 851 |
| 795 self.tool.builders = BuilderList({ | 852 self.tool.builders = BuilderList({ |
| 796 "MOCK Mac10.10": {"port_name": "test-mac-mac10.10", "specifiers": ["
Mac10.10", "Release"]}, | 853 "MOCK Mac10.10": {"port_name": "test-mac-mac10.10", "specifiers": ["
Mac10.10", "Release"]}, |
| 797 "MOCK Mac10.11": {"port_name": "test-mac-mac10.11", "specifiers": ["
Mac10.11", "Release"]}, | 854 "MOCK Mac10.11": {"port_name": "test-mac-mac10.11", "specifiers": ["
Mac10.11", "Release"]}, |
| 798 }) | 855 }) |
| 799 | 856 |
| 800 self.command.execute(self.options, [], self.tool) | 857 self.command.execute(self.options(), [], self.tool) |
| 801 self.assertEqual(self.tool.executive.calls, []) | 858 self.assertEqual(self.tool.executive.calls, []) |
| 802 | 859 |
| 803 # The mac ports should both be removed since they're the only ones in th
e builder list. | 860 # The mac ports should both be removed since they're the only ones in th
e builder list. |
| 804 self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_g
eneric_test_expectations_file()), """ | 861 self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_g
eneric_test_expectations_file()), """ |
| 805 Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ Rebaseline ] | 862 Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ Rebaseline ] |
| 806 """) | 863 """) |
| 807 | 864 |
| 808 def test_rebaseline_missing(self): | 865 def test_rebaseline_missing(self): |
| 809 self.tool.buildbot.set_results(Build('MOCK Mac10.10'), LayoutTestResults
({ | 866 self.tool.buildbot.set_results(Build('MOCK Mac10.10'), LayoutTestResults
({ |
| 810 "tests": { | 867 "tests": { |
| (...skipping 26 matching lines...) Expand all Loading... |
| 837 self._write('fast/dom/missing-text.html', "Dummy test contents") | 894 self._write('fast/dom/missing-text.html', "Dummy test contents") |
| 838 self._write('fast/dom/missing-text-and-image.html', "Dummy test contents
") | 895 self._write('fast/dom/missing-text-and-image.html', "Dummy test contents
") |
| 839 self._write('fast/dom/missing-image.html', "Dummy test contents") | 896 self._write('fast/dom/missing-image.html', "Dummy test contents") |
| 840 | 897 |
| 841 self.command._tests_to_rebaseline = lambda port: { | 898 self.command._tests_to_rebaseline = lambda port: { |
| 842 'fast/dom/missing-text.html': set(['txt', 'png']), | 899 'fast/dom/missing-text.html': set(['txt', 'png']), |
| 843 'fast/dom/missing-text-and-image.html': set(['txt', 'png']), | 900 'fast/dom/missing-text-and-image.html': set(['txt', 'png']), |
| 844 'fast/dom/missing-image.html': set(['txt', 'png']), | 901 'fast/dom/missing-image.html': set(['txt', 'png']), |
| 845 } | 902 } |
| 846 | 903 |
| 847 self.command.execute(self.options, [], self.tool) | 904 self.command.execute(self.options(), [], self.tool) |
| 848 | 905 |
| 849 self.assertEqual(self.tool.executive.calls, [ | 906 self.assertEqual(self.tool.executive.calls, [ |
| 850 [ | 907 [ |
| 851 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'txt', | 908 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'txt', |
| 852 '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text.
html'], | 909 '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text.
html'], |
| 853 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'txt,png', | 910 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'txt,png', |
| 854 '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text-
and-image.html'], | 911 '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text-
and-image.html'], |
| 855 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'png', | 912 ['python', 'echo', 'copy-existing-baselines-internal', '--suffix
es', 'png', |
| 856 '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-image
.html'], | 913 '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-image
.html'], |
| 857 ], | 914 ], |
| (...skipping 18 matching lines...) Expand all Loading... |
| 876 for cmd_line, cwd in commands: | 933 for cmd_line, cwd in commands: |
| 877 out = self.run_command(cmd_line, cwd=cwd) | 934 out = self.run_command(cmd_line, cwd=cwd) |
| 878 if 'rebaseline-test-internal' in cmd_line: | 935 if 'rebaseline-test-internal' in cmd_line: |
| 879 out = '{"add": [], "remove-lines": [{"test": "%s", "builder": "%
s"}], "delete": []}\n' % (cmd_line[8], cmd_line[6]) | 936 out = '{"add": [], "remove-lines": [{"test": "%s", "builder": "%
s"}], "delete": []}\n' % (cmd_line[8], cmd_line[6]) |
| 880 command_outputs.append([0, out, '']) | 937 command_outputs.append([0, out, '']) |
| 881 | 938 |
| 882 new_calls = self.calls[num_previous_calls:] | 939 new_calls = self.calls[num_previous_calls:] |
| 883 self.calls = self.calls[:num_previous_calls] | 940 self.calls = self.calls[:num_previous_calls] |
| 884 self.calls.append(new_calls) | 941 self.calls.append(new_calls) |
| 885 return command_outputs | 942 return command_outputs |
| OLD | NEW |