Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(6)

Side by Side Diff: third_party/WebKit/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py

Issue 2456073002: Rename test port baseline directories in unit tests. (Closed)
Patch Set: Whitespace / comments Created 4 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2016 The Chromium Authors. All rights reserved. 1 # Copyright 2016 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import optparse 5 import optparse
6 import unittest 6 import unittest
7 7
8 from webkitpy.common.net.buildbot import Build 8 from webkitpy.common.net.buildbot import Build
9 from webkitpy.common.net.layouttestresults import LayoutTestResults 9 from webkitpy.common.net.layouttestresults import LayoutTestResults
10 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 10 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2
(...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after
116 'verbose': False, 116 'verbose': False,
117 } 117 }
118 options_dict.update(kwargs) 118 options_dict.update(kwargs)
119 return optparse.Values(options_dict) 119 return optparse.Values(options_dict)
120 120
121 def test_copy_baseline_mac(self): 121 def test_copy_baseline_mac(self):
122 port = self.tool.port_factory.get('test-mac-mac10.11') 122 port = self.tool.port_factory.get('test-mac-mac10.11')
123 self._write( 123 self._write(
124 port.host.filesystem.join( 124 port.host.filesystem.join(
125 port.layout_tests_dir(), 125 port.layout_tests_dir(),
126 'platform/test-mac-mac10.11/failures/expected/image-expected.txt '), 126 'platform/mac-baselines/failures/expected/image-expected.txt'),
127 'original mac10.11 result') 127 'original mac10.11 result')
128 self.assertFalse(self.tool.filesystem.exists( 128 self.assertFalse(self.tool.filesystem.exists(
129 self.tool.filesystem.join( 129 self.tool.filesystem.join(
130 port.layout_tests_dir(), 130 port.layout_tests_dir(),
131 'platform/test-mac-mac10.10/failures/expected/image-expected.txt '))) 131 'platform/mac10.10-baselines/failures/expected/image-expected.tx t')))
132 132
133 self.command.execute(self.options(builder='MOCK Mac10.11', test='failure s/expected/image.html'), [], self.tool) 133 self.command.execute(self.options(builder='MOCK Mac10.11', test='failure s/expected/image.html'), [], self.tool)
134 134
135 # The Mac 10.11 baseline is copied over to the Mac 10.10 directory, 135 # The Mac 10.11 baseline is copied over to the Mac 10.10 directory,
136 # because Mac10.10 is the "immediate predecessor" in the fallback tree. 136 # because Mac10.10 is the "immediate predecessor" in the fallback tree.
137 # That means that normally for Mac10.10 if there's no Mac10.10-specific 137 # That means that normally for Mac10.10 if there's no Mac10.10-specific
138 # baseline, then we fall back to the Mac10.11 baseline. 138 # baseline, then we fall back to the Mac10.11 baseline.
139 # The idea is, if in the next step we download new baselines for Mac10.1 1 139 # The idea is, if in the next step we download new baselines for Mac10.1 1
140 # but not Mac10.10, then mac10.10 will still have the correct baseline. 140 # but not Mac10.10, then mac10.10 will still have the correct baseline.
141 self.assertEqual( 141 self.assertEqual(
142 self._read(self.tool.filesystem.join( 142 self._read(self.tool.filesystem.join(
143 port.layout_tests_dir(), 143 port.layout_tests_dir(),
144 'platform/test-mac-mac10.11/failures/expected/image-expected.txt ')), 144 'platform/mac-baselines/failures/expected/image-expected.txt')),
145 'original mac10.11 result') 145 'original mac10.11 result')
146 self.assertEqual( 146 self.assertEqual(
147 self._read(self.tool.filesystem.join( 147 self._read(self.tool.filesystem.join(
148 port.layout_tests_dir(), 148 port.layout_tests_dir(),
149 'platform/test-mac-mac10.10/failures/expected/image-expected.txt ')), 149 'platform/mac10.10-baselines/failures/expected/image-expected.tx t')),
150 'original mac10.11 result') 150 'original mac10.11 result')
151 151
152 def test_copying_overwritten_baseline_to_multiple_locations(self): 152 def test_copying_overwritten_baseline_to_multiple_locations(self):
153 self.tool.executive = MockExecutive2() 153 self.tool.executive = MockExecutive2()
154 154
155 def test_copy_baseline_win7_to_linux_trusty(self): 155 def test_copy_baseline_win_to_linux(self):
156 port = self.tool.port_factory.get('test-win-win7') 156 port = self.tool.port_factory.get('test-win-win7')
157 self._write( 157 self._write(
158 self.tool.filesystem.join( 158 self.tool.filesystem.join(
159 port.layout_tests_dir(), 159 port.layout_tests_dir(),
160 'platform/test-win-win7/failures/expected/image-expected.txt'), 160 'platform/win7-baselines/failures/expected/image-expected.txt'),
161 'original win7 result') 161 'original win7 result')
162 self.assertFalse(self.tool.filesystem.exists( 162 self.assertFalse(self.tool.filesystem.exists(
163 self.tool.filesystem.join( 163 self.tool.filesystem.join(
164 port.layout_tests_dir(), 164 port.layout_tests_dir(),
165 'platform/test-linux-trusty/failures/expected/image-expected.txt '))) 165 'platform/linux-baselines/failures/expected/image-expected.txt') ))
166 166
167 self.command.execute(self.options(builder='MOCK Win7', test='failures/ex pected/image.html'), [], self.tool) 167 self.command.execute(self.options(builder='MOCK Win7', test='failures/ex pected/image.html'), [], self.tool)
168 168
169 # The Mac Win7 baseline is copied over to the Linux Trusty directory, 169 # The Mac Win7 baseline is copied over to the Linux Trusty directory,
170 # because Linux Trusty is the baseline fallback "immediate predecessor" of Win7. 170 # because Linux Trusty is the baseline fallback "immediate predecessor" of Win7.
171 self.assertEqual( 171 self.assertEqual(
172 self._read(self.tool.filesystem.join( 172 self._read(self.tool.filesystem.join(
173 port.layout_tests_dir(), 173 port.layout_tests_dir(),
174 'platform/test-win-win7/failures/expected/image-expected.txt')), 174 'platform/win7-baselines/failures/expected/image-expected.txt')) ,
175 'original win7 result') 175 'original win7 result')
176 self.assertEqual( 176 self.assertEqual(
177 self._read(self.tool.filesystem.join( 177 self._read(self.tool.filesystem.join(
178 port.layout_tests_dir(), 178 port.layout_tests_dir(),
179 'platform/test-linux-trusty/failures/expected/image-expected.txt ')), 179 'platform/linux-baselines/failures/expected/image-expected.txt') ),
180 'original win7 result') 180 'original win7 result')
181 181
182 def test_no_copy_existing_baseline(self): 182 def test_no_copy_existing_baseline(self):
183 port = self.tool.port_factory.get('test-win-win7') 183 port = self.tool.port_factory.get('test-win-win7')
184 self._write( 184 self._write(
185 self.tool.filesystem.join( 185 self.tool.filesystem.join(
186 port.layout_tests_dir(), 186 port.layout_tests_dir(),
187 'platform/test-win-win7/failures/expected/image-expected.txt'), 187 'platform/win7-baselines/failures/expected/image-expected.txt'),
188 'original win7 result') 188 'original win7 result')
189 self._write( 189 self._write(
190 self.tool.filesystem.join( 190 self.tool.filesystem.join(
191 port.layout_tests_dir(), 191 port.layout_tests_dir(),
192 'platform/test-linux-trusty/failures/expected/image-expected.txt '), 192 'platform/linux-baselines/failures/expected/image-expected.txt') ,
193 'original linux trusty result') 193 'original linux trusty result')
194 194
195 self.command.execute(self.options(builder='MOCK Win7', test='failures/ex pected/image.html'), [], self.tool) 195 self.command.execute(self.options(builder='MOCK Win7', test='failures/ex pected/image.html'), [], self.tool)
196 196
197 # Since a baseline existed already for Linux Trusty, the Win7 baseline i s not copied over. 197 # Since a baseline existed already for Linux Trusty, the Win7 baseline i s not copied over.
198 self.assertEqual( 198 self.assertEqual(
199 self._read(self.tool.filesystem.join( 199 self._read(self.tool.filesystem.join(
200 port.layout_tests_dir(), 200 port.layout_tests_dir(),
201 'platform/test-win-win7/failures/expected/image-expected.txt')), 201 'platform/win7-baselines/failures/expected/image-expected.txt')) ,
202 'original win7 result') 202 'original win7 result')
203 self.assertEqual( 203 self.assertEqual(
204 self._read(self.tool.filesystem.join( 204 self._read(self.tool.filesystem.join(
205 port.layout_tests_dir(), 205 port.layout_tests_dir(),
206 'platform/test-linux-trusty/failures/expected/image-expected.txt ')), 206 'platform/linux-baselines/failures/expected/image-expected.txt') ),
207 'original linux trusty result') 207 'original linux trusty result')
208 208
209 def test_no_copy_skipped_test(self): 209 def test_no_copy_skipped_test(self):
210 port = self.tool.port_factory.get('test-win-win7') 210 port = self.tool.port_factory.get('test-win-win7')
211 self._write( 211 self._write(
212 self.tool.filesystem.join( 212 self.tool.filesystem.join(
213 port.layout_tests_dir(), 213 port.layout_tests_dir(),
214 'platform/test-win-win7/failures/expected/image-expected.txt'), 214 'platform/win7-baselines/failures/expected/image-expected.txt'),
215 'original win7 result') 215 'original win7 result')
216 self._write( 216 self._write(
217 port.path_to_generic_test_expectations_file(), 217 port.path_to_generic_test_expectations_file(),
218 ("[ Win ] failures/expected/image.html [ Failure ]\n" 218 ("[ Win ] failures/expected/image.html [ Failure ]\n"
219 "[ Linux ] failures/expected/image.html [ Skip ]\n")) 219 "[ Linux ] failures/expected/image.html [ Skip ]\n"))
220 220
221 self.command.execute(self.options(builder='MOCK Win7', test='failures/ex pected/image.html'), [], self.tool) 221 self.command.execute(self.options(builder='MOCK Win7', test='failures/ex pected/image.html'), [], self.tool)
222 222
223 # The Win7 baseline is not copied over to the Linux Trusty directory 223 # The Win7 baseline is not copied over to the Linux Trusty directory
224 # because the test is skipped on linux. 224 # because the test is skipped on linux.
225 self.assertFalse( 225 self.assertFalse(
226 self.tool.filesystem.exists(self.tool.filesystem.join( 226 self.tool.filesystem.exists(self.tool.filesystem.join(
227 port.layout_tests_dir(), 227 port.layout_tests_dir(),
228 'platform/test-linux-trusty/failures/expected/image-expected.txt '))) 228 'platform/linux-baselines/failures/expected/image-expected.txt') ))
229 229
230 230
231 class TestRebaselineTest(BaseTestCase): 231 class TestRebaselineTest(BaseTestCase):
232 command_constructor = RebaselineTest # AKA webkit-patch rebaseline-test-int ernal 232 command_constructor = RebaselineTest # AKA webkit-patch rebaseline-test-int ernal
233 233
234 def setUp(self): 234 def setUp(self):
235 super(TestRebaselineTest, self).setUp() 235 super(TestRebaselineTest, self).setUp()
236 236
237 @staticmethod 237 @staticmethod
238 def options(**kwargs): 238 def options(**kwargs):
239 return optparse.Values(dict({ 239 return optparse.Values(dict({
240 'builder': "MOCK Mac10.11", 240 'builder': "MOCK Mac10.11",
241 'test': "userscripts/another-test.html", 241 'test': "userscripts/another-test.html",
242 'suffixes': "txt", 242 'suffixes': "txt",
243 'results_directory': None, 243 'results_directory': None,
244 'build_number': None 244 'build_number': None
245 }, **kwargs)) 245 }, **kwargs))
246 246
247 def test_baseline_directory(self): 247 def test_baseline_directory(self):
248 command = self.command 248 command = self.command
249 self.assertMultiLineEqual(command._baseline_directory("MOCK Mac10.11"), 249 self.assertMultiLineEqual(command._baseline_directory("MOCK Mac10.11"),
250 "/test.checkout/LayoutTests/platform/test-mac- mac10.11") 250 "/test.checkout/LayoutTests/platform/mac-basel ines")
251 self.assertMultiLineEqual(command._baseline_directory("MOCK Mac10.10"), 251 self.assertMultiLineEqual(command._baseline_directory("MOCK Mac10.10"),
252 "/test.checkout/LayoutTests/platform/test-mac- mac10.10") 252 "/test.checkout/LayoutTests/platform/mac10.10- baselines")
253 self.assertMultiLineEqual(command._baseline_directory("MOCK Trusty"), 253 self.assertMultiLineEqual(command._baseline_directory("MOCK Trusty"),
254 "/test.checkout/LayoutTests/platform/test-linu x-trusty") 254 "/test.checkout/LayoutTests/platform/linux-bas elines")
255 self.assertMultiLineEqual(command._baseline_directory("MOCK Precise"), 255 self.assertMultiLineEqual(command._baseline_directory("MOCK Precise"),
256 "/test.checkout/LayoutTests/platform/test-linu x-precise") 256 "/test.checkout/LayoutTests/platform/linux-pre cise-baselines")
257 257
258 def test_rebaseline_updates_expectations_file_noop(self): 258 def test_rebaseline_updates_expectations_file_noop(self):
259 self._zero_out_test_expectations() 259 self._zero_out_test_expectations()
260 self._write( 260 self._write(
261 self.mac_expectations_path, 261 self.mac_expectations_path,
262 ("Bug(B) [ Mac Linux Win7 Debug ] fast/dom/Window/window-postmessage -clone-really-deep-array.html [ Pass ]\n" 262 ("Bug(B) [ Mac Linux Win7 Debug ] fast/dom/Window/window-postmessage -clone-really-deep-array.html [ Pass ]\n"
263 "Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failu re ]\n")) 263 "Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failu re ]\n"))
264 self._write("fast/dom/Window/window-postmessage-clone-really-deep-array. html", "Dummy test contents") 264 self._write("fast/dom/Window/window-postmessage-clone-really-deep-array. html", "Dummy test contents")
265 self._write("fast/css/large-list-of-rules-crash.html", "Dummy test conte nts") 265 self._write("fast/css/large-list-of-rules-crash.html", "Dummy test conte nts")
266 self._write("userscripts/another-test.html", "Dummy test contents") 266 self._write("userscripts/another-test.html", "Dummy test contents")
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
298 expected_logs="Cannot rebaseline image result for reftest: userscrip ts/another-test.html\n") 298 expected_logs="Cannot rebaseline image result for reftest: userscrip ts/another-test.html\n")
299 self.assertDictEqual(self.command.expectation_line_changes.to_dict(), {' remove-lines': []}) 299 self.assertDictEqual(self.command.expectation_line_changes.to_dict(), {' remove-lines': []})
300 300
301 def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self): 301 def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self):
302 self.tool.executive = MockExecutive2() 302 self.tool.executive = MockExecutive2()
303 303
304 port = self.tool.port_factory.get('test-win-win7') 304 port = self.tool.port_factory.get('test-win-win7')
305 self._write( 305 self._write(
306 port.host.filesystem.join( 306 port.host.filesystem.join(
307 port.layout_tests_dir(), 307 port.layout_tests_dir(),
308 'platform/test-win-win10/failures/expected/image-expected.txt'), 308 'platform/win-baselines/failures/expected/image-expected.txt'),
309 'original win10 result') 309 'original win10 result')
310 310
311 oc = OutputCapture() 311 oc = OutputCapture()
312 try: 312 try:
313 options = optparse.Values({ 313 options = optparse.Values({
314 'optimize': True, 314 'optimize': True,
315 'builder': "MOCK Win10", 315 'builder': "MOCK Win10",
316 'suffixes': "txt", 316 'suffixes': "txt",
317 'verbose': True, 317 'verbose': True,
318 'test': "failures/expected/image.html", 318 'test': "failures/expected/image.html",
319 'results_directory': None, 319 'results_directory': None,
320 'build_number': None 320 'build_number': None
321 }) 321 })
322 oc.capture_output() 322 oc.capture_output()
323 self.command.execute(options, [], self.tool) 323 self.command.execute(options, [], self.tool)
324 finally: 324 finally:
325 out, _, _ = oc.restore_output() 325 out, _, _ = oc.restore_output()
326 326
327 self.assertMultiLineEqual( 327 self.assertMultiLineEqual(
328 self._read(self.tool.filesystem.join( 328 self._read(self.tool.filesystem.join(
329 port.layout_tests_dir(), 329 port.layout_tests_dir(),
330 'platform/test-win-win10/failures/expected/image-expected.txt')) , 330 'platform/win-baselines/failures/expected/image-expected.txt')),
331 'MOCK Web result, convert 404 to None=True') 331 'MOCK Web result, convert 404 to None=True')
332 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( 332 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(
333 port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/i mage-expected.txt'))) 333 port.layout_tests_dir(), 'platform/win7-baselines/failures/expected/ image-expected.txt')))
334 self.assertMultiLineEqual( 334 self.assertMultiLineEqual(
335 out, '{"remove-lines": [{"test": "failures/expected/image.html", "bu ilder": "MOCK Win10"}]}\n') 335 out, '{"remove-lines": [{"test": "failures/expected/image.html", "bu ilder": "MOCK Win10"}]}\n')
336 336
337 337
338 class TestAbstractParallelRebaselineCommand(BaseTestCase): 338 class TestAbstractParallelRebaselineCommand(BaseTestCase):
339 command_constructor = AbstractParallelRebaselineCommand 339 command_constructor = AbstractParallelRebaselineCommand
340 340
341 def test_builders_to_fetch_from(self): 341 def test_builders_to_fetch_from(self):
342 builders_to_fetch = self.command._builders_to_fetch_from( 342 builders_to_fetch = self.command._builders_to_fetch_from(
343 ["MOCK Win10", "MOCK Win7 (dbg)(1)", "MOCK Win7 (dbg)(2)", "MOCK Win 7"]) 343 ["MOCK Win10", "MOCK Win7 (dbg)(1)", "MOCK Win7 (dbg)(2)", "MOCK Win 7"])
344 self.assertEqual(builders_to_fetch, ["MOCK Win7", "MOCK Win10"]) 344 self.assertEqual(builders_to_fetch, ["MOCK Win7", "MOCK Win10"])
345 345
346 def test_all_baseline_paths(self): 346 def test_all_baseline_paths(self):
347 test_prefix_list = { 347 test_prefix_list = {
348 'passes/text.html': { 348 'passes/text.html': {
349 Build('MOCK Win7'): ('txt', 'png'), 349 Build('MOCK Win7'): ('txt', 'png'),
350 Build('MOCK Win10'): ('txt',), 350 Build('MOCK Win10'): ('txt',),
351 } 351 }
352 } 352 }
353 # pylint: disable=protected-access 353 # pylint: disable=protected-access
354 baseline_paths = self.command._all_baseline_paths(test_prefix_list) 354 baseline_paths = self.command._all_baseline_paths(test_prefix_list)
355 self.assertEqual(baseline_paths, [ 355 self.assertEqual(baseline_paths, [
356 '/test.checkout/LayoutTests/passes/text-expected.png', 356 '/test.checkout/LayoutTests/passes/text-expected.png',
357 '/test.checkout/LayoutTests/passes/text-expected.txt', 357 '/test.checkout/LayoutTests/passes/text-expected.txt',
358 '/test.checkout/LayoutTests/platform/test-win-win10/passes/text-expe cted.txt', 358 '/test.checkout/LayoutTests/platform/win-baselines/passes/text-expec ted.txt',
359 '/test.checkout/LayoutTests/platform/test-win-win7/passes/text-expec ted.png', 359 '/test.checkout/LayoutTests/platform/win7-baselines/passes/text-expe cted.png',
360 '/test.checkout/LayoutTests/platform/test-win-win7/passes/text-expec ted.txt', 360 '/test.checkout/LayoutTests/platform/win7-baselines/passes/text-expe cted.txt',
361 ]) 361 ])
362 362
363 def test_remove_all_pass_testharness_baselines(self): 363 def test_remove_all_pass_testharness_baselines(self):
364 self.tool.filesystem.write_text_file( 364 self.tool.filesystem.write_text_file(
365 '/test.checkout/LayoutTests/passes/text-expected.txt', 365 '/test.checkout/LayoutTests/passes/text-expected.txt',
366 ('This is a testharness.js-based test.\n' 366 ('This is a testharness.js-based test.\n'
367 'PASS: foo\n' 367 'PASS: foo\n'
368 'Harness: the test ran to completion.\n')) 368 'Harness: the test ran to completion.\n'))
369 test_prefix_list = { 369 test_prefix_list = {
370 'passes/text.html': { 370 'passes/text.html': {
(...skipping 556 matching lines...) Expand 10 before | Expand all | Expand 10 after
927 for cmd_line, cwd in commands: 927 for cmd_line, cwd in commands:
928 out = self.run_command(cmd_line, cwd=cwd) 928 out = self.run_command(cmd_line, cwd=cwd)
929 if 'rebaseline-test-internal' in cmd_line: 929 if 'rebaseline-test-internal' in cmd_line:
930 out = '{"remove-lines": [{"test": "%s", "builder": "%s"}]}\n' % (cmd_line[8], cmd_line[6]) 930 out = '{"remove-lines": [{"test": "%s", "builder": "%s"}]}\n' % (cmd_line[8], cmd_line[6])
931 command_outputs.append([0, out, '']) 931 command_outputs.append([0, out, ''])
932 932
933 new_calls = self.calls[num_previous_calls:] 933 new_calls = self.calls[num_previous_calls:]
934 self.calls = self.calls[:num_previous_calls] 934 self.calls = self.calls[:num_previous_calls]
935 self.calls.append(new_calls) 935 self.calls.append(new_calls)
936 return command_outputs 936 return command_outputs
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698