OLD | NEW |
1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import optparse | 5 import optparse |
6 import unittest | 6 import unittest |
7 | 7 |
8 from webkitpy.common.net.buildbot import Build | 8 from webkitpy.common.net.buildbot import Build |
9 from webkitpy.common.net.layouttestresults import LayoutTestResults | 9 from webkitpy.common.net.layouttestresults import LayoutTestResults |
10 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 | 10 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 |
(...skipping 10 matching lines...) Expand all Loading... |
21 class BaseTestCase(unittest.TestCase): | 21 class BaseTestCase(unittest.TestCase): |
22 MOCK_WEB_RESULT = 'MOCK Web result, convert 404 to None=True' | 22 MOCK_WEB_RESULT = 'MOCK Web result, convert 404 to None=True' |
23 WEB_PREFIX = 'https://storage.googleapis.com/chromium-layout-test-archives/M
OCK_Mac10_11/results/layout-test-results' | 23 WEB_PREFIX = 'https://storage.googleapis.com/chromium-layout-test-archives/M
OCK_Mac10_11/results/layout-test-results' |
24 | 24 |
25 command_constructor = None | 25 command_constructor = None |
26 | 26 |
27 def setUp(self): | 27 def setUp(self): |
28 self.tool = MockWebKitPatch() | 28 self.tool = MockWebKitPatch() |
29 # lint warns that command_constructor might not be set, but this is inte
ntional; pylint: disable=E1102 | 29 # lint warns that command_constructor might not be set, but this is inte
ntional; pylint: disable=E1102 |
30 self.command = self.command_constructor() | 30 self.command = self.command_constructor() |
| 31 self.command._tool = self.tool |
31 self.tool.builders = BuilderList({ | 32 self.tool.builders = BuilderList({ |
32 "MOCK Mac10.10 (dbg)": {"port_name": "test-mac-mac10.10", "specifier
s": ["Mac10.10", "Debug"]}, | 33 "MOCK Mac10.10 (dbg)": {"port_name": "test-mac-mac10.10", "specifier
s": ["Mac10.10", "Debug"]}, |
33 "MOCK Mac10.10": {"port_name": "test-mac-mac10.10", "specifiers": ["
Mac10.10", "Release"]}, | 34 "MOCK Mac10.10": {"port_name": "test-mac-mac10.10", "specifiers": ["
Mac10.10", "Release"]}, |
34 "MOCK Mac10.11 (dbg)": {"port_name": "test-mac-mac10.11", "specifier
s": ["Mac10.11", "Debug"]}, | 35 "MOCK Mac10.11 (dbg)": {"port_name": "test-mac-mac10.11", "specifier
s": ["Mac10.11", "Debug"]}, |
35 "MOCK Mac10.11 ASAN": {"port_name": "test-mac-mac10.11", "specifiers
": ["Mac10.11", "Release"]}, | 36 "MOCK Mac10.11 ASAN": {"port_name": "test-mac-mac10.11", "specifiers
": ["Mac10.11", "Release"]}, |
36 "MOCK Mac10.11": {"port_name": "test-mac-mac10.11", "specifiers": ["
Mac10.11", "Release"]}, | 37 "MOCK Mac10.11": {"port_name": "test-mac-mac10.11", "specifiers": ["
Mac10.11", "Release"]}, |
37 "MOCK Precise": {"port_name": "test-linux-precise", "specifiers": ["
Precise", "Release"]}, | 38 "MOCK Precise": {"port_name": "test-linux-precise", "specifiers": ["
Precise", "Release"]}, |
38 "MOCK Trusty": {"port_name": "test-linux-trusty", "specifiers": ["Tr
usty", "Release"]}, | 39 "MOCK Trusty": {"port_name": "test-linux-trusty", "specifiers": ["Tr
usty", "Release"]}, |
39 "MOCK Win10": {"port_name": "test-win-win10", "specifiers": ["Win10"
, "Release"]}, | 40 "MOCK Win10": {"port_name": "test-win-win10", "specifiers": ["Win10"
, "Release"]}, |
40 "MOCK Win7 (dbg)": {"port_name": "test-win-win7", "specifiers": ["Wi
n7", "Debug"]}, | 41 "MOCK Win7 (dbg)": {"port_name": "test-win-win7", "specifiers": ["Wi
n7", "Debug"]}, |
41 "MOCK Win7 (dbg)(1)": {"port_name": "test-win-win7", "specifiers": [
"Win7", "Debug"]}, | 42 "MOCK Win7 (dbg)(1)": {"port_name": "test-win-win7", "specifiers": [
"Win7", "Debug"]}, |
42 "MOCK Win7 (dbg)(2)": {"port_name": "test-win-win7", "specifiers": [
"Win7", "Debug"]}, | 43 "MOCK Win7 (dbg)(2)": {"port_name": "test-win-win7", "specifiers": [
"Win7", "Debug"]}, |
43 "MOCK Win7": {"port_name": "test-win-win7", "specifiers": ["Win7", "
Release"]}, | 44 "MOCK Win7": {"port_name": "test-win-win7", "specifiers": ["Win7", "
Release"]}, |
44 }) | 45 }) |
45 self.command.bind_to_tool(self.tool) | |
46 self.mac_port = self.tool.port_factory.get_from_builder_name("MOCK Mac10
.11") | 46 self.mac_port = self.tool.port_factory.get_from_builder_name("MOCK Mac10
.11") |
47 | 47 |
48 self.mac_expectations_path = self.mac_port.path_to_generic_test_expectat
ions_file() | 48 self.mac_expectations_path = self.mac_port.path_to_generic_test_expectat
ions_file() |
49 self.tool.filesystem.write_text_file( | 49 self.tool.filesystem.write_text_file( |
50 self.tool.filesystem.join(self.mac_port.layout_tests_dir(), "Virtual
TestSuites"), '[]') | 50 self.tool.filesystem.join(self.mac_port.layout_tests_dir(), "Virtual
TestSuites"), '[]') |
51 | 51 |
52 # In AbstractParallelRebaselineCommand._rebaseline_commands, a default p
ort | 52 # In AbstractParallelRebaselineCommand._rebaseline_commands, a default p
ort |
53 # object is gotten using self.tool.port_factory.get(), which is used to
get | 53 # object is gotten using self.tool.port_factory.get(), which is used to
get |
54 # test paths -- and the layout tests directory may be different for the
"test" | 54 # test paths -- and the layout tests directory may be different for the
"test" |
55 # ports and real ports. Since only "test" ports are used in this class, | 55 # ports and real ports. Since only "test" ports are used in this class, |
(...skipping 343 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
399 "first-test.html": { | 399 "first-test.html": { |
400 "expected": "NEEDSREBASELINE", | 400 "expected": "NEEDSREBASELINE", |
401 "actual": "PASS" | 401 "actual": "PASS" |
402 } | 402 } |
403 } | 403 } |
404 } | 404 } |
405 })) | 405 })) |
406 | 406 |
407 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") | 407 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") |
408 self._write("userscripts/first-test.html", "Dummy test contents") | 408 self._write("userscripts/first-test.html", "Dummy test contents") |
409 | |
410 self.command._rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) | 409 self.command._rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) |
411 | 410 |
412 self.assertEqual(self.tool.executive.calls, []) | 411 self.assertEqual(self.tool.executive.calls, []) |
413 | 412 |
414 def test_rebaseline_all(self): | 413 def test_rebaseline_all(self): |
415 self._setup_mock_build_data() | 414 self._setup_mock_build_data() |
416 | 415 |
417 self._write("userscripts/first-test.html", "Dummy test contents") | 416 self._write("userscripts/first-test.html", "Dummy test contents") |
418 self.command._rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) | 417 self.command._rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) |
419 | 418 |
(...skipping 22 matching lines...) Expand all Loading... |
442 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 441 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
443 '--builder', 'MOCK Win7 (dbg)', '--test', 'userscripts/first-t
est.html', '--verbose']], | 442 '--builder', 'MOCK Win7 (dbg)', '--test', 'userscripts/first-t
est.html', '--verbose']], |
444 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', '--builder', | 443 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', '--builder', |
445 'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '-
-verbose']], | 444 'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '-
-verbose']], |
446 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', | 445 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', |
447 'userscripts/first-test.html', '--verbose']] | 446 'userscripts/first-test.html', '--verbose']] |
448 ]) | 447 ]) |
449 | 448 |
450 def test_no_optimize(self): | 449 def test_no_optimize(self): |
451 self._setup_mock_build_data() | 450 self._setup_mock_build_data() |
452 print self.tool.buildbot._canned_results | |
453 | |
454 self._write("userscripts/first-test.html", "Dummy test contents") | 451 self._write("userscripts/first-test.html", "Dummy test contents") |
455 self.command._rebaseline( | 452 self.command._rebaseline( |
456 self.options(optimize=False), | 453 self.options(optimize=False), |
457 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) | 454 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) |
458 | 455 |
459 # Note that we have only one run_in_parallel() call | 456 # Note that we have only one run_in_parallel() call |
460 self.assertEqual( | 457 self.assertEqual( |
461 self.tool.executive.calls, | 458 self.tool.executive.calls, |
462 [ | 459 [ |
463 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 460 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
464 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], | 461 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], |
465 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', | 462 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', |
466 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']] | 463 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']] |
467 ]) | 464 ]) |
468 | 465 |
469 def test_results_directory(self): | 466 def test_results_directory(self): |
470 self._setup_mock_build_data() | 467 self._setup_mock_build_data() |
471 | |
472 self._write("userscripts/first-test.html", "Dummy test contents") | 468 self._write("userscripts/first-test.html", "Dummy test contents") |
473 self.command._rebaseline( | 469 self.command._rebaseline( |
474 self.options(optimize=False, results_directory='/tmp'), | 470 self.options(optimize=False, results_directory='/tmp'), |
475 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) | 471 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) |
476 | 472 |
477 # Note that we have only one run_in_parallel() call | 473 # Note that we have only one run_in_parallel() call |
478 self.assertEqual( | 474 self.assertEqual( |
479 self.tool.executive.calls, | 475 self.tool.executive.calls, |
480 [ | 476 [ |
481 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 477 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
519 new_expectations = self._read(self.mac_expectations_path) | 515 new_expectations = self._read(self.mac_expectations_path) |
520 self.assertMultiLineEqual( | 516 self.assertMultiLineEqual( |
521 new_expectations, | 517 new_expectations, |
522 ("Bug(x) [ Mac10.10 ] userscripts/first-test.html [ Failure ]\n" | 518 ("Bug(x) [ Mac10.10 ] userscripts/first-test.html [ Failure ]\n" |
523 "bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n")) | 519 "bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n")) |
524 | 520 |
525 def test_rebaseline_updates_expectations_file_all_platforms(self): | 521 def test_rebaseline_updates_expectations_file_all_platforms(self): |
526 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") | 522 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") |
527 self._write("userscripts/first-test.html", "Dummy test contents") | 523 self._write("userscripts/first-test.html", "Dummy test contents") |
528 self._setup_mock_build_data() | 524 self._setup_mock_build_data() |
529 | |
530 self.command._rebaseline( | 525 self.command._rebaseline( |
531 self.options(), | 526 self.options(), |
532 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) | 527 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
533 | |
534 new_expectations = self._read(self.mac_expectations_path) | 528 new_expectations = self._read(self.mac_expectations_path) |
535 self.assertMultiLineEqual( | 529 self.assertMultiLineEqual( |
536 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") | 530 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") |
537 | 531 |
538 def test_rebaseline_handles_platform_skips(self): | 532 def test_rebaseline_handles_platform_skips(self): |
539 # This test is just like test_rebaseline_updates_expectations_file_all_p
latforms(), | 533 # This test is just like test_rebaseline_updates_expectations_file_all_p
latforms(), |
540 # except that if a particular port happens to SKIP a test in an override
s file, | 534 # except that if a particular port happens to SKIP a test in an override
s file, |
541 # we count that as passing, and do not think that we still need to rebas
eline it. | 535 # we count that as passing, and do not think that we still need to rebas
eline it. |
542 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") | 536 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") |
543 self._write("NeverFixTests", "Bug(y) [ Android ] userscripts [ WontFix ]
\n") | 537 self._write("NeverFixTests", "Bug(y) [ Android ] userscripts [ WontFix ]
\n") |
(...skipping 385 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
929 for cmd_line, cwd in commands: | 923 for cmd_line, cwd in commands: |
930 out = self.run_command(cmd_line, cwd=cwd) | 924 out = self.run_command(cmd_line, cwd=cwd) |
931 if 'rebaseline-test-internal' in cmd_line: | 925 if 'rebaseline-test-internal' in cmd_line: |
932 out = '{"add": [], "remove-lines": [{"test": "%s", "builder": "%
s"}], "delete": []}\n' % (cmd_line[8], cmd_line[6]) | 926 out = '{"add": [], "remove-lines": [{"test": "%s", "builder": "%
s"}], "delete": []}\n' % (cmd_line[8], cmd_line[6]) |
933 command_outputs.append([0, out, '']) | 927 command_outputs.append([0, out, '']) |
934 | 928 |
935 new_calls = self.calls[num_previous_calls:] | 929 new_calls = self.calls[num_previous_calls:] |
936 self.calls = self.calls[:num_previous_calls] | 930 self.calls = self.calls[:num_previous_calls] |
937 self.calls.append(new_calls) | 931 self.calls.append(new_calls) |
938 return command_outputs | 932 return command_outputs |
OLD | NEW |