OLD | NEW |
1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Sze
ged | 2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Sze
ged |
3 # Copyright (C) 2011 Apple Inc. All rights reserved. | 3 # Copyright (C) 2011 Apple Inc. All rights reserved. |
4 # | 4 # |
5 # Redistribution and use in source and binary forms, with or without | 5 # Redistribution and use in source and binary forms, with or without |
6 # modification, are permitted provided that the following conditions are | 6 # modification, are permitted provided that the following conditions are |
7 # met: | 7 # met: |
8 # | 8 # |
9 # * Redistributions of source code must retain the above copyright | 9 # * Redistributions of source code must retain the above copyright |
10 # notice, this list of conditions and the following disclaimer. | 10 # notice, this list of conditions and the following disclaimer. |
(...skipping 1068 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1079 | 1079 |
1080 def test_reset_results(self): | 1080 def test_reset_results(self): |
1081 # Test that we update expectations in place. If the expectation | 1081 # Test that we update expectations in place. If the expectation |
1082 # is missing, update the expected generic location. | 1082 # is missing, update the expected generic location. |
1083 host = MockHost() | 1083 host = MockHost() |
1084 details, err, _ = logging_run( | 1084 details, err, _ = logging_run( |
1085 ['--pixel-tests', '--reset-results', 'passes/image.html'], | 1085 ['--pixel-tests', '--reset-results', 'passes/image.html'], |
1086 tests_included=True, host=host, new_results=True) | 1086 tests_included=True, host=host, new_results=True) |
1087 file_list = host.filesystem.written_files.keys() | 1087 file_list = host.filesystem.written_files.keys() |
1088 self.assertEqual(details.exit_code, 0) | 1088 self.assertEqual(details.exit_code, 0) |
1089 self.assertEqual(len(file_list), 7) | 1089 self.assertEqual(len(file_list), 8) |
1090 self.assert_baselines(file_list, "passes/image", [".txt", ".png"], err) | 1090 self.assert_baselines(file_list, "passes/image", [".txt", ".png"], err) |
1091 | 1091 |
1092 def test_missing_results(self): | 1092 def test_missing_results(self): |
1093 # Test that we update expectations in place. If the expectation | 1093 # Test that we update expectations in place. If the expectation |
1094 # is missing, update the expected generic location. | 1094 # is missing, update the expected generic location. |
1095 host = MockHost() | 1095 host = MockHost() |
1096 details, err, _ = logging_run(['--no-show-results', | 1096 details, err, _ = logging_run(['--no-show-results', |
1097 'failures/unexpected/missing_text.html', | 1097 'failures/unexpected/missing_text.html', |
1098 'failures/unexpected/missing_image.html', | 1098 'failures/unexpected/missing_image.html', |
1099 'failures/unexpected/missing_render_tree_
dump.html'], | 1099 'failures/unexpected/missing_render_tree_
dump.html'], |
1100 tests_included=True, host=host, new_result
s=True) | 1100 tests_included=True, host=host, new_result
s=True) |
1101 file_list = host.filesystem.written_files.keys() | 1101 file_list = host.filesystem.written_files.keys() |
1102 self.assertEqual(details.exit_code, 3) | 1102 self.assertEqual(details.exit_code, 3) |
1103 self.assertEqual(len(file_list), 11) | 1103 self.assertEqual(len(file_list), 12) |
1104 self.assert_baselines(file_list, "failures/unexpected/missing_text", [".
txt"], err) | 1104 self.assert_baselines(file_list, "failures/unexpected/missing_text", [".
txt"], err) |
1105 self.assert_baselines(file_list, "platform/test/failures/unexpected/miss
ing_image", [".png"], err) | 1105 self.assert_baselines(file_list, "platform/test/failures/unexpected/miss
ing_image", [".png"], err) |
1106 self.assert_baselines(file_list, "platform/test/failures/unexpected/miss
ing_render_tree_dump", [".txt"], err) | 1106 self.assert_baselines(file_list, "platform/test/failures/unexpected/miss
ing_render_tree_dump", [".txt"], err) |
1107 | 1107 |
1108 def test_new_baseline(self): | 1108 def test_new_baseline(self): |
1109 # Test that we update the platform expectations in the version-specific
directories | 1109 # Test that we update the platform expectations in the version-specific
directories |
1110 # for both existing and new baselines. | 1110 # for both existing and new baselines. |
1111 host = MockHost() | 1111 host = MockHost() |
1112 details, err, _ = logging_run( | 1112 details, err, _ = logging_run( |
1113 ['--pixel-tests', '--new-baseline', 'passes/image.html'], | 1113 ['--pixel-tests', '--new-baseline', 'passes/image.html'], |
1114 tests_included=True, host=host, new_results=True) | 1114 tests_included=True, host=host, new_results=True) |
1115 file_list = host.filesystem.written_files.keys() | 1115 file_list = host.filesystem.written_files.keys() |
1116 self.assertEqual(details.exit_code, 0) | 1116 self.assertEqual(details.exit_code, 0) |
1117 self.assertEqual(len(file_list), 7) | 1117 self.assertEqual(len(file_list), 8) |
1118 self.assert_baselines(file_list, | 1118 self.assert_baselines(file_list, |
1119 "platform/test-mac-mac10.10/passes/image", [".txt",
".png"], err) | 1119 "platform/test-mac-mac10.10/passes/image", [".txt",
".png"], err) |
1120 | 1120 |
1121 def test_reftest_reset_results(self): | 1121 def test_reftest_reset_results(self): |
1122 # Test rebaseline of reftests. | 1122 # Test rebaseline of reftests. |
1123 # Should ignore reftests without text expectations. | 1123 # Should ignore reftests without text expectations. |
1124 host = MockHost() | 1124 host = MockHost() |
1125 details, err, _ = logging_run(['--reset-results', 'passes/reftest.html']
, tests_included=True, host=host) | 1125 details, err, _ = logging_run(['--reset-results', 'passes/reftest.html']
, tests_included=True, host=host) |
1126 file_list = host.filesystem.written_files.keys() | 1126 file_list = host.filesystem.written_files.keys() |
1127 self.assertEqual(details.exit_code, 0) | 1127 self.assertEqual(details.exit_code, 0) |
1128 self.assertEqual(len(file_list), 5) | 1128 self.assertEqual(len(file_list), 6) |
1129 self.assert_baselines(file_list, '', [], err) | 1129 self.assert_baselines(file_list, '', [], err) |
1130 | 1130 |
1131 host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest-
expected.txt', '') | 1131 host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest-
expected.txt', '') |
1132 host.filesystem.clear_written_files() | 1132 host.filesystem.clear_written_files() |
1133 details, err, _ = logging_run(['--reset-results', 'passes/reftest.html']
, tests_included=True, host=host) | 1133 details, err, _ = logging_run(['--reset-results', 'passes/reftest.html']
, tests_included=True, host=host) |
1134 file_list = host.filesystem.written_files.keys() | 1134 file_list = host.filesystem.written_files.keys() |
1135 self.assertEqual(details.exit_code, 0) | 1135 self.assertEqual(details.exit_code, 0) |
1136 self.assertEqual(len(file_list), 6) | 1136 self.assertEqual(len(file_list), 6) |
1137 self.assert_baselines(file_list, 'passes/reftest', ['.txt'], err) | 1137 self.assert_baselines(file_list, 'passes/reftest', ['.txt'], err) |
1138 | 1138 |
1139 def test_reftest_new_baseline(self): | 1139 def test_reftest_new_baseline(self): |
1140 # Test rebaseline of reftests. | 1140 # Test rebaseline of reftests. |
1141 # Should ignore reftests without text expectations. | 1141 # Should ignore reftests without text expectations. |
1142 host = MockHost() | 1142 host = MockHost() |
1143 details, err, _ = logging_run(['--new-baseline', 'passes/reftest.html'],
tests_included=True, host=host) | 1143 details, err, _ = logging_run(['--new-baseline', 'passes/reftest.html'],
tests_included=True, host=host) |
1144 file_list = host.filesystem.written_files.keys() | 1144 file_list = host.filesystem.written_files.keys() |
1145 self.assertEqual(details.exit_code, 0) | 1145 self.assertEqual(details.exit_code, 0) |
1146 self.assertEqual(len(file_list), 5) | 1146 self.assertEqual(len(file_list), 6) |
1147 self.assert_baselines(file_list, '', [], err) | 1147 self.assert_baselines(file_list, '', [], err) |
1148 | 1148 |
1149 host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest-
expected.txt', '') | 1149 host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest-
expected.txt', '') |
1150 host.filesystem.clear_written_files() | 1150 host.filesystem.clear_written_files() |
1151 details, err, _ = logging_run(['--new-baseline', 'passes/reftest.html'],
tests_included=True, host=host) | 1151 details, err, _ = logging_run(['--new-baseline', 'passes/reftest.html'],
tests_included=True, host=host) |
1152 file_list = host.filesystem.written_files.keys() | 1152 file_list = host.filesystem.written_files.keys() |
1153 self.assertEqual(details.exit_code, 0) | 1153 self.assertEqual(details.exit_code, 0) |
1154 self.assertEqual(len(file_list), 6) | 1154 self.assertEqual(len(file_list), 6) |
1155 self.assert_baselines(file_list, 'platform/test-mac-mac10.10/passes/reft
est', ['.txt'], err) | 1155 self.assert_baselines(file_list, 'platform/test-mac-mac10.10/passes/reft
est', ['.txt'], err) |
1156 | 1156 |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1194 | 1194 |
1195 run_webkit_tests._run_tests = successful_run | 1195 run_webkit_tests._run_tests = successful_run |
1196 res = run_webkit_tests.main(['--platform', 'test'], stdout, stderr) | 1196 res = run_webkit_tests.main(['--platform', 'test'], stdout, stderr) |
1197 self.assertEqual(res, test_run_results.UNEXPECTED_ERROR_EXIT_STATUS) | 1197 self.assertEqual(res, test_run_results.UNEXPECTED_ERROR_EXIT_STATUS) |
1198 | 1198 |
1199 run_webkit_tests._run_tests = exception_raising_run | 1199 run_webkit_tests._run_tests = exception_raising_run |
1200 res = run_webkit_tests.main([], stdout, stderr) | 1200 res = run_webkit_tests.main([], stdout, stderr) |
1201 self.assertEqual(res, test_run_results.UNEXPECTED_ERROR_EXIT_STATUS) | 1201 self.assertEqual(res, test_run_results.UNEXPECTED_ERROR_EXIT_STATUS) |
1202 finally: | 1202 finally: |
1203 run_webkit_tests._run_tests = orig_run_fn | 1203 run_webkit_tests._run_tests = orig_run_fn |
OLD | NEW |