Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(158)

Side by Side Diff: third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py

Issue 2644783003: Regenerate MANIFEST.json when WPT tests are run (Closed)
Patch Set: Regenerate MANIFEST.json from template in run-webkit-tests Created 3 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright (C) 2010 Google Inc. All rights reserved. 1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Sze ged 2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Sze ged
3 # Copyright (C) 2011 Apple Inc. All rights reserved. 3 # Copyright (C) 2011 Apple Inc. All rights reserved.
4 # 4 #
5 # Redistribution and use in source and binary forms, with or without 5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are 6 # modification, are permitted provided that the following conditions are
7 # met: 7 # met:
8 # 8 #
9 # * Redistributions of source code must retain the above copyright 9 # * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer. 10 # notice, this list of conditions and the following disclaimer.
(...skipping 1068 matching lines...) Expand 10 before | Expand all | Expand 10 after
1079 1079
1080 def test_reset_results(self): 1080 def test_reset_results(self):
1081 # Test that we update expectations in place. If the expectation 1081 # Test that we update expectations in place. If the expectation
1082 # is missing, update the expected generic location. 1082 # is missing, update the expected generic location.
1083 host = MockHost() 1083 host = MockHost()
1084 details, err, _ = logging_run( 1084 details, err, _ = logging_run(
1085 ['--pixel-tests', '--reset-results', 'passes/image.html'], 1085 ['--pixel-tests', '--reset-results', 'passes/image.html'],
1086 tests_included=True, host=host, new_results=True) 1086 tests_included=True, host=host, new_results=True)
1087 file_list = host.filesystem.written_files.keys() 1087 file_list = host.filesystem.written_files.keys()
1088 self.assertEqual(details.exit_code, 0) 1088 self.assertEqual(details.exit_code, 0)
1089 self.assertEqual(len(file_list), 7) 1089 self.assertEqual(len(file_list), 8)
1090 self.assert_baselines(file_list, "passes/image", [".txt", ".png"], err) 1090 self.assert_baselines(file_list, "passes/image", [".txt", ".png"], err)
1091 1091
1092 def test_missing_results(self): 1092 def test_missing_results(self):
1093 # Test that we update expectations in place. If the expectation 1093 # Test that we update expectations in place. If the expectation
1094 # is missing, update the expected generic location. 1094 # is missing, update the expected generic location.
1095 host = MockHost() 1095 host = MockHost()
1096 details, err, _ = logging_run(['--no-show-results', 1096 details, err, _ = logging_run(['--no-show-results',
1097 'failures/unexpected/missing_text.html', 1097 'failures/unexpected/missing_text.html',
1098 'failures/unexpected/missing_image.html', 1098 'failures/unexpected/missing_image.html',
1099 'failures/unexpected/missing_render_tree_ dump.html'], 1099 'failures/unexpected/missing_render_tree_ dump.html'],
1100 tests_included=True, host=host, new_result s=True) 1100 tests_included=True, host=host, new_result s=True)
1101 file_list = host.filesystem.written_files.keys() 1101 file_list = host.filesystem.written_files.keys()
1102 self.assertEqual(details.exit_code, 3) 1102 self.assertEqual(details.exit_code, 3)
1103 self.assertEqual(len(file_list), 11) 1103 self.assertEqual(len(file_list), 12)
1104 self.assert_baselines(file_list, "failures/unexpected/missing_text", [". txt"], err) 1104 self.assert_baselines(file_list, "failures/unexpected/missing_text", [". txt"], err)
1105 self.assert_baselines(file_list, "platform/test/failures/unexpected/miss ing_image", [".png"], err) 1105 self.assert_baselines(file_list, "platform/test/failures/unexpected/miss ing_image", [".png"], err)
1106 self.assert_baselines(file_list, "platform/test/failures/unexpected/miss ing_render_tree_dump", [".txt"], err) 1106 self.assert_baselines(file_list, "platform/test/failures/unexpected/miss ing_render_tree_dump", [".txt"], err)
1107 1107
1108 def test_new_baseline(self): 1108 def test_new_baseline(self):
1109 # Test that we update the platform expectations in the version-specific directories 1109 # Test that we update the platform expectations in the version-specific directories
1110 # for both existing and new baselines. 1110 # for both existing and new baselines.
1111 host = MockHost() 1111 host = MockHost()
1112 details, err, _ = logging_run( 1112 details, err, _ = logging_run(
1113 ['--pixel-tests', '--new-baseline', 'passes/image.html'], 1113 ['--pixel-tests', '--new-baseline', 'passes/image.html'],
1114 tests_included=True, host=host, new_results=True) 1114 tests_included=True, host=host, new_results=True)
1115 file_list = host.filesystem.written_files.keys() 1115 file_list = host.filesystem.written_files.keys()
1116 self.assertEqual(details.exit_code, 0) 1116 self.assertEqual(details.exit_code, 0)
1117 self.assertEqual(len(file_list), 7) 1117 self.assertEqual(len(file_list), 8)
1118 self.assert_baselines(file_list, 1118 self.assert_baselines(file_list,
1119 "platform/test-mac-mac10.10/passes/image", [".txt", ".png"], err) 1119 "platform/test-mac-mac10.10/passes/image", [".txt", ".png"], err)
1120 1120
1121 def test_reftest_reset_results(self): 1121 def test_reftest_reset_results(self):
1122 # Test rebaseline of reftests. 1122 # Test rebaseline of reftests.
1123 # Should ignore reftests without text expectations. 1123 # Should ignore reftests without text expectations.
1124 host = MockHost() 1124 host = MockHost()
1125 details, err, _ = logging_run(['--reset-results', 'passes/reftest.html'] , tests_included=True, host=host) 1125 details, err, _ = logging_run(['--reset-results', 'passes/reftest.html'] , tests_included=True, host=host)
1126 file_list = host.filesystem.written_files.keys() 1126 file_list = host.filesystem.written_files.keys()
1127 self.assertEqual(details.exit_code, 0) 1127 self.assertEqual(details.exit_code, 0)
1128 self.assertEqual(len(file_list), 5) 1128 self.assertEqual(len(file_list), 6)
1129 self.assert_baselines(file_list, '', [], err) 1129 self.assert_baselines(file_list, '', [], err)
1130 1130
1131 host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest- expected.txt', '') 1131 host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest- expected.txt', '')
1132 host.filesystem.clear_written_files() 1132 host.filesystem.clear_written_files()
1133 details, err, _ = logging_run(['--reset-results', 'passes/reftest.html'] , tests_included=True, host=host) 1133 details, err, _ = logging_run(['--reset-results', 'passes/reftest.html'] , tests_included=True, host=host)
1134 file_list = host.filesystem.written_files.keys() 1134 file_list = host.filesystem.written_files.keys()
1135 self.assertEqual(details.exit_code, 0) 1135 self.assertEqual(details.exit_code, 0)
1136 self.assertEqual(len(file_list), 6) 1136 self.assertEqual(len(file_list), 6)
1137 self.assert_baselines(file_list, 'passes/reftest', ['.txt'], err) 1137 self.assert_baselines(file_list, 'passes/reftest', ['.txt'], err)
1138 1138
1139 def test_reftest_new_baseline(self): 1139 def test_reftest_new_baseline(self):
1140 # Test rebaseline of reftests. 1140 # Test rebaseline of reftests.
1141 # Should ignore reftests without text expectations. 1141 # Should ignore reftests without text expectations.
1142 host = MockHost() 1142 host = MockHost()
1143 details, err, _ = logging_run(['--new-baseline', 'passes/reftest.html'], tests_included=True, host=host) 1143 details, err, _ = logging_run(['--new-baseline', 'passes/reftest.html'], tests_included=True, host=host)
1144 file_list = host.filesystem.written_files.keys() 1144 file_list = host.filesystem.written_files.keys()
1145 self.assertEqual(details.exit_code, 0) 1145 self.assertEqual(details.exit_code, 0)
1146 self.assertEqual(len(file_list), 5) 1146 self.assertEqual(len(file_list), 6)
1147 self.assert_baselines(file_list, '', [], err) 1147 self.assert_baselines(file_list, '', [], err)
1148 1148
1149 host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest- expected.txt', '') 1149 host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest- expected.txt', '')
1150 host.filesystem.clear_written_files() 1150 host.filesystem.clear_written_files()
1151 details, err, _ = logging_run(['--new-baseline', 'passes/reftest.html'], tests_included=True, host=host) 1151 details, err, _ = logging_run(['--new-baseline', 'passes/reftest.html'], tests_included=True, host=host)
1152 file_list = host.filesystem.written_files.keys() 1152 file_list = host.filesystem.written_files.keys()
1153 self.assertEqual(details.exit_code, 0) 1153 self.assertEqual(details.exit_code, 0)
1154 self.assertEqual(len(file_list), 6) 1154 self.assertEqual(len(file_list), 6)
1155 self.assert_baselines(file_list, 'platform/test-mac-mac10.10/passes/reft est', ['.txt'], err) 1155 self.assert_baselines(file_list, 'platform/test-mac-mac10.10/passes/reft est', ['.txt'], err)
1156 1156
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after
1194 1194
1195 run_webkit_tests._run_tests = successful_run 1195 run_webkit_tests._run_tests = successful_run
1196 res = run_webkit_tests.main(['--platform', 'test'], stdout, stderr) 1196 res = run_webkit_tests.main(['--platform', 'test'], stdout, stderr)
1197 self.assertEqual(res, test_run_results.UNEXPECTED_ERROR_EXIT_STATUS) 1197 self.assertEqual(res, test_run_results.UNEXPECTED_ERROR_EXIT_STATUS)
1198 1198
1199 run_webkit_tests._run_tests = exception_raising_run 1199 run_webkit_tests._run_tests = exception_raising_run
1200 res = run_webkit_tests.main([], stdout, stderr) 1200 res = run_webkit_tests.main([], stdout, stderr)
1201 self.assertEqual(res, test_run_results.UNEXPECTED_ERROR_EXIT_STATUS) 1201 self.assertEqual(res, test_run_results.UNEXPECTED_ERROR_EXIT_STATUS)
1202 finally: 1202 finally:
1203 run_webkit_tests._run_tests = orig_run_fn 1203 run_webkit_tests._run_tests = orig_run_fn
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698