OLD | NEW |
1 # Copyright (C) 2012 Google Inc. All rights reserved. | 1 # Copyright (C) 2012 Google Inc. All rights reserved. |
2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Sze
ged | 2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Sze
ged |
3 # | 3 # |
4 # Redistribution and use in source and binary forms, with or without | 4 # Redistribution and use in source and binary forms, with or without |
5 # modification, are permitted provided that the following conditions are | 5 # modification, are permitted provided that the following conditions are |
6 # met: | 6 # met: |
7 # | 7 # |
8 # * Redistributions of source code must retain the above copyright | 8 # * Redistributions of source code must retain the above copyright |
9 # notice, this list of conditions and the following disclaimer. | 9 # notice, this list of conditions and the following disclaimer. |
10 # * Redistributions in binary form must reproduce the above | 10 # * Redistributions in binary form must reproduce the above |
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
64 pass | 64 pass |
65 | 65 |
66 def write_update(self, msg): | 66 def write_update(self, msg): |
67 pass | 67 pass |
68 | 68 |
69 def flush(self): | 69 def flush(self): |
70 pass | 70 pass |
71 | 71 |
72 | 72 |
73 class LockCheckingRunner(LayoutTestRunner): | 73 class LockCheckingRunner(LayoutTestRunner): |
| 74 |
74 def __init__(self, port, options, printer, tester, http_lock): | 75 def __init__(self, port, options, printer, tester, http_lock): |
75 super(LockCheckingRunner, self).__init__(options, port, printer, port.re
sults_directory(), lambda test_name: False) | 76 super(LockCheckingRunner, self).__init__(options, port, printer, port.re
sults_directory(), lambda test_name: False) |
76 self._finished_list_called = False | 77 self._finished_list_called = False |
77 self._tester = tester | 78 self._tester = tester |
78 self._should_have_http_lock = http_lock | 79 self._should_have_http_lock = http_lock |
79 | 80 |
80 def handle_finished_list(self, source, list_name, num_tests, elapsed_time): | 81 def handle_finished_list(self, source, list_name, num_tests, elapsed_time): |
81 if not self._finished_list_called: | 82 if not self._finished_list_called: |
82 self._tester.assertEqual(list_name, 'locked_tests') | 83 self._tester.assertEqual(list_name, 'locked_tests') |
83 self._tester.assertTrue(self._remaining_locked_shards) | 84 self._tester.assertTrue(self._remaining_locked_shards) |
84 self._tester.assertTrue(self._has_http_lock is self._should_have_htt
p_lock) | 85 self._tester.assertTrue(self._has_http_lock is self._should_have_htt
p_lock) |
85 | 86 |
86 super(LockCheckingRunner, self).handle_finished_list(source, list_name,
num_tests, elapsed_time) | 87 super(LockCheckingRunner, self).handle_finished_list(source, list_name,
num_tests, elapsed_time) |
87 | 88 |
88 if not self._finished_list_called: | 89 if not self._finished_list_called: |
89 self._tester.assertEqual(self._remaining_locked_shards, []) | 90 self._tester.assertEqual(self._remaining_locked_shards, []) |
90 self._tester.assertFalse(self._has_http_lock) | 91 self._tester.assertFalse(self._has_http_lock) |
91 self._finished_list_called = True | 92 self._finished_list_called = True |
92 | 93 |
93 | 94 |
94 class LayoutTestRunnerTests(unittest.TestCase): | 95 class LayoutTestRunnerTests(unittest.TestCase): |
| 96 |
95 def _runner(self, port=None): | 97 def _runner(self, port=None): |
96 # FIXME: we shouldn't have to use run_webkit_tests.py to get the options
we need. | 98 # FIXME: we shouldn't have to use run_webkit_tests.py to get the options
we need. |
97 options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopa
rd'])[0] | 99 options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopa
rd'])[0] |
98 options.child_processes = '1' | 100 options.child_processes = '1' |
99 | 101 |
100 host = MockHost() | 102 host = MockHost() |
101 port = port or host.port_factory.get(options.platform, options=options) | 103 port = port or host.port_factory.get(options.platform, options=options) |
102 return LockCheckingRunner(port, options, FakePrinter(), self, True) | 104 return LockCheckingRunner(port, options, FakePrinter(), self, True) |
103 | 105 |
104 def _run_tests(self, runner, tests): | 106 def _run_tests(self, runner, tests): |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
152 run_results = TestRunResults(expectations, 1) | 154 run_results = TestRunResults(expectations, 1) |
153 result = TestResult(test_name=test, failures=[], reftest_type=['==']) | 155 result = TestResult(test_name=test, failures=[], reftest_type=['==']) |
154 runner._update_summary_with_result(run_results, result) | 156 runner._update_summary_with_result(run_results, result) |
155 self.assertEqual(0, run_results.expected) | 157 self.assertEqual(0, run_results.expected) |
156 self.assertEqual(1, run_results.unexpected) | 158 self.assertEqual(1, run_results.unexpected) |
157 | 159 |
158 | 160 |
159 class SharderTests(unittest.TestCase): | 161 class SharderTests(unittest.TestCase): |
160 | 162 |
161 test_list = [ | 163 test_list = [ |
162 "http/tests/websocket/tests/unicode.htm", | 164 'http/tests/websocket/tests/unicode.htm', |
163 "animations/keyframes.html", | 165 'animations/keyframes.html', |
164 "http/tests/security/view-source-no-refresh.html", | 166 'http/tests/security/view-source-no-refresh.html', |
165 "http/tests/websocket/tests/websocket-protocol-ignored.html", | 167 'http/tests/websocket/tests/websocket-protocol-ignored.html', |
166 "fast/css/display-none-inline-style-change-crash.html", | 168 'fast/css/display-none-inline-style-change-crash.html', |
167 "http/tests/xmlhttprequest/supported-xml-content-types.html", | 169 'http/tests/xmlhttprequest/supported-xml-content-types.html', |
168 "dom/html/level2/html/HTMLAnchorElement03.html", | 170 'dom/html/level2/html/HTMLAnchorElement03.html', |
169 "ietestcenter/Javascript/11.1.5_4-4-c-1.html", | 171 'ietestcenter/Javascript/11.1.5_4-4-c-1.html', |
170 "dom/html/level2/html/HTMLAnchorElement06.html", | 172 'dom/html/level2/html/HTMLAnchorElement06.html', |
171 "perf/object-keys.html", | 173 'perf/object-keys.html', |
172 "virtual/threaded/dir/test.html", | 174 'virtual/threaded/dir/test.html', |
173 "virtual/threaded/fast/foo/test.html", | 175 'virtual/threaded/fast/foo/test.html', |
174 ] | 176 ] |
175 | 177 |
176 def get_test_input(self, test_file): | 178 def get_test_input(self, test_file): |
177 return TestInput(test_file, requires_lock=(test_file.startswith('http')
or test_file.startswith('perf'))) | 179 return TestInput(test_file, requires_lock=(test_file.startswith('http')
or test_file.startswith('perf'))) |
178 | 180 |
179 def get_shards(self, num_workers, fully_parallel, run_singly, test_list=None
, max_locked_shards=1): | 181 def get_shards(self, num_workers, fully_parallel, run_singly, test_list=None
, max_locked_shards=1): |
180 port = TestPort(MockSystemHost()) | 182 port = TestPort(MockSystemHost()) |
181 self.sharder = Sharder(port.split_test, max_locked_shards) | 183 self.sharder = Sharder(port.split_test, max_locked_shards) |
182 test_list = test_list or self.test_list | 184 test_list = test_list or self.test_list |
183 return self.sharder.shard_tests([self.get_test_input(test) for test in t
est_list], | 185 return self.sharder.shard_tests([self.get_test_input(test) for test in t
est_list], |
184 num_workers, fully_parallel, run_singly) | 186 num_workers, fully_parallel, run_singly) |
185 | 187 |
186 def assert_shards(self, actual_shards, expected_shard_names): | 188 def assert_shards(self, actual_shards, expected_shard_names): |
187 self.assertEqual(len(actual_shards), len(expected_shard_names)) | 189 self.assertEqual(len(actual_shards), len(expected_shard_names)) |
188 for i, shard in enumerate(actual_shards): | 190 for i, shard in enumerate(actual_shards): |
189 expected_shard_name, expected_test_names = expected_shard_names[i] | 191 expected_shard_name, expected_test_names = expected_shard_names[i] |
190 self.assertEqual(shard.name, expected_shard_name) | 192 self.assertEqual(shard.name, expected_shard_name) |
191 self.assertEqual([test_input.test_name for test_input in shard.test_
inputs], | 193 self.assertEqual([test_input.test_name for test_input in shard.test_
inputs], |
192 expected_test_names) | 194 expected_test_names) |
193 | 195 |
194 def test_shard_by_dir(self): | 196 def test_shard_by_dir(self): |
195 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False,
run_singly=False) | 197 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False,
run_singly=False) |
196 | 198 |
197 # Note that although there are tests in multiple dirs that need locks, | 199 # Note that although there are tests in multiple dirs that need locks, |
198 # they are crammed into a single shard in order to reduce the # of | 200 # they are crammed into a single shard in order to reduce the # of |
199 # workers hitting the server at once. | 201 # workers hitting the server at once. |
200 self.assert_shards(locked, | 202 self.assert_shards(locked, |
201 [('locked_shard_1', | 203 [('locked_shard_1', |
202 ['http/tests/security/view-source-no-refresh.html', | 204 ['http/tests/security/view-source-no-refresh.html', |
203 'http/tests/websocket/tests/unicode.htm', | 205 'http/tests/websocket/tests/unicode.htm', |
204 'http/tests/websocket/tests/websocket-protocol-ignored.html', | 206 'http/tests/websocket/tests/websocket-protocol-ign
ored.html', |
205 'http/tests/xmlhttprequest/supported-xml-content-types.html', | 207 'http/tests/xmlhttprequest/supported-xml-content-t
ypes.html', |
206 'perf/object-keys.html'])]) | 208 'perf/object-keys.html'])]) |
207 self.assert_shards(unlocked, | 209 self.assert_shards(unlocked, |
208 [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']), | 210 [('virtual/threaded/dir', ['virtual/threaded/dir/test
.html']), |
209 ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html
']), | 211 ('virtual/threaded/fast/foo', ['virtual/threaded/fas
t/foo/test.html']), |
210 ('animations', ['animations/keyframes.html']), | 212 ('animations', ['animations/keyframes.html']), |
211 ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03
.html', | 213 ('dom/html/level2/html', ['dom/html/level2/html/H
TMLAnchorElement03.html', |
212 'dom/html/level2/html/HTMLAnchorElement06.
html']), | 214 'dom/html/level2/html/H
TMLAnchorElement06.html']), |
213 ('fast/css', ['fast/css/display-none-inline-style-change-crash.html
']), | 215 ('fast/css', ['fast/css/display-none-inline-style
-change-crash.html']), |
214 ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-
1.html'])]) | 216 ('ietestcenter/Javascript', ['ietestcenter/Javasc
ript/11.1.5_4-4-c-1.html'])]) |
215 | 217 |
216 def test_shard_every_file(self): | 218 def test_shard_every_file(self): |
217 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, m
ax_locked_shards=2, run_singly=False) | 219 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, m
ax_locked_shards=2, run_singly=False) |
218 self.assert_shards(locked, | 220 self.assert_shards(locked, |
219 [('locked_shard_1', | 221 [('locked_shard_1', |
220 ['http/tests/websocket/tests/unicode.htm', | 222 ['http/tests/websocket/tests/unicode.htm', |
221 'http/tests/security/view-source-no-refresh.html', | 223 'http/tests/security/view-source-no-refresh.html', |
222 'http/tests/websocket/tests/websocket-protocol-ignored.html']), | 224 'http/tests/websocket/tests/websocket-protocol-ign
ored.html']), |
223 ('locked_shard_2', | 225 ('locked_shard_2', |
224 ['http/tests/xmlhttprequest/supported-xml-content-types.html', | 226 ['http/tests/xmlhttprequest/supported-xml-conten
t-types.html', |
225 'perf/object-keys.html'])]), | 227 'perf/object-keys.html'])]), |
226 self.assert_shards(unlocked, | 228 self.assert_shards(unlocked, |
227 [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']), | 229 [('virtual/threaded/dir', ['virtual/threaded/dir/test
.html']), |
228 ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html
']), | 230 ('virtual/threaded/fast/foo', ['virtual/threaded/fas
t/foo/test.html']), |
229 ('.', ['animations/keyframes.html']), | 231 ('.', ['animations/keyframes.html']), |
230 ('.', ['fast/css/display-none-inline-style-change-crash.html']), | 232 ('.', ['fast/css/display-none-inline-style-change
-crash.html']), |
231 ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']), | 233 ('.', ['dom/html/level2/html/HTMLAnchorElement03.
html']), |
232 ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']), | 234 ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.ht
ml']), |
233 ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])]) | 235 ('.', ['dom/html/level2/html/HTMLAnchorElement06.
html'])]) |
234 | 236 |
235 def test_shard_in_two(self): | 237 def test_shard_in_two(self): |
236 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
run_singly=False) | 238 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
run_singly=False) |
237 self.assert_shards(locked, | 239 self.assert_shards(locked, |
238 [('locked_tests', | 240 [('locked_tests', |
239 ['http/tests/websocket/tests/unicode.htm', | 241 ['http/tests/websocket/tests/unicode.htm', |
240 'http/tests/security/view-source-no-refresh.html', | 242 'http/tests/security/view-source-no-refresh.html', |
241 'http/tests/websocket/tests/websocket-protocol-ignored.html', | 243 'http/tests/websocket/tests/websocket-protocol-ign
ored.html', |
242 'http/tests/xmlhttprequest/supported-xml-content-types.html', | 244 'http/tests/xmlhttprequest/supported-xml-content-t
ypes.html', |
243 'perf/object-keys.html'])]) | 245 'perf/object-keys.html'])]) |
244 self.assert_shards(unlocked, | 246 self.assert_shards(unlocked, |
245 [('unlocked_tests', | 247 [('unlocked_tests', |
246 ['animations/keyframes.html', | 248 ['animations/keyframes.html', |
247 'fast/css/display-none-inline-style-change-crash.html', | 249 'fast/css/display-none-inline-style-change-crash.h
tml', |
248 'dom/html/level2/html/HTMLAnchorElement03.html', | 250 'dom/html/level2/html/HTMLAnchorElement03.html', |
249 'ietestcenter/Javascript/11.1.5_4-4-c-1.html', | 251 'ietestcenter/Javascript/11.1.5_4-4-c-1.html', |
250 'dom/html/level2/html/HTMLAnchorElement06.html', | 252 'dom/html/level2/html/HTMLAnchorElement06.html', |
251 'virtual/threaded/dir/test.html', | 253 'virtual/threaded/dir/test.html', |
252 'virtual/threaded/fast/foo/test.html'])]) | 254 'virtual/threaded/fast/foo/test.html'])]) |
253 | 255 |
254 def test_shard_in_two_has_no_locked_shards(self): | 256 def test_shard_in_two_has_no_locked_shards(self): |
255 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
run_singly=False, | 257 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
run_singly=False, |
256 test_list=['animations/keyframe.html']) | 258 test_list=['animations/keyframe.html'
]) |
257 self.assertEqual(len(locked), 0) | 259 self.assertEqual(len(locked), 0) |
258 self.assertEqual(len(unlocked), 1) | 260 self.assertEqual(len(unlocked), 1) |
259 | 261 |
260 def test_shard_in_two_has_no_unlocked_shards(self): | 262 def test_shard_in_two_has_no_unlocked_shards(self): |
261 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
run_singly=False, | 263 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
run_singly=False, |
262 test_list=['http/tests/websocket/tests/unicode.htm']) | 264 test_list=['http/tests/websocket/test
s/unicode.htm']) |
263 self.assertEqual(len(locked), 1) | 265 self.assertEqual(len(locked), 1) |
264 self.assertEqual(len(unlocked), 0) | 266 self.assertEqual(len(unlocked), 0) |
265 | 267 |
266 def test_multiple_locked_shards(self): | 268 def test_multiple_locked_shards(self): |
267 locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False,
max_locked_shards=2, run_singly=False) | 269 locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False,
max_locked_shards=2, run_singly=False) |
268 self.assert_shards(locked, | 270 self.assert_shards(locked, |
269 [('locked_shard_1', | 271 [('locked_shard_1', |
270 ['http/tests/security/view-source-no-refresh.html', | 272 ['http/tests/security/view-source-no-refresh.html', |
271 'http/tests/websocket/tests/unicode.htm', | 273 'http/tests/websocket/tests/unicode.htm', |
272 'http/tests/websocket/tests/websocket-protocol-ignored.html']), | 274 'http/tests/websocket/tests/websocket-protocol-ign
ored.html']), |
273 ('locked_shard_2', | 275 ('locked_shard_2', |
274 ['http/tests/xmlhttprequest/supported-xml-content-types.html', | 276 ['http/tests/xmlhttprequest/supported-xml-conten
t-types.html', |
275 'perf/object-keys.html'])]) | 277 'perf/object-keys.html'])]) |
276 | 278 |
277 locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False,
run_singly=False) | 279 locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False,
run_singly=False) |
278 self.assert_shards(locked, | 280 self.assert_shards(locked, |
279 [('locked_shard_1', | 281 [('locked_shard_1', |
280 ['http/tests/security/view-source-no-refresh.html', | 282 ['http/tests/security/view-source-no-refresh.html', |
281 'http/tests/websocket/tests/unicode.htm', | 283 'http/tests/websocket/tests/unicode.htm', |
282 'http/tests/websocket/tests/websocket-protocol-ignored.html', | 284 'http/tests/websocket/tests/websocket-protocol-ign
ored.html', |
283 'http/tests/xmlhttprequest/supported-xml-content-types.html', | 285 'http/tests/xmlhttprequest/supported-xml-content-t
ypes.html', |
284 'perf/object-keys.html'])]) | 286 'perf/object-keys.html'])]) |
285 | 287 |
286 def test_virtual_shards(self): | 288 def test_virtual_shards(self): |
287 # With run_singly=False, we try to keep all of the tests in a virtual su
ite together even | 289 # With run_singly=False, we try to keep all of the tests in a virtual su
ite together even |
288 # when fully_parallel=True, so that we don't restart every time the comm
and line args change. | 290 # when fully_parallel=True, so that we don't restart every time the comm
and line args change. |
289 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, m
ax_locked_shards=2, run_singly=False, | 291 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, m
ax_locked_shards=2, run_singly=False, |
290 test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html']) | 292 test_list=['virtual/foo/bar1.html', '
virtual/foo/bar2.html']) |
291 self.assert_shards(unlocked, | 293 self.assert_shards(unlocked, |
292 [('virtual/foo', ['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])
]) | 294 [('virtual/foo', ['virtual/foo/bar1.html', 'virtual/f
oo/bar2.html'])]) |
293 | 295 |
294 # But, with run_singly=True, we have to restart every time anyway, so we
want full parallelism. | 296 # But, with run_singly=True, we have to restart every time anyway, so we
want full parallelism. |
295 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, m
ax_locked_shards=2, run_singly=True, | 297 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, m
ax_locked_shards=2, run_singly=True, |
296 test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html']) | 298 test_list=['virtual/foo/bar1.html', '
virtual/foo/bar2.html']) |
297 self.assert_shards(unlocked, | 299 self.assert_shards(unlocked, |
298 [('.', ['virtual/foo/bar1.html']), | 300 [('.', ['virtual/foo/bar1.html']), |
299 ('.', ['virtual/foo/bar2.html'])]) | 301 ('.', ['virtual/foo/bar2.html'])]) |
OLD | NEW |