| OLD | NEW |
| 1 # Copyright (C) 2012 Google Inc. All rights reserved. | 1 # Copyright (C) 2012 Google Inc. All rights reserved. |
| 2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Sze
ged | 2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Sze
ged |
| 3 # | 3 # |
| 4 # Redistribution and use in source and binary forms, with or without | 4 # Redistribution and use in source and binary forms, with or without |
| 5 # modification, are permitted provided that the following conditions are | 5 # modification, are permitted provided that the following conditions are |
| 6 # met: | 6 # met: |
| 7 # | 7 # |
| 8 # * Redistributions of source code must retain the above copyright | 8 # * Redistributions of source code must retain the above copyright |
| 9 # notice, this list of conditions and the following disclaimer. | 9 # notice, this list of conditions and the following disclaimer. |
| 10 # * Redistributions in binary form must reproduce the above | 10 # * Redistributions in binary form must reproduce the above |
| (...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 64 pass | 64 pass |
| 65 | 65 |
| 66 def write_update(self, msg): | 66 def write_update(self, msg): |
| 67 pass | 67 pass |
| 68 | 68 |
| 69 def flush(self): | 69 def flush(self): |
| 70 pass | 70 pass |
| 71 | 71 |
| 72 | 72 |
| 73 class LockCheckingRunner(LayoutTestRunner): | 73 class LockCheckingRunner(LayoutTestRunner): |
| 74 |
| 74 def __init__(self, port, options, printer, tester, http_lock): | 75 def __init__(self, port, options, printer, tester, http_lock): |
| 75 super(LockCheckingRunner, self).__init__(options, port, printer, port.re
sults_directory(), lambda test_name: False) | 76 super(LockCheckingRunner, self).__init__(options, port, printer, port.re
sults_directory(), lambda test_name: False) |
| 76 self._finished_list_called = False | 77 self._finished_list_called = False |
| 77 self._tester = tester | 78 self._tester = tester |
| 78 self._should_have_http_lock = http_lock | 79 self._should_have_http_lock = http_lock |
| 79 | 80 |
| 80 def handle_finished_list(self, source, list_name, num_tests, elapsed_time): | 81 def handle_finished_list(self, source, list_name, num_tests, elapsed_time): |
| 81 if not self._finished_list_called: | 82 if not self._finished_list_called: |
| 82 self._tester.assertEqual(list_name, 'locked_tests') | 83 self._tester.assertEqual(list_name, 'locked_tests') |
| 83 self._tester.assertTrue(self._remaining_locked_shards) | 84 self._tester.assertTrue(self._remaining_locked_shards) |
| 84 self._tester.assertTrue(self._has_http_lock is self._should_have_htt
p_lock) | 85 self._tester.assertTrue(self._has_http_lock is self._should_have_htt
p_lock) |
| 85 | 86 |
| 86 super(LockCheckingRunner, self).handle_finished_list(source, list_name,
num_tests, elapsed_time) | 87 super(LockCheckingRunner, self).handle_finished_list(source, list_name,
num_tests, elapsed_time) |
| 87 | 88 |
| 88 if not self._finished_list_called: | 89 if not self._finished_list_called: |
| 89 self._tester.assertEqual(self._remaining_locked_shards, []) | 90 self._tester.assertEqual(self._remaining_locked_shards, []) |
| 90 self._tester.assertFalse(self._has_http_lock) | 91 self._tester.assertFalse(self._has_http_lock) |
| 91 self._finished_list_called = True | 92 self._finished_list_called = True |
| 92 | 93 |
| 93 | 94 |
| 94 class LayoutTestRunnerTests(unittest.TestCase): | 95 class LayoutTestRunnerTests(unittest.TestCase): |
| 96 |
| 95 def _runner(self, port=None): | 97 def _runner(self, port=None): |
| 96 # FIXME: we shouldn't have to use run_webkit_tests.py to get the options
we need. | 98 # FIXME: we shouldn't have to use run_webkit_tests.py to get the options
we need. |
| 97 options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopa
rd'])[0] | 99 options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopa
rd'])[0] |
| 98 options.child_processes = '1' | 100 options.child_processes = '1' |
| 99 | 101 |
| 100 host = MockHost() | 102 host = MockHost() |
| 101 port = port or host.port_factory.get(options.platform, options=options) | 103 port = port or host.port_factory.get(options.platform, options=options) |
| 102 return LockCheckingRunner(port, options, FakePrinter(), self, True) | 104 return LockCheckingRunner(port, options, FakePrinter(), self, True) |
| 103 | 105 |
| 104 def _run_tests(self, runner, tests): | 106 def _run_tests(self, runner, tests): |
| (...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 152 run_results = TestRunResults(expectations, 1) | 154 run_results = TestRunResults(expectations, 1) |
| 153 result = TestResult(test_name=test, failures=[], reftest_type=['==']) | 155 result = TestResult(test_name=test, failures=[], reftest_type=['==']) |
| 154 runner._update_summary_with_result(run_results, result) | 156 runner._update_summary_with_result(run_results, result) |
| 155 self.assertEqual(0, run_results.expected) | 157 self.assertEqual(0, run_results.expected) |
| 156 self.assertEqual(1, run_results.unexpected) | 158 self.assertEqual(1, run_results.unexpected) |
| 157 | 159 |
| 158 | 160 |
| 159 class SharderTests(unittest.TestCase): | 161 class SharderTests(unittest.TestCase): |
| 160 | 162 |
| 161 test_list = [ | 163 test_list = [ |
| 162 "http/tests/websocket/tests/unicode.htm", | 164 'http/tests/websocket/tests/unicode.htm', |
| 163 "animations/keyframes.html", | 165 'animations/keyframes.html', |
| 164 "http/tests/security/view-source-no-refresh.html", | 166 'http/tests/security/view-source-no-refresh.html', |
| 165 "http/tests/websocket/tests/websocket-protocol-ignored.html", | 167 'http/tests/websocket/tests/websocket-protocol-ignored.html', |
| 166 "fast/css/display-none-inline-style-change-crash.html", | 168 'fast/css/display-none-inline-style-change-crash.html', |
| 167 "http/tests/xmlhttprequest/supported-xml-content-types.html", | 169 'http/tests/xmlhttprequest/supported-xml-content-types.html', |
| 168 "dom/html/level2/html/HTMLAnchorElement03.html", | 170 'dom/html/level2/html/HTMLAnchorElement03.html', |
| 169 "ietestcenter/Javascript/11.1.5_4-4-c-1.html", | 171 'ietestcenter/Javascript/11.1.5_4-4-c-1.html', |
| 170 "dom/html/level2/html/HTMLAnchorElement06.html", | 172 'dom/html/level2/html/HTMLAnchorElement06.html', |
| 171 "perf/object-keys.html", | 173 'perf/object-keys.html', |
| 172 "virtual/threaded/dir/test.html", | 174 'virtual/threaded/dir/test.html', |
| 173 "virtual/threaded/fast/foo/test.html", | 175 'virtual/threaded/fast/foo/test.html', |
| 174 ] | 176 ] |
| 175 | 177 |
| 176 def get_test_input(self, test_file): | 178 def get_test_input(self, test_file): |
| 177 return TestInput(test_file, requires_lock=(test_file.startswith('http')
or test_file.startswith('perf'))) | 179 return TestInput(test_file, requires_lock=(test_file.startswith('http')
or test_file.startswith('perf'))) |
| 178 | 180 |
| 179 def get_shards(self, num_workers, fully_parallel, run_singly, test_list=None
, max_locked_shards=1): | 181 def get_shards(self, num_workers, fully_parallel, run_singly, test_list=None
, max_locked_shards=1): |
| 180 port = TestPort(MockSystemHost()) | 182 port = TestPort(MockSystemHost()) |
| 181 self.sharder = Sharder(port.split_test, max_locked_shards) | 183 self.sharder = Sharder(port.split_test, max_locked_shards) |
| 182 test_list = test_list or self.test_list | 184 test_list = test_list or self.test_list |
| 183 return self.sharder.shard_tests([self.get_test_input(test) for test in t
est_list], | 185 return self.sharder.shard_tests([self.get_test_input(test) for test in t
est_list], |
| 184 num_workers, fully_parallel, run_singly) | 186 num_workers, fully_parallel, run_singly) |
| 185 | 187 |
| 186 def assert_shards(self, actual_shards, expected_shard_names): | 188 def assert_shards(self, actual_shards, expected_shard_names): |
| 187 self.assertEqual(len(actual_shards), len(expected_shard_names)) | 189 self.assertEqual(len(actual_shards), len(expected_shard_names)) |
| 188 for i, shard in enumerate(actual_shards): | 190 for i, shard in enumerate(actual_shards): |
| 189 expected_shard_name, expected_test_names = expected_shard_names[i] | 191 expected_shard_name, expected_test_names = expected_shard_names[i] |
| 190 self.assertEqual(shard.name, expected_shard_name) | 192 self.assertEqual(shard.name, expected_shard_name) |
| 191 self.assertEqual([test_input.test_name for test_input in shard.test_
inputs], | 193 self.assertEqual([test_input.test_name for test_input in shard.test_
inputs], |
| 192 expected_test_names) | 194 expected_test_names) |
| 193 | 195 |
| 194 def test_shard_by_dir(self): | 196 def test_shard_by_dir(self): |
| 195 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False,
run_singly=False) | 197 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False,
run_singly=False) |
| 196 | 198 |
| 197 # Note that although there are tests in multiple dirs that need locks, | 199 # Note that although there are tests in multiple dirs that need locks, |
| 198 # they are crammed into a single shard in order to reduce the # of | 200 # they are crammed into a single shard in order to reduce the # of |
| 199 # workers hitting the server at once. | 201 # workers hitting the server at once. |
| 200 self.assert_shards(locked, | 202 self.assert_shards(locked, |
| 201 [('locked_shard_1', | 203 [('locked_shard_1', |
| 202 ['http/tests/security/view-source-no-refresh.html', | 204 ['http/tests/security/view-source-no-refresh.html', |
| 203 'http/tests/websocket/tests/unicode.htm', | 205 'http/tests/websocket/tests/unicode.htm', |
| 204 'http/tests/websocket/tests/websocket-protocol-ignored.html', | 206 'http/tests/websocket/tests/websocket-protocol-ign
ored.html', |
| 205 'http/tests/xmlhttprequest/supported-xml-content-types.html', | 207 'http/tests/xmlhttprequest/supported-xml-content-t
ypes.html', |
| 206 'perf/object-keys.html'])]) | 208 'perf/object-keys.html'])]) |
| 207 self.assert_shards(unlocked, | 209 self.assert_shards(unlocked, |
| 208 [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']), | 210 [('virtual/threaded/dir', ['virtual/threaded/dir/test
.html']), |
| 209 ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html
']), | 211 ('virtual/threaded/fast/foo', ['virtual/threaded/fas
t/foo/test.html']), |
| 210 ('animations', ['animations/keyframes.html']), | 212 ('animations', ['animations/keyframes.html']), |
| 211 ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03
.html', | 213 ('dom/html/level2/html', ['dom/html/level2/html/H
TMLAnchorElement03.html', |
| 212 'dom/html/level2/html/HTMLAnchorElement06.
html']), | 214 'dom/html/level2/html/H
TMLAnchorElement06.html']), |
| 213 ('fast/css', ['fast/css/display-none-inline-style-change-crash.html
']), | 215 ('fast/css', ['fast/css/display-none-inline-style
-change-crash.html']), |
| 214 ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-
1.html'])]) | 216 ('ietestcenter/Javascript', ['ietestcenter/Javasc
ript/11.1.5_4-4-c-1.html'])]) |
| 215 | 217 |
| 216 def test_shard_every_file(self): | 218 def test_shard_every_file(self): |
| 217 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, m
ax_locked_shards=2, run_singly=False) | 219 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, m
ax_locked_shards=2, run_singly=False) |
| 218 self.assert_shards(locked, | 220 self.assert_shards(locked, |
| 219 [('locked_shard_1', | 221 [('locked_shard_1', |
| 220 ['http/tests/websocket/tests/unicode.htm', | 222 ['http/tests/websocket/tests/unicode.htm', |
| 221 'http/tests/security/view-source-no-refresh.html', | 223 'http/tests/security/view-source-no-refresh.html', |
| 222 'http/tests/websocket/tests/websocket-protocol-ignored.html']), | 224 'http/tests/websocket/tests/websocket-protocol-ign
ored.html']), |
| 223 ('locked_shard_2', | 225 ('locked_shard_2', |
| 224 ['http/tests/xmlhttprequest/supported-xml-content-types.html', | 226 ['http/tests/xmlhttprequest/supported-xml-conten
t-types.html', |
| 225 'perf/object-keys.html'])]), | 227 'perf/object-keys.html'])]), |
| 226 self.assert_shards(unlocked, | 228 self.assert_shards(unlocked, |
| 227 [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']), | 229 [('virtual/threaded/dir', ['virtual/threaded/dir/test
.html']), |
| 228 ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html
']), | 230 ('virtual/threaded/fast/foo', ['virtual/threaded/fas
t/foo/test.html']), |
| 229 ('.', ['animations/keyframes.html']), | 231 ('.', ['animations/keyframes.html']), |
| 230 ('.', ['fast/css/display-none-inline-style-change-crash.html']), | 232 ('.', ['fast/css/display-none-inline-style-change
-crash.html']), |
| 231 ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']), | 233 ('.', ['dom/html/level2/html/HTMLAnchorElement03.
html']), |
| 232 ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']), | 234 ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.ht
ml']), |
| 233 ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])]) | 235 ('.', ['dom/html/level2/html/HTMLAnchorElement06.
html'])]) |
| 234 | 236 |
| 235 def test_shard_in_two(self): | 237 def test_shard_in_two(self): |
| 236 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
run_singly=False) | 238 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
run_singly=False) |
| 237 self.assert_shards(locked, | 239 self.assert_shards(locked, |
| 238 [('locked_tests', | 240 [('locked_tests', |
| 239 ['http/tests/websocket/tests/unicode.htm', | 241 ['http/tests/websocket/tests/unicode.htm', |
| 240 'http/tests/security/view-source-no-refresh.html', | 242 'http/tests/security/view-source-no-refresh.html', |
| 241 'http/tests/websocket/tests/websocket-protocol-ignored.html', | 243 'http/tests/websocket/tests/websocket-protocol-ign
ored.html', |
| 242 'http/tests/xmlhttprequest/supported-xml-content-types.html', | 244 'http/tests/xmlhttprequest/supported-xml-content-t
ypes.html', |
| 243 'perf/object-keys.html'])]) | 245 'perf/object-keys.html'])]) |
| 244 self.assert_shards(unlocked, | 246 self.assert_shards(unlocked, |
| 245 [('unlocked_tests', | 247 [('unlocked_tests', |
| 246 ['animations/keyframes.html', | 248 ['animations/keyframes.html', |
| 247 'fast/css/display-none-inline-style-change-crash.html', | 249 'fast/css/display-none-inline-style-change-crash.h
tml', |
| 248 'dom/html/level2/html/HTMLAnchorElement03.html', | 250 'dom/html/level2/html/HTMLAnchorElement03.html', |
| 249 'ietestcenter/Javascript/11.1.5_4-4-c-1.html', | 251 'ietestcenter/Javascript/11.1.5_4-4-c-1.html', |
| 250 'dom/html/level2/html/HTMLAnchorElement06.html', | 252 'dom/html/level2/html/HTMLAnchorElement06.html', |
| 251 'virtual/threaded/dir/test.html', | 253 'virtual/threaded/dir/test.html', |
| 252 'virtual/threaded/fast/foo/test.html'])]) | 254 'virtual/threaded/fast/foo/test.html'])]) |
| 253 | 255 |
| 254 def test_shard_in_two_has_no_locked_shards(self): | 256 def test_shard_in_two_has_no_locked_shards(self): |
| 255 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
run_singly=False, | 257 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
run_singly=False, |
| 256 test_list=['animations/keyframe.html']) | 258 test_list=['animations/keyframe.html'
]) |
| 257 self.assertEqual(len(locked), 0) | 259 self.assertEqual(len(locked), 0) |
| 258 self.assertEqual(len(unlocked), 1) | 260 self.assertEqual(len(unlocked), 1) |
| 259 | 261 |
| 260 def test_shard_in_two_has_no_unlocked_shards(self): | 262 def test_shard_in_two_has_no_unlocked_shards(self): |
| 261 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
run_singly=False, | 263 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
run_singly=False, |
| 262 test_list=['http/tests/websocket/tests/unicode.htm']) | 264 test_list=['http/tests/websocket/test
s/unicode.htm']) |
| 263 self.assertEqual(len(locked), 1) | 265 self.assertEqual(len(locked), 1) |
| 264 self.assertEqual(len(unlocked), 0) | 266 self.assertEqual(len(unlocked), 0) |
| 265 | 267 |
| 266 def test_multiple_locked_shards(self): | 268 def test_multiple_locked_shards(self): |
| 267 locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False,
max_locked_shards=2, run_singly=False) | 269 locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False,
max_locked_shards=2, run_singly=False) |
| 268 self.assert_shards(locked, | 270 self.assert_shards(locked, |
| 269 [('locked_shard_1', | 271 [('locked_shard_1', |
| 270 ['http/tests/security/view-source-no-refresh.html', | 272 ['http/tests/security/view-source-no-refresh.html', |
| 271 'http/tests/websocket/tests/unicode.htm', | 273 'http/tests/websocket/tests/unicode.htm', |
| 272 'http/tests/websocket/tests/websocket-protocol-ignored.html']), | 274 'http/tests/websocket/tests/websocket-protocol-ign
ored.html']), |
| 273 ('locked_shard_2', | 275 ('locked_shard_2', |
| 274 ['http/tests/xmlhttprequest/supported-xml-content-types.html', | 276 ['http/tests/xmlhttprequest/supported-xml-conten
t-types.html', |
| 275 'perf/object-keys.html'])]) | 277 'perf/object-keys.html'])]) |
| 276 | 278 |
| 277 locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False,
run_singly=False) | 279 locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False,
run_singly=False) |
| 278 self.assert_shards(locked, | 280 self.assert_shards(locked, |
| 279 [('locked_shard_1', | 281 [('locked_shard_1', |
| 280 ['http/tests/security/view-source-no-refresh.html', | 282 ['http/tests/security/view-source-no-refresh.html', |
| 281 'http/tests/websocket/tests/unicode.htm', | 283 'http/tests/websocket/tests/unicode.htm', |
| 282 'http/tests/websocket/tests/websocket-protocol-ignored.html', | 284 'http/tests/websocket/tests/websocket-protocol-ign
ored.html', |
| 283 'http/tests/xmlhttprequest/supported-xml-content-types.html', | 285 'http/tests/xmlhttprequest/supported-xml-content-t
ypes.html', |
| 284 'perf/object-keys.html'])]) | 286 'perf/object-keys.html'])]) |
| 285 | 287 |
| 286 def test_virtual_shards(self): | 288 def test_virtual_shards(self): |
| 287 # With run_singly=False, we try to keep all of the tests in a virtual su
ite together even | 289 # With run_singly=False, we try to keep all of the tests in a virtual su
ite together even |
| 288 # when fully_parallel=True, so that we don't restart every time the comm
and line args change. | 290 # when fully_parallel=True, so that we don't restart every time the comm
and line args change. |
| 289 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, m
ax_locked_shards=2, run_singly=False, | 291 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, m
ax_locked_shards=2, run_singly=False, |
| 290 test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html']) | 292 test_list=['virtual/foo/bar1.html', '
virtual/foo/bar2.html']) |
| 291 self.assert_shards(unlocked, | 293 self.assert_shards(unlocked, |
| 292 [('virtual/foo', ['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])
]) | 294 [('virtual/foo', ['virtual/foo/bar1.html', 'virtual/f
oo/bar2.html'])]) |
| 293 | 295 |
| 294 # But, with run_singly=True, we have to restart every time anyway, so we
want full parallelism. | 296 # But, with run_singly=True, we have to restart every time anyway, so we
want full parallelism. |
| 295 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, m
ax_locked_shards=2, run_singly=True, | 297 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, m
ax_locked_shards=2, run_singly=True, |
| 296 test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html']) | 298 test_list=['virtual/foo/bar1.html', '
virtual/foo/bar2.html']) |
| 297 self.assert_shards(unlocked, | 299 self.assert_shards(unlocked, |
| 298 [('.', ['virtual/foo/bar1.html']), | 300 [('.', ['virtual/foo/bar1.html']), |
| 299 ('.', ['virtual/foo/bar2.html'])]) | 301 ('.', ['virtual/foo/bar2.html'])]) |
| OLD | NEW |