OLD | NEW |
1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import copy | 5 import copy |
6 | 6 |
7 from webkitpy.common.host_mock import MockHost | 7 from webkitpy.common.host_mock import MockHost |
8 from webkitpy.common.net.buildbot import Build | 8 from webkitpy.common.net.buildbot import Build |
9 from webkitpy.common.net.buildbot_mock import MockBuildBot | 9 from webkitpy.common.net.buildbot_mock import MockBuildBot |
10 from webkitpy.common.net.layout_test_results import LayoutTestResult, LayoutTest
Results | 10 from webkitpy.common.net.layout_test_results import LayoutTestResult, LayoutTest
Results |
11 from webkitpy.common.system.log_testing import LoggingTestCase | 11 from webkitpy.common.system.log_testing import LoggingTestCase |
12 from webkitpy.layout_tests.builder_list import BuilderList | 12 from webkitpy.layout_tests.builder_list import BuilderList |
| 13 from webkitpy.layout_tests.port.factory_mock import MockPortFactory |
13 from webkitpy.w3c.wpt_expectations_updater import WPTExpectationsUpdater, MARKER
_COMMENT | 14 from webkitpy.w3c.wpt_expectations_updater import WPTExpectationsUpdater, MARKER
_COMMENT |
14 | 15 |
15 | 16 |
16 class WPTExpectationsUpdaterTest(LoggingTestCase): | 17 class WPTExpectationsUpdaterTest(LoggingTestCase): |
17 | 18 |
18 def mock_host(self): | 19 def mock_host(self): |
19 super(WPTExpectationsUpdaterTest, self).setUp() | 20 super(WPTExpectationsUpdaterTest, self).setUp() |
20 host = MockHost() | 21 host = MockHost() |
| 22 host.port_factory = MockPortFactory(host) |
21 host.builders = BuilderList({ | 23 host.builders = BuilderList({ |
22 'MOCK Try Mac10.10': { | 24 'MOCK Try Mac10.10': { |
23 'port_name': 'test-mac-mac10.10', | 25 'port_name': 'test-mac-mac10.10', |
24 'specifiers': ['Mac10.10', 'Release'], | 26 'specifiers': ['Mac10.10', 'Release'], |
25 'is_try_builder': True, | 27 'is_try_builder': True, |
26 }, | 28 }, |
27 'MOCK Try Mac10.11': { | 29 'MOCK Try Mac10.11': { |
28 'port_name': 'test-mac-mac10.11', | 30 'port_name': 'test-mac-mac10.11', |
29 'specifiers': ['Mac10.11', 'Release'], | 31 'specifiers': ['Mac10.11', 'Release'], |
30 'is_try_builder': True, | 32 'is_try_builder': True, |
31 }, | 33 }, |
32 'MOCK Try Trusty': { | 34 'MOCK Try Trusty': { |
33 'port_name': 'test-linux-trusty', | 35 'port_name': 'test-linux-trusty', |
34 'specifiers': ['Trusty', 'Release'], | 36 'specifiers': ['Trusty', 'Release'], |
35 'is_try_builder': True, | 37 'is_try_builder': True, |
36 }, | 38 }, |
| 39 'MOCK Try Precise': { |
| 40 'port_name': 'test-linux-precise', |
| 41 'specifiers': ['Precise', 'Release'], |
| 42 'is_try_builder': True, |
| 43 }, |
37 'MOCK Try Win10': { | 44 'MOCK Try Win10': { |
38 'port_name': 'test-win-win10', | 45 'port_name': 'test-win-win10', |
39 'specifiers': ['Win10', 'Release'], | 46 'specifiers': ['Win10', 'Release'], |
40 'is_try_builder': True, | 47 'is_try_builder': True, |
41 }, | 48 }, |
42 'MOCK Try Win7': { | 49 'MOCK Try Win7': { |
43 'port_name': 'test-win-win7', | 50 'port_name': 'test-win-win7', |
44 'specifiers': ['Win7', 'Release'], | 51 'specifiers': ['Win7', 'Release'], |
45 'is_try_builder': True, | 52 'is_try_builder': True, |
46 }, | 53 }, |
(...skipping 109 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
156 'test-mac-mac10.10': {'expected': 'PASS', 'actual': 'FAIL', 'bug
': 'crbug.com/test'}, | 163 'test-mac-mac10.10': {'expected': 'PASS', 'actual': 'FAIL', 'bug
': 'crbug.com/test'}, |
157 }, | 164 }, |
158 'external/fake/test/path.html': { | 165 'external/fake/test/path.html': { |
159 'test-linux-trusty': {'expected': 'FAIL', 'actual': 'PASS', 'bug
': 'crbug.com/test'}, | 166 'test-linux-trusty': {'expected': 'FAIL', 'actual': 'PASS', 'bug
': 'crbug.com/test'}, |
160 'test-mac-mac10.11': {'expected': 'FAIL', 'actual': 'TIMEOUT', '
bug': 'crbug.com/test'}, | 167 'test-mac-mac10.11': {'expected': 'FAIL', 'actual': 'TIMEOUT', '
bug': 'crbug.com/test'}, |
161 }, | 168 }, |
162 } | 169 } |
163 self.assertEqual( | 170 self.assertEqual( |
164 updater.create_line_list(results), | 171 updater.create_line_list(results), |
165 [ | 172 [ |
166 'crbug.com/test [ Linux ] external/fake/test/path.html [ Pass ]'
, | 173 'crbug.com/test [ Trusty ] external/fake/test/path.html [ Pass ]
', |
167 'crbug.com/test [ Mac10.11 ] external/fake/test/path.html [ Time
out ]', | 174 'crbug.com/test [ Mac10.11 ] external/fake/test/path.html [ Time
out ]', |
168 'crbug.com/test [ Mac10.10 ] external/fake/test/zzzz.html [ Fail
ure ]', | 175 'crbug.com/test [ Mac10.10 ] external/fake/test/zzzz.html [ Fail
ure ]', |
169 ]) | 176 ]) |
170 | 177 |
171 def test_specifier_part(self): | 178 def test_specifier_part(self): |
172 updater = WPTExpectationsUpdater(self.mock_host()) | 179 updater = WPTExpectationsUpdater(self.mock_host()) |
173 self.assertEqual(updater.specifier_part(['test-mac-mac10.10'], 'x/y.html
'), '[ Mac10.10 ]') | 180 self.assertEqual(updater.specifier_part(['test-mac-mac10.10'], 'x/y.html
'), '[ Mac10.10 ]') |
| 181 self.assertEqual(updater.specifier_part([], 'x/y.html'), '') |
174 | 182 |
175 def test_skipped_specifiers_when_test_is_wontfix(self): | 183 def test_skipped_specifiers_when_test_is_wontfix(self): |
176 host = self.mock_host() | 184 host = self.mock_host() |
177 expectations_path = '/test.checkout/LayoutTests/NeverFixTests' | 185 expectations_path = '/test.checkout/LayoutTests/NeverFixTests' |
178 host.filesystem.files[expectations_path] = 'crbug.com/111 [ Trusty ] ext
ernal/wpt/test.html [ WontFix ]\n' | 186 host.filesystem.files[expectations_path] = 'crbug.com/111 [ Trusty ] ext
ernal/wpt/test.html [ WontFix ]\n' |
179 host.filesystem.files['/test.checkout/LayoutTests/external/wpt/test.html
'] = '' | 187 host.filesystem.files['/test.checkout/LayoutTests/external/wpt/test.html
'] = '' |
180 updater = WPTExpectationsUpdater(host) | 188 updater = WPTExpectationsUpdater(host) |
181 self.assertEqual(updater.skipped_specifiers('external/wpt/test.html'), [
'Trusty']) | 189 self.assertEqual(updater.skipped_specifiers('external/wpt/test.html'), [
'Trusty']) |
182 | 190 |
183 def test_simplify_specifiers(self): | 191 def test_simplify_specifiers(self): |
184 macros = { | 192 macros = { |
185 'mac': ['Mac10.10', 'mac10.11'], | 193 'mac': ['Mac10.10', 'mac10.11'], |
186 'win': ['Win7', 'win10'], | 194 'win': ['Win7', 'win10'], |
187 'Linux': ['TRUSTY'], | 195 'Linux': ['Trusty'], |
188 } | 196 } |
189 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['mac10.10',
'mac10.11'], macros), ['Mac']) | 197 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['mac10.10',
'mac10.11'], macros), ['Mac']) |
190 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['Mac10.10',
'Mac10.11', 'Trusty'], macros), ['Linux', 'Mac']) | 198 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['Mac10.10',
'Mac10.11', 'Trusty'], macros), ['Linux', 'Mac']) |
191 self.assertEqual( | 199 self.assertEqual( |
192 WPTExpectationsUpdater.simplify_specifiers(['Mac10.10', 'Mac10.11',
'Trusty', 'Win7', 'Win10'], macros), []) | 200 WPTExpectationsUpdater.simplify_specifiers(['Mac10.10', 'Mac10.11',
'Trusty', 'Win7', 'Win10'], macros), []) |
193 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['a', 'b', '
c'], {}), ['A', 'B', 'C']) | 201 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['a', 'b', '
c'], {}), ['A', 'B', 'C']) |
194 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['Mac', 'Win
', 'Linux'], macros), []) | 202 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['Mac', 'Win
', 'Linux'], macros), []) |
195 | 203 |
| 204 def test_specifier_part_with_skipped_test(self): |
| 205 host = self.mock_host() |
| 206 expectations_path = '/test.checkout/LayoutTests/NeverFixTests' |
| 207 host.filesystem.files[expectations_path] = 'crbug.com/111 [ Linux Mac10.
11 ] external/wpt/test.html [ WontFix ]\n' |
| 208 host.filesystem.files['/test.checkout/LayoutTests/external/wpt/test.html
'] = '' |
| 209 updater = WPTExpectationsUpdater(host) |
| 210 self.assertEqual( |
| 211 updater.specifier_part(['test-mac-mac10.10', 'test-win-win7', 'test-
win-win10'], 'external/wpt/test.html'), '') |
| 212 self.assertEqual( |
| 213 updater.specifier_part(['test-win-win7', 'test-win-win10'], 'externa
l/wpt/another.html'), '[ Win ]') |
| 214 |
196 def test_merge_dicts_with_conflict_raise_exception(self): | 215 def test_merge_dicts_with_conflict_raise_exception(self): |
197 updater = WPTExpectationsUpdater(self.mock_host()) | 216 updater = WPTExpectationsUpdater(self.mock_host()) |
198 # Both dicts here have the key "one", and the value is not equal. | 217 # Both dicts here have the key "one", and the value is not equal. |
199 with self.assertRaises(ValueError): | 218 with self.assertRaises(ValueError): |
200 updater.merge_dicts( | 219 updater.merge_dicts( |
201 { | 220 { |
202 'external/fake/test/path.html': { | 221 'external/fake/test/path.html': { |
203 'one': {'expected': 'FAIL', 'actual': 'PASS'}, | 222 'one': {'expected': 'FAIL', 'actual': 'PASS'}, |
204 'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'}, | 223 'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'}, |
205 'three': {'expected': 'FAIL', 'actual': 'PASS'}, | 224 'three': {'expected': 'FAIL', 'actual': 'PASS'}, |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
253 'test-mac-mac10.10': { | 272 'test-mac-mac10.10': { |
254 'expected': 'bar', | 273 'expected': 'bar', |
255 'actual': 'foo', | 274 'actual': 'foo', |
256 'bug': 'crbug.com/626703', | 275 'bug': 'crbug.com/626703', |
257 } | 276 } |
258 } | 277 } |
259 }) | 278 }) |
260 | 279 |
261 def test_write_to_test_expectations_with_marker_comment(self): | 280 def test_write_to_test_expectations_with_marker_comment(self): |
262 host = self.mock_host() | 281 host = self.mock_host() |
263 expectations_path = '/mock-checkout/third_party/WebKit/LayoutTests/TestE
xpectations' | 282 |
| 283 expectations_path = host.port_factory.get().path_to_generic_test_expecta
tions_file() |
264 host.filesystem.files[expectations_path] = MARKER_COMMENT + '\n' | 284 host.filesystem.files[expectations_path] = MARKER_COMMENT + '\n' |
265 updater = WPTExpectationsUpdater(host) | 285 updater = WPTExpectationsUpdater(host) |
266 line_list = ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]'] | 286 line_list = ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]'] |
267 updater.write_to_test_expectations(line_list) | 287 updater.write_to_test_expectations(line_list) |
268 value = updater.host.filesystem.read_text_file(expectations_path) | 288 value = updater.host.filesystem.read_text_file(expectations_path) |
269 self.assertMultiLineEqual( | 289 self.assertMultiLineEqual( |
270 value, | 290 value, |
271 (MARKER_COMMENT + '\n' | 291 (MARKER_COMMENT + '\n' |
272 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')) | 292 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')) |
273 | 293 |
274 def test_write_to_test_expectations_with_no_marker_comment(self): | 294 def test_write_to_test_expectations_with_no_marker_comment(self): |
275 host = self.mock_host() | 295 host = self.mock_host() |
276 expectations_path = '/mock-checkout/third_party/WebKit/LayoutTests/TestE
xpectations' | 296 expectations_path = host.port_factory.get().path_to_generic_test_expecta
tions_file() |
277 host.filesystem.files[expectations_path] = 'crbug.com/111 [ Trusty ]\n' | 297 host.filesystem.files[expectations_path] = 'crbug.com/111 [ Trusty ] foo
/bar.html [ Failure ]\n' |
278 updater = WPTExpectationsUpdater(host) | 298 updater = WPTExpectationsUpdater(host) |
279 line_list = ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]'] | 299 line_list = ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]'] |
280 updater.write_to_test_expectations(line_list) | 300 updater.write_to_test_expectations(line_list) |
281 value = host.filesystem.read_text_file(expectations_path) | 301 value = host.filesystem.read_text_file(expectations_path) |
282 self.assertMultiLineEqual( | 302 self.assertMultiLineEqual( |
283 value, | 303 value, |
284 ('crbug.com/111 [ Trusty ]\n' | 304 ('crbug.com/111 [ Trusty ] foo/bar.html [ Failure ]\n' |
285 '\n' + MARKER_COMMENT + '\n' | 305 '\n' + MARKER_COMMENT + '\n' |
286 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]')) | 306 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]')) |
287 | 307 |
288 def test_write_to_test_expectations_skips_existing_lines(self): | 308 def test_write_to_test_expectations_skips_existing_lines(self): |
289 host = self.mock_host() | 309 host = self.mock_host() |
290 expectations_path = '/mock-checkout/third_party/WebKit/LayoutTests/TestE
xpectations' | 310 expectations_path = host.port_factory.get().path_to_generic_test_expecta
tions_file() |
291 host.filesystem.files[expectations_path] = 'crbug.com/111 dont/copy/me.h
tml [ Failure ]\n' | 311 host.filesystem.files[expectations_path] = 'crbug.com/111 dont/copy/me.h
tml [ Failure ]\n' |
292 updater = WPTExpectationsUpdater(host) | 312 updater = WPTExpectationsUpdater(host) |
293 line_list = [ | 313 line_list = [ |
294 'crbug.com/111 dont/copy/me.html [ Failure ]', | 314 'crbug.com/111 dont/copy/me.html [ Failure ]', |
295 'crbug.com/222 do/copy/me.html [ Failure ]' | 315 'crbug.com/222 do/copy/me.html [ Failure ]' |
296 ] | 316 ] |
297 updater.write_to_test_expectations(line_list) | 317 updater.write_to_test_expectations(line_list) |
298 value = host.filesystem.read_text_file(expectations_path) | 318 value = host.filesystem.read_text_file(expectations_path) |
299 self.assertEqual( | 319 self.assertEqual( |
300 value, | 320 value, |
301 ('crbug.com/111 dont/copy/me.html [ Failure ]\n' | 321 ('crbug.com/111 dont/copy/me.html [ Failure ]\n' |
302 '\n' + MARKER_COMMENT + '\n' | 322 '\n' + MARKER_COMMENT + '\n' |
303 'crbug.com/222 do/copy/me.html [ Failure ]')) | 323 'crbug.com/222 do/copy/me.html [ Failure ]')) |
304 | 324 |
305 def test_write_to_test_expectations_with_marker_and_no_lines(self): | 325 def test_write_to_test_expectations_with_marker_and_no_lines(self): |
306 host = self.mock_host() | 326 host = self.mock_host() |
307 expectations_path = '/mock-checkout/third_party/WebKit/LayoutTests/TestE
xpectations' | 327 expectations_path = host.port_factory.get().path_to_generic_test_expecta
tions_file() |
308 host.filesystem.files[expectations_path] = ( | 328 host.filesystem.files[expectations_path] = ( |
309 MARKER_COMMENT + '\n' | 329 MARKER_COMMENT + '\n' |
310 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n') | 330 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n') |
311 updater = WPTExpectationsUpdater(host) | 331 updater = WPTExpectationsUpdater(host) |
312 updater.write_to_test_expectations([]) | 332 updater.write_to_test_expectations([]) |
313 value = updater.host.filesystem.read_text_file(expectations_path) | 333 value = updater.host.filesystem.read_text_file(expectations_path) |
314 self.assertMultiLineEqual( | 334 self.assertMultiLineEqual( |
315 value, | 335 value, |
316 (MARKER_COMMENT + '\n' | 336 (MARKER_COMMENT + '\n' |
317 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')) | 337 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')) |
(...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
423 updater = WPTExpectationsUpdater(self.mock_host()) | 443 updater = WPTExpectationsUpdater(self.mock_host()) |
424 updater.get_issue_number = lambda: 'None' | 444 updater.get_issue_number = lambda: 'None' |
425 self.assertEqual(1, updater.run(args=[])) | 445 self.assertEqual(1, updater.run(args=[])) |
426 self.assertLog(['ERROR: No issue on current branch.\n']) | 446 self.assertLog(['ERROR: No issue on current branch.\n']) |
427 | 447 |
428 def test_run_no_try_results(self): | 448 def test_run_no_try_results(self): |
429 updater = WPTExpectationsUpdater(self.mock_host()) | 449 updater = WPTExpectationsUpdater(self.mock_host()) |
430 updater.get_latest_try_jobs = lambda: [] | 450 updater.get_latest_try_jobs = lambda: [] |
431 self.assertEqual(1, updater.run(args=[])) | 451 self.assertEqual(1, updater.run(args=[])) |
432 self.assertLog(['ERROR: No try job information was collected.\n']) | 452 self.assertLog(['ERROR: No try job information was collected.\n']) |
OLD | NEW |