| OLD | NEW |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import copy | 5 import copy |
| 6 | 6 |
| 7 from webkitpy.common.host_mock import MockHost | 7 from webkitpy.common.host_mock import MockHost |
| 8 from webkitpy.common.net.buildbot import Build | 8 from webkitpy.common.net.buildbot import Build |
| 9 from webkitpy.common.net.buildbot_mock import MockBuildBot | 9 from webkitpy.common.net.buildbot_mock import MockBuildBot |
| 10 from webkitpy.common.net.layout_test_results import LayoutTestResult, LayoutTest
Results | 10 from webkitpy.common.net.layout_test_results import LayoutTestResult, LayoutTest
Results |
| 11 from webkitpy.common.system.log_testing import LoggingTestCase | 11 from webkitpy.common.system.log_testing import LoggingTestCase |
| 12 from webkitpy.layout_tests.builder_list import BuilderList | 12 from webkitpy.layout_tests.builder_list import BuilderList |
| 13 from webkitpy.layout_tests.port.factory_mock import MockPortFactory |
| 13 from webkitpy.w3c.wpt_expectations_updater import WPTExpectationsUpdater, MARKER
_COMMENT | 14 from webkitpy.w3c.wpt_expectations_updater import WPTExpectationsUpdater, MARKER
_COMMENT |
| 14 | 15 |
| 15 | 16 |
| 16 class WPTExpectationsUpdaterTest(LoggingTestCase): | 17 class WPTExpectationsUpdaterTest(LoggingTestCase): |
| 17 | 18 |
| 18 def mock_host(self): | 19 def mock_host(self): |
| 19 super(WPTExpectationsUpdaterTest, self).setUp() | 20 super(WPTExpectationsUpdaterTest, self).setUp() |
| 20 host = MockHost() | 21 host = MockHost() |
| 22 host.port_factory = MockPortFactory(host) |
| 21 host.builders = BuilderList({ | 23 host.builders = BuilderList({ |
| 22 'MOCK Try Mac10.10': { | 24 'MOCK Try Mac10.10': { |
| 23 'port_name': 'test-mac-mac10.10', | 25 'port_name': 'test-mac-mac10.10', |
| 24 'specifiers': ['Mac10.10', 'Release'], | 26 'specifiers': ['Mac10.10', 'Release'], |
| 25 'is_try_builder': True, | 27 'is_try_builder': True, |
| 26 }, | 28 }, |
| 27 'MOCK Try Mac10.11': { | 29 'MOCK Try Mac10.11': { |
| 28 'port_name': 'test-mac-mac10.11', | 30 'port_name': 'test-mac-mac10.11', |
| 29 'specifiers': ['Mac10.11', 'Release'], | 31 'specifiers': ['Mac10.11', 'Release'], |
| 30 'is_try_builder': True, | 32 'is_try_builder': True, |
| 31 }, | 33 }, |
| 32 'MOCK Try Trusty': { | 34 'MOCK Try Trusty': { |
| 33 'port_name': 'test-linux-trusty', | 35 'port_name': 'test-linux-trusty', |
| 34 'specifiers': ['Trusty', 'Release'], | 36 'specifiers': ['Trusty', 'Release'], |
| 35 'is_try_builder': True, | 37 'is_try_builder': True, |
| 36 }, | 38 }, |
| 39 'MOCK Try Precise': { |
| 40 'port_name': 'test-linux-precise', |
| 41 'specifiers': ['Precise', 'Release'], |
| 42 'is_try_builder': True, |
| 43 }, |
| 37 'MOCK Try Win10': { | 44 'MOCK Try Win10': { |
| 38 'port_name': 'test-win-win10', | 45 'port_name': 'test-win-win10', |
| 39 'specifiers': ['Win10', 'Release'], | 46 'specifiers': ['Win10', 'Release'], |
| 40 'is_try_builder': True, | 47 'is_try_builder': True, |
| 41 }, | 48 }, |
| 42 'MOCK Try Win7': { | 49 'MOCK Try Win7': { |
| 43 'port_name': 'test-win-win7', | 50 'port_name': 'test-win-win7', |
| 44 'specifiers': ['Win7', 'Release'], | 51 'specifiers': ['Win7', 'Release'], |
| 45 'is_try_builder': True, | 52 'is_try_builder': True, |
| 46 }, | 53 }, |
| (...skipping 109 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 156 'test-mac-mac10.10': {'expected': 'PASS', 'actual': 'FAIL', 'bug
': 'crbug.com/test'}, | 163 'test-mac-mac10.10': {'expected': 'PASS', 'actual': 'FAIL', 'bug
': 'crbug.com/test'}, |
| 157 }, | 164 }, |
| 158 'external/fake/test/path.html': { | 165 'external/fake/test/path.html': { |
| 159 'test-linux-trusty': {'expected': 'FAIL', 'actual': 'PASS', 'bug
': 'crbug.com/test'}, | 166 'test-linux-trusty': {'expected': 'FAIL', 'actual': 'PASS', 'bug
': 'crbug.com/test'}, |
| 160 'test-mac-mac10.11': {'expected': 'FAIL', 'actual': 'TIMEOUT', '
bug': 'crbug.com/test'}, | 167 'test-mac-mac10.11': {'expected': 'FAIL', 'actual': 'TIMEOUT', '
bug': 'crbug.com/test'}, |
| 161 }, | 168 }, |
| 162 } | 169 } |
| 163 self.assertEqual( | 170 self.assertEqual( |
| 164 updater.create_line_list(results), | 171 updater.create_line_list(results), |
| 165 [ | 172 [ |
| 166 'crbug.com/test [ Linux ] external/fake/test/path.html [ Pass ]'
, | 173 'crbug.com/test [ Trusty ] external/fake/test/path.html [ Pass ]
', |
| 167 'crbug.com/test [ Mac10.11 ] external/fake/test/path.html [ Time
out ]', | 174 'crbug.com/test [ Mac10.11 ] external/fake/test/path.html [ Time
out ]', |
| 168 'crbug.com/test [ Mac10.10 ] external/fake/test/zzzz.html [ Fail
ure ]', | 175 'crbug.com/test [ Mac10.10 ] external/fake/test/zzzz.html [ Fail
ure ]', |
| 169 ]) | 176 ]) |
| 170 | 177 |
| 171 def test_specifier_part(self): | 178 def test_specifier_part(self): |
| 172 updater = WPTExpectationsUpdater(self.mock_host()) | 179 updater = WPTExpectationsUpdater(self.mock_host()) |
| 173 self.assertEqual(updater.specifier_part(['test-mac-mac10.10'], 'x/y.html
'), '[ Mac10.10 ]') | 180 self.assertEqual(updater.specifier_part(['test-mac-mac10.10'], 'x/y.html
'), '[ Mac10.10 ]') |
| 181 self.assertEqual(updater.specifier_part([], 'x/y.html'), '') |
| 174 | 182 |
| 175 def test_skipped_specifiers_when_test_is_wontfix(self): | 183 def test_skipped_specifiers_when_test_is_wontfix(self): |
| 176 host = self.mock_host() | 184 host = self.mock_host() |
| 177 expectations_path = '/test.checkout/LayoutTests/NeverFixTests' | 185 expectations_path = '/test.checkout/LayoutTests/NeverFixTests' |
| 178 host.filesystem.files[expectations_path] = 'crbug.com/111 [ Trusty ] ext
ernal/wpt/test.html [ WontFix ]\n' | 186 host.filesystem.files[expectations_path] = 'crbug.com/111 [ Trusty ] ext
ernal/wpt/test.html [ WontFix ]\n' |
| 179 host.filesystem.files['/test.checkout/LayoutTests/external/wpt/test.html
'] = '' | 187 host.filesystem.files['/test.checkout/LayoutTests/external/wpt/test.html
'] = '' |
| 180 updater = WPTExpectationsUpdater(host) | 188 updater = WPTExpectationsUpdater(host) |
| 181 self.assertEqual(updater.skipped_specifiers('external/wpt/test.html'), [
'Trusty']) | 189 self.assertEqual(updater.skipped_specifiers('external/wpt/test.html'), [
'Trusty']) |
| 182 | 190 |
| 183 def test_simplify_specifiers(self): | 191 def test_simplify_specifiers(self): |
| 184 macros = { | 192 macros = { |
| 185 'mac': ['Mac10.10', 'mac10.11'], | 193 'mac': ['Mac10.10', 'mac10.11'], |
| 186 'win': ['Win7', 'win10'], | 194 'win': ['Win7', 'win10'], |
| 187 'Linux': ['TRUSTY'], | 195 'Linux': ['Trusty'], |
| 188 } | 196 } |
| 189 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['mac10.10',
'mac10.11'], macros), ['Mac']) | 197 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['mac10.10',
'mac10.11'], macros), ['Mac']) |
| 190 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['Mac10.10',
'Mac10.11', 'Trusty'], macros), ['Linux', 'Mac']) | 198 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['Mac10.10',
'Mac10.11', 'Trusty'], macros), ['Linux', 'Mac']) |
| 191 self.assertEqual( | 199 self.assertEqual( |
| 192 WPTExpectationsUpdater.simplify_specifiers(['Mac10.10', 'Mac10.11',
'Trusty', 'Win7', 'Win10'], macros), []) | 200 WPTExpectationsUpdater.simplify_specifiers(['Mac10.10', 'Mac10.11',
'Trusty', 'Win7', 'Win10'], macros), []) |
| 193 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['a', 'b', '
c'], {}), ['A', 'B', 'C']) | 201 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['a', 'b', '
c'], {}), ['A', 'B', 'C']) |
| 194 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['Mac', 'Win
', 'Linux'], macros), []) | 202 self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['Mac', 'Win
', 'Linux'], macros), []) |
| 195 | 203 |
| 204 def test_specifier_part_with_skipped_test(self): |
| 205 host = self.mock_host() |
| 206 expectations_path = '/test.checkout/LayoutTests/NeverFixTests' |
| 207 host.filesystem.files[expectations_path] = 'crbug.com/111 [ Linux Mac10.
11 ] external/wpt/test.html [ WontFix ]\n' |
| 208 host.filesystem.files['/test.checkout/LayoutTests/external/wpt/test.html
'] = '' |
| 209 updater = WPTExpectationsUpdater(host) |
| 210 self.assertEqual( |
| 211 updater.specifier_part(['test-mac-mac10.10', 'test-win-win7', 'test-
win-win10'], 'external/wpt/test.html'), '') |
| 212 self.assertEqual( |
| 213 updater.specifier_part(['test-win-win7', 'test-win-win10'], 'externa
l/wpt/another.html'), '[ Win ]') |
| 214 |
| 196 def test_merge_dicts_with_conflict_raise_exception(self): | 215 def test_merge_dicts_with_conflict_raise_exception(self): |
| 197 updater = WPTExpectationsUpdater(self.mock_host()) | 216 updater = WPTExpectationsUpdater(self.mock_host()) |
| 198 # Both dicts here have the key "one", and the value is not equal. | 217 # Both dicts here have the key "one", and the value is not equal. |
| 199 with self.assertRaises(ValueError): | 218 with self.assertRaises(ValueError): |
| 200 updater.merge_dicts( | 219 updater.merge_dicts( |
| 201 { | 220 { |
| 202 'external/fake/test/path.html': { | 221 'external/fake/test/path.html': { |
| 203 'one': {'expected': 'FAIL', 'actual': 'PASS'}, | 222 'one': {'expected': 'FAIL', 'actual': 'PASS'}, |
| 204 'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'}, | 223 'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'}, |
| 205 'three': {'expected': 'FAIL', 'actual': 'PASS'}, | 224 'three': {'expected': 'FAIL', 'actual': 'PASS'}, |
| (...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 253 'test-mac-mac10.10': { | 272 'test-mac-mac10.10': { |
| 254 'expected': 'bar', | 273 'expected': 'bar', |
| 255 'actual': 'foo', | 274 'actual': 'foo', |
| 256 'bug': 'crbug.com/626703', | 275 'bug': 'crbug.com/626703', |
| 257 } | 276 } |
| 258 } | 277 } |
| 259 }) | 278 }) |
| 260 | 279 |
| 261 def test_write_to_test_expectations_with_marker_comment(self): | 280 def test_write_to_test_expectations_with_marker_comment(self): |
| 262 host = self.mock_host() | 281 host = self.mock_host() |
| 263 expectations_path = '/mock-checkout/third_party/WebKit/LayoutTests/TestE
xpectations' | 282 |
| 283 expectations_path = host.port_factory.get().path_to_generic_test_expecta
tions_file() |
| 264 host.filesystem.files[expectations_path] = MARKER_COMMENT + '\n' | 284 host.filesystem.files[expectations_path] = MARKER_COMMENT + '\n' |
| 265 updater = WPTExpectationsUpdater(host) | 285 updater = WPTExpectationsUpdater(host) |
| 266 line_list = ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]'] | 286 line_list = ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]'] |
| 267 updater.write_to_test_expectations(line_list) | 287 updater.write_to_test_expectations(line_list) |
| 268 value = updater.host.filesystem.read_text_file(expectations_path) | 288 value = updater.host.filesystem.read_text_file(expectations_path) |
| 269 self.assertMultiLineEqual( | 289 self.assertMultiLineEqual( |
| 270 value, | 290 value, |
| 271 (MARKER_COMMENT + '\n' | 291 (MARKER_COMMENT + '\n' |
| 272 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')) | 292 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')) |
| 273 | 293 |
| 274 def test_write_to_test_expectations_with_no_marker_comment(self): | 294 def test_write_to_test_expectations_with_no_marker_comment(self): |
| 275 host = self.mock_host() | 295 host = self.mock_host() |
| 276 expectations_path = '/mock-checkout/third_party/WebKit/LayoutTests/TestE
xpectations' | 296 expectations_path = host.port_factory.get().path_to_generic_test_expecta
tions_file() |
| 277 host.filesystem.files[expectations_path] = 'crbug.com/111 [ Trusty ]\n' | 297 host.filesystem.files[expectations_path] = 'crbug.com/111 [ Trusty ] foo
/bar.html [ Failure ]\n' |
| 278 updater = WPTExpectationsUpdater(host) | 298 updater = WPTExpectationsUpdater(host) |
| 279 line_list = ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]'] | 299 line_list = ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]'] |
| 280 updater.write_to_test_expectations(line_list) | 300 updater.write_to_test_expectations(line_list) |
| 281 value = host.filesystem.read_text_file(expectations_path) | 301 value = host.filesystem.read_text_file(expectations_path) |
| 282 self.assertMultiLineEqual( | 302 self.assertMultiLineEqual( |
| 283 value, | 303 value, |
| 284 ('crbug.com/111 [ Trusty ]\n' | 304 ('crbug.com/111 [ Trusty ] foo/bar.html [ Failure ]\n' |
| 285 '\n' + MARKER_COMMENT + '\n' | 305 '\n' + MARKER_COMMENT + '\n' |
| 286 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]')) | 306 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]')) |
| 287 | 307 |
| 288 def test_write_to_test_expectations_skips_existing_lines(self): | 308 def test_write_to_test_expectations_skips_existing_lines(self): |
| 289 host = self.mock_host() | 309 host = self.mock_host() |
| 290 expectations_path = '/mock-checkout/third_party/WebKit/LayoutTests/TestE
xpectations' | 310 expectations_path = host.port_factory.get().path_to_generic_test_expecta
tions_file() |
| 291 host.filesystem.files[expectations_path] = 'crbug.com/111 dont/copy/me.h
tml [ Failure ]\n' | 311 host.filesystem.files[expectations_path] = 'crbug.com/111 dont/copy/me.h
tml [ Failure ]\n' |
| 292 updater = WPTExpectationsUpdater(host) | 312 updater = WPTExpectationsUpdater(host) |
| 293 line_list = [ | 313 line_list = [ |
| 294 'crbug.com/111 dont/copy/me.html [ Failure ]', | 314 'crbug.com/111 dont/copy/me.html [ Failure ]', |
| 295 'crbug.com/222 do/copy/me.html [ Failure ]' | 315 'crbug.com/222 do/copy/me.html [ Failure ]' |
| 296 ] | 316 ] |
| 297 updater.write_to_test_expectations(line_list) | 317 updater.write_to_test_expectations(line_list) |
| 298 value = host.filesystem.read_text_file(expectations_path) | 318 value = host.filesystem.read_text_file(expectations_path) |
| 299 self.assertEqual( | 319 self.assertEqual( |
| 300 value, | 320 value, |
| 301 ('crbug.com/111 dont/copy/me.html [ Failure ]\n' | 321 ('crbug.com/111 dont/copy/me.html [ Failure ]\n' |
| 302 '\n' + MARKER_COMMENT + '\n' | 322 '\n' + MARKER_COMMENT + '\n' |
| 303 'crbug.com/222 do/copy/me.html [ Failure ]')) | 323 'crbug.com/222 do/copy/me.html [ Failure ]')) |
| 304 | 324 |
| 305 def test_write_to_test_expectations_with_marker_and_no_lines(self): | 325 def test_write_to_test_expectations_with_marker_and_no_lines(self): |
| 306 host = self.mock_host() | 326 host = self.mock_host() |
| 307 expectations_path = '/mock-checkout/third_party/WebKit/LayoutTests/TestE
xpectations' | 327 expectations_path = host.port_factory.get().path_to_generic_test_expecta
tions_file() |
| 308 host.filesystem.files[expectations_path] = ( | 328 host.filesystem.files[expectations_path] = ( |
| 309 MARKER_COMMENT + '\n' | 329 MARKER_COMMENT + '\n' |
| 310 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n') | 330 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n') |
| 311 updater = WPTExpectationsUpdater(host) | 331 updater = WPTExpectationsUpdater(host) |
| 312 updater.write_to_test_expectations([]) | 332 updater.write_to_test_expectations([]) |
| 313 value = updater.host.filesystem.read_text_file(expectations_path) | 333 value = updater.host.filesystem.read_text_file(expectations_path) |
| 314 self.assertMultiLineEqual( | 334 self.assertMultiLineEqual( |
| 315 value, | 335 value, |
| 316 (MARKER_COMMENT + '\n' | 336 (MARKER_COMMENT + '\n' |
| 317 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')) | 337 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')) |
| (...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 423 updater = WPTExpectationsUpdater(self.mock_host()) | 443 updater = WPTExpectationsUpdater(self.mock_host()) |
| 424 updater.get_issue_number = lambda: 'None' | 444 updater.get_issue_number = lambda: 'None' |
| 425 self.assertEqual(1, updater.run(args=[])) | 445 self.assertEqual(1, updater.run(args=[])) |
| 426 self.assertLog(['ERROR: No issue on current branch.\n']) | 446 self.assertLog(['ERROR: No issue on current branch.\n']) |
| 427 | 447 |
| 428 def test_run_no_try_results(self): | 448 def test_run_no_try_results(self): |
| 429 updater = WPTExpectationsUpdater(self.mock_host()) | 449 updater = WPTExpectationsUpdater(self.mock_host()) |
| 430 updater.get_latest_try_jobs = lambda: [] | 450 updater.get_latest_try_jobs = lambda: [] |
| 431 self.assertEqual(1, updater.run(args=[])) | 451 self.assertEqual(1, updater.run(args=[])) |
| 432 self.assertLog(['ERROR: No try job information was collected.\n']) | 452 self.assertLog(['ERROR: No try job information was collected.\n']) |
| OLD | NEW |