WPT importer: Use concise lists of version specifiers in expectation lines.

third_party/WebKit/Tools/Scripts/webkitpy/w3c/wpt_expectations_updater_unittest.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import json
 import copy
 
 from webkitpy.common.host_mock import MockHost
 from webkitpy.common.net.buildbot import Build
 from webkitpy.common.net.buildbot_mock import MockBuildBot
 from webkitpy.common.net.layout_test_results import LayoutTestResult, LayoutTestResults
 from webkitpy.common.net.web_mock import MockWeb
 from webkitpy.common.system.log_testing import LoggingTestCase
 from webkitpy.layout_tests.builder_list import BuilderList
 from webkitpy.w3c.wpt_expectations_updater import WPTExpectationsUpdater, MARKER_COMMENT
 
 
 class WPTExpectationsUpdaterTest(LoggingTestCase):
 
     def mock_host(self):
         super(WPTExpectationsUpdaterTest, self).setUp()
         host = MockHost()
         host.builders = BuilderList({
-            'MOCK Mac10.10': {'port_name': 'test-mac-mac10.10', 'specifiers': ['Mac10.10', 'Release']},
-            'MOCK Mac10.11': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release']},
-            'MOCK Trusty': {'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release']},
-            'MOCK Win10': {'port_name': 'test-win-win10', 'specifiers': ['Win10', 'Release']},
-            'MOCK Win7': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Release']},
-            'MOCK Android': {'port_name': 'test-android', 'specifiers': ['Android', 'Release']},
+            'MOCK Try Mac10.10': {
+                'port_name': 'test-mac-mac10.10',
+                'specifiers': ['Mac10.10', 'Release'],
+                'is_try_builder': True,
+            },
+            'MOCK Try Mac10.11': {
+                'port_name': 'test-mac-mac10.11',
+                'specifiers': ['Mac10.11', 'Release'],
+                'is_try_builder': True,
+            },
+            'MOCK Try Trusty': {
+                'port_name': 'test-linux-trusty',
+                'specifiers': ['Trusty', 'Release'],
+                'is_try_builder': True,
+            },
+            'MOCK Try Win10': {
+                'port_name': 'test-win-win10',
+                'specifiers': ['Win10', 'Release'],
+                'is_try_builder': True,
+            },
+            'MOCK Try Win7': {
+                'port_name': 'test-win-win7',
+                'specifiers': ['Win7', 'Release'],
+                'is_try_builder': True,
+            },
         })
         return host
 
     def test_get_failing_results_dict_only_passing_results(self):
         host = self.mock_host()
-        host.buildbot.set_results(Build('MOCK Mac10.10', 123), LayoutTestResults({
+        host.buildbot.set_results(Build('MOCK Try Mac10.10', 123), LayoutTestResults({
             'tests': {
                 'x': {
                     'passing-test.html': {
                         'expected': 'PASS',
                         'actual': 'PASS',
                     },
                 },
             },
         }))
         updater = WPTExpectationsUpdater(host)
-        self.assertEqual(updater.get_failing_results_dict(Build('MOCK Mac10.10', 123)), {})
+        self.assertEqual(updater.get_failing_results_dict(Build('MOCK Try Mac10.10', 123)), {})
 
     def test_get_failing_results_dict_no_results(self):
         host = self.mock_host()
         host.buildbot = MockBuildBot()
-        host.buildbot.set_results(Build('MOCK Mac10.10', 123), None)
+        host.buildbot.set_results(Build('MOCK Try Mac10.10', 123), None)
         updater = WPTExpectationsUpdater(host)
-        self.assertEqual(updater.get_failing_results_dict(Build('MOCK Mac10.10', 123)), {})
+        self.assertEqual(updater.get_failing_results_dict(Build('MOCK Try Mac10.10', 123)), {})
 
     def test_get_failing_results_dict_some_failing_results(self):
         host = self.mock_host()
-        host.buildbot.set_results(Build('MOCK Mac10.10', 123), LayoutTestResults({
+        host.buildbot.set_results(Build('MOCK Try Mac10.10', 123), LayoutTestResults({
             'tests': {
                 'x': {
                     'failing-test.html': {
                         'expected': 'PASS',
                         'actual': 'IMAGE',
                         'is_unexpected': True,
                     },
                 },
             },
         }))
         updater = WPTExpectationsUpdater(host)
-        results_dict = updater.get_failing_results_dict(Build('MOCK Mac10.10', 123))
+        results_dict = updater.get_failing_results_dict(Build('MOCK Try Mac10.10', 123))
         self.assertEqual(
             results_dict,
             {
                 'x/failing-test.html': {
                     'test-mac-mac10.10': {
                         'actual': 'IMAGE',
                         'expected': 'PASS',
                         'bug': 'crbug.com/626703',
                     },
                 },
@@ skipping 48 matching lines @@
                 'two': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'},
             }
         }
         self.assertEqual(updater.create_line_list(results), [])
 
     def test_create_line_list_new_tests(self):
         # In this example, there are unexpected non-fail results in w3c tests.
         updater = WPTExpectationsUpdater(self.mock_host())
         results = {
             'external/fake/test/path.html': {
-                'one': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'},
-                'two': {'expected': 'FAIL', 'actual': 'TIMEOUT', 'bug': 'crbug.com/test'},
-                'three': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'},
+                'test-linux-trusty': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'},
+                'test-mac-mac10.10': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'},
+                'test-mac-mac10.11': {'expected': 'FAIL', 'actual': 'TIMEOUT', 'bug': 'crbug.com/test'},
             }
         }
         self.assertEqual(
             updater.create_line_list(results),
             [
-                'crbug.com/test [ three ] external/fake/test/path.html [ Pass ]',
-                'crbug.com/test [ two ] external/fake/test/path.html [ Timeout ]',
-                'crbug.com/test [ one ] external/fake/test/path.html [ Pass ]',
+                'crbug.com/test [ Linux ] external/fake/test/path.html [ Pass ]',
+                'crbug.com/test [ Mac10.10 ] external/fake/test/path.html [ Pass ]',
+                'crbug.com/test [ Mac10.11 ] external/fake/test/path.html [ Timeout ]',
             ])
 
+    def test_specifier_part(self):
+        updater = WPTExpectationsUpdater(self.mock_host())
+        self.assertEqual(updater.specifier_part(['test-mac-mac10.10'], 'x/y.html'), '[ Mac10.10 ]')
+
+    def test_skipped_specifiers_when_test_is_wontfix(self):
+        host = self.mock_host()
+        expectations_path = '/test.checkout/LayoutTests/NeverFixTests'
+        host.filesystem.files[expectations_path] = 'crbug.com/111 [ Trusty ] external/wpt/test.html [ WontFix ]\n'
+        host.filesystem.files['/test.checkout/LayoutTests/external/wpt/test.html'] = ''
+        updater = WPTExpectationsUpdater(host)
+        self.assertEqual(updater.skipped_specifiers('external/wpt/test.html'), ['Trusty'])
+
+    def test_simplify_specifiers(self):
+        macros = {
+            'mac': ['mac10.10', 'mac10.11'],
+            'win': ['win7', 'win10'],
+            'linux': ['trusty'],
+        }
+        updater = WPTExpectationsUpdater(self.mock_host())
+        self.assertEqual(updater.simplify_specifiers(['Mac10.10', 'Mac10.11'], macros), ['Mac'])
+        self.assertEqual(updater.simplify_specifiers(['Mac10.10', 'Mac10.11', 'Trusty'], macros), ['Linux', 'Mac'])
+        self.assertEqual(updater.simplify_specifiers(['Mac10.10', 'Mac10.11', 'Trusty', 'Win7', 'Win10'], macros), [])

[jeffcarp, 2017/02/13 20:42:18]

Can you also test the case when there are more versions than specifiers? (I'm
trying to figure out what would happen on line 337 of
wpt_expectations_updater.py if that is the case)

[qyearsley, 2017/02/14 00:00:51]

A few points which are possibly confusing:
  The "versions" are also "specifiers", and vice versa -- they're all "version
specifiers". This should be made clearer.
  The <= operator is overridden for sets, so a <= b is equivalent to
a.issubset(b) here. Maybe it would be more explicit to actually call that
method? So anyway, if the set on the left side is larger than the set on the
right side, then it's definitely not a subset, so it's false, so no change is
made to the specifier list.

+
     def test_merge_dicts_with_conflict_raise_exception(self):
         updater = WPTExpectationsUpdater(self.mock_host())
         # Both dicts here have the key "one", and the value is not equal.
         with self.assertRaises(ValueError):
             updater.merge_dicts(
                 {
                     'external/fake/test/path.html': {
                         'one': {'expected': 'FAIL', 'actual': 'PASS'},
                         'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'},
                         'three': {'expected': 'FAIL', 'actual': 'PASS'},
@@ skipping 50 matching lines @@
                     'bug': 'crbug.com/626703',
                 }
             }
         })
 
     def test_write_to_test_expectations_with_marker_comment(self):
         host = self.mock_host()
         expectations_path = '/mock-checkout/third_party/WebKit/LayoutTests/TestExpectations'
         host.filesystem.files[expectations_path] = MARKER_COMMENT + '\n'
         updater = WPTExpectationsUpdater(host)
-        line_list = ['crbug.com/123 [ FakePlatform ] fake/file/path.html [ Pass ]']
+        line_list = ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]']
         updater.write_to_test_expectations(line_list)
         value = updater.host.filesystem.read_text_file(expectations_path)
         self.assertMultiLineEqual(
             value,
             (MARKER_COMMENT + '\n'
-             'crbug.com/123 [ FakePlatform ] fake/file/path.html [ Pass ]\n'))
+             'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n'))
 
     def test_write_to_test_expectations_with_no_marker_comment(self):
         host = self.mock_host()
         expectations_path = '/mock-checkout/third_party/WebKit/LayoutTests/TestExpectations'
-        host.filesystem.files[expectations_path] = 'crbug.com/111 [ FakePlatform ]\n'
+        host.filesystem.files[expectations_path] = 'crbug.com/111 [ Trusty ]\n'
         updater = WPTExpectationsUpdater(host)
-        line_list = ['crbug.com/123 [ FakePlatform ] fake/file/path.html [ Pass ]']
+        line_list = ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]']
         updater.write_to_test_expectations(line_list)
         value = host.filesystem.read_text_file(expectations_path)
         self.assertMultiLineEqual(
             value,
-            ('crbug.com/111 [ FakePlatform ]\n'
+            ('crbug.com/111 [ Trusty ]\n'
              '\n' + MARKER_COMMENT + '\n'
-             'crbug.com/123 [ FakePlatform ] fake/file/path.html [ Pass ]'))
+             'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]'))
 
     def test_write_to_test_expectations_skips_existing_lines(self):
         host = self.mock_host()
         expectations_path = '/mock-checkout/third_party/WebKit/LayoutTests/TestExpectations'
         host.filesystem.files[expectations_path] = 'crbug.com/111 dont/copy/me.html [ Failure ]\n'
         updater = WPTExpectationsUpdater(host)
         line_list = [
             'crbug.com/111 dont/copy/me.html [ Failure ]',
             'crbug.com/222 do/copy/me.html [ Failure ]'
         ]
         updater.write_to_test_expectations(line_list)
         value = host.filesystem.read_text_file(expectations_path)
         self.assertEqual(
             value,
             ('crbug.com/111 dont/copy/me.html [ Failure ]\n'
              '\n' + MARKER_COMMENT + '\n'
              'crbug.com/222 do/copy/me.html [ Failure ]'))
 
     def test_write_to_test_expectations_with_marker_and_no_lines(self):
         host = self.mock_host()
         expectations_path = '/mock-checkout/third_party/WebKit/LayoutTests/TestExpectations'
         host.filesystem.files[expectations_path] = (
             MARKER_COMMENT + '\n'
-            'crbug.com/123 [ FakePlatform ] fake/file/path.html [ Pass ]\n')
+            'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')
         updater = WPTExpectationsUpdater(host)
         updater.write_to_test_expectations([])
         value = updater.host.filesystem.read_text_file(expectations_path)
         self.assertMultiLineEqual(
             value,
             (MARKER_COMMENT + '\n'
-             'crbug.com/123 [ FakePlatform ] fake/file/path.html [ Pass ]\n'))
+             'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n'))
 
     def test_is_js_test_true(self):
         host = self.mock_host()
         host.filesystem.files['/mock-checkout/third_party/WebKit/LayoutTests/foo/bar.html'] = (
             '<script src="/resources/testharness.js"></script>')
         updater = WPTExpectationsUpdater(host)
         self.assertTrue(updater.is_js_test('foo/bar.html'))
 
     def test_is_js_test_false(self):
         host = self.mock_host()
@@ skipping 109 matching lines @@
         updater.get_issue_number = lambda: '11112222'
         updater.get_try_bots = lambda: ['test-builder-name']
         self.assertEqual(1, updater.run(args=[]))
         self.assertEqual(
             host.web.urls_fetched,
             [
                 'https://codereview.chromium.org/api/11112222',
                 'https://codereview.chromium.org/api/11112222/1'
             ])
         self.assertLog(['ERROR: No try job information was collected.\n'])