Create test expectations line automatically from failing test results

third_party/WebKit/Tools/Scripts/webkitpy/w3c/update_w3c_test_expectations_unittest.py

Index: third_party/WebKit/Tools/Scripts/webkitpy/w3c/update_w3c_test_expectations_unittest.py
diff --git a/third_party/WebKit/Tools/Scripts/webkitpy/w3c/update_w3c_test_expectations_unittest.py b/third_party/WebKit/Tools/Scripts/webkitpy/w3c/update_w3c_test_expectations_unittest.py
new file mode 100644
index 0000000000000000000000000000000000000000..3ff94850cef5ddcbd37b7e2c48108fe5e4966d33
--- /dev/null
+++ b/third_party/WebKit/Tools/Scripts/webkitpy/w3c/update_w3c_test_expectations_unittest.py
@@ -0,0 +1,112 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import logging
+import unittest
+
+from webkitpy.common.host_mock import MockHost
+from webkitpy.common.net.layouttestresults import LayoutTestResult
+from webkitpy.common.net.rietveld import TryJob
+from webkitpy.common.net.web_mock import MockWeb
+from webkitpy.w3c.update_w3c_test_expectations import W3CExpectationsLineAdder
+
+_log = logging.getLogger(__name__)
+
+
+class UpdateW3CTestExpectationsTest(unittest.TestCase, W3CExpectationsLineAdder):
+
+    def setUp(self):
+        self.web = MockWeb(urls={
+            'https://codereview.chromium.org/api/11112222': json.dumps({
+                'patchsets': [1],
+            }),
+            'https://codereview.chromium.org/api/11112222/1': json.dumps({
+                'try_job_results': [
+                    {
+                        'builder': 'foo-builder',
+                        'buildnumber': 20,
+                        'result': 1
+                    },
+                    {
+                        'builder': 'bar-builder',
+                        'buildnumber': 60,
+                        'result': 0
+                    },
+                ],
+            }),
+        })
+        self.mock_dict_one = {
+            'fake/test/path.html': {
+                'one': {'expected': u'FAIL', 'actual': u'PASS', 'bug': 'crbug.com/626703'},
+                'two': {'expected': "FAIL", 'actual': 'PASS', 'bug': 'crbug.com/626703'}

[qyearsley, 2016/07/19 17:59:48]

The u can be taken off of the front of the string literals, and the quote type
(double or single) could be made consistent.

+            }
+        }
+        self.mock_dict_two = {
+            'fake/test/path.html': {
+                'one': {'expected': u'FAIL', 'actual': u'PASS', 'bug': 'crbug.com/626703'},
+                'two': {'expected': u'FAIL', 'actual': u'TIMEOUT', 'bug': 'crbug.com/626703'},
+                'three': {'expected': u'FAIL', 'actual': u'PASS', 'bug': 'crbug.com/626703'}
+            }
+        }
+        self.mock_dict_three = {
+            'fake/test/path.html': {
+                'four': {'expected': u'FAIL', 'actual': u'PASS', 'bug': 'crbug.com/626703'}}
+        }
+        self.mock_builder_dict = {
+            'foo-builder': {'port_name': 'port-a', 'specifiers': ['A', 'Release'], "is_try_builder": True},
+            'bar-builder': {'port_name': 'port-b', 'specifiers': ['B', 'Release'], "is_try_builder": True}
+        }
+
+    def test_merge_same_valued_keys(self):
+        self.assertEqual(self.merge_same_valued_keys(self.mock_dict_one['fake/test/path.html']), {
+            ('two', 'one'): {'expected': u'FAIL', 'actual': u'PASS', 'bug': 'crbug.com/626703'}
+        })
+        self.assertEqual(self.merge_same_valued_keys(self.mock_dict_two['fake/test/path.html']), {
+            ('three', 'one'): {'expected': u'FAIL', 'actual': u'PASS', 'bug': 'crbug.com/626703'},
+            'two': {'expected': u'FAIL', 'actual': u'TIMEOUT', 'bug': 'crbug.com/626703'}
+        })
+
+    def test_get_expectations(self):
+        self.assertEqual(self.get_expectations({'expected': 'FAIL', 'actual': 'PASS'}), ['Pass'])
+        self.assertEqual(self.get_expectations({'expected': 'FAIL', 'actual': 'TIMEOUT'}), ['Timeout'])
+        self.assertEqual(self.get_expectations({'expected': 'TIMEOUT', 'actual': 'PASS'}), ['Pass', 'Timeout'])
+
+    def test_create_line_list(self):
+        self.assertEqual(self.create_line_list(self.mock_dict_one),
+                         ['crbug.com/626703 [ two ] fake/test/path.html [ Pass ]',
+                          'crbug.com/626703 [ one ] fake/test/path.html [ Pass ]'])
+        self.assertEqual(self.create_line_list(self.mock_dict_two),
+                         ['crbug.com/626703 [ three ] fake/test/path.html [ Pass ]',
+                          'crbug.com/626703 [ two ] fake/test/path.html [ Timeout ]',
+                          'crbug.com/626703 [ one ] fake/test/path.html [ Pass ]'])
+
+    def test_merge_dicts(self):
+        self.assertRaises(Exception, self.merge_dicts, self.mock_dict_one, self.mock_dict_two)
+        self.assertTrue('four' in self.merge_dicts(self.mock_dict_one, self.mock_dict_three)['fake/test/path.html'])
+
+    def test_get_try_job_information(self):
+        W3C = W3CExpectationsLineAdder(MockHost())

[qyearsley, 2016/07/19 17:59:48]

For a short name for this object, you could consider calling it `adder` or
`line_adder`. In general, variable names should be lower case.

+        W3C._host.web = self.web
+        self.assertEqual(W3C.get_try_jobs_information(11112222, ('bar-builder', 'other-builder')), [TryJob('bar-builder', 60)])
+
+    def test_generate_results_dict(self):
+        W3C = W3CExpectationsLineAdder(MockHost())
+        layout_test_list = [LayoutTestResult('test/name.html', {'expected': 'bar', 'actual': 'foo', 'is_unexpected': True, 'has_stderr': True})]
+        platform = 'dummy_platform'
+        self.assertEqual(W3C._generate_results_dict(platform, layout_test_list), {'test/name.html': {'dummy_platform': {'expected': 'bar', 'actual': 'foo', 'bug': 'crbug.com/626703'}}})

[qyearsley, 2016/07/19 17:59:48]

Nit: long lines like this can be made more readable by breaking up across
multiple lines.

+
+    def test_write_to_testexpectations(self):

[qyearsley, 2016/07/19 17:59:48]

"testexpectations" -> "test_expectations"

+        W3C = W3CExpectationsLineAdder(MockHost())
+        host = MockHost()
+        host.filesystem.files = {'TestExpectations': '# Tests added from W3C auto import bot\n'}
+        line_list = ['fake crbug [foo] fake/file/path.html [Pass]']
+        path = 'TestExpectations'
+        W3C.write_to_test_expectations(host, path, line_list)
+        value = host.filesystem.read_text_file(path)
+        self.assertEqual(value, '# Tests added from W3C auto import bot\nfake crbug [foo] fake/file/path.html [Pass]\n')
+        host.filesystem.files['TestExpectations'] = 'not empty\n'
+        W3C.write_to_test_expectations(host, path, line_list)
+        value = host.filesystem.read_text_file(path)
+        self.assertEqual(value, 'not empty\n\n\n# Tests added from W3C auto import bot\nfake crbug [foo] fake/file/path.html [Pass]')