Create test expectations line automatically from failing test results

third_party/WebKit/Tools/Scripts/webkitpy/w3c/update_w3c_test_expectations.py

+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+'''A script to modify TestExpectations lines based layout test failures in try jobs.

[qyearsley, 2016/07/18 22:55:29]

Usually triple-double-quotes (""") are used for docstrings (same below).

+
+This script outputs a list of test expectation lines to add to a 'TestExpectations' file
+by retrieving the try job results for the current CL.
+'''
+
+import logging
+
+from webkitpy.common.net import buildbot
+from webkitpy.common.net import rietveld
+
+
+_log = logging.getLogger(__name__)
+
+
+def main(host, port):
+    expectations_file = port.path_to_generic_test_expectations_file()
+    expectations_line_adder = W3CExpectationsLineAdder(host)
+    issue_number = expectations_line_adder.get_issue_number()
+    try_bots = expectations_line_adder.get_try_bots()
+    try_jobs_info = expectations_line_adder.get_try_jobs_information(issue_number, try_bots)
+    line_expectations_dict = {}
+    if not try_jobs_info:
+        print 'No Try Job information was collected.'
+        return 1
+    for try_job in try_jobs_info:
+        builder_name = try_job[0]
+        build_number = try_job[1]
+        builder = buildbot.Builder(builder_name, expectations_line_adder.get_build_bot)
+        build = buildbot.Build(builder, build_number)
+        platform_results_dict = expectations_line_adder.get_failing_results_dict(builder, build)
+        line_expectations_dict = expectations_line_adder.merge_dicts(line_expectations_dict, platform_results_dict)
+    for platform_results_dicts in line_expectations_dict.values():
+        platform_results_dicts = expectations_line_adder.merge_same_valued_keys(platform_results_dicts)
+    line_list = expectations_line_adder.create_line_list(line_expectations_dict)
+    expectations_line_adder.write_to_test_expectations(host, expectations_file, line_list)
+
+
+class W3CExpectationsLineAdder(object):
+
+    def __init__(self, host):
+        self._host = host
+
+    def get_build_bot(self):
+        return self._host.buildbot
+
+    def get_try_jobs_information(self, issue_number, try_bots):
+        return rietveld.latest_try_jobs(issue_number, try_bots, self._host.web)
+
+    def get_issue_number(self):
+        return self._host._scm.get_issue_number()
+
+    def get_try_bots(self):
+        return self._host.builders.all_try_builder_names()
+
+    def _generate_results_dict(self, platform, result_list):
+        test_dict = {}
+        if '-' in platform:
+            platform = platform[platform.find('-') + 1:].capitalize()
+        for result in result_list:
+            test_dict[result.test_name()] = {
+                platform: {
+                    'expected': result.expected_results(),
+                    'actual': result.actual_results(),
+                    'bug': 'crbug.com/626703'
+                }}
+        return test_dict
+
+    def get_failing_results_dict(self, builder, build):
+        ''' returns a dict of dicts with the format
+        {'key': {'expected': results, 'actual': results, 'bug': bug_url, ...}}
+        '''
+        layout_test_results = builder.fetch_layout_test_results(build.results_url())
+        builder_name = layout_test_results.builder_name()
+        platform = self._host.builders.port_name_for_builder_name(builder_name)
+        result_list = layout_test_results.didnt_run_as_expected_results()
+        failing_results_dict = self._generate_results_dict(platform, result_list)
+        return failing_results_dict
+
+    def merge_dicts(self, final, temp, path=None):
+        path = path or []
+        for key in temp:
+            if key in final:
+                if (isinstance(final[key], dict)) and isinstance(temp[key], dict):
+                    self.merge_dicts(final[key], temp[key], path + [str(key)])
+                elif final[key] == temp[key]:
+                    pass
+                else:
+                    raise Exception('conflict at %s' % '.'.join(path))
+            else:
+                final[key] = temp[key]
+        return final
+
+    def merge_same_valued_keys(self, dictionary):
+        '''This function takes a dictionary of dictionaries and creates a new tuple key

[qyearsley, 2016/07/18 22:55:29]

You could omit "This function", and just start with "Takes a dictionary of
dictionaries ...

+        if two or more values match. Example: {

[qyearsley, 2016/07/18 22:55:28]

You should add a blank line after the first sentence of the docstring (same
below).

+        'one': {'foo': 'bar'},
+        'two': {'foo': 'bar'},
+        'three': {'foo': bar'}
+        }     is converted to
+        {('one', 'two', 'three'): {'foo': 'bar'}}
+        '''
+        matching_value_keys = set()
+        keys = dictionary.keys()
+        isLastItem = False
+        for index, item in enumerate(keys):
+            if isLastItem:
+                break
+            for i in range(index + 1, len(keys)):
+                next_item = keys[i]
+                if dictionary[item] == dictionary[next_item]:
+                    matching_value_keys.update([item, next_item])
+                    dictionary[tuple(matching_value_keys)] = dictionary[item]
+                    isLastItem = next_item == keys[-1]
+                    del dictionary[item]
+                    del dictionary[next_item]
+        return dictionary
+
+    def get_expectations(self, results):
+        expectations = []
+        failure_expectations = ['TEXT', 'FAIL', 'IMAGE+TEXT', 'IMAGE']
+        pass_crash_timeout = ['TIMEOUT', 'CRASH', 'PASS']
+        if results['expected'] in pass_crash_timeout and results['actual'] in failure_expectations:
+            expectations.append('Failure')
+        if results['expected'] in failure_expectations and results['actual'] in pass_crash_timeout:
+            expectations.append(results['actual'].capitalize())
+        if results['expected'] in pass_crash_timeout and results['actual'] in pass_crash_timeout:
+            expectations.append(results['actual'].capitalize())
+            expectations.append(results['expected'].capitalize())
+        return expectations
+
+    def create_line_list(self, dictionary):
+        '''Returns a list of test expectations lines with the format
+        ['BUG_URL [PLATFORM(S)] TEST_MAME [EXPECTATION(S)]']
+        '''
+        line_list = []
+        for key, value in dictionary.iteritems():
+            test_name = key
+            for key2 in value:
+                platform = []
+                bug = []
+                expectations = []
+                if isinstance(key2, tuple):
+                    platform = list(key2)
+                else:
+                    platform.append(key2)
+                bug.append(value[key2]['bug'])
+                expectations = self.get_expectations(value[key2])
+                line = '%s [ %s ] %s [ %s ]' % (bug[0], ' '.join(platform), test_name, ' '.join(expectations))
+                line_list.append(str(line))
+        return line_list
+
+    def write_to_test_expectations(self, host, path, line_list):
+        '''Writes to test expectations file on the filesystem. Checks the file for
+        '#Tests added from W3C auto import bot' and writes expectation lines directly under it. If not found,
+        it writes to the end of the file.
+        '''
+        file_contents = host.filesystem.read_text_file(path)
+        w3c_comment_line_index = file_contents.find('# Tests added from W3C auto import bot')
+        all_lines = ''
+        for line in line_list:
+            all_lines += str(line) + '\n'
+        all_lines = all_lines[:-1]
+        if w3c_comment_line_index == -1:
+            file_contents += '\n\n# Tests added from W3C auto import bot\n'
+            file_contents += all_lines
+        else:
+            end_of_comment_line = (file_contents[w3c_comment_line_index:].find('\n')) + w3c_comment_line_index
+            new_data = file_contents[: end_of_comment_line + 1] + all_lines + file_contents[end_of_comment_line:]
+            file_contents = new_data
+        host.filesystem.write_text_file(path, file_contents)