Added support for versioning of layout test results of run-webkit-tests runs

Tools/Scripts/webkitpy/layout_tests/generate_results_dashboard.py

+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import json
+import logging
+import string
+
+from webkitpy.common.system.filesystem import FileSystem

[Dirk Pranke, 2014/06/17 17:58:58]

there's no need to import FileSystem directly.

[patro, 2014/07/15 10:36:57]

Done.

+from webkitpy.layout_tests.port import configuration_options, platform_options
+
+_log = logging.getLogger(__name__)
+
+
+class GenerateDashBoard:

[Dirk Pranke, 2014/06/17 17:58:58]

nit: this should inherit from (object).

[patro, 2014/07/15 10:36:57]

Done.

+
+    "A class for generating the Dashboard from the list of archived results"
+
+    def __init__(self, port):
+        self._port = port
+        self._filesystem = port.host.filesystem
+        self._results_directory = self._port.results_directory()
+        self._release_directory = self._filesystem.join(self._filesystem.dirname(self._results_directory), '')
+        self._input_json_data = ""
+        self._old_failing_results_list = []
+        self._old_full_results_list = []
+        self._final_result = []
+
+    def _add_individual_result_links(self, file_list):
+        file_list = [(file + '/results.html') for file in file_list]
+        self._input_json_data['result_links'] = file_list
+
+    def _copy_dashboard_html(self):
+        dashboard_file = self._filesystem.join(self._release_directory, 'dashboard.html')
+        dashboard_html_file_path = self._filesystem.join(self._port.layout_tests_dir(), 'fast/harness/dashboard.html')
+        if ~(self._filesystem.exists(dashboard_file)):
+            if self._filesystem.exists(dashboard_html_file_path):
+                self._filesystem.copyfile(dashboard_html_file_path, dashboard_file)
+
+    def _initialize(self):
+        file_list = self._filesystem.listdir(self._release_directory)
+        json_file_list = []
+        for dir in file_list:
+            if self._filesystem.isdir(self._filesystem.join(self._release_directory, dir)):
+                json_file_list.append(self._filesystem.join(self._release_directory, dir))
+        json_file_list.sort(reverse=True)
+        #Read the current Layout Test Results
+        with open(self._filesystem.join(json_file_list[0], 'failing_results.json'), "r") as file:
+            input_json_string = file.readline()
+        input_json_string = input_json_string[12:-2]   # Remove preceeding string ADD_RESULTS( and ); at the end
+        self._input_json_data = json.loads(input_json_string)
+        #To add hyperlink to individual results.html
+        self._add_individual_result_links(json_file_list)
+        json_file_list = json_file_list[1:]
+        #Load the remaining stale layout test results Json's to create the dashboard
+        for json_file in json_file_list:
+            with open(self._filesystem.join(json_file, 'failing_results.json'), "r") as file:
+                json_string = file.readline()
+            json_string = json_string[12:-2]   # Remove preceeding string ADD_RESULTS( and ); at the end
+            self._old_failing_results_list.append(json.loads(json_string))
+
+            with open(self._filesystem.join(json_file, 'full_results.json'), "r") as full_file:
+                json_string_full_result = full_file.readline()
+            self._old_full_results_list.append(json.loads(json_string_full_result))
+        self._copy_dashboard_html()
+
+    #To safely get the value if key doesn't exit then it is a syntax error
+    def _get_value(self, json_object, key):
+        try:
+            value = json_object[key]
+            return value
+        except KeyError, e:
+            print("Syntax error: Could not find the key ", key)
+            exit()
+
+    #To process the final dict
+    def _process_json_result(self, json_object):
+        actual = self._get_value(json_object, "actual")
+        expected = self._get_value(json_object, "expected")
+        if actual == 'SKIP':
+            return actual
+        if actual == expected:
+            hasStderr = 'false'
+            try:
+                hasStderr = json_object["has_stderr"]
+            except KeyError, e:
+                pass
+            if hasStderr == 'true':
+                return 'HASSTDERR'
+            return 'PASS'
+        else:
+            return actual
+
+    def _recurse_json_object(self, json_object, key_list):
+        for key in key_list:
+            try:
+                json_object = json_object[key]
+            except KeyError:
+                return 'NOTFOUND'
+        return self._process_json_result(json_object)
+
+    def _process_previous_json_results(self, key_list):
+        row = []
+        length = len(self._old_failing_results_list)
+        for index in range(0, length - 1):
+            result = self._recurse_json_object(self._old_failing_results_list[index]["tests"], key_list)
+            if result == 'NOTFOUND':
+                result = self._recurse_json_object(self._old_full_results_list[index]["tests"], key_list)
+            row.append(result)
+        return row
+
+    def _add_archived_results(self, json_object, result):
+        json_object['archived_results'] = result
+
+    def _process_json_object(self, json_object, keyList):
+        flag = 0
+        for key, subdict in json_object.iteritems():
+            if type(subdict) == dict:
+                self._process_json_object(subdict, keyList + [key])
+            else:
+                flag = 1
+                break
+        if flag == 1:
+            row = [self._process_json_result(json_object)]
+            row += self._process_previous_json_results(keyList)
+            self._add_archived_results(json_object, row)
+
+    def _process_json_data(self):
+        for key in self._input_json_data["tests"]:
+            self._process_json_object(self._input_json_data["tests"][key], [key])
+
+    def generate(self):
+        self._initialize()
+        self._process_json_data()
+        final_json = json.dumps(self._input_json_data)
+        final_json = 'ADD_RESULTS(' + final_json + ');'
+        with open(self._filesystem.join(self._release_directory, 'archived_results.json'), "w") as file:
+            file.write(final_json)