Added support for versioning of layout test results of run-webkit-tests runs

Tools/Scripts/webkitpy/layout_tests/generate_results_dashboard.py

+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import json
+
+
+class GenerateDashBoard(object):
+
+    def __init__(self, port):
+        self._port = port
+        self._filesystem = port.host.filesystem
+        self._results_directory = self._port.results_directory()
+        self._release_directory = self._filesystem.dirname(self._results_directory)
+        self._current_result_json_dict = ""
+        self._old_failing_results_list = []
+        self._old_full_results_list = []
+        self._final_result = []
+
+    def _add_individual_result_links(self, results_directories):
+        archived_results_file_list = [(file + '/results.html') for file in results_directories]
+        self._current_result_json_dict['result_links'] = archived_results_file_list

[Dirk Pranke, 2014/07/16 19:32:44]

So, if I'm following this, _current_result_json_dict() is basically
failing_results.json  + result_links, right?

And result_links is only used when you write the value back out in generate()?

This approach seems a little hard to follow; can you store result_links in a
separate variable, and then add it to current_result_json_dict() only in
generate()? I think that might be a bit clearer ...

Also, if you do end up combining dashboard.html and results.html to share more
code, it's probably better (if possible) to leave failing_results.html alone;
so, I'd consider just writing the list of results out to a separate file and
loading that instead. Make sense?

[patro, 2014/07/18 06:53:25]

_current_result_json_dict() = failing_results.json + result_links + old results

For example 
consider one test result in failing_list.json
"2d.canvas.readonly.html": {
    "actual": "PASS", 
    "expected": "PASS", 
    "has_stderr": true, 
    "time": 4.0
}
in archived_results.json the content will be the above data plus pass/fail
status in previous iterations
"2d.canvas.readonly.html": {
    "actual": "PASS", 
    "archived_results": [
        "PASS", 
        "PASS", 
        "PASS", 
        "PASS"
    ], 
    "expected": "PASS", 
    "has_stderr": true, 
    "time": 4.0
}

Also we can combine failing_results.json and archived_results.json into one file
and results.html and dashboard.html will extract the required data separately.

results_links are used by the js in dashboard.html to provide links to old
results.

+
+    def _copy_dashboard_html(self):
+        dashboard_file = self._filesystem.join(self._release_directory, 'dashboard.html')
+        dashboard_html_file_path = self._filesystem.join(self._port.layout_tests_dir(), 'fast/harness/dashboard.html')
+        if not self._filesystem.exists(dashboard_file):
+            if self._filesystem.exists(dashboard_html_file_path):
+                self._filesystem.copyfile(dashboard_html_file_path, dashboard_file)
+
+    def _initialize(self):
+        file_list = self._filesystem.listdir(self._release_directory)
+        results_directories = []
+        for dir in file_list:
+            if self._filesystem.isdir(self._filesystem.join(self._release_directory, dir)):
+                results_directories.append(self._filesystem.join(self._release_directory, dir))
+        results_directories.sort(reverse=True)
+        with open(self._filesystem.join(results_directories[0], 'failing_results.json'), "r") as file:
+            input_json_string = file.readline()
+        input_json_string = input_json_string[12:-2]   # Remove preceeding string ADD_RESULTS( and ); at the end
+        self._current_result_json_dict = json.loads(input_json_string)
+
+        # To add hyperlink to individual results.html
+        self._add_individual_result_links(results_directories)
+        results_directories = results_directories[1:]
+
+        # Load the remaining stale layout test results Json's to create the dashboard
+        for json_file in results_directories:
+            with open(self._filesystem.join(json_file, 'failing_results.json'), "r") as file:
+                json_string = file.readline()
+            json_string = json_string[12:-2]   # Remove preceeding string ADD_RESULTS( and ); at the end
+            self._old_failing_results_list.append(json.loads(json_string))
+
+            with open(self._filesystem.join(json_file, 'full_results.json'), "r") as full_file:
+                json_string_full_result = full_file.readline()
+            self._old_full_results_list.append(json.loads(json_string_full_result))
+        self._copy_dashboard_html()
+
+    # To process the final dict
+    def _get_test_result(self, test_result_data):
+        actual = test_result_data['actual']
+        expected = test_result_data['expected']
+        if actual == 'SKIP':
+            return actual
+        if actual == expected:
+            return 'HASSTDERR' if test_result_data.get('has_stderr') == 'true' else 'PASS'
+        else:
+            return actual
+
+    def _recurse_json_object(self, json_object, key_list):
+        for key in key_list:
+            try:
+                json_object = json_object[key]
+            except KeyError:
+                return 'NOTFOUND'
+        return self._get_test_result(json_object)
+
+    def _process_previous_json_results(self, key_list):
+        row = []
+        length = len(self._old_failing_results_list)
+        for index in range(0, length - 1):
+            result = self._recurse_json_object(self._old_failing_results_list[index]["tests"], key_list)
+            if result == 'NOTFOUND':
+                result = self._recurse_json_object(self._old_full_results_list[index]["tests"], key_list)
+            row.append(result)
+        return row
+
+    def _add_archived_results(self, json_object, result):
+        json_object['archived_results'] = result
+
+    def _process_json_object(self, json_object, keyList):
+        for key, subdict in json_object.iteritems():
+            if type(subdict) == dict:
+                self._process_json_object(subdict, keyList + [key])
+            else:
+                row = [self._get_test_result(json_object)]
+                row += self._process_previous_json_results(keyList)
+                self._add_archived_results(json_object, row)
+                return
+
+    def _process_json_data(self):
+        for key in self._current_result_json_dict["tests"]:
+            self._process_json_object(self._current_result_json_dict["tests"][key], [key])
+
+    def generate(self):
+        self._initialize()
+        self._process_json_data()
+        final_json = json.dumps(self._current_result_json_dict)
+        final_json = 'ADD_RESULTS(' + final_json + ');'
+        with open(self._filesystem.join(self._release_directory, 'archived_results.json'), "w") as file:
+            file.write(final_json)