Index: gm/rebaseline_server/results.py |
=================================================================== |
--- gm/rebaseline_server/results.py (revision 0) |
+++ gm/rebaseline_server/results.py (revision 0) |
@@ -0,0 +1,136 @@ |
+#!/usr/bin/python |
+ |
+''' |
+Copyright 2013 Google Inc. |
+ |
+Use of this source code is governed by a BSD-style license that can be |
+found in the LICENSE file. |
+''' |
+ |
+''' |
+Repackage expected/actual GM results as needed by our HTML rebaseline viewer. |
+''' |
+ |
+# System-level imports |
+import fnmatch |
+import json |
+import os |
+import re |
+import sys |
+ |
+# Imports from within Skia |
+# |
+# We need to add the 'gm' directory, so that we can import gm_json.py within |
+# that directory. That script allows us to parse the actual-results.json file |
+# written out by the GM tool. |
+# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* |
+# so any dirs that are already in the PYTHONPATH will be preferred. |
+GM_DIRECTORY = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) |
+if GM_DIRECTORY not in sys.path: |
+ sys.path.append(GM_DIRECTORY) |
+import gm_json |
+ |
+IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) |
+ |
+class Results(object): |
+ """ Loads actual and expected results from all builders, supplying combined |
+ reports as requested. """ |
+ |
+ def __init__(self, actuals_root, expected_root): |
+ """ |
+ params: |
+ actuals_root: root directory containing all actual-results.json files |
+ expected_root: root directory containing all expected-results.json files |
+ """ |
+ self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root) |
+ self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root) |
+ |
+ @staticmethod |
+ def _GetDictsFromRoot(root, pattern='*.json'): |
+ """Read all JSON dictionaries within a directory tree, returning them within |
+ a meta-dictionary (keyed by the builder type for each dictionary). |
+ |
+ params: |
+ root: path to root of directory tree |
+ pattern: which files to read within root (fnmatch-style pattern) |
+ """ |
+ meta_dict = {} |
+ for dirpath, dirnames, filenames in os.walk(root): |
+ for matching_filename in fnmatch.filter(filenames, pattern): |
+ builder = os.path.basename(dirpath) |
+ # EPOGER: make this check configurable? |
+ if builder.endswith('-Trybot'): |
+ continue |
+ fullpath = os.path.join(dirpath, matching_filename) |
+ meta_dict[builder] = gm_json.LoadFromFile(fullpath) |
+ return meta_dict |
+ |
+ def OfType(self, result_type): |
+ """Returns an array of all tests, across all builders, of a certain |
+ result_type. Returns the array in this form: |
+ |
+ [ |
+ { |
+ "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug", |
+ "test": "bigmatrix", |
+ "config": "8888", |
+ "expectedHashType": "bitmap-64bitMD5", |
+ "expectedHashDigest": "10894408024079689926", |
+ "actualHashType": "bitmap-64bitMD5", |
+ "actualHashDigest": "2409857384569", |
+ }, |
+ ... |
+ ] |
+ |
+ params: |
+ result_type: string describing result type, such as 'failed'; should be |
+ one of the gm_json.JSONKEY_ACTUALRESULTS_* strings. |
+ """ |
+ # EPOGER: validate that result_type is one of the gm_json.JSONKEY_ACTUALRESULTS_* strings? |
+ array = [] |
+ for builder in sorted(self._actual_builder_dicts.keys()): |
+ results_of_this_type = self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS][result_type] |
+ if not results_of_this_type: |
+ continue |
+ for image_name in sorted(results_of_this_type.keys()): |
+ actual_image = results_of_this_type[image_name] |
+ try: |
+ # EPOGER: assumes a single allowed digest per test |
+ expected_image = self._expected_builder_dicts[builder][gm_json.JSONKEY_EXPECTEDRESULTS][image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0] |
+ except (KeyError, TypeError): |
+ # It's OK to find no expectations, if this is a NOCOMPARISON test! |
+ # But we still want to look for expectations, in case someone has |
+ # committed expectations and the bots just haven't tested them yet. |
+ if result_type != gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON: |
+ raise |
+ expected_image = [None, None] |
+ |
+ # If this test was recently rebaselined, it will remain in the "failed" |
+ # set of actuals until all the bots have cycled (although the |
+ # expectations have indeed been set from the most recent actuals). |
+ # Weed out such cases. |
+ # EPOGER: make this optional? |
+ if expected_image == actual_image: |
+ continue |
+ |
+ # EPOGER: call re.match just once for both test and config? |
+ array.append({ |
+ "builder": builder, |
+ "test": IMAGE_FILENAME_RE.match(image_name).group(1), |
+ "config": IMAGE_FILENAME_RE.match(image_name).group(2), |
+ "actualHashType": actual_image[0], |
+ "actualHashDigest": str(actual_image[1]), |
+ "expectedHashType": expected_image[0], |
+ "expectedHashDigest": str(expected_image[1]), |
+ }) |
+ return array |
+ |
+ |
+if __name__ == '__main__': |
+ # EPOGER: actual_root should point at a separate checkout of gm-actual |
+ # EPOGER: expected_root should point at a separately checked-out expectations/gm workspace |
+ results = Results( |
+ actuals_root='/usr/local/google/home/epoger/src/rebaseline-mockup/gm-actual', |
+ expected_root=os.path.join(GM_DIRECTORY, os.pardir, 'expectations', 'gm')) |
+ for result_type in ['failed', 'failure-ignored', 'no-comparison']: |
+ gm_json.WriteToFile(results.OfType(result_type), os.path.join('EPOGER-results', result_type + '.json')) |
Property changes on: gm/rebaseline_server/results.py |
___________________________________________________________________ |
Added: svn:executable |
+ * |