Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(532)

Side by Side Diff: gm/rebaseline_server/results.py

Issue 24274003: Create HTTP-based GM results viewer. (Closed) Base URL: http://skia.googlecode.com/svn/trunk/
Patch Set: linewrap Created 7 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | gm/rebaseline_server/server.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Property Changes:
Added: svn:executable
+ *
OLDNEW
(Empty)
1 #!/usr/bin/python
2
3 '''
4 Copyright 2013 Google Inc.
5
6 Use of this source code is governed by a BSD-style license that can be
7 found in the LICENSE file.
8 '''
9
10 '''
11 Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
12 '''
13
14 # System-level imports
15 import fnmatch
16 import json
17 import os
18 import re
19 import sys
20
21 # Imports from within Skia
22 #
23 # We need to add the 'gm' directory, so that we can import gm_json.py within
24 # that directory. That script allows us to parse the actual-results.json file
25 # written out by the GM tool.
26 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
27 # so any dirs that are already in the PYTHONPATH will be preferred.
28 GM_DIRECTORY = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
29 if GM_DIRECTORY not in sys.path:
30 sys.path.append(GM_DIRECTORY)
31 import gm_json
32
33 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
34
35 class Results(object):
36 """ Loads actual and expected results from all builders, supplying combined
37 reports as requested. """
38
39 def __init__(self, actuals_root, expected_root):
40 """
41 params:
42 actuals_root: root directory containing all actual-results.json files
43 expected_root: root directory containing all expected-results.json files
44 """
45 self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root)
46 self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root)
47 self._all_results = self._Combine()
48
49 def GetAll(self):
50 """Return results of all tests, as a list in this form:
51
52 [
53 {
54 "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug",
55 "test": "bigmatrix",
56 "config": "8888",
57 "resultType": "failed",
58 "expectedHashType": "bitmap-64bitMD5",
59 "expectedHashDigest": "10894408024079689926",
60 "actualHashType": "bitmap-64bitMD5",
61 "actualHashDigest": "2409857384569",
62 },
63 ...
64 ]
65 """
66 return self._all_results
67
68 @staticmethod
69 def _GetDictsFromRoot(root, pattern='*.json'):
70 """Read all JSON dictionaries within a directory tree, returning them within
71 a meta-dictionary (keyed by the builder name for each dictionary).
72
73 params:
74 root: path to root of directory tree
75 pattern: which files to read within root (fnmatch-style pattern)
76 """
77 meta_dict = {}
78 for dirpath, dirnames, filenames in os.walk(root):
79 for matching_filename in fnmatch.filter(filenames, pattern):
80 builder = os.path.basename(dirpath)
81 if builder.endswith('-Trybot'):
82 continue
83 fullpath = os.path.join(dirpath, matching_filename)
84 meta_dict[builder] = gm_json.LoadFromFile(fullpath)
85 return meta_dict
86
87 def _Combine(self):
88 """Returns a list of all tests, across all builders, based on the
89 contents of self._actual_builder_dicts and self._expected_builder_dicts .
90 Returns the list in the same form needed for GetAllResults().
91 """
92 all_tests = []
93 for builder in sorted(self._actual_builder_dicts.keys()):
94 actual_results_for_this_builder = (
95 self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
96 for result_type in sorted(actual_results_for_this_builder.keys()):
97 results_of_this_type = actual_results_for_this_builder[result_type]
98 if not results_of_this_type:
99 continue
100 for image_name in sorted(results_of_this_type.keys()):
101 actual_image = results_of_this_type[image_name]
102 try:
103 # TODO(epoger): assumes a single allowed digest per test
104 expected_image = (
105 self._expected_builder_dicts
106 [builder][gm_json.JSONKEY_EXPECTEDRESULTS]
107 [image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]
108 [0])
109 except (KeyError, TypeError):
110 # There are several cases in which we would expect to find
111 # no expectations for a given test:
112 #
113 # 1. result_type == NOCOMPARISON
114 # There are no expectations for this test yet!
115 #
116 # 2. ignore-tests.txt
117 # If a test has been listed in ignore-tests.txt, then its status
118 # may show as FAILUREIGNORED even if it doesn't have any
119 # expectations yet.
120 #
121 # 3. alternate rendering mode failures (e.g. serialized)
122 # In cases like
123 # https://code.google.com/p/skia/issues/detail?id=1684
124 # ('tileimagefilter GM test failing in serialized render mode'),
125 # the gm-actuals will list a failure for the alternate
126 # rendering mode even though we don't have explicit expectations
127 # for the test (the implicit expectation is that it must
128 # render the same in all rendering modes).
129 #
130 # Don't log types 1 or 2, because they are common.
131 # Log other types, because they are rare and we should know about
132 # them, but don't throw an exception, because we need to keep our
133 # tools working in the meanwhile!
134 if result_type not in [
135 gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON,
136 gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED] :
137 print 'WARNING: No expectations found for test: %s' % {
138 'builder': builder,
139 'image_name': image_name,
140 'result_type': result_type,
141 }
142 expected_image = [None, None]
143
144 # If this test was recently rebaselined, it will remain in
145 # the "failed" set of actuals until all the bots have
146 # cycled (although the expectations have indeed been set
147 # from the most recent actuals). Treat these as successes
148 # instead of failures.
149 #
150 # TODO(epoger): Do we need to do something similar in
151 # other cases, such as when we have recently marked a test
152 # as ignoreFailure but it still shows up in the "failed"
153 # category? Maybe we should not rely on the result_type
154 # categories recorded within the gm_actuals AT ALL, and
155 # instead evaluate the result_type ourselves based on what
156 # we see in expectations vs actual checksum?
157 if expected_image == actual_image:
158 updated_result_type = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
159 else:
160 updated_result_type = result_type
161
162 # TODO(epoger): For now, don't include succeeded results.
163 # There are so many of them that they make the client too slow.
164 if updated_result_type == gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
165 continue
166
167 (test, config) = IMAGE_FILENAME_RE.match(image_name).groups()
168 all_tests.append({
169 "builder": builder,
170 "test": test,
171 "config": config,
172 "resultType": updated_result_type,
173 "actualHashType": actual_image[0],
174 "actualHashDigest": str(actual_image[1]),
175 "expectedHashType": expected_image[0],
176 "expectedHashDigest": str(expected_image[1]),
177 })
178 return all_tests
OLDNEW
« no previous file with comments | « no previous file | gm/rebaseline_server/server.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698