|
OLD | NEW |
---|---|
(Empty) | |
1 #!/usr/bin/python | |
2 | |
3 ''' | |
4 Copyright 2013 Google Inc. | |
5 | |
6 Use of this source code is governed by a BSD-style license that can be | |
7 found in the LICENSE file. | |
8 ''' | |
9 | |
10 ''' | |
11 Repackage expected/actual GM results as needed by our HTML rebaseline viewer. | |
12 ''' | |
13 | |
14 # System-level imports | |
15 import fnmatch | |
16 import json | |
17 import os | |
18 import re | |
19 import sys | |
20 | |
21 # Imports from within Skia | |
22 # | |
23 # We need to add the 'gm' directory, so that we can import gm_json.py within | |
24 # that directory. That script allows us to parse the actual-results.json file | |
25 # written out by the GM tool. | |
26 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* | |
27 # so any dirs that are already in the PYTHONPATH will be preferred. | |
28 GM_DIRECTORY = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) | |
29 if GM_DIRECTORY not in sys.path: | |
30 sys.path.append(GM_DIRECTORY) | |
31 import gm_json | |
32 | |
33 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) | |
34 | |
35 class Results(object): | |
36 """ Loads actual and expected results from all builders, supplying combined | |
37 reports as requested. """ | |
38 | |
39 def __init__(self, actuals_root, expected_root): | |
40 """ | |
41 params: | |
42 actuals_root: root directory containing all actual-results.json files | |
43 expected_root: root directory containing all expected-results.json files | |
44 """ | |
45 self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root) | |
46 self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root) | |
47 | |
48 @staticmethod | |
49 def _GetDictsFromRoot(root, pattern='*.json'): | |
50 """Read all JSON dictionaries within a directory tree, returning them within | |
51 a meta-dictionary (keyed by the builder type for each dictionary). | |
borenet
2013/09/25 15:08:08
nit: "builder name"
epoger
2013/09/25 16:21:18
Done.
| |
52 | |
53 params: | |
54 root: path to root of directory tree | |
55 pattern: which files to read within root (fnmatch-style pattern) | |
56 """ | |
57 meta_dict = {} | |
58 for dirpath, dirnames, filenames in os.walk(root): | |
59 for matching_filename in fnmatch.filter(filenames, pattern): | |
60 builder = os.path.basename(dirpath) | |
61 if builder.endswith('-Trybot'): | |
62 continue | |
63 fullpath = os.path.join(dirpath, matching_filename) | |
64 meta_dict[builder] = gm_json.LoadFromFile(fullpath) | |
65 return meta_dict | |
66 | |
67 def OfType(self, result_type): | |
borenet
2013/09/25 15:08:08
Is this the only method which gets called on Resul
epoger
2013/09/25 16:21:18
That's a good point. Actually, it opens the wider
epoger
2013/09/25 18:26:33
Brian and I agree that this is the way to go (in a
borenet
2013/09/25 19:28:17
I agree. The page will be more interactive that w
| |
68 """Returns an array of all tests, across all builders, of a certain | |
69 result_type. Returns the array in this form: | |
70 | |
71 [ | |
72 { | |
73 "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug", | |
74 "test": "bigmatrix", | |
75 "config": "8888", | |
76 "expectedHashType": "bitmap-64bitMD5", | |
77 "expectedHashDigest": "10894408024079689926", | |
78 "actualHashType": "bitmap-64bitMD5", | |
79 "actualHashDigest": "2409857384569", | |
80 }, | |
81 ... | |
82 ] | |
83 | |
84 params: | |
85 result_type: string describing result type, such as 'failed'; should be | |
86 one of the gm_json.JSONKEY_ACTUALRESULTS_* strings. | |
87 """ | |
88 # TODO(epoger): validate that result_type is one of the | |
89 # gm_json.JSONKEY_ACTUALRESULTS_* strings? | |
borenet
2013/09/25 15:08:08
If not, please change line 93 to use .get() instea
epoger
2013/09/25 16:21:18
Done.
| |
90 array = [] | |
91 for builder in sorted(self._actual_builder_dicts.keys()): | |
92 results_of_this_type = ( | |
93 self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS] | |
94 [result_type]) | |
95 if not results_of_this_type: | |
96 continue | |
97 for image_name in sorted(results_of_this_type.keys()): | |
98 actual_image = results_of_this_type[image_name] | |
99 try: | |
100 # TODO(epoger): assumes a single allowed digest per test | |
101 expected_image = ( | |
102 self._expected_builder_dicts | |
103 [builder][gm_json.JSONKEY_EXPECTEDRESULTS] | |
104 [image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS] | |
105 [0]) | |
106 except (KeyError, TypeError): | |
107 # It's OK to find no expectations, if this is a NOCOMPARISON test! | |
108 # But we still want to look for expectations, in case someone has | |
109 # committed expectations and the bots just haven't tested them yet. | |
110 if result_type != gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON: | |
111 raise | |
112 expected_image = [None, None] | |
113 | |
114 # If this test was recently rebaselined, it will remain in the "failed" | |
115 # set of actuals until all the bots have cycled (although the | |
116 # expectations have indeed been set from the most recent actuals). | |
117 # Weed out such cases. | |
118 if expected_image == actual_image: | |
119 continue | |
120 | |
121 (test, config) = IMAGE_FILENAME_RE.match(image_name).groups() | |
122 array.append({ | |
123 "builder": builder, | |
124 "test": test, | |
125 "config": config, | |
126 "actualHashType": actual_image[0], | |
127 "actualHashDigest": str(actual_image[1]), | |
128 "expectedHashType": expected_image[0], | |
129 "expectedHashDigest": str(expected_image[1]), | |
130 }) | |
131 return array | |
OLD | NEW |