OLD | NEW |
| (Empty) |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | |
2 # Use of this source code is governed by a BSD-style license that can be | |
3 # found in the LICENSE file. | |
4 | |
5 # | |
6 # Most of this file was ported over from Blink's | |
7 # webkitpy/layout_tests/layout_package/json_results_generator_unittest.py | |
8 # | |
9 | |
10 import unittest | |
11 import json | |
12 | |
13 from pylib.utils import json_results_generator | |
14 | |
15 | |
16 class JSONGeneratorTest(unittest.TestCase): | |
17 | |
18 def setUp(self): | |
19 self.builder_name = 'DUMMY_BUILDER_NAME' | |
20 self.build_name = 'DUMMY_BUILD_NAME' | |
21 self.build_number = 'DUMMY_BUILDER_NUMBER' | |
22 | |
23 # For archived results. | |
24 self._json = None | |
25 self._num_runs = 0 | |
26 self._tests_set = set([]) | |
27 self._test_timings = {} | |
28 self._failed_count_map = {} | |
29 | |
30 self._PASS_count = 0 | |
31 self._DISABLED_count = 0 | |
32 self._FLAKY_count = 0 | |
33 self._FAILS_count = 0 | |
34 self._fixable_count = 0 | |
35 | |
36 self._orig_write_json = json_results_generator.WriteJSON | |
37 | |
38 # unused arguments ... pylint: disable=W0613 | |
39 def _WriteJSONStub(json_object, file_path, callback=None): | |
40 pass | |
41 | |
42 json_results_generator.WriteJSON = _WriteJSONStub | |
43 | |
44 def tearDown(self): | |
45 json_results_generator.WriteJSON = self._orig_write_json | |
46 | |
47 def _TestJSONGeneration(self, passed_tests_list, failed_tests_list): | |
48 tests_set = set(passed_tests_list) | set(failed_tests_list) | |
49 | |
50 DISABLED_tests = set([t for t in tests_set | |
51 if t.startswith('DISABLED_')]) | |
52 FLAKY_tests = set([t for t in tests_set | |
53 if t.startswith('FLAKY_')]) | |
54 FAILS_tests = set([t for t in tests_set | |
55 if t.startswith('FAILS_')]) | |
56 PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests) | |
57 | |
58 failed_tests = set(failed_tests_list) - DISABLED_tests | |
59 failed_count_map = dict([(t, 1) for t in failed_tests]) | |
60 | |
61 test_timings = {} | |
62 i = 0 | |
63 for test in tests_set: | |
64 test_timings[test] = float(self._num_runs * 100 + i) | |
65 i += 1 | |
66 | |
67 test_results_map = dict() | |
68 for test in tests_set: | |
69 test_results_map[test] = json_results_generator.TestResult( | |
70 test, failed=(test in failed_tests), | |
71 elapsed_time=test_timings[test]) | |
72 | |
73 generator = json_results_generator.JSONResultsGeneratorBase( | |
74 self.builder_name, self.build_name, self.build_number, | |
75 '', | |
76 None, # don't fetch past json results archive | |
77 test_results_map) | |
78 | |
79 failed_count_map = dict([(t, 1) for t in failed_tests]) | |
80 | |
81 # Test incremental json results | |
82 incremental_json = generator.GetJSON() | |
83 self._VerifyJSONResults( | |
84 tests_set, | |
85 test_timings, | |
86 failed_count_map, | |
87 len(PASS_tests), | |
88 len(DISABLED_tests), | |
89 len(FLAKY_tests), | |
90 len(DISABLED_tests | failed_tests), | |
91 incremental_json, | |
92 1) | |
93 | |
94 # We don't verify the results here, but at least we make sure the code | |
95 # runs without errors. | |
96 generator.GenerateJSONOutput() | |
97 generator.GenerateTimesMSFile() | |
98 | |
99 def _VerifyJSONResults(self, tests_set, test_timings, failed_count_map, | |
100 PASS_count, DISABLED_count, FLAKY_count, | |
101 fixable_count, json_obj, num_runs): | |
102 # Aliasing to a short name for better access to its constants. | |
103 JRG = json_results_generator.JSONResultsGeneratorBase | |
104 | |
105 self.assertIn(JRG.VERSION_KEY, json_obj) | |
106 self.assertIn(self.builder_name, json_obj) | |
107 | |
108 buildinfo = json_obj[self.builder_name] | |
109 self.assertIn(JRG.FIXABLE, buildinfo) | |
110 self.assertIn(JRG.TESTS, buildinfo) | |
111 self.assertEqual(len(buildinfo[JRG.BUILD_NUMBERS]), num_runs) | |
112 self.assertEqual(buildinfo[JRG.BUILD_NUMBERS][0], self.build_number) | |
113 | |
114 if tests_set or DISABLED_count: | |
115 fixable = {} | |
116 for fixable_items in buildinfo[JRG.FIXABLE]: | |
117 for (result_type, count) in fixable_items.iteritems(): | |
118 if result_type in fixable: | |
119 fixable[result_type] = fixable[result_type] + count | |
120 else: | |
121 fixable[result_type] = count | |
122 | |
123 if PASS_count: | |
124 self.assertEqual(fixable[JRG.PASS_RESULT], PASS_count) | |
125 else: | |
126 self.assertTrue(JRG.PASS_RESULT not in fixable or | |
127 fixable[JRG.PASS_RESULT] == 0) | |
128 if DISABLED_count: | |
129 self.assertEqual(fixable[JRG.SKIP_RESULT], DISABLED_count) | |
130 else: | |
131 self.assertTrue(JRG.SKIP_RESULT not in fixable or | |
132 fixable[JRG.SKIP_RESULT] == 0) | |
133 if FLAKY_count: | |
134 self.assertEqual(fixable[JRG.FLAKY_RESULT], FLAKY_count) | |
135 else: | |
136 self.assertTrue(JRG.FLAKY_RESULT not in fixable or | |
137 fixable[JRG.FLAKY_RESULT] == 0) | |
138 | |
139 if failed_count_map: | |
140 tests = buildinfo[JRG.TESTS] | |
141 for test_name in failed_count_map.iterkeys(): | |
142 test = self._FindTestInTrie(test_name, tests) | |
143 | |
144 failed = 0 | |
145 for result in test[JRG.RESULTS]: | |
146 if result[1] == JRG.FAIL_RESULT: | |
147 failed += result[0] | |
148 self.assertEqual(failed_count_map[test_name], failed) | |
149 | |
150 timing_count = 0 | |
151 for timings in test[JRG.TIMES]: | |
152 if timings[1] == test_timings[test_name]: | |
153 timing_count = timings[0] | |
154 self.assertEqual(1, timing_count) | |
155 | |
156 if fixable_count: | |
157 self.assertEqual(sum(buildinfo[JRG.FIXABLE_COUNT]), fixable_count) | |
158 | |
159 def _FindTestInTrie(self, path, trie): | |
160 nodes = path.split('/') | |
161 sub_trie = trie | |
162 for node in nodes: | |
163 self.assertIn(node, sub_trie) | |
164 sub_trie = sub_trie[node] | |
165 return sub_trie | |
166 | |
167 def testJSONGeneration(self): | |
168 self._TestJSONGeneration([], []) | |
169 self._TestJSONGeneration(['A1', 'B1'], []) | |
170 self._TestJSONGeneration([], ['FAILS_A2', 'FAILS_B2']) | |
171 self._TestJSONGeneration(['DISABLED_A3', 'DISABLED_B3'], []) | |
172 self._TestJSONGeneration(['A4'], ['B4', 'FAILS_C4']) | |
173 self._TestJSONGeneration(['DISABLED_C5', 'DISABLED_D5'], ['A5', 'B5']) | |
174 self._TestJSONGeneration( | |
175 ['A6', 'B6', 'FAILS_C6', 'DISABLED_E6', 'DISABLED_F6'], | |
176 ['FAILS_D6']) | |
177 | |
178 # Generate JSON with the same test sets. (Both incremental results and | |
179 # archived results must be updated appropriately.) | |
180 self._TestJSONGeneration( | |
181 ['A', 'FLAKY_B', 'DISABLED_C'], | |
182 ['FAILS_D', 'FLAKY_E']) | |
183 self._TestJSONGeneration( | |
184 ['A', 'DISABLED_C', 'FLAKY_E'], | |
185 ['FLAKY_B', 'FAILS_D']) | |
186 self._TestJSONGeneration( | |
187 ['FLAKY_B', 'DISABLED_C', 'FAILS_D'], | |
188 ['A', 'FLAKY_E']) | |
189 | |
190 def testHierarchicalJSNGeneration(self): | |
191 # FIXME: Re-work tests to be more comprehensible and comprehensive. | |
192 self._TestJSONGeneration(['foo/A'], ['foo/B', 'bar/C']) | |
193 | |
194 def testTestTimingsTrie(self): | |
195 individual_test_timings = [] | |
196 individual_test_timings.append( | |
197 json_results_generator.TestResult( | |
198 'foo/bar/baz.html', | |
199 elapsed_time=1.2)) | |
200 individual_test_timings.append( | |
201 json_results_generator.TestResult('bar.html', elapsed_time=0.0001)) | |
202 trie = json_results_generator.TestTimingsTrie(individual_test_timings) | |
203 | |
204 expected_trie = { | |
205 'bar.html': 0, | |
206 'foo': { | |
207 'bar': { | |
208 'baz.html': 1200, | |
209 } | |
210 } | |
211 } | |
212 | |
213 self.assertEqual(json.dumps(trie), json.dumps(expected_trie)) | |
OLD | NEW |