OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Utility to confirm that a JSON summary written by GM contains no failures. | 6 """Utility to display a summary of JSON-format GM results, and exit with |
| 7 a nonzero errorcode if there were non-ignored failures in the GM results. |
7 | 8 |
8 Usage: | 9 Usage: |
9 python confirm_no_failures_in_json.py <filename> | 10 python display_json_results.py <filename> |
| 11 |
| 12 TODO(epoger): We may want to add flags to set the following: |
| 13 - which error types cause a nonzero return code |
| 14 - maximum number of tests to list for any one ResultAccumulator |
| 15 (to keep the output reasonably short) |
10 """ | 16 """ |
11 | 17 |
12 __author__ = 'Elliot Poger' | 18 __author__ = 'Elliot Poger' |
13 | 19 |
14 | 20 |
15 import json | 21 import json |
16 import sys | 22 import sys |
17 | 23 |
18 | 24 |
19 # These constants must be kept in sync with the kJsonKey_ constants in | 25 # These constants must be kept in sync with the kJsonKey_ constants in |
20 # gm_expectations.cpp ! | 26 # gm_expectations.cpp ! |
21 JSONKEY_ACTUALRESULTS = 'actual-results' | 27 JSONKEY_ACTUALRESULTS = 'actual-results' |
22 JSONKEY_ACTUALRESULTS_FAILED = 'failed' | 28 JSONKEY_ACTUALRESULTS_FAILED = 'failed' |
23 | 29 JSONKEY_ACTUALRESULTS_FAILUREIGNORED = 'failure-ignored' |
24 # This is the same indent level as used by jsoncpp, just for consistency. | 30 JSONKEY_ACTUALRESULTS_NOCOMPARISON = 'no-comparison' |
25 JSON_INDENTLEVEL = 3 | 31 JSONKEY_ACTUALRESULTS_SUCCEEDED = 'succeeded' |
26 | 32 |
27 | 33 |
28 def Assert(filepath): | 34 class ResultAccumulator(object): |
29 """Raises an exception if the JSON summary at filepath contains any failed | 35 """Object that accumulates results of a given type, and can generate a |
30 tests, or if we were unable to read the JSON summary.""" | 36 summary upon request.""" |
31 failed_tests = GetFailedTests(filepath) | 37 |
32 if failed_tests: | 38 def __init__(self, name, do_list, do_fail): |
33 raise Exception('JSON file %s contained these test failures...\n%s' % ( | 39 """name: name of the category this result type falls into |
34 filepath, json.dumps(failed_tests, indent=JSON_INDENTLEVEL))) | 40 do_list: whether to list all of the tests with this results type |
| 41 do_fail: whether to return with nonzero exit code if there are any |
| 42 results of this type |
| 43 """ |
| 44 self._name = name |
| 45 self._do_list = do_list |
| 46 self._do_fail = do_fail |
| 47 self._testnames = [] |
| 48 |
| 49 def AddResult(self, testname): |
| 50 """Adds a result of this particular type. |
| 51 testname: (string) name of the test""" |
| 52 self._testnames.append(testname) |
| 53 |
| 54 def ShouldSignalFailure(self): |
| 55 """Returns true if this result type is serious (self._do_fail is True) |
| 56 and there were any results of this type.""" |
| 57 if self._do_fail and self._testnames: |
| 58 return True |
| 59 else: |
| 60 return False |
| 61 |
| 62 def GetSummaryLine(self): |
| 63 """Returns a single-line string summary of all results added to this |
| 64 accumulator so far.""" |
| 65 summary = '' |
| 66 if self._do_fail: |
| 67 summary += '[*] ' |
| 68 else: |
| 69 summary += '[ ] ' |
| 70 summary += str(len(self._testnames)) |
| 71 summary += ' ' |
| 72 summary += self._name |
| 73 if self._do_list: |
| 74 summary += ': ' |
| 75 for testname in self._testnames: |
| 76 summary += testname |
| 77 summary += ' ' |
| 78 return summary |
35 | 79 |
36 | 80 |
37 def GetFailedTests(filepath): | 81 def Display(filepath): |
38 """Returns the dictionary of failed tests from the JSON file at filepath.""" | 82 """Displays a summary of the results in a JSON file. |
| 83 Returns True if the results are free of any significant failures. |
| 84 filepath: (string) path to JSON file""" |
| 85 |
| 86 # Map labels within the JSON file to the ResultAccumulator for each label. |
| 87 results_map = { |
| 88 JSONKEY_ACTUALRESULTS_FAILED: |
| 89 ResultAccumulator(name='ExpectationsMismatch', |
| 90 do_list=True, do_fail=True), |
| 91 JSONKEY_ACTUALRESULTS_FAILUREIGNORED: |
| 92 ResultAccumulator(name='IgnoredExpectationsMismatch', |
| 93 do_list=True, do_fail=False), |
| 94 JSONKEY_ACTUALRESULTS_NOCOMPARISON: |
| 95 ResultAccumulator(name='MissingExpectations', |
| 96 do_list=False, do_fail=False), |
| 97 JSONKEY_ACTUALRESULTS_SUCCEEDED: |
| 98 ResultAccumulator(name='Passed', |
| 99 do_list=False, do_fail=False), |
| 100 } |
| 101 |
| 102 success = True |
39 json_dict = json.load(open(filepath)) | 103 json_dict = json.load(open(filepath)) |
40 actual_results = json_dict[JSONKEY_ACTUALRESULTS] | 104 actual_results = json_dict[JSONKEY_ACTUALRESULTS] |
41 return actual_results[JSONKEY_ACTUALRESULTS_FAILED] | 105 for label, accumulator in results_map.iteritems(): |
| 106 results = actual_results[label] |
| 107 if results: |
| 108 for result in results: |
| 109 accumulator.AddResult(result) |
| 110 print accumulator.GetSummaryLine() |
| 111 if accumulator.ShouldSignalFailure(): |
| 112 success = False |
| 113 print '(results marked with [*] will cause nonzero return value)' |
| 114 return success |
42 | 115 |
43 | 116 |
44 if '__main__' == __name__: | 117 if '__main__' == __name__: |
45 if len(sys.argv) != 2: | 118 if len(sys.argv) != 2: |
46 raise Exception('usage: %s <input-json-filepath>' % sys.argv[0]) | 119 raise Exception('usage: %s <input-json-filepath>' % sys.argv[0]) |
47 Assert(sys.argv[1]) | 120 sys.exit(0 if Display(sys.argv[1]) else 1) |
OLD | NEW |