Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(497)

Side by Side Diff: Tools/Scripts/webkitpy/layout_tests/layout_package/bot_test_expectations_unittest.py

Issue 296423002: TestResultServer: Add failure 'LEAK' (Closed) Base URL: svn://svn.chromium.org/blink/trunk
Patch Set: Modify jsonresults_unittest.py Created 6 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | Tools/TestResultServer/model/jsonresults.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright (C) 2013 Google Inc. All rights reserved. 1 # Copyright (C) 2013 Google Inc. All rights reserved.
2 # 2 #
3 # Redistribution and use in source and binary forms, with or without 3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are 4 # modification, are permitted provided that the following conditions are
5 # met: 5 # met:
6 # 6 #
7 # * Redistributions of source code must retain the above copyright 7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer. 8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above 9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer 10 # copyright notice, this list of conditions and the following disclaimer
(...skipping 17 matching lines...) Expand all
28 28
29 import webkitpy.thirdparty.unittest2 as unittest 29 import webkitpy.thirdparty.unittest2 as unittest
30 30
31 from webkitpy.layout_tests.layout_package import bot_test_expectations 31 from webkitpy.layout_tests.layout_package import bot_test_expectations
32 from webkitpy.layout_tests.models import test_expectations 32 from webkitpy.layout_tests.models import test_expectations
33 33
34 34
35 class BotTestExpectationsTest(unittest.TestCase): 35 class BotTestExpectationsTest(unittest.TestCase):
36 # FIXME: Find a way to import this map from Tools/TestResultServer/model/jso nresults.py. 36 # FIXME: Find a way to import this map from Tools/TestResultServer/model/jso nresults.py.
37 FAILURE_MAP = {"A": "AUDIO", "C": "CRASH", "F": "TEXT", "I": "IMAGE", "O": " MISSING", 37 FAILURE_MAP = {"A": "AUDIO", "C": "CRASH", "F": "TEXT", "I": "IMAGE", "O": " MISSING",
38 "N": "NO DATA", "P": "PASS", "T": "TIMEOUT", "Y": "NOTRUN", "X": "SKIP", "Z": "IMAGE+TEXT"} 38 "N": "NO DATA", "P": "PASS", "T": "TIMEOUT", "Y": "NOTRUN", "X": "SKIP", "Z": "IMAGE+TEXT", "K": "LEAK"}
39 39
40 # All result_string's in this file expect newest result 40 # All result_string's in this file expect newest result
41 # on left: "PFF", means it just passed after 2 failures. 41 # on left: "PFF", means it just passed after 2 failures.
42 42
43 def _assert_is_flaky(self, results_string, should_be_flaky): 43 def _assert_is_flaky(self, results_string, should_be_flaky):
44 results_json = self._results_json_from_test_data({}) 44 results_json = self._results_json_from_test_data({})
45 expectations = bot_test_expectations.BotTestExpectations(results_json) 45 expectations = bot_test_expectations.BotTestExpectations(results_json)
46 length_encoded = self._results_from_string(results_string)['results'] 46 length_encoded = self._results_from_string(results_string)['results']
47 num_actual_results = len(expectations._flaky_types_in_results(length_enc oded, only_ignore_very_flaky=True)) 47 num_actual_results = len(expectations._flaky_types_in_results(length_enc oded, only_ignore_very_flaky=True))
48 if should_be_flaky: 48 if should_be_flaky:
(...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after
105 105
106 self._assert_expectations(test_data, { 106 self._assert_expectations(test_data, {
107 'foo/veryflaky.html': sorted(["TEXT", "PASS"]), 107 'foo/veryflaky.html': sorted(["TEXT", "PASS"]),
108 'foo/maybeflaky.html': sorted(["TEXT", "PASS"]), 108 'foo/maybeflaky.html': sorted(["TEXT", "PASS"]),
109 }, only_ignore_very_flaky=False) 109 }, only_ignore_very_flaky=False)
110 110
111 def test_all_failure_types(self): 111 def test_all_failure_types(self):
112 test_data = { 112 test_data = {
113 'tests': { 113 'tests': {
114 'foo': { 114 'foo': {
115 'allfailures.html': self._results_from_string('FPFPCNCNTXTXI ZIZOCYOCY'), 115 'allfailures.html': self._results_from_string('FPFPCNCNTXTXI ZIZOCOCYKYK'),
116 'imageplustextflake.html': self._results_from_string('ZPZPPP PPPPPPPPPPPPPP'), 116 'imageplustextflake.html': self._results_from_string('ZPZPPP PPPPPPPPPPPPPP'),
117 } 117 }
118 } 118 }
119 } 119 }
120 self._assert_expectations(test_data, { 120 self._assert_expectations(test_data, {
121 'foo/imageplustextflake.html': sorted(["IMAGE+TEXT", "PASS"]), 121 'foo/imageplustextflake.html': sorted(["IMAGE+TEXT", "PASS"]),
122 'foo/allfailures.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEO UT", "CRASH", "IMAGE", "MISSING"]), 122 'foo/allfailures.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEO UT", "CRASH", "IMAGE", "MISSING", "LEAK"]),
123 }, only_ignore_very_flaky=True) 123 }, only_ignore_very_flaky=True)
124 124
125 def test_unexpected_results_no_unexpected(self): 125 def test_unexpected_results_no_unexpected(self):
126 test_data = { 126 test_data = {
127 'tests': { 127 'tests': {
128 'foo': { 128 'foo': {
129 'pass1.html': {'results': [[4, 'P']]}, 129 'pass1.html': {'results': [[4, 'P']]},
130 'pass2.html': {'results': [[2, 'Z']], 'expected': 'PASS FAIL '}, 130 'pass2.html': {'results': [[2, 'Z']], 'expected': 'PASS FAIL '},
131 'fail.html': {'results': [[2, 'P'], [1, 'F']], 'expected': ' PASS FAIL'}, 131 'fail.html': {'results': [[2, 'P'], [1, 'F']], 'expected': ' PASS FAIL'},
132 'not_run.html': {'results': []}, 132 'not_run.html': {'results': []},
133 'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected': 'CRASH FAIL WONTFIX'}, 133 'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected': 'CRASH FAIL WONTFIX'},
134 } 134 }
135 } 135 }
136 } 136 }
137 self._assert_unexpected_results(test_data, {}) 137 self._assert_unexpected_results(test_data, {})
138 138
139 def test_unexpected_results_all_unexpected(self): 139 def test_unexpected_results_all_unexpected(self):
140 test_data = { 140 test_data = {
141 'tests': { 141 'tests': {
142 'foo': { 142 'foo': {
143 'pass1.html': {'results': [[4, 'P']], 'expected': 'FAIL'}, 143 'pass1.html': {'results': [[4, 'P']], 'expected': 'FAIL'},
144 'pass2.html': {'results': [[2, 'P']], 'expected': 'IMAGE'}, 144 'pass2.html': {'results': [[2, 'P']], 'expected': 'IMAGE'},
145 'fail.html': {'results': [[4, 'F']]}, 145 'fail.html': {'results': [[4, 'F']]},
146 'f_p.html': {'results': [[1, 'F'], [2, 'P']]}, 146 'f_p.html': {'results': [[1, 'F'], [2, 'P']]},
147 'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected': 'WONTFIX'}, 147 'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected': 'WONTFIX'},
148 'image.html': {'results': [[2, 'F'], [1, 'I']], 'expected': 'CRASH FAIL'}, 148 'image.html': {'results': [[2, 'F'], [1, 'I']], 'expected': 'CRASH FAIL'},
149 'i_f.html': {'results': [[1, 'F'], [5, 'I']], 'expected': 'P ASS'}, 149 'i_f.html': {'results': [[1, 'F'], [5, 'I']], 'expected': 'P ASS'},
150 'all.html': self._results_from_string('FPFPCNCNTXTXIZIZOCYOC Y'), 150 'all.html': self._results_from_string('FPFPCNCNTXTXIZIZOCOCY KYK'),
151 } 151 }
152 } 152 }
153 } 153 }
154 self.maxDiff = None 154 self.maxDiff = None
155 self._assert_unexpected_results(test_data, { 155 self._assert_unexpected_results(test_data, {
156 'foo/pass1.html': sorted(["FAIL", "PASS"]), 156 'foo/pass1.html': sorted(["FAIL", "PASS"]),
157 'foo/pass2.html': sorted(["IMAGE", "PASS"]), 157 'foo/pass2.html': sorted(["IMAGE", "PASS"]),
158 'foo/fail.html': sorted(["TEXT", "PASS"]), 158 'foo/fail.html': sorted(["TEXT", "PASS"]),
159 'foo/f_p.html': sorted(["TEXT", "PASS"]), 159 'foo/f_p.html': sorted(["TEXT", "PASS"]),
160 'foo/crash.html': sorted(["WONTFIX", "CRASH", "TEXT"]), 160 'foo/crash.html': sorted(["WONTFIX", "CRASH", "TEXT"]),
161 'foo/image.html': sorted(["CRASH", "FAIL", "IMAGE"]), 161 'foo/image.html': sorted(["CRASH", "FAIL", "IMAGE"]),
162 'foo/i_f.html': sorted(["PASS", "IMAGE", "TEXT"]), 162 'foo/i_f.html': sorted(["PASS", "IMAGE", "TEXT"]),
163 'foo/all.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEOUT", "CR ASH", "IMAGE", "MISSING"]), 163 'foo/all.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEOUT", "CR ASH", "IMAGE", "MISSING", "LEAK"]),
164 }) 164 })
OLDNEW
« no previous file with comments | « no previous file | Tools/TestResultServer/model/jsonresults.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698