Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(16)

Side by Side Diff: Tools/Scripts/webkitpy/layout_tests/layout_package/bot_test_expectations_unittest.py

Issue 301853003: Make update-flaky-tests to work again and support all bots (Closed) Base URL: svn://svn.chromium.org/blink/trunk
Patch Set: Removed use of operator.attrgetter Created 6 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 # Copyright (C) 2013 Google Inc. All rights reserved. 1 # Copyright (C) 2013 Google Inc. All rights reserved.
2 # 2 #
3 # Redistribution and use in source and binary forms, with or without 3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are 4 # modification, are permitted provided that the following conditions are
5 # met: 5 # met:
6 # 6 #
7 # * Redistributions of source code must retain the above copyright 7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer. 8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above 9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer 10 # copyright notice, this list of conditions and the following disclaimer
(...skipping 24 matching lines...) Expand all
35 class BotTestExpectationsTest(unittest.TestCase): 35 class BotTestExpectationsTest(unittest.TestCase):
36 # FIXME: Find a way to import this map from Tools/TestResultServer/model/jso nresults.py. 36 # FIXME: Find a way to import this map from Tools/TestResultServer/model/jso nresults.py.
37 FAILURE_MAP = {"A": "AUDIO", "C": "CRASH", "F": "TEXT", "I": "IMAGE", "O": " MISSING", 37 FAILURE_MAP = {"A": "AUDIO", "C": "CRASH", "F": "TEXT", "I": "IMAGE", "O": " MISSING",
38 "N": "NO DATA", "P": "PASS", "T": "TIMEOUT", "Y": "NOTRUN", "X": "SKIP", "Z": "IMAGE+TEXT", "K": "LEAK"} 38 "N": "NO DATA", "P": "PASS", "T": "TIMEOUT", "Y": "NOTRUN", "X": "SKIP", "Z": "IMAGE+TEXT", "K": "LEAK"}
39 39
40 # All result_string's in this file expect newest result 40 # All result_string's in this file expect newest result
41 # on left: "PFF", means it just passed after 2 failures. 41 # on left: "PFF", means it just passed after 2 failures.
42 42
43 def _assert_is_flaky(self, results_string, should_be_flaky): 43 def _assert_is_flaky(self, results_string, should_be_flaky):
44 results_json = self._results_json_from_test_data({}) 44 results_json = self._results_json_from_test_data({})
45 expectations = bot_test_expectations.BotTestExpectations(results_json) 45 expectations = bot_test_expectations.BotTestExpectations(results_json, s et('test'))
46 length_encoded = self._results_from_string(results_string)['results'] 46 length_encoded = self._results_from_string(results_string)['results']
47 num_actual_results = len(expectations._flaky_types_in_results(length_enc oded, only_ignore_very_flaky=True)) 47 num_actual_results = len(expectations._flaky_types_in_results(length_enc oded, only_ignore_very_flaky=True))
48 if should_be_flaky: 48 if should_be_flaky:
49 self.assertGreater(num_actual_results, 1) 49 self.assertGreater(num_actual_results, 1)
50 else: 50 else:
51 self.assertEqual(num_actual_results, 1) 51 self.assertEqual(num_actual_results, 1)
52 52
53 def test_basic_flaky(self): 53 def test_basic_flaky(self):
54 self._assert_is_flaky('PFF', False) # Used to fail, but now passes. 54 self._assert_is_flaky('PFF', False) # Used to fail, but now passes.
55 self._assert_is_flaky('FFP', False) # Just started failing. 55 self._assert_is_flaky('FFP', False) # Just started failing.
(...skipping 17 matching lines...) Expand all
73 last_char = None 73 last_char = None
74 for char in results_string: 74 for char in results_string:
75 if char != last_char: 75 if char != last_char:
76 results_list.insert(0, [1, char]) 76 results_list.insert(0, [1, char])
77 else: 77 else:
78 results_list[0][0] += 1 78 results_list[0][0] += 1
79 return {'results': results_list} 79 return {'results': results_list}
80 80
81 def _assert_expectations(self, test_data, expectations_string, only_ignore_v ery_flaky): 81 def _assert_expectations(self, test_data, expectations_string, only_ignore_v ery_flaky):
82 results_json = self._results_json_from_test_data(test_data) 82 results_json = self._results_json_from_test_data(test_data)
83 expectations = bot_test_expectations.BotTestExpectations(results_json) 83 expectations = bot_test_expectations.BotTestExpectations(results_json, s et('test'))
84 self.assertEqual(expectations.flakes_by_path(only_ignore_very_flaky), ex pectations_string) 84 self.assertEqual(expectations.flakes_by_path(only_ignore_very_flaky), ex pectations_string)
85 85
86 def _assert_unexpected_results(self, test_data, expectations_string): 86 def _assert_unexpected_results(self, test_data, expectations_string):
87 results_json = self._results_json_from_test_data(test_data) 87 results_json = self._results_json_from_test_data(test_data)
88 expectations = bot_test_expectations.BotTestExpectations(results_json) 88 expectations = bot_test_expectations.BotTestExpectations(results_json, s et('test'))
89 self.assertEqual(expectations.unexpected_results_by_path(), expectations _string) 89 self.assertEqual(expectations.unexpected_results_by_path(), expectations _string)
90 90
91 def test_basic(self): 91 def test_basic(self):
92 test_data = { 92 test_data = {
93 'tests': { 93 'tests': {
94 'foo': { 94 'foo': {
95 'veryflaky.html': self._results_from_string('FPFP'), 95 'veryflaky.html': self._results_from_string('FPFP'),
96 'maybeflaky.html': self._results_from_string('PPFP'), 96 'maybeflaky.html': self._results_from_string('PPFP'),
97 'notflakypass.html': self._results_from_string('PPPP'), 97 'notflakypass.html': self._results_from_string('PPPP'),
98 'notflakyfail.html': self._results_from_string('FFFF'), 98 'notflakyfail.html': self._results_from_string('FFFF'),
(...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after
155 self._assert_unexpected_results(test_data, { 155 self._assert_unexpected_results(test_data, {
156 'foo/pass1.html': sorted(["FAIL", "PASS"]), 156 'foo/pass1.html': sorted(["FAIL", "PASS"]),
157 'foo/pass2.html': sorted(["IMAGE", "PASS"]), 157 'foo/pass2.html': sorted(["IMAGE", "PASS"]),
158 'foo/fail.html': sorted(["TEXT", "PASS"]), 158 'foo/fail.html': sorted(["TEXT", "PASS"]),
159 'foo/f_p.html': sorted(["TEXT", "PASS"]), 159 'foo/f_p.html': sorted(["TEXT", "PASS"]),
160 'foo/crash.html': sorted(["WONTFIX", "CRASH", "TEXT"]), 160 'foo/crash.html': sorted(["WONTFIX", "CRASH", "TEXT"]),
161 'foo/image.html': sorted(["CRASH", "FAIL", "IMAGE"]), 161 'foo/image.html': sorted(["CRASH", "FAIL", "IMAGE"]),
162 'foo/i_f.html': sorted(["PASS", "IMAGE", "TEXT"]), 162 'foo/i_f.html': sorted(["PASS", "IMAGE", "TEXT"]),
163 'foo/all.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEOUT", "CR ASH", "IMAGE", "MISSING", "LEAK"]), 163 'foo/all.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEOUT", "CR ASH", "IMAGE", "MISSING", "LEAK"]),
164 }) 164 })
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698