OLD | NEW |
---|---|
1 // Copyright (c) 2009 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2010 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #ifndef BASE_TEST_TEST_SUITE_H_ | 5 #include "base/test/test_suite.h" |
6 #define BASE_TEST_TEST_SUITE_H_ | |
7 #pragma once | |
8 | |
9 // Defines a basic test suite framework for running gtest based tests. You can | |
10 // instantiate this class in your main function and call its Run method to run | |
11 // any gtest based tests that are linked into your executable. | |
12 | 6 |
13 #include "base/at_exit.h" | 7 #include "base/at_exit.h" |
14 #include "base/base_paths.h" | 8 #include "base/base_paths.h" |
9 #include "base/base_switches.h" | |
10 #include "base/command_line.h" | |
15 #include "base/debug_on_start.h" | 11 #include "base/debug_on_start.h" |
16 #include "base/debug_util.h" | 12 #include "base/debug_util.h" |
17 #include "base/file_path.h" | 13 #include "base/file_path.h" |
18 #include "base/i18n/icu_util.h" | 14 #include "base/i18n/icu_util.h" |
19 #include "base/multiprocess_test.h" | 15 #include "base/logging.h" |
20 #include "base/nss_util.h" | 16 #include "base/nss_util.h" |
21 #include "base/path_service.h" | 17 #include "base/path_service.h" |
22 #include "base/process_util.h" | 18 #include "base/process_util.h" |
23 #include "base/scoped_nsautorelease_pool.h" | 19 #include "base/scoped_nsautorelease_pool.h" |
24 #include "base/scoped_ptr.h" | 20 #include "base/scoped_ptr.h" |
21 #include "base/test/multiprocess_test.h" | |
25 #include "base/time.h" | 22 #include "base/time.h" |
26 #include "testing/gtest/include/gtest/gtest.h" | 23 #include "testing/gtest/include/gtest/gtest.h" |
27 #include "testing/multiprocess_func_list.h" | 24 #include "testing/multiprocess_func_list.h" |
28 | 25 |
29 #if defined(OS_POSIX) && !defined(OS_MACOSX) | 26 #if defined(OS_POSIX) && !defined(OS_MACOSX) |
30 #include <gtk/gtk.h> | 27 #include <gtk/gtk.h> |
31 #endif | 28 #endif |
32 | 29 |
30 namespace base { | |
31 | |
32 namespace { | |
33 | |
33 // A command-line flag that makes a test failure always result in a non-zero | 34 // A command-line flag that makes a test failure always result in a non-zero |
34 // process exit code. | 35 // process exit code. |
35 const char kStrictFailureHandling[] = "strict_failure_handling"; | 36 const char kStrictFailureHandling[] = "strict_failure_handling"; |
36 | 37 |
37 // Match function used by the GetTestCount method. | |
38 typedef bool (*TestMatch)(const testing::TestInfo&); | |
39 | |
40 // By setting up a shadow AtExitManager, this test event listener ensures that | 38 // By setting up a shadow AtExitManager, this test event listener ensures that |
41 // no state is carried between tests (like singletons, lazy instances, etc). | 39 // no state is carried between tests (like singletons, lazy instances, etc). |
42 // Of course it won't help if the code under test corrupts memory. | 40 // Of course it won't help if the code under test corrupts memory. |
43 class TestIsolationEnforcer : public testing::EmptyTestEventListener { | 41 class TestIsolationEnforcer : public testing::EmptyTestEventListener { |
44 public: | 42 public: |
45 virtual void OnTestStart(const testing::TestInfo& test_info) { | 43 virtual void OnTestStart(const testing::TestInfo& test_info) { |
46 ASSERT_FALSE(exit_manager_.get()); | 44 ASSERT_FALSE(exit_manager_.get()); |
47 exit_manager_.reset(new base::ShadowingAtExitManager()); | 45 exit_manager_.reset(new ShadowingAtExitManager()); |
48 } | 46 } |
49 | 47 |
50 virtual void OnTestEnd(const testing::TestInfo& test_info) { | 48 virtual void OnTestEnd(const testing::TestInfo& test_info) { |
51 ASSERT_TRUE(exit_manager_.get()); | 49 ASSERT_TRUE(exit_manager_.get()); |
52 exit_manager_.reset(); | 50 exit_manager_.reset(); |
53 } | 51 } |
54 | 52 |
55 private: | 53 private: |
56 scoped_ptr<base::ShadowingAtExitManager> exit_manager_; | 54 scoped_ptr<ShadowingAtExitManager> exit_manager_; |
57 }; | 55 }; |
58 | 56 |
59 class TestSuite { | 57 class MaybeTestDisabler : public testing::EmptyTestEventListener { |
60 public: | 58 public: |
61 TestSuite(int argc, char** argv) { | 59 virtual void OnTestStart(const testing::TestInfo& test_info) { |
62 base::EnableTerminationOnHeapCorruption(); | 60 ASSERT_FALSE(TestSuite::IsMarkedMaybe(test_info)) |
63 CommandLine::Init(argc, argv); | 61 << "Probably the OS #ifdefs don't include all of the necessary " |
Mark Mentovai
2010/08/05 22:23:25
I guess the \n is there to keep things from going
| |
64 testing::InitGoogleTest(&argc, argv); | 62 "platforms.\nPlease ensure that no tests have the MAYBE_ prefix " |
63 "after the code is preprocessed."; | |
64 } | |
65 }; | |
66 | |
67 } // namespace | |
68 | |
69 TestSuite::TestSuite(int argc, char** argv) { | |
70 EnableTerminationOnHeapCorruption(); | |
71 CommandLine::Init(argc, argv); | |
72 testing::InitGoogleTest(&argc, argv); | |
65 #if defined(OS_POSIX) && !defined(OS_MACOSX) | 73 #if defined(OS_POSIX) && !defined(OS_MACOSX) |
66 g_thread_init(NULL); | 74 g_thread_init(NULL); |
67 gtk_init_check(&argc, &argv); | 75 gtk_init_check(&argc, &argv); |
68 #endif // defined(OS_LINUX) | 76 #endif // defined(OS_LINUX) |
69 // Don't add additional code to this constructor. Instead add it to | 77 // Don't add additional code to this constructor. Instead add it to |
70 // Initialize(). See bug 6436. | 78 // Initialize(). See bug 6436. |
79 } | |
80 | |
81 TestSuite::~TestSuite() { | |
82 CommandLine::Reset(); | |
83 } | |
84 | |
85 // static | |
86 bool TestSuite::IsMarkedFlaky(const testing::TestInfo& test) { | |
87 return strncmp(test.name(), "FLAKY_", 6) == 0; | |
88 } | |
89 | |
90 // static | |
91 bool TestSuite::IsMarkedFailing(const testing::TestInfo& test) { | |
92 return strncmp(test.name(), "FAILS_", 6) == 0; | |
93 } | |
94 | |
95 // static | |
96 bool TestSuite::IsMarkedMaybe(const testing::TestInfo& test) { | |
97 return strncmp(test.name(), "MAYBE_", 6) == 0; | |
98 } | |
99 | |
100 // static | |
101 bool TestSuite::ShouldIgnoreFailure(const testing::TestInfo& test) { | |
102 if (CommandLine::ForCurrentProcess()->HasSwitch(kStrictFailureHandling)) | |
103 return false; | |
104 return IsMarkedFlaky(test) || IsMarkedFailing(test); | |
105 } | |
106 | |
107 // static | |
108 bool TestSuite::NonIgnoredFailures(const testing::TestInfo& test) { | |
109 return test.should_run() && test.result()->Failed() && | |
110 !ShouldIgnoreFailure(test); | |
111 } | |
112 | |
113 int TestSuite::GetTestCount(TestMatch test_match) { | |
114 testing::UnitTest* instance = testing::UnitTest::GetInstance(); | |
115 int count = 0; | |
116 | |
117 for (int i = 0; i < instance->total_test_case_count(); ++i) { | |
118 const testing::TestCase& test_case = *instance->GetTestCase(i); | |
119 for (int j = 0; j < test_case.total_test_count(); ++j) { | |
120 if (test_match(*test_case.GetTestInfo(j))) { | |
121 count++; | |
122 } | |
123 } | |
71 } | 124 } |
72 | 125 |
73 virtual ~TestSuite() { | 126 return count; |
74 CommandLine::Reset(); | 127 } |
128 | |
129 void TestSuite::EnforceTestIsolation() { | |
130 testing::TestEventListeners& listeners = | |
131 testing::UnitTest::GetInstance()->listeners(); | |
132 listeners.Append(new TestIsolationEnforcer); | |
133 } | |
134 | |
135 void TestSuite::CatchMaybeTests() { | |
136 testing::TestEventListeners& listeners = | |
137 testing::UnitTest::GetInstance()->listeners(); | |
138 listeners.Append(new MaybeTestDisabler); | |
139 } | |
140 | |
141 // Don't add additional code to this method. Instead add it to | |
142 // Initialize(). See bug 6436. | |
143 int TestSuite::Run() { | |
144 ScopedNSAutoreleasePool scoped_pool; | |
145 | |
146 Initialize(); | |
147 std::string client_func = | |
148 CommandLine::ForCurrentProcess()->GetSwitchValueASCII( | |
149 switches::kTestChildProcess); | |
150 // Check to see if we are being run as a client process. | |
151 if (!client_func.empty()) | |
152 return multi_process_function_list::InvokeChildProcessTest(client_func); | |
153 int result = RUN_ALL_TESTS(); | |
154 | |
155 // If there are failed tests, see if we should ignore the failures. | |
156 if (result != 0 && GetTestCount(&TestSuite::NonIgnoredFailures) == 0) | |
157 result = 0; | |
158 | |
159 // Display the number of flaky tests. | |
160 int flaky_count = GetTestCount(&TestSuite::IsMarkedFlaky); | |
161 if (flaky_count) { | |
162 printf(" YOU HAVE %d FLAKY %s\n\n", flaky_count, | |
163 flaky_count == 1 ? "TEST" : "TESTS"); | |
75 } | 164 } |
76 | 165 |
77 // Returns true if the test is marked as flaky. | 166 // Display the number of tests with ignored failures (FAILS). |
78 static bool IsMarkedFlaky(const testing::TestInfo& test) { | 167 int failing_count = GetTestCount(&TestSuite::IsMarkedFailing); |
79 return strncmp(test.name(), "FLAKY_", 6) == 0; | 168 if (failing_count) { |
169 printf(" YOU HAVE %d %s with ignored failures (FAILS prefix)\n\n", | |
170 failing_count, failing_count == 1 ? "test" : "tests"); | |
80 } | 171 } |
81 | 172 |
82 // Returns true if the test is marked as failing. | 173 // This MUST happen before Shutdown() since Shutdown() tears down |
83 static bool IsMarkedFailing(const testing::TestInfo& test) { | 174 // objects (such as NotificationService::current()) that Cocoa |
84 return strncmp(test.name(), "FAILS_", 6) == 0; | 175 // objects use to remove themselves as observers. |
176 scoped_pool.Recycle(); | |
177 | |
178 Shutdown(); | |
179 | |
180 return result; | |
181 } | |
182 | |
183 // static | |
184 void TestSuite::UnitTestAssertHandler(const std::string& str) { | |
185 RAW_LOG(FATAL, str.c_str()); | |
186 } | |
187 | |
188 void TestSuite::SuppressErrorDialogs() { | |
189 #if defined(OS_WIN) | |
190 UINT new_flags = SEM_FAILCRITICALERRORS | | |
191 SEM_NOGPFAULTERRORBOX | | |
192 SEM_NOOPENFILEERRORBOX; | |
193 | |
194 // Preserve existing error mode, as discussed at | |
195 // http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx | |
196 UINT existing_flags = SetErrorMode(new_flags); | |
197 SetErrorMode(existing_flags | new_flags); | |
198 #endif // defined(OS_WIN) | |
199 } | |
200 | |
201 void TestSuite::Initialize() { | |
202 // Initialize logging. | |
203 FilePath exe; | |
204 PathService::Get(FILE_EXE, &exe); | |
205 FilePath log_filename = exe.ReplaceExtension(FILE_PATH_LITERAL("log")); | |
206 logging::InitLogging(log_filename.value().c_str(), | |
207 logging::LOG_TO_BOTH_FILE_AND_SYSTEM_DEBUG_LOG, | |
208 logging::LOCK_LOG_FILE, | |
209 logging::DELETE_OLD_LOG_FILE); | |
210 // We want process and thread IDs because we may have multiple processes. | |
211 // Note: temporarily enabled timestamps in an effort to catch bug 6361. | |
212 logging::SetLogItems(true, true, true, true); | |
213 | |
214 CHECK(EnableInProcessStackDumping()); | |
215 #if defined(OS_WIN) | |
216 // Make sure we run with high resolution timer to minimize differences | |
217 // between production code and test code. | |
218 Time::EnableHighResolutionTimer(true); | |
219 #endif // defined(OS_WIN) | |
220 | |
221 // In some cases, we do not want to see standard error dialogs. | |
222 if (!DebugUtil::BeingDebugged() && | |
223 !CommandLine::ForCurrentProcess()->HasSwitch("show-error-dialogs")) { | |
224 SuppressErrorDialogs(); | |
225 DebugUtil::SuppressDialogs(); | |
226 logging::SetLogAssertHandler(UnitTestAssertHandler); | |
85 } | 227 } |
86 | 228 |
87 // Returns true if the test is marked as "MAYBE_". | 229 icu_util::Initialize(); |
88 // When using different prefixes depending on platform, we use MAYBE_ and | |
89 // preprocessor directives to replace MAYBE_ with the target prefix. | |
90 static bool IsMarkedMaybe(const testing::TestInfo& test) { | |
91 return strncmp(test.name(), "MAYBE_", 6) == 0; | |
92 } | |
93 | |
94 // Returns true if the test failure should be ignored. | |
95 static bool ShouldIgnoreFailure(const testing::TestInfo& test) { | |
96 if (CommandLine::ForCurrentProcess()->HasSwitch(kStrictFailureHandling)) | |
97 return false; | |
98 return IsMarkedFlaky(test) || IsMarkedFailing(test); | |
99 } | |
100 | |
101 // Returns true if the test failed and the failure shouldn't be ignored. | |
102 static bool NonIgnoredFailures(const testing::TestInfo& test) { | |
103 return test.should_run() && test.result()->Failed() && | |
104 !ShouldIgnoreFailure(test); | |
105 } | |
106 | |
107 // Returns the number of tests where the match function returns true. | |
108 int GetTestCount(TestMatch test_match) { | |
109 testing::UnitTest* instance = testing::UnitTest::GetInstance(); | |
110 int count = 0; | |
111 | |
112 for (int i = 0; i < instance->total_test_case_count(); ++i) { | |
113 const testing::TestCase& test_case = *instance->GetTestCase(i); | |
114 for (int j = 0; j < test_case.total_test_count(); ++j) { | |
115 if (test_match(*test_case.GetTestInfo(j))) { | |
116 count++; | |
117 } | |
118 } | |
119 } | |
120 | |
121 return count; | |
122 } | |
123 | |
124 // TODO(phajdan.jr): Enforce isolation for all tests once it's stable. | |
125 void EnforceTestIsolation() { | |
126 testing::TestEventListeners& listeners = | |
127 testing::UnitTest::GetInstance()->listeners(); | |
128 listeners.Append(new TestIsolationEnforcer); | |
129 } | |
130 | |
131 void CatchMaybeTests() { | |
132 testing::TestEventListeners& listeners = | |
133 testing::UnitTest::GetInstance()->listeners(); | |
134 listeners.Append(new MaybeTestDisabler); | |
135 } | |
136 | |
137 // Don't add additional code to this method. Instead add it to | |
138 // Initialize(). See bug 6436. | |
139 int Run() { | |
140 base::ScopedNSAutoreleasePool scoped_pool; | |
141 | |
142 Initialize(); | |
143 std::string client_func = | |
144 CommandLine::ForCurrentProcess()->GetSwitchValueASCII( | |
145 kRunClientProcess); | |
146 // Check to see if we are being run as a client process. | |
147 if (!client_func.empty()) | |
148 return multi_process_function_list::InvokeChildProcessTest(client_func); | |
149 int result = RUN_ALL_TESTS(); | |
150 | |
151 // If there are failed tests, see if we should ignore the failures. | |
152 if (result != 0 && GetTestCount(&TestSuite::NonIgnoredFailures) == 0) | |
153 result = 0; | |
154 | |
155 // Display the number of flaky tests. | |
156 int flaky_count = GetTestCount(&TestSuite::IsMarkedFlaky); | |
157 if (flaky_count) { | |
158 printf(" YOU HAVE %d FLAKY %s\n\n", flaky_count, | |
159 flaky_count == 1 ? "TEST" : "TESTS"); | |
160 } | |
161 | |
162 // Display the number of tests with ignored failures (FAILS). | |
163 int failing_count = GetTestCount(&TestSuite::IsMarkedFailing); | |
164 if (failing_count) { | |
165 printf(" YOU HAVE %d %s with ignored failures (FAILS prefix)\n\n", | |
166 failing_count, failing_count == 1 ? "test" : "tests"); | |
167 } | |
168 | |
169 // This MUST happen before Shutdown() since Shutdown() tears down | |
170 // objects (such as NotificationService::current()) that Cocoa | |
171 // objects use to remove themselves as observers. | |
172 scoped_pool.Recycle(); | |
173 | |
174 Shutdown(); | |
175 | |
176 return result; | |
177 } | |
178 | |
179 protected: | |
180 class MaybeTestDisabler : public testing::EmptyTestEventListener { | |
181 public: | |
182 virtual void OnTestStart(const testing::TestInfo& test_info) { | |
183 ASSERT_FALSE(TestSuite::IsMarkedMaybe(test_info)) | |
184 << "Probably the OS #ifdefs don't include all of the necessary " | |
185 "platforms.\nPlease ensure that no tests have the MAYBE_ prefix " | |
186 "after the code is preprocessed."; | |
187 } | |
188 }; | |
189 | |
190 // By default fatal log messages (e.g. from DCHECKs) result in error dialogs | |
191 // which gum up buildbots. Use a minimalistic assert handler which just | |
192 // terminates the process. | |
193 static void UnitTestAssertHandler(const std::string& str) { | |
194 RAW_LOG(FATAL, str.c_str()); | |
195 } | |
196 | |
197 // Disable crash dialogs so that it doesn't gum up the buildbot | |
198 virtual void SuppressErrorDialogs() { | |
199 #if defined(OS_WIN) | |
200 UINT new_flags = SEM_FAILCRITICALERRORS | | |
201 SEM_NOGPFAULTERRORBOX | | |
202 SEM_NOOPENFILEERRORBOX; | |
203 | |
204 // Preserve existing error mode, as discussed at | |
205 // http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx | |
206 UINT existing_flags = SetErrorMode(new_flags); | |
207 SetErrorMode(existing_flags | new_flags); | |
208 #endif // defined(OS_WIN) | |
209 } | |
210 | |
211 // Override these for custom initialization and shutdown handling. Use these | |
212 // instead of putting complex code in your constructor/destructor. | |
213 | |
214 virtual void Initialize() { | |
215 // Initialize logging. | |
216 FilePath exe; | |
217 PathService::Get(base::FILE_EXE, &exe); | |
218 FilePath log_filename = exe.ReplaceExtension(FILE_PATH_LITERAL("log")); | |
219 logging::InitLogging(log_filename.value().c_str(), | |
220 logging::LOG_TO_BOTH_FILE_AND_SYSTEM_DEBUG_LOG, | |
221 logging::LOCK_LOG_FILE, | |
222 logging::DELETE_OLD_LOG_FILE); | |
223 // We want process and thread IDs because we may have multiple processes. | |
224 // Note: temporarily enabled timestamps in an effort to catch bug 6361. | |
225 logging::SetLogItems(true, true, true, true); | |
226 | |
227 CHECK(base::EnableInProcessStackDumping()); | |
228 #if defined(OS_WIN) | |
229 // Make sure we run with high resolution timer to minimize differences | |
230 // between production code and test code. | |
231 base::Time::EnableHighResolutionTimer(true); | |
232 #endif // defined(OS_WIN) | |
233 | |
234 // In some cases, we do not want to see standard error dialogs. | |
235 if (!DebugUtil::BeingDebugged() && | |
236 !CommandLine::ForCurrentProcess()->HasSwitch("show-error-dialogs")) { | |
237 SuppressErrorDialogs(); | |
238 DebugUtil::SuppressDialogs(); | |
239 logging::SetLogAssertHandler(UnitTestAssertHandler); | |
240 } | |
241 | |
242 icu_util::Initialize(); | |
243 | 230 |
244 #if defined(USE_NSS) | 231 #if defined(USE_NSS) |
245 // Trying to repeatedly initialize and cleanup NSS and NSPR may result in | 232 // Trying to repeatedly initialize and cleanup NSS and NSPR may result in |
246 // a deadlock. Such repeated initialization will happen when using test | 233 // a deadlock. Such repeated initialization will happen when using test |
247 // isolation. Prevent problems by initializing NSS here, so that the cleanup | 234 // isolation. Prevent problems by initializing NSS here, so that the cleanup |
248 // will be done only on process exit. | 235 // will be done only on process exit. |
249 base::EnsureNSSInit(); | 236 EnsureNSSInit(); |
250 #endif // defined(USE_NSS) | 237 #endif // defined(USE_NSS) |
251 | 238 |
252 CatchMaybeTests(); | 239 CatchMaybeTests(); |
253 } | 240 } |
254 | 241 |
255 virtual void Shutdown() { | 242 void TestSuite::Shutdown() { |
256 } | 243 } |
257 | 244 |
258 // Make sure that we setup an AtExitManager so Singleton objects will be | 245 } // namespace base |
259 // destroyed. | |
260 base::AtExitManager at_exit_manager_; | |
261 }; | |
262 | |
263 #endif // BASE_TEST_TEST_SUITE_H_ | |
OLD | NEW |