Stop running tests if more than 5 time out.

content/test/test_launcher.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/test/test_launcher.h"
 
 #include <string>
 #include <vector>
 
 #include "base/command_line.h"
@@ skipping 20 matching lines @@
 #include "content/common/sandbox_policy.h"
 #include "sandbox/src/dep.h"
 #include "sandbox/src/sandbox_factory.h"
 #include "sandbox/src/sandbox_types.h"
 #elif defined(OS_MACOSX)
 #include "base/mac/scoped_nsautorelease_pool.h"
 #endif
 
 namespace test_launcher {
 
-namespace {
-
 // The environment variable name for the total number of test shards.
-static const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS";
+const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS";
 // The environment variable name for the test shard index.
-static const char kTestShardIndex[] = "GTEST_SHARD_INDEX";
+const char kTestShardIndex[] = "GTEST_SHARD_INDEX";
 
 // The default output file for XML output.
-static const FilePath::CharType kDefaultOutputFile[] = FILE_PATH_LITERAL(
+const FilePath::CharType kDefaultOutputFile[] = FILE_PATH_LITERAL(
     "test_detail.xml");
 
 // Name of the empty test below.
-static const char kEmptyTestName[] = "InProcessBrowserTest.Empty";
+const char kEmptyTestName[] = "InProcessBrowserTest.Empty";
+
+// Quit test execution after this number of tests has timed out.
+const int kMaxTimeouts = 5;  // 45s timeout * (5 + 1) = 270s max run time.
+
+namespace {
 
 // An empty test (it starts up and shuts down the browser as part of its
 // setup and teardown) used to prefetch all of the browser code into memory.
 IN_PROC_BROWSER_TEST_F(InProcessBrowserTest, Empty) {
-}
+
+}  // namespace
 
 // Parses the environment variable var as an Int32.  If it is unset, returns
 // default_val.  If it is set, unsets it then converts it to Int32 before
 // returning it.  If unsetting or converting to an Int32 fails, print an
 // error and exit with failure.
 int32 Int32FromEnvOrDie(const char* const var, int32 default_val) {
   scoped_ptr<base::Environment> env(base::Environment::Create());
   std::string str_val;
   int32 result;
   if (!env->GetVar(var, &str_val))
@@ skipping 226 matching lines @@
   if (test_name.find("SLOW_") != std::string::npos)
     timeout_ms *= kSlowTestTimeoutMultiplier;
 
   return timeout_ms;
 }
 
 // Runs test specified by |test_name| in a child process,
 // and returns the exit code.
 int RunTest(TestLauncherDelegate* launcher_delegate,
             const std::string& test_name,
-            int default_timeout_ms) {
+            int default_timeout_ms,
+            bool* was_timeout) {
+  if (was_timeout)
+    *was_timeout = false;
+
 #if defined(OS_MACOSX)
   // Some of the below method calls will leak objects if there is no
   // autorelease pool in place.
   base::mac::ScopedNSAutoreleasePool pool;
 #endif
 
   const CommandLine* cmd_line = CommandLine::ForCurrentProcess();
   CommandLine new_cmd_line(cmd_line->GetProgram());
   CommandLine::SwitchMap switches = cmd_line->GetSwitches();
 
@@ skipping 52 matching lines @@
 
   int timeout_ms = GetTestTerminationTimeout(test_name,
                                              default_timeout_ms);
 
   int exit_code = 0;
   if (!base::WaitForExitCodeWithTimeout(process_handle, &exit_code,
                                         timeout_ms)) {
     LOG(ERROR) << "Test timeout (" << timeout_ms
                << " ms) exceeded for " << test_name;
 
+    if (was_timeout)
+      *was_timeout = true;
     exit_code = -1;  // Set a non-zero exit code to signal a failure.
 
     // Ensure that the process terminates.
     base::KillProcess(process_handle, -1, true);
   }
 
 #if defined(OS_POSIX)
   if (exit_code != 0) {
     // On POSIX, in case the test does not exit cleanly, either due to a crash
     // or due to it timing out, we need to clean up any child processes that
@@ skipping 26 matching lines @@
   std::string negative_filter = "";
   size_t dash_pos = filter.find('-');
   if (dash_pos != std::string::npos) {
     positive_filter = filter.substr(0, dash_pos);  // Everything up to the dash.
     negative_filter = filter.substr(dash_pos + 1); // Everything after the dash.
   }
 
   int num_runnable_tests = 0;
   int test_run_count = 0;
   int ignored_failure_count = 0;
+  int timeout_count = 0;
   std::vector<std::string> failed_tests;
 
   ResultsPrinter printer(*command_line);
   for (int i = 0; i < unit_test->total_test_case_count(); ++i) {
     const testing::TestCase* test_case = unit_test->GetTestCase(i);
     TestCasePrinterHelper helper(printer, test_case->name(),
                                  test_case->total_test_count());
     for (int j = 0; j < test_case->total_test_count(); ++j) {
       const testing::TestInfo* test_info = test_case->GetTestInfo(j);
       std::string test_name = test_info->test_case_name();
@@ skipping 29 matching lines @@
                                           num_runnable_tests);
       }
       num_runnable_tests += 1;
       // If sharding is enabled and the test should not be run, skip it.
       if (!should_run) {
         continue;
       }
 
       base::Time start_time = base::Time::Now();
       ++test_run_count;
+      bool was_timeout = false;
       int exit_code = RunTest(launcher_delegate,
                               test_name,
-                              TestTimeouts::action_max_timeout_ms());
+                              TestTimeouts::action_max_timeout_ms(),
+                              &was_timeout);
       if (exit_code == 0) {
         // Test passed.
         printer.OnTestEnd(test_info->name(), test_case->name(), true, false,
                           false,
                           (base::Time::Now() - start_time).InMillisecondsF());
       } else {
         failed_tests.push_back(test_name);
 
         bool ignore_failure = false;
 
         // -1 exit code means a crash or hang. Never ignore those failures,
         // they are serious and should always be visible.
         if (exit_code != -1)
           ignore_failure = base::TestSuite::ShouldIgnoreFailure(*test_info);
 
         printer.OnTestEnd(test_info->name(), test_case->name(), true, true,
                           ignore_failure,
                           (base::Time::Now() - start_time).InMillisecondsF());
         if (ignore_failure)
           ++ignored_failure_count;
+
+        if (was_timeout)
+          ++timeout_count;
       }
+
+      if (timeout_count > kMaxTimeouts) {
+        printf("More than %d timeouts, aborting test case\n", kMaxTimeouts);
+        break;
+      }
+    }
+    if (timeout_count > kMaxTimeouts) {
+      printf("More than %d timeouts, aborting test\n", kMaxTimeouts);
+      break;
     }
   }
 
   printf("%d test%s run\n", test_run_count, test_run_count > 1 ? "s" : "");
   printf("%d test%s failed (%d ignored)\n",
          static_cast<int>(failed_tests.size()),
          failed_tests.size() > 1 ? "s" : "", ignored_failure_count);
   if (failed_tests.size() - ignored_failure_count == 0)
     return true;
 
@@ skipping 101 matching lines @@
 
   testing::InitGoogleTest(&argc, argv);
   TestTimeouts::Initialize();
 
   // Make sure the entire browser code is loaded into memory. Reading it
   // from disk may be slow on a busy bot, and can easily exceed the default
   // timeout causing flaky test failures. Use an empty test that only starts
   // and closes a browser with a long timeout to avoid those problems.
   int exit_code = RunTest(launcher_delegate,
                           kEmptyTestName,
-                          TestTimeouts::large_test_timeout_ms());
+                          TestTimeouts::large_test_timeout_ms(),
+                          NULL);
   if (exit_code != 0)
     return exit_code;
 
   int cycles = 1;
   if (command_line->HasSwitch(kGTestRepeatFlag)) {
     base::StringToInt(command_line->GetSwitchValueASCII(kGTestRepeatFlag),
                       &cycles);
   }
 
   while (cycles != 0) {
     if (!RunTests(launcher_delegate,
                   should_shard,
                   total_shards,
                   shard_index)) {
       exit_code = 1;
       break;
     }
 
     // Special value "-1" means "repeat indefinitely".
     if (cycles != -1)
       cycles--;
   }
   return exit_code;
 }
 
 }  // namespace test_launcher