Let test runner rerun failures to test for flakes.

tools/run-tests.py

Index: tools/run-tests.py
diff --git a/tools/run-tests.py b/tools/run-tests.py
index 5cf49049f48e2729985c9e03de8d1c98c1277d06..82b766bf4ab963d3986ff676cdc0aabe86fa6c29 100755
--- a/tools/run-tests.py
+++ b/tools/run-tests.py
@@ -152,6 +152,8 @@ def BuildOptions():
   result.add_option("--no-presubmit", "--nopresubmit",
                     help='Skip presubmit checks',
                     default=False, dest="no_presubmit", action="store_true")
+  result.add_option("--no-repeat-failures", help="Don't rerun failed tests",
+                    default=False, action="store_true")
   result.add_option("--no-snap", "--nosnap",
                     help='Test a build compiled without snapshot.',
                     default=False, dest="no_snap", action="store_true")
@@ -372,6 +374,53 @@ def Main():
   return exit_code
 
 
+def RunTests(suites, num_tests, ctx, options, workspace):
+  # Run the tests, either locally or distributed on the network.
+  try:
+    start_time = time.time()
+    progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
+    if options.junitout:
+      progress_indicator = progress.JUnitTestProgressIndicator(
+          progress_indicator, options.junitout, options.junittestsuite)
+
+    run_networked = not options.no_network
+    if not run_networked:
+      print("Network distribution disabled, running tests locally.")
+    elif utils.GuessOS() != "linux":
+      print("Network distribution is only supported on Linux, sorry!")
+      run_networked = False
+    peers = []
+    if run_networked:
+      peers = network_execution.GetPeers()
+      if not peers:
+        print("No connection to distribution server; running tests locally.")
+        run_networked = False
+      elif len(peers) == 1:
+        print("No other peers on the network; running tests locally.")
+        run_networked = False
+      elif num_tests <= 100:
+        print("Less than 100 tests, running them locally.")
+        run_networked = False
+
+    if run_networked:
+      runner = network_execution.NetworkedRunner(suites, progress_indicator,
+                                                 ctx, peers, workspace)
+    else:
+      runner = execution.Runner(suites, progress_indicator, ctx)
+
+    # TODO(machenbach): Maybe use single thread for reruns?
+    exit_code = runner.Run(options.j)
+    if runner.terminate:
+      return [], exit_code
+    overall_duration = time.time() - start_time
+  except KeyboardInterrupt:
+    raise
+
+  if options.time:
+    verbose.PrintTestDurations(suites, overall_duration)
+  return runner.FailedTests(), exit_code
+
+
 def Execute(arch, mode, args, options, suites, workspace):
   print(">>> Running tests for %s.%s" % (arch, mode))
 
@@ -458,48 +507,25 @@ def Execute(arch, mode, args, options, suites, workspace):
     print "No tests to run."
     return 0
 
-  # Run the tests, either locally or distributed on the network.
-  try:
-    start_time = time.time()
-    progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
-    if options.junitout:
-      progress_indicator = progress.JUnitTestProgressIndicator(
-          progress_indicator, options.junitout, options.junittestsuite)
+  failed_tests, exit_code = RunTests(
+      suites, num_tests, ctx, options, workspace)
+  if not failed_tests or options.no_repeat_failures:
+    return exit_code
 
-    run_networked = not options.no_network
-    if not run_networked:
-      print("Network distribution disabled, running tests locally.")
-    elif utils.GuessOS() != "linux":
-      print("Network distribution is only supported on Linux, sorry!")
-      run_networked = False
-    peers = []
-    if run_networked:
-      peers = network_execution.GetPeers()
-      if not peers:
-        print("No connection to distribution server; running tests locally.")
-        run_networked = False
-      elif len(peers) == 1:
-        print("No other peers on the network; running tests locally.")
-        run_networked = False
-      elif num_tests <= 100:
-        print("Less than 100 tests, running them locally.")
-        run_networked = False
-
-    if run_networked:
-      runner = network_execution.NetworkedRunner(suites, progress_indicator,
-                                                 ctx, peers, workspace)
-    else:
-      runner = execution.Runner(suites, progress_indicator, ctx)
-
-    exit_code = runner.Run(options.j)
-    if runner.terminate:
-      return exit_code
-    overall_duration = time.time() - start_time
-  except KeyboardInterrupt:
-    raise
+  # TODO(machenbach): Additional features: Rerun count, max rerun test cases,
+  # rerun dependent on test timing (i.e. don't rerun very slow tests), group
+  # flakes, e.g. flaky crashes, flaky failures.
+  print(">>> Rerunning failures for %s.%s" % (arch, mode))
+  num_tests = 0
+  for s in suites:
+    s.tests = [t.Copy() for t in failed_tests if t in s.tests]
+    num_tests += len(s.tests)
 
-  if options.time:
-    verbose.PrintTestDurations(suites, overall_duration)
+  # TODO(machenbach): Use exit_code of final result as soon as flakes can be
+  # displayed as warnings. For now, the rerun is just FYI.
+  # TODO(machenbach): Never hide failures in gc-stress mode.
+  failed_tests, _ = RunTests(
+      suites, num_tests, ctx, options, workspace)
   return exit_code