Let test runner rerun failures to test for flakes.

tools/run-tests.py

 #!/usr/bin/env python
 #
 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 134 matching lines @@
   result.add_option("--no-i18n", "--noi18n",
                     help="Skip internationalization tests",
                     default=False, action="store_true")
   result.add_option("--no-network", "--nonetwork",
                     help="Don't distribute tests on the network",
                     default=(utils.GuessOS() != "linux"),
                     dest="no_network", action="store_true")
   result.add_option("--no-presubmit", "--nopresubmit",
                     help='Skip presubmit checks',
                     default=False, dest="no_presubmit", action="store_true")
+  result.add_option("--no-repeat-failures", help="Don't rerun failed tests",
+                    default=False, action="store_true")
   result.add_option("--no-snap", "--nosnap",
                     help='Test a build compiled without snapshot.',
                     default=False, dest="no_snap", action="store_true")
   result.add_option("--no-stress", "--nostress",
                     help="Don't run crankshaft --always-opt --stress-op test",
                     default=False, dest="no_stress", action="store_true")
   result.add_option("--no-variants", "--novariants",
                     help="Don't run any testing variants",
                     default=False, dest="no_variants", action="store_true")
   result.add_option("--variants",
@@ skipping 200 matching lines @@
 
   for (arch, mode) in options.arch_and_mode:
     try:
       code = Execute(arch, mode, args, options, suites, workspace)
     except KeyboardInterrupt:
       return 2
     exit_code = exit_code or code
   return exit_code
 
 
+def RunTests(suites, num_tests, ctx, options, workspace):
+  # Run the tests, either locally or distributed on the network.
+  try:
+    start_time = time.time()
+    progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
+    if options.junitout:
+      progress_indicator = progress.JUnitTestProgressIndicator(
+          progress_indicator, options.junitout, options.junittestsuite)
+
+    run_networked = not options.no_network
+    if not run_networked:
+      print("Network distribution disabled, running tests locally.")
+    elif utils.GuessOS() != "linux":
+      print("Network distribution is only supported on Linux, sorry!")
+      run_networked = False
+    peers = []
+    if run_networked:
+      peers = network_execution.GetPeers()
+      if not peers:
+        print("No connection to distribution server; running tests locally.")
+        run_networked = False
+      elif len(peers) == 1:
+        print("No other peers on the network; running tests locally.")
+        run_networked = False
+      elif num_tests <= 100:
+        print("Less than 100 tests, running them locally.")
+        run_networked = False
+
+    if run_networked:
+      runner = network_execution.NetworkedRunner(suites, progress_indicator,
+                                                 ctx, peers, workspace)
+    else:
+      runner = execution.Runner(suites, progress_indicator, ctx)
+
+    # TODO(machenbach): Maybe use single thread for reruns?
+    exit_code = runner.Run(options.j)
+    if runner.terminate:
+      return [], exit_code
+    overall_duration = time.time() - start_time
+  except KeyboardInterrupt:
+    raise
+
+  if options.time:
+    verbose.PrintTestDurations(suites, overall_duration)
+  return runner.FailedTests(), exit_code
+
+
 def Execute(arch, mode, args, options, suites, workspace):
   print(">>> Running tests for %s.%s" % (arch, mode))
 
   shell_dir = options.shell_dir
   if not shell_dir:
     if options.buildbot:
       shell_dir = os.path.join(workspace, options.outdir, mode)
       mode = mode.lower()
     else:
       shell_dir = os.path.join(workspace, options.outdir,
@@ skipping 66 matching lines @@
   if options.cat:
     return 0  # We're done here.
 
   if options.report:
     verbose.PrintReport(all_tests)
 
   if num_tests == 0:
     print "No tests to run."
     return 0
 
-  # Run the tests, either locally or distributed on the network.
-  try:
-    start_time = time.time()
-    progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
-    if options.junitout:
-      progress_indicator = progress.JUnitTestProgressIndicator(
-          progress_indicator, options.junitout, options.junittestsuite)
+  failed_tests, exit_code = RunTests(
+      suites, num_tests, ctx, options, workspace)
+  if not failed_tests or options.no_repeat_failures:
+    return exit_code
 
-    run_networked = not options.no_network
-    if not run_networked:
-      print("Network distribution disabled, running tests locally.")
-    elif utils.GuessOS() != "linux":
-      print("Network distribution is only supported on Linux, sorry!")
-      run_networked = False
-    peers = []
-    if run_networked:
-      peers = network_execution.GetPeers()
-      if not peers:
-        print("No connection to distribution server; running tests locally.")
-        run_networked = False
-      elif len(peers) == 1:
-        print("No other peers on the network; running tests locally.")
-        run_networked = False
-      elif num_tests <= 100:
-        print("Less than 100 tests, running them locally.")
-        run_networked = False
+  # TODO(machenbach): Additional features: Rerun count, max rerun test cases,
+  # rerun dependent on test timing (i.e. don't rerun very slow tests), group
+  # flakes, e.g. flaky crashes, flaky failures.
+  print(">>> Rerunning failures for %s.%s" % (arch, mode))
+  num_tests = 0
+  for s in suites:
+    s.tests = [t.Copy() for t in failed_tests if t in s.tests]
+    num_tests += len(s.tests)
 
-    if run_networked:
-      runner = network_execution.NetworkedRunner(suites, progress_indicator,
-                                                 ctx, peers, workspace)
-    else:
-      runner = execution.Runner(suites, progress_indicator, ctx)
-
-    exit_code = runner.Run(options.j)
-    if runner.terminate:
-      return exit_code
-    overall_duration = time.time() - start_time
-  except KeyboardInterrupt:
-    raise
-
-  if options.time:
-    verbose.PrintTestDurations(suites, overall_duration)
+  # TODO(machenbach): Use exit_code of final result as soon as flakes can be
+  # displayed as warnings. For now, the rerun is just FYI.
+  # TODO(machenbach): Never hide failures in gc-stress mode.
+  failed_tests, _ = RunTests(
+      suites, num_tests, ctx, options, workspace)
   return exit_code
 
 
 if __name__ == "__main__":
   sys.exit(Main())