Let test runner rerun failures to test for flakes.

tools/run-tests.py

 #!/usr/bin/env python
 #
 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 163 matching lines @@
   result.add_option("-p", "--progress",
                     help=("The style of progress indicator"
                           " (verbose, dots, color, mono)"),
                     choices=progress.PROGRESS_INDICATORS.keys(), default="mono")
   result.add_option("--quickcheck", default=False, action="store_true",
                     help=("Quick check mode (skip slow/flaky tests)"))
   result.add_option("--report", help="Print a summary of the tests to be run",
                     default=False, action="store_true")
   result.add_option("--json-test-results",
                     help="Path to a file for storing json results.")
+  result.add_option("--rerun-failures-count",
+                    help=("Number of times to rerun failures. Very slow tests"
+                          "will be rerun only once."),
+                    default=0, type="int")
+  result.add_option("--rerun-failures-max",
+                    help="Maximum number of failures to rerun.",

[Jakob Kummerow, 2014/07/02 07:27:26]

nit: for better clarity, s/failures/failing test cases/. And maybe accordingly
change the other option's description to "Number of times to rerun each failing
test case".

[Michael Achenbach, 2014/07/02 08:06:33]

Done.

+                    default=100, type="int")
   result.add_option("--shard-count",
                     help="Split testsuites into this number of shards",
                     default=1, type="int")
   result.add_option("--shard-run",
                     help="Run this shard from the split up tests.",
                     default=1, type="int")
   result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="")
   result.add_option("--shell-dir", help="Directory containing executables",
                     default="")
   result.add_option("--dont-skip-slow-simulator-tests",
@@ skipping 215 matching lines @@
       timeout = TIMEOUT_DEFAULT;
 
   timeout *= TIMEOUT_SCALEFACTOR[mode]
   ctx = context.Context(arch, mode, shell_dir,
                         mode_flags, options.verbose,
                         timeout, options.isolates,
                         options.command_prefix,
                         options.extra_flags,
                         options.no_i18n,
                         options.random_seed,
-                        options.no_sorting)
+                        options.no_sorting,
+                        options.rerun_failures_count,
+                        options.rerun_failures_max)
 
   # TODO(all): Combine "simulator" and "simulator_run".
   simulator_run = not options.dont_skip_simulator_slow_tests and \
       arch in ['arm64', 'arm', 'mips'] and ARCH_GUESS and arch != ARCH_GUESS
   # Find available test suites and read test cases from them.
   variables = {
     "arch": arch,
     "asan": options.asan,
     "deopt_fuzzer": False,
     "gc_stress": options.gc_stress,
@@ skipping 77 matching lines @@
   exit_code = runner.Run(options.j)
   overall_duration = time.time() - start_time
 
   if options.time:
     verbose.PrintTestDurations(suites, overall_duration)
   return exit_code
 
 
 if __name__ == "__main__":
   sys.exit(Main())