Let test runner rerun failures to test for flakes.

tools/testrunner/local/execution.py

 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 63 matching lines @@
 
   def _CommonInit(self, num_tests, progress_indicator, context):
     self.indicator = progress_indicator
     progress_indicator.runner = self
     self.context = context
     self.succeeded = 0
     self.total = num_tests
     self.remaining = num_tests
     self.failed = []
     self.crashed = 0
+    self.reran_tests = 0
 
   def _RunPerfSafe(self, fun):
     try:
       fun()
     except Exception, e:
       print("PerfData exception: %s" % e)
       self.perf_failures = True
 
+  def _GetJob(self, test):
+    command = self.GetCommand(test)
+    timeout = self.context.timeout
+    if ("--stress-opt" in test.flags or
+        "--stress-opt" in self.context.mode_flags or
+        "--stress-opt" in self.context.extra_flags):
+      timeout *= 4
+    if test.dependency is not None:
+      dep_command = [ c.replace(test.path, test.dependency) for c in command ]
+    else:
+      dep_command = None
+    return Job(command, dep_command, test.id, timeout, self.context.verbose)
+
+  def _MaybeRerun(self, pool, test):
+    if test.run <= self.context.rerun_failures_count:
+      # Possibly rerun this test if its run count is below the maximum per
+      # test.
+      if test.run == 1:
+        # Count the overall number of reran tests on the first rerun.
+        if self.reran_tests < self.context.rerun_failures_max:
+          self.reran_tests += 1
+        else:
+          # Don't rerun this if the overall number of rerun tests has been
+          # reached.
+          return
+      if test.run >= 2 and test.duration > self.context.timeout / 20:
+        # Rerun slow tests at most once.
+        return
+
+      # Rerun this test.
+      test.duration = None

[Jakob Kummerow, 2014/07/02 07:27:26]

I don't think we need to reset duration and output (but it shouldn't hurt if we
do).

[Michael Achenbach, 2014/07/02 08:06:33]

OK - keeping it.

+      test.output = None
+      test.run += 1
+      pool.add([self._GetJob(test)])
+      self.remaining += 1
+
   def Run(self, jobs):
     self.indicator.Starting()
     self._RunInternal(jobs)
     self.indicator.Done()
     if self.failed or self.remaining:
       return 1
     return 0
 
   def _RunInternal(self, jobs):
     pool = Pool(jobs)
     test_map = {}
     # TODO(machenbach): Instead of filling the queue completely before
     # pool.imap_unordered, make this a generator that already starts testing
     # while the queue is filled.
     queue = []
     queued_exception = None
     for test in self.tests:
       assert test.id >= 0
       test_map[test.id] = test
       try:
-        command = self.GetCommand(test)
+        queue.append([self._GetJob(test)])

[Michael Achenbach, 2014/07/01 15:21:11]

For simplicity, I put it all within the try block. Shouldn't make a big
difference as there is not much that can fail besides GetCommand.

       except Exception, e:
         # If this failed, save the exception and re-raise it later (after
         # all other tests have had a chance to run).
         queued_exception = e
         continue
-      timeout = self.context.timeout
-      if ("--stress-opt" in test.flags or
-          "--stress-opt" in self.context.mode_flags or
-          "--stress-opt" in self.context.extra_flags):
-        timeout *= 4
-      if test.dependency is not None:
-        dep_command = [ c.replace(test.path, test.dependency) for c in command ]
-      else:
-        dep_command = None
-      job = Job(command, dep_command, test.id, timeout, self.context.verbose)
-      queue.append([job])
     try:
       it = pool.imap_unordered(RunTest, queue)
       for result in it:
         test = test_map[result[0]]
         self.indicator.AboutToRun(test)
         test.output = result[1]
         test.duration = result[2]
         has_unexpected_output = test.suite.HasUnexpectedOutput(test)
         if has_unexpected_output:
           self.failed.append(test)
           if test.output.HasCrashed():
             self.crashed += 1
         else:
           self._RunPerfSafe(lambda: self.perfdata.UpdatePerfData(test))
           self.succeeded += 1
         self.remaining -= 1
         self.indicator.HasRun(test, has_unexpected_output)
+        if has_unexpected_output:
+          # Rerun test failures after the indicator has processed the results.
+          self._MaybeRerun(pool, test)
     finally:
       pool.terminate()
       self._RunPerfSafe(lambda: self.perf_data_manager.close())
       if self.perf_failures:
         # Nuke perf data in case of failures. This might not work on windows as
         # some files might still be open.
         print "Deleting perf test data due to db corruption."
         shutil.rmtree(self.datapath)
     if queued_exception:
       raise queued_exception
@@ skipping 13 matching lines @@
            test.suite.GetFlagsForTestCase(test, self.context) +
            self.context.extra_flags)
     return cmd
 
 
 class BreakNowException(Exception):
   def __init__(self, value):
     self.value = value
   def __str__(self):
     return repr(self.value)