Implemented a flaky test runner for auto-bisect bot.

tools/flakiness/is_flaky_test.py

Index: tools/flakiness/is_flaky_test.py
diff --git a/tools/flakiness/is_flaky_test.py b/tools/flakiness/is_flaky_test.py
new file mode 100755
index 0000000000000000000000000000000000000000..e748ec8c203e024d8290bbeaa948e0e187be63c8
--- /dev/null
+++ b/tools/flakiness/is_flaky_test.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Runs a test repeatedly to measure its flakiness. The return code is non-zero
+if the failure rate is higher than the specified threshold, but is not 100%."""
+
+import argparse
+import subprocess
+import sys
+import time
+
+def load_options():
+  parser = argparse.ArgumentParser(description=__doc__)
+  parser.add_argument('--retries', default=1000, type=int,
+                      help='Number of test retries to measure flakiness.')
+  parser.add_argument('--threshold', default=0.05, type=float,
+                      help='Minimum flakiness level at which test is '
+                           'considered flaky.')
+  parser.add_argument('--jobs', '-j', type=int, default=1,
+                      help='Number of parallel jobs to run tests.')
+  parser.add_argument('command', nargs='+', help='Command to run test.')
+  return parser.parse_args()
+
+
+def process_finished(running, num_passed, num_failed):
+  finished = [p for p in running if p.poll() is not None]
+  running[:] = [p for p in running if p.poll() is None]

[qyearsley, 2014/09/15 16:02:28]

What's the difference between using slice assignment here and not? i.e. what's
the difference between the following two lines?

  running[:] = [p for p in running if p.poll() is None]
  running = [p for p in running if p.poll() is None]

[Sergiy Byelozyorov, 2014/09/15 16:27:29]

This is to modify the value outside of the function too. Just assignment would
just replace the parameter with a different object, while in main the list of
running jobs would still contain jobs that are finished (and thus loop will
never quit).

+  num_passed += len([p for p in finished if p.returncode == 0])
+  num_failed += len([p for p in finished if p.returncode != 0])
+  print '%d processed finished. Total passed: %d. Total failed: %d' % (
+      len(finished), num_passed, num_failed)
+  return num_passed, num_failed
+
+
+def main():
+  options = load_options()
+  num_passed = num_failed = 0
+  running = []
+
+  # Start all retries, while limiting total number of running processes.
+  for attempt in range(options.retries):
+    print 'Starting retry %d out of %d\n' % (attempt + 1, options.retries)
+    running.append(subprocess.Popen(options.command, stdout=subprocess.PIPE,
+                                    stderr=subprocess.STDOUT))
+    while len(running) >= options.jobs:
+      print 'Waiting for previous retries to finish before starting new ones...'
+      time.sleep(0.1)
+      num_passed, num_failed = process_finished(running, num_passed, num_failed)
+
+
+  # Wait for the remaining retries to finish.
+  print 'Waiting for the remaining retries to finish...'
+  for process in running:
+    process.wait()
+
+  num_passed, num_failed = process_finished(running, num_passed, num_failed)
+  if num_passed == 0 or num_failed == 0:
+    flakiness = 0
+  else:
+    flakiness = num_failed / (options.retries * 1.0)

[qyearsley, 2014/09/15 16:02:28]

Equivalently, float(options.retries) would do the same thing.

[Sergiy Byelozyorov, 2014/09/15 16:27:29]

Done.

+
+  print 'Flakiness is %.2f' % flakiness
+  if flakiness > options.threshold:
+    return 1
+  else:
+    return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())