Added tools for finding and purging flaky tests

tools/purge_flakies.py

+#!/usr/bin/env python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Runs all the test cases in a given test in parallel with itself, to get at

[Paweł Hajdan Jr., 2011/08/19 18:35:50]

Would it make sense to combine this in a single file with the previous tool?
Requiring people to run two scripts is more complicated than saying "run this
one script, it will take a lot of time but will finally print the useful
results". It also makes running it overnight more effective.

[clee, 2011/08/19 21:06:35]

Ideally yes, but the command line options to the two scripts are different, and
the formats of the output files are different. It's very easy to write something
that just runs the two procedures with default options, but quite confusing to
allow argument specification. I'll just do it with default arguments here, and
then you can call the functions directly to change them. 

+those that hold on to shared resources. The idea is that if a test uses a
+unary resource, then running many instances of this test will purge out some
+of them as failures or timeouts.
+"""
+
+
+import optparse
+import os
+import re
+import subprocess
+import time
+
+
+PF_USAGE = 'python %prog [options] path/to/test'
+PF_DEFAULT_OUTPUT_SUFFIX = '_purges'
+PF_DEFAULT_NUM_PROCS = 20
+PF_DEFAULT_NUM_REPEATS = 10
+PF_DEFAULT_TIMEOUT = 600
+
+
+def PurgeFlakies(test_path, output_path, num_procs, num_repeats, timeout):
+  test_name_regex = r'((\w+/)?\w+\.\w+(/\d+)?)'
+  test_start = re.compile('\[\s+RUN\s+\] ' + test_name_regex)
+  test_list = []
+
+  # run the test to discover all the test cases
+  proc = subprocess.Popen([test_path], stdout=subprocess.PIPE)
+  while True:
+    line = proc.stdout.readline()
+    if not line:
+      if proc.poll() is not None:
+        break
+      continue
+    print line.rstrip()
+    results = test_start.search(line)
+    if results:
+      test_list.append(results.group(1))
+
+  failures = []
+  index = 0
+  total = len(test_list)
+
+  # run each test case in parallel with itself
+  for test_name in test_list:
+    num_fails = 0
+    num_terminated = 0
+    procs = []
+    args = [test_path, '--gtest_filter=' + test_name,
+            '--gtest_repeat=%i' % num_repeats]
+    while len(procs) < num_procs:
+      procs.append(subprocess.Popen(args))
+    seconds = 0
+    while procs:
+      for proc in procs:
+        if proc.poll() is not None:
+          if proc.returncode != 0:
+            ++num_fails
+          procs.remove(proc)
+      # timeout exceeded, kill the remaining processes and make a note
+      if seconds > timeout:
+        num_fails += len(procs)
+        num_terminated = len(procs)
+        while procs:
+          procs.pop().terminate()
+      time.sleep(1.0)
+      seconds += 1
+    if num_fails:
+      line = '%s: %i failed' % (test_name, num_fails)
+      if num_terminated:
+        line += ' (%i terminated)' % num_terminated
+      failures.append(line)
+    print '%s (%i / %i): %i failed' % (test_name, index, total, num_fails)
+    index += 1
+    time.sleep(1.0)
+
+  # print the results and write the data file
+  print failures
+  data_file = open(output_path, 'w')
+  for line in failures:
+    data_file.write(line + '\n')
+  data_file.close()
+
+
+def main():
+  parser = optparse.OptionParser(usage=PF_USAGE)
+  parser.add_option(
+      '--output-path',
+      help='path to output file (default is to append "%s" to the end of the'
+      ' test name in the current directory)' % PF_DEFAULT_OUTPUT_SUFFIX)
+  parser.add_option(
+      '--procs', type='int', default=PF_DEFAULT_NUM_PROCS,
+      help='number of parallel test processes to start up'
+      ' (default = %s)' % PF_DEFAULT_NUM_PROCS)
+  parser.add_option(
+      '--repeats', type='int', default=PF_DEFAULT_NUM_REPEATS,
+      help='number of times to repeat each test in each process'
+      ' (default = %s)' % PF_DEFAULT_NUM_REPEATS)
+  parser.add_option(
+      '--timeout', type='int', default=PF_DEFAULT_TIMEOUT,
+      help='test processes are killed if their runtimes exceed the timeout,'
+      ' useful for some tests (default = %s)' % PF_DEFAULT_TIMEOUT)
+  (options, args) = parser.parse_args()
+
+  if not args:
+    parser.error('You must specify a path to test!')
+  if not os.path.exists(args[0]):
+    parser.error('%s does not exist!' % args[0])
+
+  if not options.output_path:
+    options.output_path = os.path.basename(args[0]) + PF_DEFAULT_OUTPUT_SUFFIX
+
+  PurgeFlakies(args[0], options.output_path, options.procs, options.repeats,
+               options.timeout)
+
+
+if __name__ == '__main__':
+  main()