Added tools for finding and purging flaky tests

tools/find_flakies.py

+#!/usr/bin/env python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Finds flaky test cases by sharding and running a test for the specified
+number of times. The data file is read at the beginning of each run to find
+the last known counts and is overwritten at the end of each run with the new
+counts. There is an optional sleep interval between each run so the script can
+be killed without losing the data, useful for overnight (or weekend!) runs.
+"""
+
+
+import optparse
+import os
+import random
+import subprocess
+import sys
+import time
+
+
+FF_USAGE = 'python %prog [options] path/to/test [sharding_supervisor_args]'
+FF_DEFAULT_DATA_SUFFIX = '_flakies'
+FF_DEFAULT_SLEEP_INTERVAL = 10.0
+FF_DEFAULT_NUM_ITERATIONS = 100
+FF_DEFAULT_SUPERVISOR_ARGS = ['-r3', '--random-seed']
+
+
+def FindFlakies(test_path, data_path, supervisor_args):
+  failed_tests = {}
+  if os.path.exists(data_path):
+    data_file = open(data_path, 'r')
+    num_runs = int(data_file.readline().split(' ')[0])
+    num_passes = int(data_file.readline().split(' ')[0])
+    for line in data_file:
+      if line:
+        split_line = line.split(' -> ')
+        failed_tests[split_line[0]] = int(split_line[1])
+    data_file.close()
+  else:
+    num_runs = 0
+    num_passes = 0
+  log_lines = False
+  args = ['python', 'sharding_supervisor/sharding_supervisor.py']
+  args.extend(supervisor_args + [test_path])
+  proc = subprocess.Popen(args, stderr=subprocess.PIPE)
+  while True:
+    line = proc.stderr.readline()
+    if not line:
+      if proc.poll() is not None:
+        break
+      continue
+    sys.stderr.write(line)
+    if log_lines:
+      line = line.rstrip()
+      if line in failed_tests:
+        failed_tests[line] += 1
+      else:
+        failed_tests[line] = 1
+    elif line.find('FAILED TESTS:') >= 0:
+      log_lines = True
+  num_runs += 1
+  if proc.returncode == 0:
+    num_passes += 1
+  data_file = open(data_path, 'w')
+  print '%i runs' % num_runs
+  data_file.write('%i runs\n' % num_runs)
+  print '%i passes' % num_passes
+  data_file.write('%i passes\n' % num_passes)
+  for (test, count) in failed_tests.iteritems():
+    print '%s -> %i' % (test, count)
+    data_file.write('%s -> %i\n' % (test, count))
+  data_file.close()
+
+
+def main():
+  parser = optparse.OptionParser(usage=FF_USAGE)
+  parser.add_option(
+      '--data-path',
+      help='path to i/o file (default is to append "%s" to the end of the'
+      ' test name in the current directory)' % FF_DEFAULT_DATA_SUFFIX)
+  parser.add_option(
+      '--sleep', type='float', default=FF_DEFAULT_SLEEP_INTERVAL,
+      help='sleep interval between each run, useful for overnight runs'
+      ' (default = %i)' % FF_DEFAULT_SLEEP_INTERVAL)
+  parser.add_option(
+      '--iterations', type='int', default=FF_DEFAULT_NUM_ITERATIONS,
+      help='number of times to run the sharded test'
+      ' (default = %i)' % FF_DEFAULT_NUM_ITERATIONS)
+  parser.disable_interspersed_args()
+  (options, args) = parser.parse_args()
+
+  if not args:
+    parser.error('You must specify a path to test!')
+  if not os.path.exists(args[0]):
+    parser.error('%s does not exist!' % args[0])
+
+  if not options.data_path:
+    options.data_path = os.path.basename(args[0]) + FF_DEFAULT_DATA_SUFFIX
+
+  supervisor_args = FF_DEFAULT_SUPERVISOR_ARGS + args[1:]
+
+  for i in range(options.iterations):
+    FindFlakies(args[0], options.data_path, supervisor_args)
+    print 'That was just iteration %i of %i' % (i + 1, options.iterations)
+    time.sleep(options.sleep)
+
+
+if __name__ == '__main__':
+  main()