Add everything from src/tools/isolate r159537.

fix_test_cases.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Runs a test, grab the failures and trace them."""
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+
+import run_test_cases
+
+
+XVFB_PATH = os.path.join('..', '..', 'testing', 'xvfb.py')
+
+
+if sys.platform == 'win32':
+  import msvcrt  # pylint: disable=F0401
+
+  def get_keyboard():
+    """Returns a letter from the keyboard if any.
+
+    This function returns immediately.
+    """
+    if msvcrt.kbhit():
+      return ord(msvcrt.getch())
+
+else:
+  import select
+
+  def get_keyboard():
+    """Returns a letter from the keyboard if any, as soon as he pressed enter.
+
+    This function returns (almost) immediately.
+
+    The library doesn't give a way to just get the initial letter.
+    """
+    if select.select([sys.stdin], [], [], 0.00001)[0]:
+      return sys.stdin.read(1)
+
+
+def trace_and_merge(result, test):
+  """Traces a single test case and merges the result back into .isolate."""
+  env = os.environ.copy()
+  env['RUN_TEST_CASES_RUN_ALL'] = '1'
+
+  print 'Starting trace of %s' % test
+  subprocess.call(
+      [
+        sys.executable, 'isolate.py', 'trace', '-r', result,
+        '--', '--gtest_filter=' + test,
+      ],
+      env=env)
+
+  print 'Starting merge of %s' % test
+  return not subprocess.call(
+      [sys.executable, 'isolate.py', 'merge', '-r', result])
+
+
+def run_all(result, shard_index, shard_count):
+  """Runs all the tests. Returns the tests that failed or None on failure.
+
+  Assumes run_test_cases.py is implicitly called.
+  """
+  handle, result_file = tempfile.mkstemp(prefix='run_test_cases')
+  os.close(handle)
+  env = os.environ.copy()
+  env['RUN_TEST_CASES_RESULT_FILE'] = result_file
+  env['RUN_TEST_CASES_RUN_ALL'] = '1'
+  env['GTEST_SHARD_INDEX'] = str(shard_index)
+  env['GTEST_TOTAL_SHARDS'] = str(shard_count)
+  cmd = [sys.executable, 'isolate.py', 'run', '-r', result]
+  subprocess.call(cmd, env=env)
+  if not os.path.isfile(result_file):
+    print >> sys.stderr, 'Failed to find %s' % result_file
+    return None
+  with open(result_file) as f:
+    try:
+      data = json.load(f)
+    except ValueError as e:
+      print >> sys.stderr, ('Unable to load json file, %s: %s' %
+                            (result_file, str(e)))
+      return None
+  os.remove(result_file)
+  return [
+    test for test, runs in data.iteritems()
+    if not any(not run['returncode'] for run in runs)
+  ]
+
+
+def run(result, test):
+  """Runs a single test case in an isolated environment.
+
+  Returns True if the test passed.
+  """
+  return not subprocess.call([
+    sys.executable, 'isolate.py', 'run', '-r', result,
+    '--', '--gtest_filter=' + test,
+  ])
+
+
+def run_normally(executable, test):
+  return not subprocess.call([
+    sys.executable, XVFB_PATH, os.path.dirname(executable), executable,
+    '--gtest_filter=' + test])
+
+
+def diff_and_commit(test):
+  """Prints the diff and commit."""
+  subprocess.call(['git', 'diff'])
+  subprocess.call(['git', 'commit', '-a', '-m', test])
+
+
+def trace_and_verify(result, test):
+  """Traces a test case, updates .isolate and makes sure it passes afterward.
+
+  Return None if the test was already passing,  True on success.
+  """
+  trace_and_merge(result, test)
+  diff_and_commit(test)
+  print 'Verifying trace...'
+  return run(result, test)
+
+
+def fix_all(result, shard_index, shard_count, executable):
+  """Runs all the test cases in a gtest executable and trace the failing tests.
+
+  Returns True on success.
+
+  Makes sure the test passes afterward.
+  """
+  # These could have adverse side-effects.
+  # TODO(maruel): Be more intelligent about it, for now be safe.
+  run_test_cases_env = ['RUN_TEST_CASES_RESULT_FILE', 'RUN_TEST_CASES_RUN_ALL']
+  for i in run_test_cases.KNOWN_GTEST_ENV_VARS + run_test_cases_env:
+    if i in os.environ:
+      print >> 'Please unset %s' % i
+      return False
+
+  test_cases = run_all(result, shard_index, shard_count)
+  if test_cases is None:
+    return False
+
+  print '\nFound %d broken test cases.' % len(test_cases)
+  if not test_cases:
+    return True
+
+  failed_alone = []
+  failures = []
+  fixed_tests = []
+  try:
+    for index, test_case in enumerate(test_cases):
+      if get_keyboard():
+        # Return early.
+        return True
+
+      try:
+        # Check if the test passes normally, because otherwise there is no
+        # reason to trace its failure.
+        if not run_normally(executable, test_case):
+          print '%s is broken when run alone, please fix the test.' % test_case
+          failed_alone.append(test_case)
+          continue
+
+        if not trace_and_verify(result, test_case):
+          failures.append(test_case)
+          print 'Failed to fix %s' % test_case
+        else:
+          fixed_tests.append(test_case)
+      except:  # pylint: disable=W0702
+        failures.append(test_case)
+        print 'Failed to fix %s' % test_case
+      print '%d/%d' % (index+1, len(test_cases))
+  finally:
+    print 'Test cases fixed (%d):' % len(fixed_tests)
+    for fixed_test in fixed_tests:
+      print '  %s' % fixed_test
+    print ''
+
+    print 'Test cases still failing (%d):' % len(failures)
+    for failure in failures:
+      print '  %s' % failure
+
+    if failed_alone:
+      print ('Test cases that failed normally when run alone (%d):' %
+             len(failed_alone))
+      for failed in failed_alone:
+        print failed
+  return not failures
+
+
+def main():
+  parser = run_test_cases.OptionParserWithTestSharding(
+      usage='%prog <option> [test]')
+  parser.add_option('-d', '--dir', default='../../out/Release',
+                    help='The directory containing the the test executable and '
+                    'result file. Defaults to %default')
+  options, args = parser.parse_args()
+
+  if len(args) != 1:
+    parser.error('Use with the name of the test only, e.g. unit_tests')
+
+  basename = args[0]
+  executable = os.path.join(options.dir, basename)
+  result = '%s.results' % executable
+  if sys.platform in('win32', 'cygwin'):
+    executable += '.exe'
+  if not os.path.isfile(executable):
+    print >> sys.stderr, (
+        '%s doesn\'t exist, please build %s_run' % (executable, basename))
+    return 1
+  if not os.path.isfile(result):
+    print >> sys.stderr, (
+        '%s doesn\'t exist, please build %s_run' % (result, basename))
+    return 1
+
+  return not fix_all(result, options.index, options.shards, executable)
+
+
+if __name__ == '__main__':
+  sys.exit(main())