| Index: swarm_client/googletest/fix_test_cases.py
|
| ===================================================================
|
| --- swarm_client/googletest/fix_test_cases.py (revision 235167)
|
| +++ swarm_client/googletest/fix_test_cases.py (working copy)
|
| @@ -1,243 +0,0 @@
|
| -#!/usr/bin/env python
|
| -# Copyright 2013 The Chromium Authors. All rights reserved.
|
| -# Use of this source code is governed by a BSD-style license that can be
|
| -# found in the LICENSE file.
|
| -
|
| -"""Runs through isolate_test_cases.py all the tests cases in a google-test
|
| -executable, grabs the failures and traces them to generate a new .isolate.
|
| -
|
| -This scripts requires a .isolated file. This file is generated from a .isolate
|
| -file. You can use 'GYP_DEFINES=test_isolation_mode=check ninja foo_test_run' to
|
| -generate it.
|
| -"""
|
| -
|
| -import json
|
| -import logging
|
| -import os
|
| -import subprocess
|
| -import sys
|
| -import tempfile
|
| -
|
| -ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
| -if not ROOT_DIR in sys.path:
|
| - sys.path.insert(0, ROOT_DIR)
|
| -
|
| -import isolate
|
| -import isolate_test_cases
|
| -import run_test_cases
|
| -from utils import tools
|
| -
|
| -
|
| -def with_tempfile(function):
|
| - """Creates a temporary file and calls the inner function."""
|
| - def hook(*args, **kwargs):
|
| - handle, tempfilepath = tempfile.mkstemp(prefix='fix_test_cases')
|
| - os.close(handle)
|
| - try:
|
| - return function(tempfilepath, *args, **kwargs)
|
| - finally:
|
| - try:
|
| - os.remove(tempfilepath)
|
| - except OSError, e:
|
| - print >> sys.stderr, 'Failed to remove %s: %s' % (tempfilepath, e)
|
| - return hook
|
| -
|
| -
|
| -def load_run_test_cases_results(run_test_cases_file):
|
| - """Loads a .run_test_cases result file.
|
| -
|
| - Returns a tuple of two lists, (success, failures).
|
| - """
|
| - if not os.path.isfile(run_test_cases_file):
|
| - print >> sys.stderr, 'Failed to find %s' % run_test_cases_file
|
| - return None, None
|
| - with open(run_test_cases_file) as f:
|
| - try:
|
| - data = json.load(f)
|
| - except ValueError as e:
|
| - print >> sys.stderr, ('Unable to load json file, %s: %s' %
|
| - (run_test_cases_file, str(e)))
|
| - return None, None
|
| - failure = [
|
| - test for test, runs in data['test_cases'].iteritems()
|
| - if not any(run['returncode'] == 0 for run in runs)
|
| - ]
|
| - success = [
|
| - test for test, runs in data['test_cases'].iteritems()
|
| - if any(run['returncode'] == 0 for run in runs)
|
| - ]
|
| - return success, failure
|
| -
|
| -
|
| -def add_verbosity(cmd, verbosity):
|
| - """Adds --verbose flags to |cmd| depending on verbosity."""
|
| - if verbosity:
|
| - cmd.append('--verbose')
|
| - if verbosity > 1:
|
| - cmd.append('--verbose')
|
| -
|
| -
|
| -# This function requires 2 temporary files.
|
| -@with_tempfile
|
| -@with_tempfile
|
| -def run_tests(
|
| - tempfilepath_cases, tempfilepath_result, isolated, test_cases, verbosity):
|
| - """Runs all the test cases in an isolated environment."""
|
| - with open(tempfilepath_cases, 'w') as f:
|
| - f.write('\n'.join(test_cases))
|
| - cmd = [
|
| - sys.executable, os.path.join(ROOT_DIR, 'isolate.py'),
|
| - 'run',
|
| - '--isolated', isolated,
|
| - ]
|
| - # Make sure isolate.py is verbose.
|
| - add_verbosity(cmd, verbosity)
|
| - cmd += [
|
| - '--',
|
| - # This assumes run_test_cases.py is used.
|
| - '--result', tempfilepath_result,
|
| - '--test-case-file', tempfilepath_cases,
|
| - # Do not retry; it's faster to trace flaky test than retrying each failing
|
| - # tests 3 times.
|
| - # Do not use --run-all, iterate multiple times instead.
|
| - '--retries', '0',
|
| - # Trace at most 25 test cases at a time. While this may seem particularly
|
| - # small, it is because the tracer doesn't scale well on Windows and tends to
|
| - # generate multi-gigabytes data files, that needs to be read N-times the
|
| - # number of test cases. On linux and OSX it's not that much a big deal but
|
| - # if there's 25 test cases that are broken, it's likely that there's a core
|
| - # file missing anyway.
|
| - '--max-failures', '25',
|
| - ]
|
| - # Make sure run_test_cases.py is verbose.
|
| - add_verbosity(cmd, verbosity)
|
| - logging.debug(cmd)
|
| - retcode = subprocess.call(cmd)
|
| - success, failures = load_run_test_cases_results(tempfilepath_result)
|
| - # Returning non-zero must match having failures.
|
| - assert bool(retcode) == (failures is None or bool(failures))
|
| - return success, failures
|
| -
|
| -
|
| -@with_tempfile
|
| -def trace_some(tempfilepath, isolated, test_cases, trace_blacklist, verbosity):
|
| - """Traces the test cases."""
|
| - with open(tempfilepath, 'w') as f:
|
| - f.write('\n'.join(test_cases))
|
| - cmd = [
|
| - sys.executable, os.path.join(
|
| - ROOT_DIR, 'googletest', 'isolate_test_cases.py'),
|
| - '--isolated', isolated,
|
| - '--test-case-file', tempfilepath,
|
| - # Do not use --run-all here, we assume the test cases will pass inside the
|
| - # checkout.
|
| - ]
|
| - for i in trace_blacklist:
|
| - cmd.extend(('--trace-blacklist', i))
|
| - add_verbosity(cmd, verbosity)
|
| - logging.debug(cmd)
|
| - return subprocess.call(cmd)
|
| -
|
| -
|
| -def fix_all(isolated, all_test_cases, trace_blacklist, verbosity):
|
| - """Runs all the test cases in a gtest executable and trace the failing tests.
|
| -
|
| - Returns True on success.
|
| -
|
| - Makes sure the test passes afterward.
|
| - """
|
| - # These environment variables could have adverse side-effects.
|
| - # TODO(maruel): Be more intelligent about it, for now be safe.
|
| - env_blacklist = set(run_test_cases.KNOWN_GTEST_ENV_VARS) - set([
|
| - 'GTEST_SHARD_INDEX', 'GTEST_TOTAL_SHARDS'])
|
| - for i in env_blacklist:
|
| - if i in os.environ:
|
| - print >> sys.stderr, 'Please unset %s' % i
|
| - return False
|
| -
|
| - # Run until test cases remain to be tested.
|
| - remaining_test_cases = all_test_cases[:]
|
| - if not remaining_test_cases:
|
| - print >> sys.stderr, 'Didn\'t find any test case to run'
|
| - return 1
|
| -
|
| - previous_failures = set()
|
| - had_failure = False
|
| - while remaining_test_cases:
|
| - # pylint is confused about with_tempfile.
|
| - # pylint: disable=E1120
|
| - print(
|
| - '\nTotal: %5d; Remaining: %5d' % (
|
| - len(all_test_cases), len(remaining_test_cases)))
|
| - success, failures = run_tests(isolated, remaining_test_cases, verbosity)
|
| - if success is None:
|
| - if had_failure:
|
| - print >> sys.stderr, 'Failed to run test cases'
|
| - return 1
|
| - # Maybe there's even enough things mapped to start the child process.
|
| - logging.info('Failed to run, trace one test case.')
|
| - had_failure = True
|
| - success = []
|
| - failures = [remaining_test_cases[0]]
|
| - print(
|
| - '\nTotal: %5d; Tried to run: %5d; Ran: %5d; Succeeded: %5d; Failed: %5d'
|
| - % (
|
| - len(all_test_cases),
|
| - len(remaining_test_cases),
|
| - len(success) + len(failures),
|
| - len(success),
|
| - len(failures)))
|
| -
|
| - if not failures:
|
| - print('I\'m done. Have a nice day!')
|
| - return True
|
| -
|
| - previous_failures.difference_update(success)
|
| - # If all the failures had already failed at least once.
|
| - if previous_failures.issuperset(failures):
|
| - print('The last trace didn\'t help, aborting.')
|
| - return False
|
| -
|
| - # Test cases that passed to not need to be retried anymore.
|
| - remaining_test_cases = [
|
| - i for i in remaining_test_cases if i not in success and i not in failures
|
| - ]
|
| - # Make sure the failures at put at the end. This way if some tests fails
|
| - # simply because they are broken, and not because of test isolation, the
|
| - # other tests will still be traced.
|
| - remaining_test_cases.extend(failures)
|
| -
|
| - # Trace the test cases and update the .isolate file.
|
| - print('\nTracing the %d failing tests.' % len(failures))
|
| - if trace_some(isolated, failures, trace_blacklist, verbosity):
|
| - logging.info('The tracing itself failed.')
|
| - return False
|
| - previous_failures.update(failures)
|
| -
|
| -
|
| -def main():
|
| - tools.disable_buffering()
|
| - parser = run_test_cases.OptionParserTestCases(
|
| - usage='%prog <options> -s <something.isolated>')
|
| - isolate.add_trace_option(parser)
|
| - parser.add_option(
|
| - '-s', '--isolated',
|
| - help='The isolated file')
|
| - options, args = parser.parse_args()
|
| -
|
| - if args:
|
| - parser.error('Unsupported arg: %s' % args)
|
| - isolate.parse_isolated_option(parser, options, os.getcwd(), True)
|
| -
|
| - _, command, test_cases = isolate_test_cases.safely_load_isolated(
|
| - parser, options)
|
| - if not command:
|
| - parser.error('A command must be defined')
|
| - if not test_cases:
|
| - parser.error('No test case to run')
|
| - return not fix_all(
|
| - options.isolated, test_cases, options.trace_blacklist, options.verbose)
|
| -
|
| -
|
| -if __name__ == '__main__':
|
| - sys.exit(main())
|
|
|