Index: swarm_client/googletest/fix_test_cases.py |
=================================================================== |
--- swarm_client/googletest/fix_test_cases.py (revision 235167) |
+++ swarm_client/googletest/fix_test_cases.py (working copy) |
@@ -1,243 +0,0 @@ |
-#!/usr/bin/env python |
-# Copyright 2013 The Chromium Authors. All rights reserved. |
-# Use of this source code is governed by a BSD-style license that can be |
-# found in the LICENSE file. |
- |
-"""Runs through isolate_test_cases.py all the tests cases in a google-test |
-executable, grabs the failures and traces them to generate a new .isolate. |
- |
-This scripts requires a .isolated file. This file is generated from a .isolate |
-file. You can use 'GYP_DEFINES=test_isolation_mode=check ninja foo_test_run' to |
-generate it. |
-""" |
- |
-import json |
-import logging |
-import os |
-import subprocess |
-import sys |
-import tempfile |
- |
-ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
-if not ROOT_DIR in sys.path: |
- sys.path.insert(0, ROOT_DIR) |
- |
-import isolate |
-import isolate_test_cases |
-import run_test_cases |
-from utils import tools |
- |
- |
-def with_tempfile(function): |
- """Creates a temporary file and calls the inner function.""" |
- def hook(*args, **kwargs): |
- handle, tempfilepath = tempfile.mkstemp(prefix='fix_test_cases') |
- os.close(handle) |
- try: |
- return function(tempfilepath, *args, **kwargs) |
- finally: |
- try: |
- os.remove(tempfilepath) |
- except OSError, e: |
- print >> sys.stderr, 'Failed to remove %s: %s' % (tempfilepath, e) |
- return hook |
- |
- |
-def load_run_test_cases_results(run_test_cases_file): |
- """Loads a .run_test_cases result file. |
- |
- Returns a tuple of two lists, (success, failures). |
- """ |
- if not os.path.isfile(run_test_cases_file): |
- print >> sys.stderr, 'Failed to find %s' % run_test_cases_file |
- return None, None |
- with open(run_test_cases_file) as f: |
- try: |
- data = json.load(f) |
- except ValueError as e: |
- print >> sys.stderr, ('Unable to load json file, %s: %s' % |
- (run_test_cases_file, str(e))) |
- return None, None |
- failure = [ |
- test for test, runs in data['test_cases'].iteritems() |
- if not any(run['returncode'] == 0 for run in runs) |
- ] |
- success = [ |
- test for test, runs in data['test_cases'].iteritems() |
- if any(run['returncode'] == 0 for run in runs) |
- ] |
- return success, failure |
- |
- |
-def add_verbosity(cmd, verbosity): |
- """Adds --verbose flags to |cmd| depending on verbosity.""" |
- if verbosity: |
- cmd.append('--verbose') |
- if verbosity > 1: |
- cmd.append('--verbose') |
- |
- |
-# This function requires 2 temporary files. |
-@with_tempfile |
-@with_tempfile |
-def run_tests( |
- tempfilepath_cases, tempfilepath_result, isolated, test_cases, verbosity): |
- """Runs all the test cases in an isolated environment.""" |
- with open(tempfilepath_cases, 'w') as f: |
- f.write('\n'.join(test_cases)) |
- cmd = [ |
- sys.executable, os.path.join(ROOT_DIR, 'isolate.py'), |
- 'run', |
- '--isolated', isolated, |
- ] |
- # Make sure isolate.py is verbose. |
- add_verbosity(cmd, verbosity) |
- cmd += [ |
- '--', |
- # This assumes run_test_cases.py is used. |
- '--result', tempfilepath_result, |
- '--test-case-file', tempfilepath_cases, |
- # Do not retry; it's faster to trace flaky test than retrying each failing |
- # tests 3 times. |
- # Do not use --run-all, iterate multiple times instead. |
- '--retries', '0', |
- # Trace at most 25 test cases at a time. While this may seem particularly |
- # small, it is because the tracer doesn't scale well on Windows and tends to |
- # generate multi-gigabytes data files, that needs to be read N-times the |
- # number of test cases. On linux and OSX it's not that much a big deal but |
- # if there's 25 test cases that are broken, it's likely that there's a core |
- # file missing anyway. |
- '--max-failures', '25', |
- ] |
- # Make sure run_test_cases.py is verbose. |
- add_verbosity(cmd, verbosity) |
- logging.debug(cmd) |
- retcode = subprocess.call(cmd) |
- success, failures = load_run_test_cases_results(tempfilepath_result) |
- # Returning non-zero must match having failures. |
- assert bool(retcode) == (failures is None or bool(failures)) |
- return success, failures |
- |
- |
-@with_tempfile |
-def trace_some(tempfilepath, isolated, test_cases, trace_blacklist, verbosity): |
- """Traces the test cases.""" |
- with open(tempfilepath, 'w') as f: |
- f.write('\n'.join(test_cases)) |
- cmd = [ |
- sys.executable, os.path.join( |
- ROOT_DIR, 'googletest', 'isolate_test_cases.py'), |
- '--isolated', isolated, |
- '--test-case-file', tempfilepath, |
- # Do not use --run-all here, we assume the test cases will pass inside the |
- # checkout. |
- ] |
- for i in trace_blacklist: |
- cmd.extend(('--trace-blacklist', i)) |
- add_verbosity(cmd, verbosity) |
- logging.debug(cmd) |
- return subprocess.call(cmd) |
- |
- |
-def fix_all(isolated, all_test_cases, trace_blacklist, verbosity): |
- """Runs all the test cases in a gtest executable and trace the failing tests. |
- |
- Returns True on success. |
- |
- Makes sure the test passes afterward. |
- """ |
- # These environment variables could have adverse side-effects. |
- # TODO(maruel): Be more intelligent about it, for now be safe. |
- env_blacklist = set(run_test_cases.KNOWN_GTEST_ENV_VARS) - set([ |
- 'GTEST_SHARD_INDEX', 'GTEST_TOTAL_SHARDS']) |
- for i in env_blacklist: |
- if i in os.environ: |
- print >> sys.stderr, 'Please unset %s' % i |
- return False |
- |
- # Run until test cases remain to be tested. |
- remaining_test_cases = all_test_cases[:] |
- if not remaining_test_cases: |
- print >> sys.stderr, 'Didn\'t find any test case to run' |
- return 1 |
- |
- previous_failures = set() |
- had_failure = False |
- while remaining_test_cases: |
- # pylint is confused about with_tempfile. |
- # pylint: disable=E1120 |
- print( |
- '\nTotal: %5d; Remaining: %5d' % ( |
- len(all_test_cases), len(remaining_test_cases))) |
- success, failures = run_tests(isolated, remaining_test_cases, verbosity) |
- if success is None: |
- if had_failure: |
- print >> sys.stderr, 'Failed to run test cases' |
- return 1 |
- # Maybe there's even enough things mapped to start the child process. |
- logging.info('Failed to run, trace one test case.') |
- had_failure = True |
- success = [] |
- failures = [remaining_test_cases[0]] |
- print( |
- '\nTotal: %5d; Tried to run: %5d; Ran: %5d; Succeeded: %5d; Failed: %5d' |
- % ( |
- len(all_test_cases), |
- len(remaining_test_cases), |
- len(success) + len(failures), |
- len(success), |
- len(failures))) |
- |
- if not failures: |
- print('I\'m done. Have a nice day!') |
- return True |
- |
- previous_failures.difference_update(success) |
- # If all the failures had already failed at least once. |
- if previous_failures.issuperset(failures): |
- print('The last trace didn\'t help, aborting.') |
- return False |
- |
- # Test cases that passed to not need to be retried anymore. |
- remaining_test_cases = [ |
- i for i in remaining_test_cases if i not in success and i not in failures |
- ] |
- # Make sure the failures at put at the end. This way if some tests fails |
- # simply because they are broken, and not because of test isolation, the |
- # other tests will still be traced. |
- remaining_test_cases.extend(failures) |
- |
- # Trace the test cases and update the .isolate file. |
- print('\nTracing the %d failing tests.' % len(failures)) |
- if trace_some(isolated, failures, trace_blacklist, verbosity): |
- logging.info('The tracing itself failed.') |
- return False |
- previous_failures.update(failures) |
- |
- |
-def main(): |
- tools.disable_buffering() |
- parser = run_test_cases.OptionParserTestCases( |
- usage='%prog <options> -s <something.isolated>') |
- isolate.add_trace_option(parser) |
- parser.add_option( |
- '-s', '--isolated', |
- help='The isolated file') |
- options, args = parser.parse_args() |
- |
- if args: |
- parser.error('Unsupported arg: %s' % args) |
- isolate.parse_isolated_option(parser, options, os.getcwd(), True) |
- |
- _, command, test_cases = isolate_test_cases.safely_load_isolated( |
- parser, options) |
- if not command: |
- parser.error('A command must be defined') |
- if not test_cases: |
- parser.error('No test case to run') |
- return not fix_all( |
- options.isolated, test_cases, options.trace_blacklist, options.verbose) |
- |
- |
-if __name__ == '__main__': |
- sys.exit(main()) |