| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # | 2 # |
| 3 # Copyright 2013 The Chromium Authors. All rights reserved. | 3 # Copyright 2013 The Chromium Authors. All rights reserved. |
| 4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
| 5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
| 6 | 6 |
| 7 """Runs all types of tests from one unified interface. | 7 """Runs all types of tests from one unified interface. |
| 8 | 8 |
| 9 TODO(gkanwar): | 9 TODO(gkanwar): |
| 10 * Add options to run Monkey tests. | 10 * Add options to run Monkey tests. |
| 11 """ | 11 """ |
| 12 | 12 |
| 13 import collections | 13 import collections |
| 14 import optparse | 14 import optparse |
| 15 import os | 15 import os |
| 16 import shutil | 16 import shutil |
| 17 import sys | 17 import sys |
| 18 | 18 |
| 19 from pylib import cmd_helper | 19 from pylib import cmd_helper |
| 20 from pylib import constants | 20 from pylib import constants |
| 21 from pylib import ports | 21 from pylib import ports |
| 22 from pylib.base import base_test_result | 22 from pylib.base import base_test_result |
| 23 from pylib.base import test_dispatcher | 23 from pylib.base import test_dispatcher |
| 24 from pylib.browsertests import setup as browsertests_setup | |
| 25 from pylib.gtest import setup as gtest_setup | 24 from pylib.gtest import setup as gtest_setup |
| 26 from pylib.gtest import gtest_config | 25 from pylib.gtest import gtest_config |
| 27 from pylib.host_driven import run_python_tests as python_dispatch | 26 from pylib.host_driven import run_python_tests as python_dispatch |
| 28 from pylib.instrumentation import setup as instrumentation_setup | 27 from pylib.instrumentation import setup as instrumentation_setup |
| 29 from pylib.uiautomator import setup as uiautomator_setup | 28 from pylib.uiautomator import setup as uiautomator_setup |
| 30 from pylib.utils import report_results | 29 from pylib.utils import report_results |
| 31 from pylib.utils import run_tests_helper | 30 from pylib.utils import run_tests_helper |
| 32 | 31 |
| 33 | 32 |
| 34 _SDK_OUT_DIR = os.path.join(constants.DIR_SOURCE_ROOT, 'out') | 33 _SDK_OUT_DIR = os.path.join(constants.DIR_SOURCE_ROOT, 'out') |
| (...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 90 option_parser.add_option('-d', '--device', dest='test_device', | 89 option_parser.add_option('-d', '--device', dest='test_device', |
| 91 help=('Target device for the test suite ' | 90 help=('Target device for the test suite ' |
| 92 'to run on.')) | 91 'to run on.')) |
| 93 | 92 |
| 94 | 93 |
| 95 def ProcessCommonOptions(options): | 94 def ProcessCommonOptions(options): |
| 96 """Processes and handles all common options.""" | 95 """Processes and handles all common options.""" |
| 97 run_tests_helper.SetLogLevel(options.verbose_count) | 96 run_tests_helper.SetLogLevel(options.verbose_count) |
| 98 | 97 |
| 99 | 98 |
| 100 def AddCoreGTestOptions(option_parser): | |
| 101 """Add options specific to the gtest framework to |option_parser|.""" | |
| 102 | |
| 103 # TODO(gkanwar): Consolidate and clean up test filtering for gtests and | |
| 104 # content_browsertests. | |
| 105 option_parser.add_option('-f', '--gtest_filter', dest='test_filter', | |
| 106 help='googletest-style filter string.') | |
| 107 option_parser.add_option('-a', '--test_arguments', dest='test_arguments', | |
| 108 help='Additional arguments to pass to the test.') | |
| 109 option_parser.add_option('--exe', action='store_true', | |
| 110 help='If set, use the exe test runner instead of ' | |
| 111 'the APK.') | |
| 112 option_parser.add_option('-t', dest='timeout', | |
| 113 help='Timeout to wait for each test', | |
| 114 type='int', | |
| 115 default=60) | |
| 116 | |
| 117 | |
| 118 def AddContentBrowserTestOptions(option_parser): | |
| 119 """Adds Content Browser test options to |option_parser|.""" | |
| 120 | |
| 121 option_parser.usage = '%prog content_browsertests [options]' | |
| 122 option_parser.command_list = [] | |
| 123 option_parser.example = '%prog content_browsertests' | |
| 124 | |
| 125 AddCoreGTestOptions(option_parser) | |
| 126 AddCommonOptions(option_parser) | |
| 127 | |
| 128 | |
| 129 def AddGTestOptions(option_parser): | 99 def AddGTestOptions(option_parser): |
| 130 """Adds gtest options to |option_parser|.""" | 100 """Adds gtest options to |option_parser|.""" |
| 131 | 101 |
| 132 option_parser.usage = '%prog gtest [options]' | 102 option_parser.usage = '%prog gtest [options]' |
| 133 option_parser.command_list = [] | 103 option_parser.command_list = [] |
| 134 option_parser.example = '%prog gtest -s base_unittests' | 104 option_parser.example = '%prog gtest -s base_unittests' |
| 135 | 105 |
| 136 # TODO(gkanwar): Make this option required | 106 # TODO(gkanwar): Make this option required |
| 137 option_parser.add_option('-s', '--suite', dest='suite_name', | 107 option_parser.add_option('-s', '--suite', dest='suite_name', |
| 138 help=('Executable name of the test suite to run ' | 108 help=('Executable name of the test suite to run ' |
| 139 '(use -s help to list them).')) | 109 '(use -s help to list them).')) |
| 140 AddCoreGTestOptions(option_parser) | 110 option_parser.add_option('-f', '--gtest_filter', dest='test_filter', |
| 111 help='googletest-style filter string.') |
| 112 option_parser.add_option('-a', '--test_arguments', dest='test_arguments', |
| 113 help='Additional arguments to pass to the test.') |
| 114 option_parser.add_option('-t', dest='timeout', |
| 115 help='Timeout to wait for each test', |
| 116 type='int', |
| 117 default=60) |
| 141 # TODO(gkanwar): Move these to Common Options once we have the plumbing | 118 # TODO(gkanwar): Move these to Common Options once we have the plumbing |
| 142 # in our other test types to handle these commands | 119 # in our other test types to handle these commands |
| 143 AddCommonOptions(option_parser) | 120 AddCommonOptions(option_parser) |
| 144 | 121 |
| 145 | 122 |
| 146 def ProcessGTestOptions(options): | 123 def ProcessGTestOptions(options): |
| 147 """Intercept test suite help to list test suites. | 124 """Intercept test suite help to list test suites. |
| 148 | 125 |
| 149 Args: | 126 Args: |
| 150 options: Command line options. | 127 options: Command line options. |
| 151 | 128 |
| 152 Returns: | 129 Returns: |
| 153 True if the command should continue. | 130 True if the command should continue. |
| 154 """ | 131 """ |
| 155 if options.suite_name == 'help': | 132 if options.suite_name == 'help': |
| 156 print 'Available test suites are:' | 133 print 'Available test suites are:' |
| 157 for test_suite in gtest_config.STABLE_TEST_SUITES: | 134 for test_suite in (gtest_config.STABLE_TEST_SUITES + |
| 158 print test_suite.name | 135 gtest_config.EXPERIMENTAL_TEST_SUITES): |
| 136 print test_suite |
| 159 return False | 137 return False |
| 160 | 138 |
| 161 # Convert to a list, assuming all test suites if nothing was specified. | 139 # Convert to a list, assuming all test suites if nothing was specified. |
| 162 # TODO(gkanwar): Require having a test suite | 140 # TODO(gkanwar): Require having a test suite |
| 163 if options.suite_name: | 141 if options.suite_name: |
| 164 options.suite_name = [options.suite_name] | 142 options.suite_name = [options.suite_name] |
| 165 else: | 143 else: |
| 166 options.suite_name = [suite.name | 144 options.suite_name = [s for s in gtest_config.STABLE_TEST_SUITES] |
| 167 for suite in gtest_config.STABLE_TEST_SUITES] | |
| 168 return True | 145 return True |
| 169 | 146 |
| 170 | 147 |
| 171 def AddJavaTestOptions(option_parser): | 148 def AddJavaTestOptions(option_parser): |
| 172 """Adds the Java test options to |option_parser|.""" | 149 """Adds the Java test options to |option_parser|.""" |
| 173 | 150 |
| 174 option_parser.add_option('-f', '--test_filter', dest='test_filter', | 151 option_parser.add_option('-f', '--test_filter', dest='test_filter', |
| 175 help=('Test filter (if not fully qualified, ' | 152 help=('Test filter (if not fully qualified, ' |
| 176 'will run all matches).')) | 153 'will run all matches).')) |
| 177 option_parser.add_option( | 154 option_parser.add_option( |
| (...skipping 165 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 343 | 320 |
| 344 | 321 |
| 345 def _RunGTests(options, error_func): | 322 def _RunGTests(options, error_func): |
| 346 """Subcommand of RunTestsCommands which runs gtests.""" | 323 """Subcommand of RunTestsCommands which runs gtests.""" |
| 347 if not ProcessGTestOptions(options): | 324 if not ProcessGTestOptions(options): |
| 348 return 0 | 325 return 0 |
| 349 | 326 |
| 350 exit_code = 0 | 327 exit_code = 0 |
| 351 for suite_name in options.suite_name: | 328 for suite_name in options.suite_name: |
| 352 runner_factory, tests = gtest_setup.Setup( | 329 runner_factory, tests = gtest_setup.Setup( |
| 353 options.exe, suite_name, options.test_arguments, | 330 suite_name, options.test_arguments, |
| 354 options.timeout, options.cleanup_test_files, options.tool, | 331 options.timeout, options.cleanup_test_files, options.tool, |
| 355 options.build_type, options.push_deps, options.test_filter) | 332 options.build_type, options.push_deps, options.test_filter) |
| 356 | 333 |
| 357 results, test_exit_code = test_dispatcher.RunTests( | 334 results, test_exit_code = test_dispatcher.RunTests( |
| 358 tests, runner_factory, False, options.test_device, | 335 tests, runner_factory, False, options.test_device, |
| 359 shard=True, | 336 shard=True, |
| 360 build_type=options.build_type, | 337 build_type=options.build_type, |
| 361 test_timeout=None, | 338 test_timeout=None, |
| 362 num_retries=options.num_retries) | 339 num_retries=options.num_retries) |
| 363 | 340 |
| 364 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: | 341 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: |
| 365 exit_code = test_exit_code | 342 exit_code = test_exit_code |
| 366 | 343 |
| 367 report_results.LogFull( | 344 report_results.LogFull( |
| 368 results=results, | 345 results=results, |
| 369 test_type='Unit test', | 346 test_type='Unit test', |
| 370 test_package=suite_name, | 347 test_package=suite_name, |
| 371 build_type=options.build_type, | 348 build_type=options.build_type, |
| 372 flakiness_server=options.flakiness_dashboard_server) | 349 flakiness_server=options.flakiness_dashboard_server) |
| 373 | 350 |
| 374 if os.path.isdir(constants.ISOLATE_DEPS_DIR): | 351 if os.path.isdir(constants.ISOLATE_DEPS_DIR): |
| 375 shutil.rmtree(constants.ISOLATE_DEPS_DIR) | 352 shutil.rmtree(constants.ISOLATE_DEPS_DIR) |
| 376 | 353 |
| 377 return exit_code | 354 return exit_code |
| 378 | 355 |
| 379 | 356 |
| 380 def _RunContentBrowserTests(options, error_func): | |
| 381 """Subcommand of RunTestsCommands which runs content_browsertests.""" | |
| 382 runner_factory, tests = browsertests_setup.Setup( | |
| 383 options.test_arguments, options.timeout, options.cleanup_test_files, | |
| 384 options.tool, options.build_type, options.push_deps, | |
| 385 options.test_filter) | |
| 386 | |
| 387 # TODO(nileshagrawal): remove this abnormally long setup timeout once fewer | |
| 388 # files are pushed to the devices for content_browsertests: crbug.com/138275 | |
| 389 setup_timeout = 20 * 60 # 20 minutes | |
| 390 results, exit_code = test_dispatcher.RunTests( | |
| 391 tests, runner_factory, False, options.test_device, | |
| 392 shard=True, | |
| 393 build_type=options.build_type, | |
| 394 test_timeout=None, | |
| 395 setup_timeout=setup_timeout, | |
| 396 num_retries=options.num_retries) | |
| 397 | |
| 398 report_results.LogFull( | |
| 399 results=results, | |
| 400 test_type='Unit test', | |
| 401 test_package=constants.BROWSERTEST_SUITE_NAME, | |
| 402 build_type=options.build_type, | |
| 403 flakiness_server=options.flakiness_dashboard_server) | |
| 404 | |
| 405 if os.path.isdir(constants.ISOLATE_DEPS_DIR): | |
| 406 shutil.rmtree(constants.ISOLATE_DEPS_DIR) | |
| 407 | |
| 408 return exit_code | |
| 409 | |
| 410 | |
| 411 def _RunInstrumentationTests(options, error_func): | 357 def _RunInstrumentationTests(options, error_func): |
| 412 """Subcommand of RunTestsCommands which runs instrumentation tests.""" | 358 """Subcommand of RunTestsCommands which runs instrumentation tests.""" |
| 413 ProcessInstrumentationOptions(options, error_func) | 359 ProcessInstrumentationOptions(options, error_func) |
| 414 | 360 |
| 415 results = base_test_result.TestRunResults() | 361 results = base_test_result.TestRunResults() |
| 416 exit_code = 0 | 362 exit_code = 0 |
| 417 | 363 |
| 418 if options.run_java_tests: | 364 if options.run_java_tests: |
| 419 runner_factory, tests = instrumentation_setup.Setup( | 365 runner_factory, tests = instrumentation_setup.Setup( |
| 420 options.test_apk_path, options.test_apk_jar_path, options.annotations, | 366 options.test_apk_path, options.test_apk_jar_path, options.annotations, |
| (...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 520 | 466 |
| 521 # Check for extra arguments | 467 # Check for extra arguments |
| 522 if len(args) > 2: | 468 if len(args) > 2: |
| 523 option_parser.error('Unrecognized arguments: %s' % (' '.join(args[2:]))) | 469 option_parser.error('Unrecognized arguments: %s' % (' '.join(args[2:]))) |
| 524 return constants.ERROR_EXIT_CODE | 470 return constants.ERROR_EXIT_CODE |
| 525 | 471 |
| 526 ProcessCommonOptions(options) | 472 ProcessCommonOptions(options) |
| 527 | 473 |
| 528 if command == 'gtest': | 474 if command == 'gtest': |
| 529 return _RunGTests(options, option_parser.error) | 475 return _RunGTests(options, option_parser.error) |
| 530 elif command == 'content_browsertests': | |
| 531 return _RunContentBrowserTests(options, option_parser.error) | |
| 532 elif command == 'instrumentation': | 476 elif command == 'instrumentation': |
| 533 return _RunInstrumentationTests(options, option_parser.error) | 477 return _RunInstrumentationTests(options, option_parser.error) |
| 534 elif command == 'uiautomator': | 478 elif command == 'uiautomator': |
| 535 return _RunUIAutomatorTests(options, option_parser.error) | 479 return _RunUIAutomatorTests(options, option_parser.error) |
| 536 else: | 480 else: |
| 537 raise Exception('Unknown test type.') | 481 raise Exception('Unknown test type.') |
| 538 | 482 |
| 539 return exit_code | 483 return exit_code |
| 540 | 484 |
| 541 | 485 |
| (...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 580 return 0 | 524 return 0 |
| 581 | 525 |
| 582 | 526 |
| 583 # Define a named tuple for the values in the VALID_COMMANDS dictionary so the | 527 # Define a named tuple for the values in the VALID_COMMANDS dictionary so the |
| 584 # syntax is a bit prettier. The tuple is two functions: (add options, run | 528 # syntax is a bit prettier. The tuple is two functions: (add options, run |
| 585 # command). | 529 # command). |
| 586 CommandFunctionTuple = collections.namedtuple( | 530 CommandFunctionTuple = collections.namedtuple( |
| 587 'CommandFunctionTuple', ['add_options_func', 'run_command_func']) | 531 'CommandFunctionTuple', ['add_options_func', 'run_command_func']) |
| 588 VALID_COMMANDS = { | 532 VALID_COMMANDS = { |
| 589 'gtest': CommandFunctionTuple(AddGTestOptions, RunTestsCommand), | 533 'gtest': CommandFunctionTuple(AddGTestOptions, RunTestsCommand), |
| 590 'content_browsertests': CommandFunctionTuple( | |
| 591 AddContentBrowserTestOptions, RunTestsCommand), | |
| 592 'instrumentation': CommandFunctionTuple( | 534 'instrumentation': CommandFunctionTuple( |
| 593 AddInstrumentationTestOptions, RunTestsCommand), | 535 AddInstrumentationTestOptions, RunTestsCommand), |
| 594 'uiautomator': CommandFunctionTuple( | 536 'uiautomator': CommandFunctionTuple( |
| 595 AddUIAutomatorTestOptions, RunTestsCommand), | 537 AddUIAutomatorTestOptions, RunTestsCommand), |
| 596 'help': CommandFunctionTuple(lambda option_parser: None, HelpCommand) | 538 'help': CommandFunctionTuple(lambda option_parser: None, HelpCommand) |
| 597 } | 539 } |
| 598 | 540 |
| 599 | 541 |
| 600 class CommandOptionParser(optparse.OptionParser): | 542 class CommandOptionParser(optparse.OptionParser): |
| 601 """Wrapper class for OptionParser to help with listing commands.""" | 543 """Wrapper class for OptionParser to help with listing commands.""" |
| (...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 634 return 0 | 576 return 0 |
| 635 command = argv[1] | 577 command = argv[1] |
| 636 VALID_COMMANDS[command].add_options_func(option_parser) | 578 VALID_COMMANDS[command].add_options_func(option_parser) |
| 637 options, args = option_parser.parse_args(argv) | 579 options, args = option_parser.parse_args(argv) |
| 638 return VALID_COMMANDS[command].run_command_func( | 580 return VALID_COMMANDS[command].run_command_func( |
| 639 command, options, args, option_parser) | 581 command, options, args, option_parser) |
| 640 | 582 |
| 641 | 583 |
| 642 if __name__ == '__main__': | 584 if __name__ == '__main__': |
| 643 sys.exit(main(sys.argv)) | 585 sys.exit(main(sys.argv)) |
| OLD | NEW |