| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # | 2 # |
| 3 # Copyright 2013 The Chromium Authors. All rights reserved. | 3 # Copyright 2013 The Chromium Authors. All rights reserved. |
| 4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
| 5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
| 6 | 6 |
| 7 """Runs all types of tests from one unified interface. | 7 """Runs all types of tests from one unified interface. |
| 8 | 8 |
| 9 TODO(gkanwar): | 9 TODO(gkanwar): |
| 10 * Add options to run Monkey tests. | 10 * Add options to run Monkey tests. |
| 11 """ | 11 """ |
| 12 | 12 |
| 13 import collections | 13 import collections |
| 14 import optparse | 14 import optparse |
| 15 import os | 15 import os |
| 16 import shutil | 16 import shutil |
| 17 import sys | 17 import sys |
| 18 | 18 |
| 19 from pylib import cmd_helper | |
| 20 from pylib import constants | 19 from pylib import constants |
| 21 from pylib import ports | 20 from pylib import ports |
| 22 from pylib.base import base_test_result | 21 from pylib.base import base_test_result |
| 23 from pylib.base import test_dispatcher | 22 from pylib.base import test_dispatcher |
| 23 from pylib.gtest import gtest_config |
| 24 from pylib.gtest import setup as gtest_setup | 24 from pylib.gtest import setup as gtest_setup |
| 25 from pylib.gtest import gtest_config | |
| 26 from pylib.host_driven import run_python_tests as python_dispatch | 25 from pylib.host_driven import run_python_tests as python_dispatch |
| 27 from pylib.instrumentation import setup as instrumentation_setup | 26 from pylib.instrumentation import setup as instrumentation_setup |
| 27 from pylib.test_options import GTestOptions |
| 28 from pylib.test_options import InstrumentationOptions |
| 29 from pylib.test_options import UIAutomatorOptions |
| 28 from pylib.uiautomator import setup as uiautomator_setup | 30 from pylib.uiautomator import setup as uiautomator_setup |
| 29 from pylib.utils import report_results | 31 from pylib.utils import report_results |
| 30 from pylib.utils import run_tests_helper | 32 from pylib.utils import run_tests_helper |
| 31 | 33 |
| 32 | 34 |
| 33 _SDK_OUT_DIR = os.path.join(constants.DIR_SOURCE_ROOT, 'out') | 35 _SDK_OUT_DIR = os.path.join(constants.DIR_SOURCE_ROOT, 'out') |
| 34 | 36 |
| 35 | 37 |
| 36 def AddBuildTypeOption(option_parser): | 38 def AddBuildTypeOption(option_parser): |
| 37 """Adds the build type option to |option_parser|.""" | 39 """Adds the build type option to |option_parser|.""" |
| (...skipping 80 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 118 # TODO(gkanwar): Move these to Common Options once we have the plumbing | 120 # TODO(gkanwar): Move these to Common Options once we have the plumbing |
| 119 # in our other test types to handle these commands | 121 # in our other test types to handle these commands |
| 120 AddCommonOptions(option_parser) | 122 AddCommonOptions(option_parser) |
| 121 | 123 |
| 122 | 124 |
| 123 def ProcessGTestOptions(options): | 125 def ProcessGTestOptions(options): |
| 124 """Intercept test suite help to list test suites. | 126 """Intercept test suite help to list test suites. |
| 125 | 127 |
| 126 Args: | 128 Args: |
| 127 options: Command line options. | 129 options: Command line options. |
| 128 | |
| 129 Returns: | |
| 130 True if the command should continue. | |
| 131 """ | 130 """ |
| 132 if options.suite_name == 'help': | 131 if options.suite_name == 'help': |
| 133 print 'Available test suites are:' | 132 print 'Available test suites are:' |
| 134 for test_suite in (gtest_config.STABLE_TEST_SUITES + | 133 for test_suite in (gtest_config.STABLE_TEST_SUITES + |
| 135 gtest_config.EXPERIMENTAL_TEST_SUITES): | 134 gtest_config.EXPERIMENTAL_TEST_SUITES): |
| 136 print test_suite | 135 print test_suite |
| 137 return False | 136 sys.exit(0) |
| 138 | 137 |
| 139 # Convert to a list, assuming all test suites if nothing was specified. | 138 # Convert to a list, assuming all test suites if nothing was specified. |
| 140 # TODO(gkanwar): Require having a test suite | 139 # TODO(gkanwar): Require having a test suite |
| 141 if options.suite_name: | 140 if options.suite_name: |
| 142 options.suite_name = [options.suite_name] | 141 options.suite_name = [options.suite_name] |
| 143 else: | 142 else: |
| 144 options.suite_name = [s for s in gtest_config.STABLE_TEST_SUITES] | 143 options.suite_name = [s for s in gtest_config.STABLE_TEST_SUITES] |
| 145 return True | |
| 146 | 144 |
| 147 | 145 |
| 148 def AddJavaTestOptions(option_parser): | 146 def AddJavaTestOptions(option_parser): |
| 149 """Adds the Java test options to |option_parser|.""" | 147 """Adds the Java test options to |option_parser|.""" |
| 150 | 148 |
| 151 option_parser.add_option('-f', '--test_filter', dest='test_filter', | 149 option_parser.add_option('-f', '--test_filter', dest='test_filter', |
| 152 help=('Test filter (if not fully qualified, ' | 150 help=('Test filter (if not fully qualified, ' |
| 153 'will run all matches).')) | 151 'will run all matches).')) |
| 154 option_parser.add_option( | 152 option_parser.add_option( |
| 155 '-A', '--annotation', dest='annotation_str', | 153 '-A', '--annotation', dest='annotation_str', |
| (...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 246 option_parser.add_option('-I', dest='install_apk', action='store_true', | 244 option_parser.add_option('-I', dest='install_apk', action='store_true', |
| 247 help='(DEPRECATED) Install the test apk.') | 245 help='(DEPRECATED) Install the test apk.') |
| 248 option_parser.add_option( | 246 option_parser.add_option( |
| 249 '--test-apk', dest='test_apk', | 247 '--test-apk', dest='test_apk', |
| 250 help=('The name of the apk containing the tests ' | 248 help=('The name of the apk containing the tests ' |
| 251 '(without the .apk extension; e.g. "ContentShellTest"). ' | 249 '(without the .apk extension; e.g. "ContentShellTest"). ' |
| 252 'Alternatively, this can be a full path to the apk.')) | 250 'Alternatively, this can be a full path to the apk.')) |
| 253 | 251 |
| 254 | 252 |
| 255 def ProcessInstrumentationOptions(options, error_func): | 253 def ProcessInstrumentationOptions(options, error_func): |
| 256 """Processes options/arguments and populate |options| with defaults.""" | 254 """Processes options/arguments and populate |options| with defaults. |
| 255 |
| 256 Args: |
| 257 options: optparse.Options object. |
| 258 error_func: Function to call with the error message in case of an error. |
| 259 |
| 260 Returns: |
| 261 An InstrumentationOptions named tuple which contains all options relevant to |
| 262 instrumentation tests. |
| 263 """ |
| 257 | 264 |
| 258 ProcessJavaTestOptions(options, error_func) | 265 ProcessJavaTestOptions(options, error_func) |
| 259 | 266 |
| 260 if not options.test_apk: | 267 if not options.test_apk: |
| 261 error_func('--test-apk must be specified.') | 268 error_func('--test-apk must be specified.') |
| 262 | 269 |
| 263 if os.path.exists(options.test_apk): | 270 if os.path.exists(options.test_apk): |
| 264 # The APK is fully qualified, assume the JAR lives along side. | 271 # The APK is fully qualified, assume the JAR lives along side. |
| 265 options.test_apk_path = options.test_apk | 272 options.test_apk_path = options.test_apk |
| 266 options.test_apk_jar_path = (os.path.splitext(options.test_apk_path)[0] + | 273 options.test_apk_jar_path = (os.path.splitext(options.test_apk_path)[0] + |
| 267 '.jar') | 274 '.jar') |
| 268 else: | 275 else: |
| 269 options.test_apk_path = os.path.join(_SDK_OUT_DIR, | 276 options.test_apk_path = os.path.join(_SDK_OUT_DIR, |
| 270 options.build_type, | 277 options.build_type, |
| 271 constants.SDK_BUILD_APKS_DIR, | 278 constants.SDK_BUILD_APKS_DIR, |
| 272 '%s.apk' % options.test_apk) | 279 '%s.apk' % options.test_apk) |
| 273 options.test_apk_jar_path = os.path.join( | 280 options.test_apk_jar_path = os.path.join( |
| 274 _SDK_OUT_DIR, options.build_type, constants.SDK_BUILD_TEST_JAVALIB_DIR, | 281 _SDK_OUT_DIR, options.build_type, constants.SDK_BUILD_TEST_JAVALIB_DIR, |
| 275 '%s.jar' % options.test_apk) | 282 '%s.jar' % options.test_apk) |
| 276 | 283 |
| 284 return InstrumentationOptions(options.build_type, |
| 285 options.tool, |
| 286 options.cleanup_test_files, |
| 287 options.push_deps, |
| 288 options.annotations, |
| 289 options.exclude_annotations, |
| 290 options.test_filter, |
| 291 options.test_data, |
| 292 options.save_perf_json, |
| 293 options.screenshot_failures, |
| 294 options.disable_assertions, |
| 295 options.wait_for_debugger, |
| 296 options.test_apk, |
| 297 options.test_apk_path, |
| 298 options.test_apk_jar_path) |
| 299 |
| 277 | 300 |
| 278 def AddUIAutomatorTestOptions(option_parser): | 301 def AddUIAutomatorTestOptions(option_parser): |
| 279 """Adds UI Automator test options to |option_parser|.""" | 302 """Adds UI Automator test options to |option_parser|.""" |
| 280 | 303 |
| 281 option_parser.usage = '%prog uiautomator [options]' | 304 option_parser.usage = '%prog uiautomator [options]' |
| 282 option_parser.command_list = [] | 305 option_parser.command_list = [] |
| 283 option_parser.example = ( | 306 option_parser.example = ( |
| 284 '%prog uiautomator --test-jar=chromium_testshell_uiautomator_tests' | 307 '%prog uiautomator --test-jar=chromium_testshell_uiautomator_tests' |
| 285 ' --package-name=org.chromium.chrome.testshell') | 308 ' --package-name=org.chromium.chrome.testshell') |
| 286 option_parser.add_option( | 309 option_parser.add_option( |
| 287 '--package-name', | 310 '--package-name', |
| 288 help='The package name used by the apk containing the application.') | 311 help='The package name used by the apk containing the application.') |
| 289 option_parser.add_option( | 312 option_parser.add_option( |
| 290 '--test-jar', dest='test_jar', | 313 '--test-jar', dest='test_jar', |
| 291 help=('The name of the dexed jar containing the tests (without the ' | 314 help=('The name of the dexed jar containing the tests (without the ' |
| 292 '.dex.jar extension). Alternatively, this can be a full path ' | 315 '.dex.jar extension). Alternatively, this can be a full path ' |
| 293 'to the jar.')) | 316 'to the jar.')) |
| 294 | 317 |
| 295 AddJavaTestOptions(option_parser) | 318 AddJavaTestOptions(option_parser) |
| 296 AddCommonOptions(option_parser) | 319 AddCommonOptions(option_parser) |
| 297 | 320 |
| 298 | 321 |
| 299 def ProcessUIAutomatorOptions(options, error_func): | 322 def ProcessUIAutomatorOptions(options, error_func): |
| 300 """Processes UIAutomator options/arguments.""" | 323 """Processes UIAutomator options/arguments. |
| 324 |
| 325 Args: |
| 326 options: optparse.Options object. |
| 327 error_func: Function to call with the error message in case of an error. |
| 328 |
| 329 Returns: |
| 330 A UIAutomatorOptions named tuple which contains all options relevant to |
| 331 instrumentation tests. |
| 332 """ |
| 301 | 333 |
| 302 ProcessJavaTestOptions(options, error_func) | 334 ProcessJavaTestOptions(options, error_func) |
| 303 | 335 |
| 304 if not options.package_name: | 336 if not options.package_name: |
| 305 error_func('--package-name must be specified.') | 337 error_func('--package-name must be specified.') |
| 306 | 338 |
| 307 if not options.test_jar: | 339 if not options.test_jar: |
| 308 error_func('--test-jar must be specified.') | 340 error_func('--test-jar must be specified.') |
| 309 | 341 |
| 310 if os.path.exists(options.test_jar): | 342 if os.path.exists(options.test_jar): |
| 311 # The dexed JAR is fully qualified, assume the info JAR lives along side. | 343 # The dexed JAR is fully qualified, assume the info JAR lives along side. |
| 312 options.uiautomator_jar = options.test_jar | 344 options.uiautomator_jar = options.test_jar |
| 313 else: | 345 else: |
| 314 options.uiautomator_jar = os.path.join( | 346 options.uiautomator_jar = os.path.join( |
| 315 _SDK_OUT_DIR, options.build_type, constants.SDK_BUILD_JAVALIB_DIR, | 347 _SDK_OUT_DIR, options.build_type, constants.SDK_BUILD_JAVALIB_DIR, |
| 316 '%s.dex.jar' % options.test_jar) | 348 '%s.dex.jar' % options.test_jar) |
| 317 options.uiautomator_info_jar = ( | 349 options.uiautomator_info_jar = ( |
| 318 options.uiautomator_jar[:options.uiautomator_jar.find('.dex.jar')] + | 350 options.uiautomator_jar[:options.uiautomator_jar.find('.dex.jar')] + |
| 319 '_java.jar') | 351 '_java.jar') |
| 320 | 352 |
| 353 return UIAutomatorOptions(options.build_type, |
| 354 options.tool, |
| 355 options.cleanup_test_files, |
| 356 options.push_deps, |
| 357 options.annotations, |
| 358 options.exclude_annotations, |
| 359 options.test_filter, |
| 360 options.test_data, |
| 361 options.save_perf_json, |
| 362 options.screenshot_failures, |
| 363 options.disable_assertions, |
| 364 options.uiautomator_jar, |
| 365 options.uiautomator_info_jar, |
| 366 options.package_name) |
| 367 |
| 321 | 368 |
| 322 def _RunGTests(options, error_func): | 369 def _RunGTests(options, error_func): |
| 323 """Subcommand of RunTestsCommands which runs gtests.""" | 370 """Subcommand of RunTestsCommands which runs gtests.""" |
| 324 if not ProcessGTestOptions(options): | 371 ProcessGTestOptions(options) |
| 325 return 0 | |
| 326 | 372 |
| 327 exit_code = 0 | 373 exit_code = 0 |
| 328 for suite_name in options.suite_name: | 374 for suite_name in options.suite_name: |
| 329 runner_factory, tests = gtest_setup.Setup( | 375 # TODO(gkanwar): Move this into ProcessGTestOptions once we require -s for |
| 330 suite_name, options.test_arguments, | 376 # the gtest command. |
| 331 options.timeout, options.cleanup_test_files, options.tool, | 377 gtest_options = GTestOptions(options.build_type, |
| 332 options.build_type, options.push_deps, options.test_filter) | 378 options.tool, |
| 379 options.cleanup_test_files, |
| 380 options.push_deps, |
| 381 options.test_filter, |
| 382 options.test_arguments, |
| 383 options.timeout, |
| 384 suite_name) |
| 385 runner_factory, tests = gtest_setup.Setup(gtest_options) |
| 333 | 386 |
| 334 results, test_exit_code = test_dispatcher.RunTests( | 387 results, test_exit_code = test_dispatcher.RunTests( |
| 335 tests, runner_factory, False, options.test_device, | 388 tests, runner_factory, False, options.test_device, |
| 336 shard=True, | 389 shard=True, |
| 337 build_type=options.build_type, | 390 build_type=options.build_type, |
| 338 test_timeout=None, | 391 test_timeout=None, |
| 339 num_retries=options.num_retries) | 392 num_retries=options.num_retries) |
| 340 | 393 |
| 341 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: | 394 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: |
| 342 exit_code = test_exit_code | 395 exit_code = test_exit_code |
| 343 | 396 |
| 344 report_results.LogFull( | 397 report_results.LogFull( |
| 345 results=results, | 398 results=results, |
| 346 test_type='Unit test', | 399 test_type='Unit test', |
| 347 test_package=suite_name, | 400 test_package=suite_name, |
| 348 build_type=options.build_type, | 401 build_type=options.build_type, |
| 349 flakiness_server=options.flakiness_dashboard_server) | 402 flakiness_server=options.flakiness_dashboard_server) |
| 350 | 403 |
| 351 if os.path.isdir(constants.ISOLATE_DEPS_DIR): | 404 if os.path.isdir(constants.ISOLATE_DEPS_DIR): |
| 352 shutil.rmtree(constants.ISOLATE_DEPS_DIR) | 405 shutil.rmtree(constants.ISOLATE_DEPS_DIR) |
| 353 | 406 |
| 354 return exit_code | 407 return exit_code |
| 355 | 408 |
| 356 | 409 |
| 357 def _RunInstrumentationTests(options, error_func): | 410 def _RunInstrumentationTests(options, error_func): |
| 358 """Subcommand of RunTestsCommands which runs instrumentation tests.""" | 411 """Subcommand of RunTestsCommands which runs instrumentation tests.""" |
| 359 ProcessInstrumentationOptions(options, error_func) | 412 instrumentation_options = ProcessInstrumentationOptions(options, error_func) |
| 360 | 413 |
| 361 results = base_test_result.TestRunResults() | 414 results = base_test_result.TestRunResults() |
| 362 exit_code = 0 | 415 exit_code = 0 |
| 363 | 416 |
| 364 if options.run_java_tests: | 417 if options.run_java_tests: |
| 365 runner_factory, tests = instrumentation_setup.Setup( | 418 runner_factory, tests = instrumentation_setup.Setup(instrumentation_options) |
| 366 options.test_apk_path, options.test_apk_jar_path, options.annotations, | |
| 367 options.exclude_annotations, options.test_filter, options.build_type, | |
| 368 options.test_data, options.save_perf_json, options.screenshot_failures, | |
| 369 options.tool, options.wait_for_debugger, options.disable_assertions, | |
| 370 options.push_deps, options.cleanup_test_files) | |
| 371 | 419 |
| 372 test_results, exit_code = test_dispatcher.RunTests( | 420 test_results, exit_code = test_dispatcher.RunTests( |
| 373 tests, runner_factory, options.wait_for_debugger, | 421 tests, runner_factory, options.wait_for_debugger, |
| 374 options.test_device, | 422 options.test_device, |
| 375 shard=True, | 423 shard=True, |
| 376 build_type=options.build_type, | 424 build_type=options.build_type, |
| 377 test_timeout=None, | 425 test_timeout=None, |
| 378 num_retries=options.num_retries) | 426 num_retries=options.num_retries) |
| 379 | 427 |
| 380 results.AddTestRunResults(test_results) | 428 results.AddTestRunResults(test_results) |
| (...skipping 14 matching lines...) Expand all Loading... |
| 395 test_package=os.path.basename(options.test_apk), | 443 test_package=os.path.basename(options.test_apk), |
| 396 annotation=options.annotations, | 444 annotation=options.annotations, |
| 397 build_type=options.build_type, | 445 build_type=options.build_type, |
| 398 flakiness_server=options.flakiness_dashboard_server) | 446 flakiness_server=options.flakiness_dashboard_server) |
| 399 | 447 |
| 400 return exit_code | 448 return exit_code |
| 401 | 449 |
| 402 | 450 |
| 403 def _RunUIAutomatorTests(options, error_func): | 451 def _RunUIAutomatorTests(options, error_func): |
| 404 """Subcommand of RunTestsCommands which runs uiautomator tests.""" | 452 """Subcommand of RunTestsCommands which runs uiautomator tests.""" |
| 405 ProcessUIAutomatorOptions(options, error_func) | 453 uiautomator_options = ProcessUIAutomatorOptions(options, error_func) |
| 406 | 454 |
| 407 results = base_test_result.TestRunResults() | 455 results = base_test_result.TestRunResults() |
| 408 exit_code = 0 | 456 exit_code = 0 |
| 409 | 457 |
| 410 if options.run_java_tests: | 458 if options.run_java_tests: |
| 411 runner_factory, tests = uiautomator_setup.Setup( | 459 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options) |
| 412 options.uiautomator_jar, options.uiautomator_info_jar, | |
| 413 options.annotations, options.exclude_annotations, options.test_filter, | |
| 414 options.package_name, options.build_type, options.test_data, | |
| 415 options.save_perf_json, options.screenshot_failures, options.tool, | |
| 416 options.disable_assertions, options.push_deps, | |
| 417 options.cleanup_test_files) | |
| 418 | 460 |
| 419 test_results, exit_code = test_dispatcher.RunTests( | 461 test_results, exit_code = test_dispatcher.RunTests( |
| 420 tests, runner_factory, False, options.test_device, | 462 tests, runner_factory, False, options.test_device, |
| 421 shard=True, | 463 shard=True, |
| 422 build_type=options.build_type, | 464 build_type=options.build_type, |
| 423 test_timeout=None, | 465 test_timeout=None, |
| 424 num_retries=options.num_retries) | 466 num_retries=options.num_retries) |
| 425 | 467 |
| 426 results.AddTestRunResults(test_results) | 468 results.AddTestRunResults(test_results) |
| 427 | 469 |
| (...skipping 148 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 576 return 0 | 618 return 0 |
| 577 command = argv[1] | 619 command = argv[1] |
| 578 VALID_COMMANDS[command].add_options_func(option_parser) | 620 VALID_COMMANDS[command].add_options_func(option_parser) |
| 579 options, args = option_parser.parse_args(argv) | 621 options, args = option_parser.parse_args(argv) |
| 580 return VALID_COMMANDS[command].run_command_func( | 622 return VALID_COMMANDS[command].run_command_func( |
| 581 command, options, args, option_parser) | 623 command, options, args, option_parser) |
| 582 | 624 |
| 583 | 625 |
| 584 if __name__ == '__main__': | 626 if __name__ == '__main__': |
| 585 sys.exit(main(sys.argv)) | 627 sys.exit(main(sys.argv)) |
| OLD | NEW |