OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # | 2 # |
3 # Copyright 2013 The Chromium Authors. All rights reserved. | 3 # Copyright 2013 The Chromium Authors. All rights reserved. |
4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
6 | 6 |
7 """Runs all types of tests from one unified interface.""" | 7 """Runs all types of tests from one unified interface.""" |
8 | 8 |
9 import argparse | 9 import argparse |
10 import collections | 10 import collections |
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
88 help=('Address of the server that is hosting the ' | 88 help=('Address of the server that is hosting the ' |
89 'Chrome for Android flakiness dashboard.')) | 89 'Chrome for Android flakiness dashboard.')) |
90 group.add_argument('--enable-platform-mode', action='store_true', | 90 group.add_argument('--enable-platform-mode', action='store_true', |
91 help=('Run the test scripts in platform mode, which ' | 91 help=('Run the test scripts in platform mode, which ' |
92 'conceptually separates the test runner from the ' | 92 'conceptually separates the test runner from the ' |
93 '"device" (local or remote, real or emulated) on ' | 93 '"device" (local or remote, real or emulated) on ' |
94 'which the tests are running. [experimental]')) | 94 'which the tests are running. [experimental]')) |
95 group.add_argument('-e', '--environment', default='local', | 95 group.add_argument('-e', '--environment', default='local', |
96 choices=constants.VALID_ENVIRONMENTS, | 96 choices=constants.VALID_ENVIRONMENTS, |
97 help='Test environment to run in (default: %(default)s).') | 97 help='Test environment to run in (default: %(default)s).') |
98 group.add_argument('--adb-path', type=os.path.abspath, | 98 group.add_argument('--adb-path', type=os.path.realpath, |
99 help=('Specify the absolute path of the adb binary that ' | 99 help=('Specify the absolute path of the adb binary that ' |
100 'should be used.')) | 100 'should be used.')) |
101 group.add_argument('--json-results-file', '--test-launcher-summary-output', | 101 group.add_argument('--json-results-file', '--test-launcher-summary-output', |
102 dest='json_results_file', | 102 dest='json_results_file', type=os.path.realpath, |
103 help='If set, will dump results in JSON form ' | 103 help='If set, will dump results in JSON form ' |
104 'to specified file.') | 104 'to specified file.') |
105 | 105 |
106 logcat_output_group = group.add_mutually_exclusive_group() | 106 logcat_output_group = group.add_mutually_exclusive_group() |
107 logcat_output_group.add_argument( | 107 logcat_output_group.add_argument( |
108 '--logcat-output-dir', | 108 '--logcat-output-dir', type=os.path.realpath, |
109 help='If set, will dump logcats recorded during test run to directory. ' | 109 help='If set, will dump logcats recorded during test run to directory. ' |
110 'File names will be the device ids with timestamps.') | 110 'File names will be the device ids with timestamps.') |
111 logcat_output_group.add_argument( | 111 logcat_output_group.add_argument( |
112 '--logcat-output-file', | 112 '--logcat-output-file', type=os.path.realpath, |
113 help='If set, will merge logcats recorded during test run and dump them ' | 113 help='If set, will merge logcats recorded during test run and dump them ' |
114 'to the specified file.') | 114 'to the specified file.') |
115 | 115 |
116 class FastLocalDevAction(argparse.Action): | 116 class FastLocalDevAction(argparse.Action): |
117 def __call__(self, parser, namespace, values, option_string=None): | 117 def __call__(self, parser, namespace, values, option_string=None): |
118 namespace.verbose_count = max(namespace.verbose_count, 1) | 118 namespace.verbose_count = max(namespace.verbose_count, 1) |
119 namespace.num_retries = 0 | 119 namespace.num_retries = 0 |
120 namespace.enable_device_cache = True | 120 namespace.enable_device_cache = True |
121 namespace.enable_concurrent_adb = True | 121 namespace.enable_concurrent_adb = True |
122 namespace.skip_clear_data = True | 122 namespace.skip_clear_data = True |
(...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
208 def AddDeviceOptions(parser): | 208 def AddDeviceOptions(parser): |
209 """Adds device options to |parser|.""" | 209 """Adds device options to |parser|.""" |
210 group = parser.add_argument_group(title='Device Options') | 210 group = parser.add_argument_group(title='Device Options') |
211 group.add_argument('--tool', | 211 group.add_argument('--tool', |
212 dest='tool', | 212 dest='tool', |
213 help=('Run the test under a tool ' | 213 help=('Run the test under a tool ' |
214 '(use --tool help to list them)')) | 214 '(use --tool help to list them)')) |
215 group.add_argument('-d', '--device', dest='test_device', | 215 group.add_argument('-d', '--device', dest='test_device', |
216 help=('Target device for the test suite ' | 216 help=('Target device for the test suite ' |
217 'to run on.')) | 217 'to run on.')) |
218 group.add_argument('--blacklist-file', help='Device blacklist file.') | 218 group.add_argument('--blacklist-file', type=os.path.realpath, |
| 219 help='Device blacklist file.') |
219 group.add_argument('--enable-device-cache', action='store_true', | 220 group.add_argument('--enable-device-cache', action='store_true', |
220 help='Cache device state to disk between runs') | 221 help='Cache device state to disk between runs') |
221 group.add_argument('--enable-concurrent-adb', action='store_true', | 222 group.add_argument('--enable-concurrent-adb', action='store_true', |
222 help='Run multiple adb commands at the same time, even ' | 223 help='Run multiple adb commands at the same time, even ' |
223 'for the same device.') | 224 'for the same device.') |
224 group.add_argument('--skip-clear-data', action='store_true', | 225 group.add_argument('--skip-clear-data', action='store_true', |
225 help='Do not wipe app data between tests. Use this to ' | 226 help='Do not wipe app data between tests. Use this to ' |
226 'speed up local development and never on bots ' | 227 'speed up local development and never on bots ' |
227 '(increases flakiness)') | 228 '(increases flakiness)') |
228 group.add_argument('--target-devices-file', | 229 group.add_argument('--target-devices-file', type=os.path.realpath, |
229 help='Path to file with json list of device serials to ' | 230 help='Path to file with json list of device serials to ' |
230 'run tests on. When not specified, all available ' | 231 'run tests on. When not specified, all available ' |
231 'devices are used.') | 232 'devices are used.') |
232 | 233 |
233 | 234 |
234 def AddGTestOptions(parser): | 235 def AddGTestOptions(parser): |
235 """Adds gtest options to |parser|.""" | 236 """Adds gtest options to |parser|.""" |
236 | 237 |
237 group = parser.add_argument_group('GTest Options') | 238 group = parser.add_argument_group('GTest Options') |
238 group.add_argument('-s', '--suite', dest='suite_name', | 239 group.add_argument('-s', '--suite', dest='suite_name', |
239 nargs='+', metavar='SUITE_NAME', required=True, | 240 nargs='+', metavar='SUITE_NAME', required=True, |
240 help='Executable name of the test suite to run.') | 241 help='Executable name of the test suite to run.') |
241 group.add_argument('--executable-dist-dir', | 242 group.add_argument('--executable-dist-dir', type=os.path.realpath, |
242 help="Path to executable's dist directory for native" | 243 help="Path to executable's dist directory for native" |
243 " (non-apk) tests.") | 244 " (non-apk) tests.") |
244 group.add_argument('--test-apk-incremental-install-script', | 245 group.add_argument('--test-apk-incremental-install-script', |
| 246 type=os.path.realpath, |
245 help='Path to install script for the test apk.') | 247 help='Path to install script for the test apk.') |
246 group.add_argument('--gtest_also_run_disabled_tests', | 248 group.add_argument('--gtest_also_run_disabled_tests', |
247 '--gtest-also-run-disabled-tests', | 249 '--gtest-also-run-disabled-tests', |
248 dest='run_disabled', action='store_true', | 250 dest='run_disabled', action='store_true', |
249 help='Also run disabled tests if applicable.') | 251 help='Also run disabled tests if applicable.') |
250 group.add_argument('-a', '--test-arguments', dest='test_arguments', | 252 group.add_argument('-a', '--test-arguments', dest='test_arguments', |
251 default='', | 253 default='', |
252 help='Additional arguments to pass to the test.') | 254 help='Additional arguments to pass to the test.') |
253 group.add_argument('-t', '--shard-timeout', | 255 group.add_argument('-t', '--shard-timeout', |
254 dest='shard_timeout', type=int, default=120, | 256 dest='shard_timeout', type=int, default=120, |
255 help='Timeout to wait for each test ' | 257 help='Timeout to wait for each test ' |
256 '(default: %(default)s).') | 258 '(default: %(default)s).') |
257 group.add_argument('--isolate_file_path', | 259 group.add_argument('--isolate_file_path', |
258 '--isolate-file-path', | 260 '--isolate-file-path', |
259 dest='isolate_file_path', | 261 dest='isolate_file_path', |
| 262 type=os.path.realpath, |
260 help='.isolate file path to override the default ' | 263 help='.isolate file path to override the default ' |
261 'path') | 264 'path') |
262 group.add_argument('--app-data-file', action='append', dest='app_data_files', | 265 group.add_argument('--app-data-file', action='append', dest='app_data_files', |
263 help='A file path relative to the app data directory ' | 266 help='A file path relative to the app data directory ' |
264 'that should be saved to the host.') | 267 'that should be saved to the host.') |
265 group.add_argument('--app-data-file-dir', | 268 group.add_argument('--app-data-file-dir', |
266 help='Host directory to which app data files will be' | 269 help='Host directory to which app data files will be' |
267 ' saved. Used with --app-data-file.') | 270 ' saved. Used with --app-data-file.') |
268 group.add_argument('--delete-stale-data', dest='delete_stale_data', | 271 group.add_argument('--delete-stale-data', dest='delete_stale_data', |
269 action='store_true', | 272 action='store_true', |
(...skipping 11 matching lines...) Expand all Loading... |
281 'tests can be determined from it, skip querying the ' | 284 'tests can be determined from it, skip querying the ' |
282 'device for the list of all tests. Speeds up local ' | 285 'device for the list of all tests. Speeds up local ' |
283 'development, but is not safe to use on bots (' | 286 'development, but is not safe to use on bots (' |
284 'http://crbug.com/549214') | 287 'http://crbug.com/549214') |
285 | 288 |
286 filter_group = group.add_mutually_exclusive_group() | 289 filter_group = group.add_mutually_exclusive_group() |
287 filter_group.add_argument('-f', '--gtest_filter', '--gtest-filter', | 290 filter_group.add_argument('-f', '--gtest_filter', '--gtest-filter', |
288 dest='test_filter', | 291 dest='test_filter', |
289 help='googletest-style filter string.') | 292 help='googletest-style filter string.') |
290 filter_group.add_argument('--gtest-filter-file', dest='test_filter_file', | 293 filter_group.add_argument('--gtest-filter-file', dest='test_filter_file', |
| 294 type=os.path.realpath, |
291 help='Path to file that contains googletest-style ' | 295 help='Path to file that contains googletest-style ' |
292 'filter strings. (Lines will be joined with ' | 296 'filter strings. (Lines will be joined with ' |
293 '":" to create a single filter string.)') | 297 '":" to create a single filter string.)') |
294 | 298 |
295 AddDeviceOptions(parser) | 299 AddDeviceOptions(parser) |
296 AddCommonOptions(parser) | 300 AddCommonOptions(parser) |
297 AddRemoteDeviceOptions(parser) | 301 AddRemoteDeviceOptions(parser) |
298 | 302 |
299 | 303 |
300 def AddLinkerTestOptions(parser): | 304 def AddLinkerTestOptions(parser): |
(...skipping 22 matching lines...) Expand all Loading... |
323 '-A', '--annotation', dest='annotation_str', | 327 '-A', '--annotation', dest='annotation_str', |
324 help=('Comma-separated list of annotations. Run only tests with any of ' | 328 help=('Comma-separated list of annotations. Run only tests with any of ' |
325 'the given annotations. An annotation can be either a key or a ' | 329 'the given annotations. An annotation can be either a key or a ' |
326 'key-values pair. A test that has no annotation is considered ' | 330 'key-values pair. A test that has no annotation is considered ' |
327 '"SmallTest".')) | 331 '"SmallTest".')) |
328 argument_group.add_argument( | 332 argument_group.add_argument( |
329 '-E', '--exclude-annotation', dest='exclude_annotation_str', | 333 '-E', '--exclude-annotation', dest='exclude_annotation_str', |
330 help=('Comma-separated list of annotations. Exclude tests with these ' | 334 help=('Comma-separated list of annotations. Exclude tests with these ' |
331 'annotations.')) | 335 'annotations.')) |
332 argument_group.add_argument( | 336 argument_group.add_argument( |
333 '--screenshot-directory', dest='screenshot_dir', | 337 '--screenshot-directory', dest='screenshot_dir', type=os.path.realpath, |
334 help='Capture screenshots of test failures') | 338 help='Capture screenshots of test failures') |
335 argument_group.add_argument( | 339 argument_group.add_argument( |
336 '--save-perf-json', action='store_true', | 340 '--save-perf-json', action='store_true', |
337 help='Saves the JSON file for each UI Perf test.') | 341 help='Saves the JSON file for each UI Perf test.') |
338 argument_group.add_argument( | 342 argument_group.add_argument( |
339 '--official-build', action='store_true', help='Run official build tests.') | 343 '--official-build', action='store_true', help='Run official build tests.') |
340 argument_group.add_argument( | 344 argument_group.add_argument( |
341 '--disable-dalvik-asserts', dest='set_asserts', action='store_false', | 345 '--disable-dalvik-asserts', dest='set_asserts', action='store_false', |
342 default=True, help='Removes the dalvik.vm.enableassertions property') | 346 default=True, help='Removes the dalvik.vm.enableassertions property') |
343 | 347 |
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
376 java_or_python_group.add_argument( | 380 java_or_python_group.add_argument( |
377 '-p', '--python-only', action='store_false', | 381 '-p', '--python-only', action='store_false', |
378 dest='run_java_tests', default=True, | 382 dest='run_java_tests', default=True, |
379 help='DEPRECATED') | 383 help='DEPRECATED') |
380 | 384 |
381 group.add_argument('--host-driven-root', | 385 group.add_argument('--host-driven-root', |
382 help='DEPRECATED') | 386 help='DEPRECATED') |
383 group.add_argument('-w', '--wait_debugger', dest='wait_for_debugger', | 387 group.add_argument('-w', '--wait_debugger', dest='wait_for_debugger', |
384 action='store_true', | 388 action='store_true', |
385 help='Wait for debugger.') | 389 help='Wait for debugger.') |
| 390 # TODO(jbudorick): Remove support for name-style APK specification once |
| 391 # bots are no longer doing it. |
386 group.add_argument('--apk-under-test', | 392 group.add_argument('--apk-under-test', |
387 help='Path or name of the apk under test.') | 393 help='Path or name of the apk under test.') |
388 group.add_argument('--apk-under-test-incremental-install-script', | 394 group.add_argument('--apk-under-test-incremental-install-script', |
389 help='Path to install script for the --apk-under-test.') | 395 help='Path to install script for the --apk-under-test.') |
390 group.add_argument('--test-apk', required=True, | 396 group.add_argument('--test-apk', required=True, |
391 help='Path or name of the apk containing the tests ' | 397 help='Path or name of the apk containing the tests ' |
392 '(name is without the .apk extension; ' | 398 '(name is without the .apk extension; ' |
393 'e.g. "ContentShellTest").') | 399 'e.g. "ContentShellTest").') |
394 group.add_argument('--test-apk-incremental-install-script', | 400 group.add_argument('--test-apk-incremental-install-script', |
| 401 type=os.path.realpath, |
395 help='Path to install script for the --test-apk.') | 402 help='Path to install script for the --test-apk.') |
396 group.add_argument('--additional-apk', action='append', | 403 group.add_argument('--additional-apk', action='append', |
397 dest='additional_apks', default=[], | 404 dest='additional_apks', default=[], |
| 405 type=os.path.realpath, |
398 help='Additional apk that must be installed on ' | 406 help='Additional apk that must be installed on ' |
399 'the device when the tests are run') | 407 'the device when the tests are run') |
400 group.add_argument('--coverage-dir', | 408 group.add_argument('--coverage-dir', type=os.path.realpath, |
401 help=('Directory in which to place all generated ' | 409 help=('Directory in which to place all generated ' |
402 'EMMA coverage files.')) | 410 'EMMA coverage files.')) |
403 group.add_argument('--device-flags', dest='device_flags', default='', | 411 group.add_argument('--device-flags', dest='device_flags', |
| 412 type=os.path.realpath, |
404 help='The relative filepath to a file containing ' | 413 help='The relative filepath to a file containing ' |
405 'command-line flags to set on the device') | 414 'command-line flags to set on the device') |
406 group.add_argument('--device-flags-file', default='', | 415 group.add_argument('--device-flags-file', type=os.path.realpath, |
407 help='The relative filepath to a file containing ' | 416 help='The relative filepath to a file containing ' |
408 'command-line flags to set on the device') | 417 'command-line flags to set on the device') |
409 group.add_argument('--isolate_file_path', | 418 group.add_argument('--isolate_file_path', |
410 '--isolate-file-path', | 419 '--isolate-file-path', |
411 dest='isolate_file_path', | 420 dest='isolate_file_path', |
| 421 type=os.path.realpath, |
412 help='.isolate file path to override the default ' | 422 help='.isolate file path to override the default ' |
413 'path') | 423 'path') |
414 group.add_argument('--delete-stale-data', dest='delete_stale_data', | 424 group.add_argument('--delete-stale-data', dest='delete_stale_data', |
415 action='store_true', | 425 action='store_true', |
416 help='Delete stale test data on the device.') | 426 help='Delete stale test data on the device.') |
417 group.add_argument('--timeout-scale', type=float, | 427 group.add_argument('--timeout-scale', type=float, |
418 help='Factor by which timeouts should be scaled.') | 428 help='Factor by which timeouts should be scaled.') |
419 group.add_argument('--strict-mode', dest='strict_mode', default='testing', | 429 group.add_argument('--strict-mode', dest='strict_mode', default='testing', |
420 help='StrictMode command-line flag set on the device, ' | 430 help='StrictMode command-line flag set on the device, ' |
421 'death/testing to kill the process, off to stop ' | 431 'death/testing to kill the process, off to stop ' |
(...skipping 25 matching lines...) Expand all Loading... |
447 group.add_argument( | 457 group.add_argument( |
448 '--package-filter', dest='package_filter', | 458 '--package-filter', dest='package_filter', |
449 help='Filters tests by package.') | 459 help='Filters tests by package.') |
450 group.add_argument( | 460 group.add_argument( |
451 '--runner-filter', dest='runner_filter', | 461 '--runner-filter', dest='runner_filter', |
452 help='Filters tests by runner class. Must be fully qualified.') | 462 help='Filters tests by runner class. Must be fully qualified.') |
453 group.add_argument( | 463 group.add_argument( |
454 '--sdk-version', dest='sdk_version', type=int, | 464 '--sdk-version', dest='sdk_version', type=int, |
455 help='The Android SDK version.') | 465 help='The Android SDK version.') |
456 group.add_argument( | 466 group.add_argument( |
457 '--coverage-dir', dest='coverage_dir', | 467 '--coverage-dir', dest='coverage_dir', type=os.path.realpath, |
458 help='Directory to store coverage info.') | 468 help='Directory to store coverage info.') |
459 AddCommonOptions(parser) | 469 AddCommonOptions(parser) |
460 | 470 |
461 | 471 |
462 def AddMonkeyTestOptions(parser): | 472 def AddMonkeyTestOptions(parser): |
463 """Adds monkey test options to |parser|.""" | 473 """Adds monkey test options to |parser|.""" |
464 | 474 |
465 group = parser.add_argument_group('Monkey Test Options') | 475 group = parser.add_argument_group('Monkey Test Options') |
466 group.add_argument( | 476 group.add_argument( |
467 '--package', required=True, choices=constants.PACKAGE_INFO.keys(), | 477 '--package', required=True, choices=constants.PACKAGE_INFO.keys(), |
(...skipping 83 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
551 help='Execute the given command with retries, but only print the result ' | 561 help='Execute the given command with retries, but only print the result ' |
552 'for the "most successful" round.') | 562 'for the "most successful" round.') |
553 step_group.add_argument( | 563 step_group.add_argument( |
554 '--steps', | 564 '--steps', |
555 help='JSON file containing the list of commands to run.') | 565 help='JSON file containing the list of commands to run.') |
556 step_group.add_argument( | 566 step_group.add_argument( |
557 '--print-step', | 567 '--print-step', |
558 help='The name of a previously executed perf step to print.') | 568 help='The name of a previously executed perf step to print.') |
559 | 569 |
560 group.add_argument( | 570 group.add_argument( |
561 '--output-json-list', | 571 '--output-json-list', type=os.path.realpath, |
562 help='Write a simple list of names from --steps into the given file.') | 572 help='Write a simple list of names from --steps into the given file.') |
563 group.add_argument( | 573 group.add_argument( |
564 '--collect-chartjson-data', | 574 '--collect-chartjson-data', |
565 action='store_true', | 575 action='store_true', |
566 help='Cache the chartjson output from each step for later use.') | 576 help='Cache the chartjson output from each step for later use.') |
567 group.add_argument( | 577 group.add_argument( |
568 '--output-chartjson-data', | 578 '--output-chartjson-data', |
569 default='', | 579 default='', |
| 580 type=os.path.realpath, |
570 help='Write out chartjson into the given file.') | 581 help='Write out chartjson into the given file.') |
571 # TODO(rnephew): Remove this when everything moves to new option in platform | 582 # TODO(rnephew): Remove this when everything moves to new option in platform |
572 # mode. | 583 # mode. |
573 group.add_argument( | 584 group.add_argument( |
574 '--get-output-dir-archive', metavar='FILENAME', | 585 '--get-output-dir-archive', metavar='FILENAME', type=os.path.realpath, |
575 help='Write the cached output directory archived by a step into the' | 586 help='Write the cached output directory archived by a step into the' |
576 ' given ZIP file.') | 587 ' given ZIP file.') |
577 group.add_argument( | 588 group.add_argument( |
578 '--output-dir-archive-path', metavar='FILENAME', | 589 '--output-dir-archive-path', metavar='FILENAME', type=os.path.realpath, |
579 help='Write the cached output directory archived by a step into the' | 590 help='Write the cached output directory archived by a step into the' |
580 ' given ZIP file.') | 591 ' given ZIP file.') |
581 group.add_argument( | 592 group.add_argument( |
582 '--flaky-steps', | 593 '--flaky-steps', type=os.path.realpath, |
583 help=('A JSON file containing steps that are flaky ' | 594 help=('A JSON file containing steps that are flaky ' |
584 'and will have its exit code ignored.')) | 595 'and will have its exit code ignored.')) |
585 group.add_argument( | 596 group.add_argument( |
586 '--no-timeout', action='store_true', | 597 '--no-timeout', action='store_true', |
587 help=('Do not impose a timeout. Each perf step is responsible for ' | 598 help=('Do not impose a timeout. Each perf step is responsible for ' |
588 'implementing the timeout logic.')) | 599 'implementing the timeout logic.')) |
589 group.add_argument( | 600 group.add_argument( |
590 '-f', '--test-filter', | 601 '-f', '--test-filter', |
591 help=('Test filter (will match against the names listed in --steps).')) | 602 help=('Test filter (will match against the names listed in --steps).')) |
592 group.add_argument( | 603 group.add_argument( |
(...skipping 346 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
939 if e.is_infra_error: | 950 if e.is_infra_error: |
940 return constants.INFRA_EXIT_CODE | 951 return constants.INFRA_EXIT_CODE |
941 return constants.ERROR_EXIT_CODE | 952 return constants.ERROR_EXIT_CODE |
942 except: # pylint: disable=W0702 | 953 except: # pylint: disable=W0702 |
943 logging.exception('Unrecognized error occurred.') | 954 logging.exception('Unrecognized error occurred.') |
944 return constants.ERROR_EXIT_CODE | 955 return constants.ERROR_EXIT_CODE |
945 | 956 |
946 | 957 |
947 if __name__ == '__main__': | 958 if __name__ == '__main__': |
948 sys.exit(main()) | 959 sys.exit(main()) |
OLD | NEW |