OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # | 2 # |
3 # Copyright 2013 The Chromium Authors. All rights reserved. | 3 # Copyright 2013 The Chromium Authors. All rights reserved. |
4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
6 | 6 |
7 """Runs all types of tests from one unified interface. | 7 """Runs all types of tests from one unified interface. |
8 | 8 |
9 TODO(gkanwar): | 9 TODO(gkanwar): |
10 * Add options to run Monkey tests. | 10 * Add options to run Monkey tests. |
11 """ | 11 """ |
12 | 12 |
13 import collections | 13 import collections |
14 import optparse | 14 import optparse |
15 import os | 15 import os |
16 import shutil | 16 import shutil |
17 import sys | 17 import sys |
18 | 18 |
19 from pylib import cmd_helper | 19 from pylib import cmd_helper |
20 from pylib import constants | 20 from pylib import constants |
21 from pylib import ports | 21 from pylib import ports |
22 from pylib.base import base_test_result | 22 from pylib.base import base_test_result |
23 from pylib.base import test_dispatcher | 23 from pylib.base import test_dispatcher |
24 from pylib.browsertests import setup as browsertests_setup | |
25 from pylib.gtest import setup as gtest_setup | 24 from pylib.gtest import setup as gtest_setup |
26 from pylib.gtest import gtest_config | 25 from pylib.gtest import gtest_config |
27 from pylib.host_driven import run_python_tests as python_dispatch | 26 from pylib.host_driven import run_python_tests as python_dispatch |
28 from pylib.instrumentation import setup as instrumentation_setup | 27 from pylib.instrumentation import setup as instrumentation_setup |
29 from pylib.uiautomator import setup as uiautomator_setup | 28 from pylib.uiautomator import setup as uiautomator_setup |
30 from pylib.utils import report_results | 29 from pylib.utils import report_results |
31 from pylib.utils import run_tests_helper | 30 from pylib.utils import run_tests_helper |
32 | 31 |
33 | 32 |
34 _SDK_OUT_DIR = os.path.join(constants.DIR_SOURCE_ROOT, 'out') | 33 _SDK_OUT_DIR = os.path.join(constants.DIR_SOURCE_ROOT, 'out') |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
90 option_parser.add_option('-d', '--device', dest='test_device', | 89 option_parser.add_option('-d', '--device', dest='test_device', |
91 help=('Target device for the test suite ' | 90 help=('Target device for the test suite ' |
92 'to run on.')) | 91 'to run on.')) |
93 | 92 |
94 | 93 |
95 def ProcessCommonOptions(options): | 94 def ProcessCommonOptions(options): |
96 """Processes and handles all common options.""" | 95 """Processes and handles all common options.""" |
97 run_tests_helper.SetLogLevel(options.verbose_count) | 96 run_tests_helper.SetLogLevel(options.verbose_count) |
98 | 97 |
99 | 98 |
100 def AddCoreGTestOptions(option_parser): | |
101 """Add options specific to the gtest framework to |option_parser|.""" | |
102 | |
103 # TODO(gkanwar): Consolidate and clean up test filtering for gtests and | |
104 # content_browsertests. | |
105 option_parser.add_option('-f', '--gtest_filter', dest='test_filter', | |
106 help='googletest-style filter string.') | |
107 option_parser.add_option('-a', '--test_arguments', dest='test_arguments', | |
108 help='Additional arguments to pass to the test.') | |
109 option_parser.add_option('--exe', action='store_true', | |
110 help='If set, use the exe test runner instead of ' | |
111 'the APK.') | |
112 option_parser.add_option('-t', dest='timeout', | |
113 help='Timeout to wait for each test', | |
114 type='int', | |
115 default=60) | |
116 | |
117 | |
118 def AddContentBrowserTestOptions(option_parser): | |
119 """Adds Content Browser test options to |option_parser|.""" | |
120 | |
121 option_parser.usage = '%prog content_browsertests [options]' | |
122 option_parser.command_list = [] | |
123 option_parser.example = '%prog content_browsertests' | |
124 | |
125 AddCoreGTestOptions(option_parser) | |
126 AddCommonOptions(option_parser) | |
127 | |
128 | |
129 def AddGTestOptions(option_parser): | 99 def AddGTestOptions(option_parser): |
130 """Adds gtest options to |option_parser|.""" | 100 """Adds gtest options to |option_parser|.""" |
131 | 101 |
132 option_parser.usage = '%prog gtest [options]' | 102 option_parser.usage = '%prog gtest [options]' |
133 option_parser.command_list = [] | 103 option_parser.command_list = [] |
134 option_parser.example = '%prog gtest -s base_unittests' | 104 option_parser.example = '%prog gtest -s base_unittests' |
135 | 105 |
136 # TODO(gkanwar): Make this option required | 106 # TODO(gkanwar): Make this option required |
137 option_parser.add_option('-s', '--suite', dest='suite_name', | 107 option_parser.add_option('-s', '--suite', dest='suite_name', |
138 help=('Executable name of the test suite to run ' | 108 help=('Executable name of the test suite to run ' |
139 '(use -s help to list them).')) | 109 '(use -s help to list them).')) |
140 AddCoreGTestOptions(option_parser) | 110 option_parser.add_option('-f', '--gtest_filter', dest='test_filter', |
| 111 help='googletest-style filter string.') |
| 112 option_parser.add_option('-a', '--test_arguments', dest='test_arguments', |
| 113 help='Additional arguments to pass to the test.') |
| 114 option_parser.add_option('-t', dest='timeout', |
| 115 help='Timeout to wait for each test', |
| 116 type='int', |
| 117 default=60) |
141 # TODO(gkanwar): Move these to Common Options once we have the plumbing | 118 # TODO(gkanwar): Move these to Common Options once we have the plumbing |
142 # in our other test types to handle these commands | 119 # in our other test types to handle these commands |
143 AddCommonOptions(option_parser) | 120 AddCommonOptions(option_parser) |
144 | 121 |
145 | 122 |
146 def ProcessGTestOptions(options): | 123 def ProcessGTestOptions(options): |
147 """Intercept test suite help to list test suites. | 124 """Intercept test suite help to list test suites. |
148 | 125 |
149 Args: | 126 Args: |
150 options: Command line options. | 127 options: Command line options. |
151 | 128 |
152 Returns: | 129 Returns: |
153 True if the command should continue. | 130 True if the command should continue. |
154 """ | 131 """ |
155 if options.suite_name == 'help': | 132 if options.suite_name == 'help': |
156 print 'Available test suites are:' | 133 print 'Available test suites are:' |
157 for test_suite in gtest_config.STABLE_TEST_SUITES: | 134 for test_suite in (gtest_config.STABLE_TEST_SUITES + |
158 print test_suite.name | 135 gtest_config.EXPERIMENTAL_TEST_SUITES): |
| 136 print test_suite |
159 return False | 137 return False |
160 | 138 |
161 # Convert to a list, assuming all test suites if nothing was specified. | 139 # Convert to a list, assuming all test suites if nothing was specified. |
162 # TODO(gkanwar): Require having a test suite | 140 # TODO(gkanwar): Require having a test suite |
163 if options.suite_name: | 141 if options.suite_name: |
164 options.suite_name = [options.suite_name] | 142 options.suite_name = [options.suite_name] |
165 else: | 143 else: |
166 options.suite_name = [suite.name | 144 options.suite_name = [s for s in gtest_config.STABLE_TEST_SUITES] |
167 for suite in gtest_config.STABLE_TEST_SUITES] | |
168 return True | 145 return True |
169 | 146 |
170 | 147 |
171 def AddJavaTestOptions(option_parser): | 148 def AddJavaTestOptions(option_parser): |
172 """Adds the Java test options to |option_parser|.""" | 149 """Adds the Java test options to |option_parser|.""" |
173 | 150 |
174 option_parser.add_option('-f', '--test_filter', dest='test_filter', | 151 option_parser.add_option('-f', '--test_filter', dest='test_filter', |
175 help=('Test filter (if not fully qualified, ' | 152 help=('Test filter (if not fully qualified, ' |
176 'will run all matches).')) | 153 'will run all matches).')) |
177 option_parser.add_option( | 154 option_parser.add_option( |
(...skipping 165 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
343 | 320 |
344 | 321 |
345 def _RunGTests(options, error_func): | 322 def _RunGTests(options, error_func): |
346 """Subcommand of RunTestsCommands which runs gtests.""" | 323 """Subcommand of RunTestsCommands which runs gtests.""" |
347 if not ProcessGTestOptions(options): | 324 if not ProcessGTestOptions(options): |
348 return 0 | 325 return 0 |
349 | 326 |
350 exit_code = 0 | 327 exit_code = 0 |
351 for suite_name in options.suite_name: | 328 for suite_name in options.suite_name: |
352 runner_factory, tests = gtest_setup.Setup( | 329 runner_factory, tests = gtest_setup.Setup( |
353 options.exe, suite_name, options.test_arguments, | 330 suite_name, options.test_arguments, |
354 options.timeout, options.cleanup_test_files, options.tool, | 331 options.timeout, options.cleanup_test_files, options.tool, |
355 options.build_type, options.push_deps, options.test_filter) | 332 options.build_type, options.push_deps, options.test_filter) |
356 | 333 |
357 results, test_exit_code = test_dispatcher.RunTests( | 334 results, test_exit_code = test_dispatcher.RunTests( |
358 tests, runner_factory, False, options.test_device, | 335 tests, runner_factory, False, options.test_device, |
359 shard=True, | 336 shard=True, |
360 build_type=options.build_type, | 337 build_type=options.build_type, |
361 test_timeout=None, | 338 test_timeout=None, |
362 num_retries=options.num_retries) | 339 num_retries=options.num_retries) |
363 | 340 |
364 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: | 341 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: |
365 exit_code = test_exit_code | 342 exit_code = test_exit_code |
366 | 343 |
367 report_results.LogFull( | 344 report_results.LogFull( |
368 results=results, | 345 results=results, |
369 test_type='Unit test', | 346 test_type='Unit test', |
370 test_package=suite_name, | 347 test_package=suite_name, |
371 build_type=options.build_type, | 348 build_type=options.build_type, |
372 flakiness_server=options.flakiness_dashboard_server) | 349 flakiness_server=options.flakiness_dashboard_server) |
373 | 350 |
374 if os.path.isdir(constants.ISOLATE_DEPS_DIR): | 351 if os.path.isdir(constants.ISOLATE_DEPS_DIR): |
375 shutil.rmtree(constants.ISOLATE_DEPS_DIR) | 352 shutil.rmtree(constants.ISOLATE_DEPS_DIR) |
376 | 353 |
377 return exit_code | 354 return exit_code |
378 | 355 |
379 | 356 |
380 def _RunContentBrowserTests(options, error_func): | |
381 """Subcommand of RunTestsCommands which runs content_browsertests.""" | |
382 runner_factory, tests = browsertests_setup.Setup( | |
383 options.test_arguments, options.timeout, options.cleanup_test_files, | |
384 options.tool, options.build_type, options.push_deps, | |
385 options.test_filter) | |
386 | |
387 # TODO(nileshagrawal): remove this abnormally long setup timeout once fewer | |
388 # files are pushed to the devices for content_browsertests: crbug.com/138275 | |
389 setup_timeout = 20 * 60 # 20 minutes | |
390 results, exit_code = test_dispatcher.RunTests( | |
391 tests, runner_factory, False, options.test_device, | |
392 shard=True, | |
393 build_type=options.build_type, | |
394 test_timeout=None, | |
395 setup_timeout=setup_timeout, | |
396 num_retries=options.num_retries) | |
397 | |
398 report_results.LogFull( | |
399 results=results, | |
400 test_type='Unit test', | |
401 test_package=constants.BROWSERTEST_SUITE_NAME, | |
402 build_type=options.build_type, | |
403 flakiness_server=options.flakiness_dashboard_server) | |
404 | |
405 if os.path.isdir(constants.ISOLATE_DEPS_DIR): | |
406 shutil.rmtree(constants.ISOLATE_DEPS_DIR) | |
407 | |
408 return exit_code | |
409 | |
410 | |
411 def _RunInstrumentationTests(options, error_func): | 357 def _RunInstrumentationTests(options, error_func): |
412 """Subcommand of RunTestsCommands which runs instrumentation tests.""" | 358 """Subcommand of RunTestsCommands which runs instrumentation tests.""" |
413 ProcessInstrumentationOptions(options, error_func) | 359 ProcessInstrumentationOptions(options, error_func) |
414 | 360 |
415 results = base_test_result.TestRunResults() | 361 results = base_test_result.TestRunResults() |
416 exit_code = 0 | 362 exit_code = 0 |
417 | 363 |
418 if options.run_java_tests: | 364 if options.run_java_tests: |
419 runner_factory, tests = instrumentation_setup.Setup( | 365 runner_factory, tests = instrumentation_setup.Setup( |
420 options.test_apk_path, options.test_apk_jar_path, options.annotations, | 366 options.test_apk_path, options.test_apk_jar_path, options.annotations, |
(...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
520 | 466 |
521 # Check for extra arguments | 467 # Check for extra arguments |
522 if len(args) > 2: | 468 if len(args) > 2: |
523 option_parser.error('Unrecognized arguments: %s' % (' '.join(args[2:]))) | 469 option_parser.error('Unrecognized arguments: %s' % (' '.join(args[2:]))) |
524 return constants.ERROR_EXIT_CODE | 470 return constants.ERROR_EXIT_CODE |
525 | 471 |
526 ProcessCommonOptions(options) | 472 ProcessCommonOptions(options) |
527 | 473 |
528 if command == 'gtest': | 474 if command == 'gtest': |
529 return _RunGTests(options, option_parser.error) | 475 return _RunGTests(options, option_parser.error) |
530 elif command == 'content_browsertests': | |
531 return _RunContentBrowserTests(options, option_parser.error) | |
532 elif command == 'instrumentation': | 476 elif command == 'instrumentation': |
533 return _RunInstrumentationTests(options, option_parser.error) | 477 return _RunInstrumentationTests(options, option_parser.error) |
534 elif command == 'uiautomator': | 478 elif command == 'uiautomator': |
535 return _RunUIAutomatorTests(options, option_parser.error) | 479 return _RunUIAutomatorTests(options, option_parser.error) |
536 else: | 480 else: |
537 raise Exception('Unknown test type.') | 481 raise Exception('Unknown test type.') |
538 | 482 |
539 return exit_code | 483 return exit_code |
540 | 484 |
541 | 485 |
(...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
580 return 0 | 524 return 0 |
581 | 525 |
582 | 526 |
583 # Define a named tuple for the values in the VALID_COMMANDS dictionary so the | 527 # Define a named tuple for the values in the VALID_COMMANDS dictionary so the |
584 # syntax is a bit prettier. The tuple is two functions: (add options, run | 528 # syntax is a bit prettier. The tuple is two functions: (add options, run |
585 # command). | 529 # command). |
586 CommandFunctionTuple = collections.namedtuple( | 530 CommandFunctionTuple = collections.namedtuple( |
587 'CommandFunctionTuple', ['add_options_func', 'run_command_func']) | 531 'CommandFunctionTuple', ['add_options_func', 'run_command_func']) |
588 VALID_COMMANDS = { | 532 VALID_COMMANDS = { |
589 'gtest': CommandFunctionTuple(AddGTestOptions, RunTestsCommand), | 533 'gtest': CommandFunctionTuple(AddGTestOptions, RunTestsCommand), |
590 'content_browsertests': CommandFunctionTuple( | |
591 AddContentBrowserTestOptions, RunTestsCommand), | |
592 'instrumentation': CommandFunctionTuple( | 534 'instrumentation': CommandFunctionTuple( |
593 AddInstrumentationTestOptions, RunTestsCommand), | 535 AddInstrumentationTestOptions, RunTestsCommand), |
594 'uiautomator': CommandFunctionTuple( | 536 'uiautomator': CommandFunctionTuple( |
595 AddUIAutomatorTestOptions, RunTestsCommand), | 537 AddUIAutomatorTestOptions, RunTestsCommand), |
596 'help': CommandFunctionTuple(lambda option_parser: None, HelpCommand) | 538 'help': CommandFunctionTuple(lambda option_parser: None, HelpCommand) |
597 } | 539 } |
598 | 540 |
599 | 541 |
600 class CommandOptionParser(optparse.OptionParser): | 542 class CommandOptionParser(optparse.OptionParser): |
601 """Wrapper class for OptionParser to help with listing commands.""" | 543 """Wrapper class for OptionParser to help with listing commands.""" |
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
634 return 0 | 576 return 0 |
635 command = argv[1] | 577 command = argv[1] |
636 VALID_COMMANDS[command].add_options_func(option_parser) | 578 VALID_COMMANDS[command].add_options_func(option_parser) |
637 options, args = option_parser.parse_args(argv) | 579 options, args = option_parser.parse_args(argv) |
638 return VALID_COMMANDS[command].run_command_func( | 580 return VALID_COMMANDS[command].run_command_func( |
639 command, options, args, option_parser) | 581 command, options, args, option_parser) |
640 | 582 |
641 | 583 |
642 if __name__ == '__main__': | 584 if __name__ == '__main__': |
643 sys.exit(main(sys.argv)) | 585 sys.exit(main(sys.argv)) |
OLD | NEW |