Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(266)

Side by Side Diff: build/android/test_runner.py

Issue 18323020: Updates the test runner script exit codes (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Changes exit_code tracking to a singleton paradigm Created 7 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « build/android/pylib/gtest/test_runner.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # 2 #
3 # Copyright 2013 The Chromium Authors. All rights reserved. 3 # Copyright 2013 The Chromium Authors. All rights reserved.
4 # Use of this source code is governed by a BSD-style license that can be 4 # Use of this source code is governed by a BSD-style license that can be
5 # found in the LICENSE file. 5 # found in the LICENSE file.
6 6
7 """Runs all types of tests from one unified interface. 7 """Runs all types of tests from one unified interface.
8 8
9 TODO(gkanwar): 9 TODO(gkanwar):
10 * Add options to run Monkey tests. 10 * Add options to run Monkey tests.
11 """ 11 """
12 12
13 import collections 13 import collections
14 import optparse 14 import optparse
15 import os 15 import os
16 import sys 16 import sys
17 17
18 from pylib import cmd_helper 18 from pylib import cmd_helper
19 from pylib import constants 19 from pylib import constants
20 from pylib import exit_code
20 from pylib import ports 21 from pylib import ports
21 from pylib.base import base_test_result 22 from pylib.base import base_test_result
22 from pylib.browsertests import dispatch as browsertests_dispatch 23 from pylib.browsertests import dispatch as browsertests_dispatch
23 from pylib.gtest import dispatch as gtest_dispatch 24 from pylib.gtest import dispatch as gtest_dispatch
24 from pylib.host_driven import run_python_tests as python_dispatch 25 from pylib.host_driven import run_python_tests as python_dispatch
25 from pylib.instrumentation import dispatch as instrumentation_dispatch 26 from pylib.instrumentation import dispatch as instrumentation_dispatch
26 from pylib.uiautomator import dispatch as uiautomator_dispatch 27 from pylib.uiautomator import dispatch as uiautomator_dispatch
27 from pylib.utils import emulator, report_results, run_tests_helper 28 from pylib.utils import emulator, report_results, run_tests_helper
28 29
29 30
(...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after
101 '(use --tool help to list them)')) 102 '(use --tool help to list them)'))
102 option_parser.add_option('--flakiness-dashboard-server', 103 option_parser.add_option('--flakiness-dashboard-server',
103 dest='flakiness_dashboard_server', 104 dest='flakiness_dashboard_server',
104 help=('Address of the server that is hosting the ' 105 help=('Address of the server that is hosting the '
105 'Chrome for Android flakiness dashboard.')) 106 'Chrome for Android flakiness dashboard.'))
106 option_parser.add_option('--skip-deps-push', dest='push_deps', 107 option_parser.add_option('--skip-deps-push', dest='push_deps',
107 action='store_false', default=True, 108 action='store_false', default=True,
108 help=('Do not push dependencies to the device. ' 109 help=('Do not push dependencies to the device. '
109 'Use this at own risk for speeding up test ' 110 'Use this at own risk for speeding up test '
110 'execution on local machine.')) 111 'execution on local machine.'))
111 # TODO(gkanwar): This option is deprecated. Remove it in the future.
112 option_parser.add_option('--exit-code', action='store_true',
113 help=('(DEPRECATED) If set, the exit code will be '
114 'total number of failures.'))
115 # TODO(gkanwar): This option is deprecated. It is currently used to run tests
116 # with the FlakyTest annotation to prevent the bots going red downstream. We
117 # should instead use exit codes and let the Buildbot scripts deal with test
118 # failures appropriately. See crbug.com/170477.
119 option_parser.add_option('--buildbot-step-failure',
120 action='store_true',
121 help=('(DEPRECATED) If present, will set the '
122 'buildbot status as STEP_FAILURE, otherwise '
123 'as STEP_WARNINGS when test(s) fail.'))
124 option_parser.add_option('-d', '--device', dest='test_device', 112 option_parser.add_option('-d', '--device', dest='test_device',
125 help=('Target device for the test suite ' 113 help=('Target device for the test suite '
126 'to run on.')) 114 'to run on.'))
127 115
128 116
129 def ProcessCommonOptions(options): 117 def ProcessCommonOptions(options):
130 """Processes and handles all common options.""" 118 """Processes and handles all common options."""
131 if options.out_directory: 119 if options.out_directory:
132 cmd_helper.OutDirectory.set(options.out_directory) 120 cmd_helper.OutDirectory.set(options.out_directory)
133 run_tests_helper.SetLogLevel(options.verbose_count) 121 run_tests_helper.SetLogLevel(options.verbose_count)
(...skipping 234 matching lines...) Expand 10 before | Expand all | Expand 10 after
368 356
369 Args: 357 Args:
370 command: String indicating the command that was received to trigger 358 command: String indicating the command that was received to trigger
371 this function. 359 this function.
372 options: optparse options dictionary. 360 options: optparse options dictionary.
373 args: List of extra args from optparse. 361 args: List of extra args from optparse.
374 option_parser: optparse.OptionParser object. 362 option_parser: optparse.OptionParser object.
375 363
376 Returns: 364 Returns:
377 Integer indicated exit code. 365 Integer indicated exit code.
366
367 Raises:
368 Exception: Unknown command name passed in.
378 """ 369 """
379 370
371 exit_code.ResetExitCode()
372
380 ProcessCommonOptions(options) 373 ProcessCommonOptions(options)
381 374
382 total_failed = 0
383 if command == 'gtest': 375 if command == 'gtest':
384 # TODO(gkanwar): See the emulator TODO above -- this call should either go 376 # TODO(gkanwar): See the emulator TODO above -- this call should either go
385 # away or become generalized. 377 # away or become generalized.
386 ProcessEmulatorOptions(options) 378 ProcessEmulatorOptions(options)
387 total_failed = gtest_dispatch.Dispatch(options) 379 gtest_dispatch.Dispatch(options)
380 returned_exit_code = exit_code.GetExitCode()
388 elif command == 'content_browsertests': 381 elif command == 'content_browsertests':
389 total_failed = browsertests_dispatch.Dispatch(options) 382 browsertests_dispatch.Dispatch(options)
383 returned_exit_code = exit_code.GetExitCode()
390 elif command == 'instrumentation': 384 elif command == 'instrumentation':
391 ProcessInstrumentationOptions(options, option_parser.error) 385 ProcessInstrumentationOptions(options, option_parser.error)
392 results = base_test_result.TestRunResults() 386 results = base_test_result.TestRunResults()
393 if options.run_java_tests: 387 if options.run_java_tests:
394 results.AddTestRunResults(instrumentation_dispatch.Dispatch(options)) 388 results.AddTestRunResults(instrumentation_dispatch.Dispatch(options))
395 if options.run_python_tests: 389 if options.run_python_tests:
396 results.AddTestRunResults(python_dispatch.DispatchPythonTests(options)) 390 results.AddTestRunResults(python_dispatch.DispatchPythonTests(options))
397 report_results.LogFull( 391 report_results.LogFull(
398 results=results, 392 results=results,
399 test_type='Instrumentation', 393 test_type='Instrumentation',
400 test_package=os.path.basename(options.test_apk), 394 test_package=os.path.basename(options.test_apk),
401 annotation=options.annotations, 395 annotation=options.annotations,
402 build_type=options.build_type, 396 build_type=options.build_type,
403 flakiness_server=options.flakiness_dashboard_server) 397 flakiness_server=options.flakiness_dashboard_server)
404 total_failed += len(results.GetNotPass()) 398 returned_exit_code = exit_code.GetExitCode()
405 elif command == 'uiautomator': 399 elif command == 'uiautomator':
406 ProcessUIAutomatorOptions(options, option_parser.error) 400 ProcessUIAutomatorOptions(options, option_parser.error)
407 results = base_test_result.TestRunResults() 401 results = base_test_result.TestRunResults()
408 if options.run_java_tests: 402 if options.run_java_tests:
409 results.AddTestRunResults(uiautomator_dispatch.Dispatch(options)) 403 results.AddTestRunResults(uiautomator_dispatch.Dispatch(options))
410 if options.run_python_tests: 404 if options.run_python_tests:
411 results.AddTestRunResults(python_dispatch.Dispatch(options)) 405 results.AddTestRunResults(python_dispatch.Dispatch(options))
412 report_results.LogFull( 406 report_results.LogFull(
413 results=results, 407 results=results,
414 test_type='UIAutomator', 408 test_type='UIAutomator',
415 test_package=os.path.basename(options.test_jar), 409 test_package=os.path.basename(options.test_jar),
416 annotation=options.annotations, 410 annotation=options.annotations,
417 build_type=options.build_type, 411 build_type=options.build_type,
418 flakiness_server=options.flakiness_dashboard_server) 412 flakiness_server=options.flakiness_dashboard_server)
419 total_failed += len(results.GetNotPass()) 413 returned_exit_code = exit_code.GetExitCode()
frankf 2013/07/04 00:46:19 Why not just return this on line 417?
gkanwar 2013/07/08 18:50:25 In the updated code we get the exit_code before pr
420 else: 414 else:
421 raise Exception('Unknown test type state') 415 raise Exception('Unknown test type state')
422 416
423 return total_failed 417 return returned_exit_code
424 418
425 419
426 def HelpCommand(command, options, args, option_parser): 420 def HelpCommand(command, options, args, option_parser):
427 """Display help for a certain command, or overall help. 421 """Display help for a certain command, or overall help.
428 422
429 Args: 423 Args:
430 command: String indicating the command that was received to trigger 424 command: String indicating the command that was received to trigger
431 this function. 425 this function.
432 options: optparse options dictionary. 426 options: optparse options dictionary.
433 args: List of extra args from optparse. 427 args: List of extra args from optparse.
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after
496 def get_command_list(self): 490 def get_command_list(self):
497 if self.command_list: 491 if self.command_list:
498 return '\nCommands:\n %s\n' % '\n '.join(sorted(self.command_list)) 492 return '\nCommands:\n %s\n' % '\n '.join(sorted(self.command_list))
499 return '' 493 return ''
500 494
501 def get_example(self): 495 def get_example(self):
502 if self.example: 496 if self.example:
503 return '\nExample:\n %s\n' % self.example 497 return '\nExample:\n %s\n' % self.example
504 return '' 498 return ''
505 499
500
506 def main(argv): 501 def main(argv):
507 option_parser = CommandOptionParser( 502 option_parser = CommandOptionParser(
508 usage='Usage: %prog <command> [options]', 503 usage='Usage: %prog <command> [options]',
509 command_list=VALID_COMMANDS.keys()) 504 command_list=VALID_COMMANDS.keys())
510 505
511 if len(argv) < 2 or argv[1] not in VALID_COMMANDS: 506 if len(argv) < 2 or argv[1] not in VALID_COMMANDS:
512 option_parser.print_help() 507 option_parser.print_help()
513 return 0 508 return 0
514 command = argv[1] 509 command = argv[1]
515 VALID_COMMANDS[command].add_options_func(option_parser) 510 VALID_COMMANDS[command].add_options_func(option_parser)
516 options, args = option_parser.parse_args(argv) 511 options, args = option_parser.parse_args(argv)
517 exit_code = VALID_COMMANDS[command].run_command_func( 512 exit_code = VALID_COMMANDS[command].run_command_func(
518 command, options, args, option_parser) 513 command, options, args, option_parser)
519 514
520 # Failures of individual test suites are communicated by printing a
521 # STEP_FAILURE message.
522 # Returning a success exit status also prevents the buildbot from incorrectly
523 # marking the last suite as failed if there were failures in other suites in
524 # the batch (this happens because the exit status is a sum of all failures
525 # from all suites, but the buildbot associates the exit status only with the
526 # most recent step).
527 return exit_code 515 return exit_code
528 516
529 517
530 if __name__ == '__main__': 518 if __name__ == '__main__':
531 sys.exit(main(sys.argv)) 519 sys.exit(main(sys.argv))
OLDNEW
« no previous file with comments | « build/android/pylib/gtest/test_runner.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698