Retry Chromoting browser-test execution if test times-out without launching browser.

testing/chromoting/browser_tests_launcher.py

 # Copyright (c) 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 
 """Utility script to launch browser-tests on the Chromoting bot."""
 import argparse
 import glob
 import hashlib
 import os
 from os.path import expanduser
 import shutil
 import socket
 import subprocess
 
 import psutil
 
 BROWSER_TEST_ID = 'browser_tests'
 PROD_DIR_ID = '#PROD_DIR#'
 HOST_HASH_VALUE = hashlib.md5(socket.gethostname()).hexdigest()
 SUCCESS_INDICATOR = 'SUCCESS: all tests passed.'
 NATIVE_MESSAGING_DIR = 'NativeMessagingHosts'
 CRD_ID = 'chrome-remote-desktop'  # Used in a few file/folder names
 CHROMOTING_HOST_PATH = '/opt/google/chrome-remote-desktop/chrome-remote-desktop'
 TEST_FAILURE = False
 FAILING_TESTS = ''
 HOST_READY_INDICATOR = 'Host ready to receive connections.'
+BROWSER_NOT_STARTED_ERROR = (
+    'Still waiting for the following processes to finish')
+TIME_OUT_INDICATOR = '(TIMED OUT)'
 
 
-def LaunchBTCommand(command):
+def LaunchBTCommand(args, command):
+  """Launches the specified browser-test command.
+
+      If the execution failed because a browser-instance was not launched, retry
+      once.
+  Args:
+    args: Command line args, used for test-case startup tasks.
+    command: Browser-test command line.
+  """
   global TEST_FAILURE, FAILING_TESTS
   results = RunCommandInSubProcess(command)
 

[joedow, 2015/06/09 23:59:32]

I think you should move the TestCaseSetup() call from line 217 up here.  It
would be cleaner if you have the original run and retry logic in a loop which
terminates when:
1.) Test passes
2.) Test fails for unexpected reason
3.) Test has failed with the time out issue the maximum number of times

This would clean up the flow and also allow you to tweak the re-run conditions
later on if needed.

[anandc, 2015/06/10 01:02:12]

Done.

+  # Sometimes, during execution of browser-tests, a browser instance is
+  # not started and the test times out. See http://crbug/480025.
+  # To work around it, check if this execution failed owing to that
+  # problem and retry.
+  # There are 2 things to look for in the results:
+  # A line saying "Still waiting for the following processes to finish",
+  # and, because sometimes that line gets logged even if the test
+  # eventually passes, we'll also look for "(TIMED OUT)", before retrying.

[joedow, 2015/06/09 23:59:32]

Just curious, is there an error code we can key off of or is string compare the
only way to tell?

[anandc, 2015/06/10 01:02:12]

There doesn't seem to be a special error code for a time-out.
See here for results when a time-out occurs (search for "Return code = "):
https://chromium-swarm.appspot.com/user/task/27f3b5c2605bbd10

And here for results when a test fails for a reason other than a time-out:
https://chromium-swarm.appspot.com/user/task/27f3ba00601c8f10
Both return an error code of 1.

+  if BROWSER_NOT_STARTED_ERROR in results and TIME_OUT_INDICATOR in results:
+    # Retry test execution once.
+    TestCaseSetup(args)
+    results = RunCommandInSubProcess(command)
+
   # Check that the test passed.
   if SUCCESS_INDICATOR not in results:
     TEST_FAILURE = True
     # Add this command-line to list of tests that failed.
     FAILING_TESTS += command
 
 
 def RunCommandInSubProcess(command):
   """Creates a subprocess with command-line that is passed in.
 
@@ skipping 122 matching lines @@
 
 def PrintRunningProcesses():
   processes = psutil.get_process_list()
   processes = sorted(processes, key=lambda process: process.name)
 
   print 'List of running processes:\n'
   for process in processes:
     print process.name
 
 
+def TestCaseSetup(args):
+  # Stop+start me2me host process.
+  if not RestartMe2MeHost():
+    # Host restart failed. Don't run any more tests.
+    raise Exception('Host restart failed.')
+
+  # Reset the user profile directory to start each test with a clean slate.
+  SetupUserProfileDir(args.me2me_manifest_file, args.it2me_manifest_file,
+                      args.user_profile_dir)
+
+
 def main(args):
 
   InitialiseTestMachineForLinux(args.cfg_file)
 
   with open(args.commands_file) as f:
     for line in f:
-      # Reset the user profile directory to start each test with a clean slate.
-      SetupUserProfileDir(args.me2me_manifest_file, args.it2me_manifest_file,
-                          args.user_profile_dir)
-
+      TestCaseSetup(args)
       # Replace the PROD_DIR value in the command-line with
       # the passed in value.
       line = line.replace(PROD_DIR_ID, args.prod_dir)
       # Launch specified command line for test.
-      LaunchBTCommand(line)
-      # After each test, stop+start me2me host process.
-      if not RestartMe2MeHost():
-        # Host restart failed. Don't run any more tests.
-        raise Exception('Host restart failed.')
-
-      # Print list of currently running processes.
-      PrintRunningProcesses()
+      LaunchBTCommand(args, line)
 
   # All tests completed. Include host-logs in the test results.
   host_log_contents = ''
   # There should be only 1 log file, as we delete logs on test completion.
   # Loop through matching files, just in case there are more.
   for log_file in glob.glob('/tmp/chrome_remote_desktop_*'):
     with open(log_file, 'r') as log:
       host_log_contents += '\nHOST LOG %s\n CONTENTS:\n%s' % (
           log_file, log.read())
   print host_log_contents
@@ skipping 20 matching lines @@
                       help='path to it2me host manifest file.')
   parser.add_argument(
       '-u', '--user_profile_dir',
       help='path to user-profile-dir, used by connect-to-host tests.')
   command_line_args = parser.parse_args()
   try:
     main(command_line_args)
   finally:
     # Stop host and cleanup user-profile-dir.
     TestMachineCleanup(command_line_args.user_profile_dir)