Retry Chromoting browser-test execution if test times-out without launching browser.

testing/chromoting/browser_tests_launcher.py

Index: testing/chromoting/browser_tests_launcher.py
diff --git a/testing/chromoting/browser_tests_launcher.py b/testing/chromoting/browser_tests_launcher.py
index 1c9652fb513c5c6b576aa214994a50c283a11ac0..2b7aed8e9f149f8ea78e7be410ccc63e8074e646 100644
--- a/testing/chromoting/browser_tests_launcher.py
+++ b/testing/chromoting/browser_tests_launcher.py
@@ -25,12 +25,36 @@ CHROMOTING_HOST_PATH = '/opt/google/chrome-remote-desktop/chrome-remote-desktop'
 TEST_FAILURE = False
 FAILING_TESTS = ''
 HOST_READY_INDICATOR = 'Host ready to receive connections.'
+BROWSER_NOT_STARTED_ERROR = (
+    'Still waiting for the following processes to finish')
+TIME_OUT_INDICATOR = '(TIMED OUT)'
 
 
-def LaunchBTCommand(command):
+def LaunchBTCommand(args, command):
+  """Launches the specified browser-test command.
+
+      If the execution failed because a browser-instance was not launched, retry
+      once.
+  Args:
+    args: Command line args, used for test-case startup tasks.
+    command: Browser-test command line.
+  """
   global TEST_FAILURE, FAILING_TESTS
   results = RunCommandInSubProcess(command)
 

[joedow, 2015/06/09 23:59:32]

I think you should move the TestCaseSetup() call from line 217 up here.  It
would be cleaner if you have the original run and retry logic in a loop which
terminates when:
1.) Test passes
2.) Test fails for unexpected reason
3.) Test has failed with the time out issue the maximum number of times

This would clean up the flow and also allow you to tweak the re-run conditions
later on if needed.

[anandc, 2015/06/10 01:02:12]

Done.

+  # Sometimes, during execution of browser-tests, a browser instance is
+  # not started and the test times out. See http://crbug/480025.
+  # To work around it, check if this execution failed owing to that
+  # problem and retry.
+  # There are 2 things to look for in the results:
+  # A line saying "Still waiting for the following processes to finish",
+  # and, because sometimes that line gets logged even if the test
+  # eventually passes, we'll also look for "(TIMED OUT)", before retrying.

[joedow, 2015/06/09 23:59:32]

Just curious, is there an error code we can key off of or is string compare the
only way to tell?

[anandc, 2015/06/10 01:02:12]

There doesn't seem to be a special error code for a time-out.
See here for results when a time-out occurs (search for "Return code = "):
https://chromium-swarm.appspot.com/user/task/27f3b5c2605bbd10

And here for results when a test fails for a reason other than a time-out:
https://chromium-swarm.appspot.com/user/task/27f3ba00601c8f10
Both return an error code of 1.

+  if BROWSER_NOT_STARTED_ERROR in results and TIME_OUT_INDICATOR in results:
+    # Retry test execution once.
+    TestCaseSetup(args)
+    results = RunCommandInSubProcess(command)
+
   # Check that the test passed.
   if SUCCESS_INDICATOR not in results:
     TEST_FAILURE = True
@@ -173,28 +197,29 @@ def PrintRunningProcesses():
     print process.name
 
 
+def TestCaseSetup(args):
+  # Stop+start me2me host process.
+  if not RestartMe2MeHost():
+    # Host restart failed. Don't run any more tests.
+    raise Exception('Host restart failed.')
+
+  # Reset the user profile directory to start each test with a clean slate.
+  SetupUserProfileDir(args.me2me_manifest_file, args.it2me_manifest_file,
+                      args.user_profile_dir)
+
+
 def main(args):
 
   InitialiseTestMachineForLinux(args.cfg_file)
 
   with open(args.commands_file) as f:
     for line in f:
-      # Reset the user profile directory to start each test with a clean slate.
-      SetupUserProfileDir(args.me2me_manifest_file, args.it2me_manifest_file,
-                          args.user_profile_dir)
-
+      TestCaseSetup(args)
       # Replace the PROD_DIR value in the command-line with
       # the passed in value.
       line = line.replace(PROD_DIR_ID, args.prod_dir)
       # Launch specified command line for test.
-      LaunchBTCommand(line)
-      # After each test, stop+start me2me host process.
-      if not RestartMe2MeHost():
-        # Host restart failed. Don't run any more tests.
-        raise Exception('Host restart failed.')
-
-      # Print list of currently running processes.
-      PrintRunningProcesses()
+      LaunchBTCommand(args, line)
 
   # All tests completed. Include host-logs in the test results.
   host_log_contents = ''