Add a retry to Chromoting tests on Linux builders.

testing/chromoting/browser_tests_launcher.py

 # Copyright (c) 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 
 """Utility script to launch browser-tests on the Chromoting bot."""
 import argparse
 import time
 
 from chromoting_test_utilities import CleanupUserProfileDir
 from chromoting_test_utilities import GetJidFromHostLog
 from chromoting_test_utilities import GetJidListFromTestResults
 from chromoting_test_utilities import InitialiseTestMachineForLinux
+from chromoting_test_utilities import MAX_RETRIES
 from chromoting_test_utilities import PrintHostLogContents
 from chromoting_test_utilities import PROD_DIR_ID
 from chromoting_test_utilities import RunCommandInSubProcess
 from chromoting_test_utilities import TestCaseSetup
 from chromoting_test_utilities import TestMachineCleanup
 
 SUCCESS_INDICATOR = 'SUCCESS: all tests passed.'
 TEST_FAILURE = False
 FAILING_TESTS = ''
 BROWSER_NOT_STARTED_ERROR = (
     'Still waiting for the following processes to finish')
 TIME_OUT_INDICATOR = '(TIMED OUT)'
-MAX_RETRIES = 1
 
 
 def LaunchBTCommand(args, command):
   """Launches the specified browser-test command.
 
     Retry if the execution failed because a browser-instance was not launched or
     because the JID used did not match the host-JID.
   Args:
     args: Command line args, used for test-case startup tasks.
     command: Browser-test command line.
@@ skipping 49 matching lines @@
       print 'JID used by test matched me2me host JID: %s' % host_jid
     else:
       # There wasn't a mismatch and no JIDs were returned. If no JIDs were
       # returned, that means the test didn't use any JIDs, so there is nothing
       # further for us to do.
       pass
 
     if SUCCESS_INDICATOR in results:
       break
 
-    # Sometimes, during execution of browser-tests, a browser instance is

[joedow, 2016/02/20 17:08:38]

I think this logic is important to keep.  In the long term we will remove the
generic retry logic but this was something you worked on quite a bit to track
down and it is likely to remain a problem (since it was never root caused).  I'd
hate to remove the retry logic and then start hitting his failure again.

It should be too hard to distinguish between this failure and some other
failure, the retry logic is ultimately the same but we would keep this
protection for later.

If you ultimately decide to rip this code out, then please remove the vars at
the top of the file that were referenced here but are no longer used.

[anandc1, 2016/02/22 20:58:08]

Done. Logging a line indicating what type of failure this potentially was and
retrying for any failure, up to MAX_RETRIES.

-    # not started and the test times out. See http://crbug/480025.
-    # To work around it, check if this execution failed owing to that
-    # problem and retry.
-    # There are 2 things to look for in the results:
-    # A line saying "Still waiting for the following processes to finish",
-    # and, because sometimes that line gets logged even if the test
-    # eventually passes, we'll also look for "(TIMED OUT)", before retrying.
-    if not (
-        BROWSER_NOT_STARTED_ERROR in results and TIME_OUT_INDICATOR in results):
-      # Test failed for some other reason. Let's not retry.
-      break
+    # Test failed, retry.
     retries += 1
 
   # Check that the test passed.
   if SUCCESS_INDICATOR not in results:
     TEST_FAILURE = True
     # Add this command-line to list of tests that failed.
     FAILING_TESTS += command
 
   return host_log_file_names
 
@@ skipping 37 matching lines @@
   try:
     host_logs = main(command_line_args)
     if TEST_FAILURE:
       print '++++++++++AT LEAST 1 TEST FAILED++++++++++'
       print FAILING_TESTS.rstrip('\n')
       print '++++++++++++++++++++++++++++++++++++++++++'
       raise Exception('At least one test failed.')
   finally:
     # Stop host and cleanup user-profile-dir.
     TestMachineCleanup(command_line_args.user_profile_dir, host_logs)