Add a retry to Chromoting tests on Linux builders.

testing/chromoting/chromoting_test_driver_launcher.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Utility script to run chromoting test driver tests on the Chromoting bot."""
 
 import argparse
 
 from chromoting_test_utilities import GetJidFromHostLog
 from chromoting_test_utilities import InitialiseTestMachineForLinux
+from chromoting_test_utilities import MAX_RETRIES
 from chromoting_test_utilities import PrintHostLogContents
 from chromoting_test_utilities import PROD_DIR_ID
 from chromoting_test_utilities import RunCommandInSubProcess
 from chromoting_test_utilities import TestCaseSetup
 from chromoting_test_utilities import TestMachineCleanup
 
 TEST_ENVIRONMENT_TEAR_DOWN_INDICATOR = 'Global test environment tear-down'
 FAILED_INDICATOR = '[  FAILED  ]'
 
 
@@ skipping 12 matching lines @@
 
   host_log_file_names.append(TestCaseSetup(args))
   # Parse the me2me host log to obtain the JID that the host registered.
   host_jid = GetJidFromHostLog(host_log_file_names[-1])
 
   if not host_jid:
     # Host-JID not found in log. Let's not attempt to run this test.
     print 'Host-JID not found in log %s.' % host_log_file_names[-1]
     return '[Command failed]: %s, %s' % (command, host_log_file_names)
 
-  # In order to ensure the host is online with the expected JID, pass in the
-  # jid obtained from the host-log as a command-line parameter.
-  command = command.replace('\n', '') + ' --hostjid=%s' % host_jid
+  retries = 0
+  failed_tests_list = []
+  while retries <= MAX_RETRIES:

[joedow, 2016/02/20 17:08:38]

Can you add a TODO/bug to track the removal of the retry logic?  I don't think
we want to keep this for more than a few months as we stabilize the waterfall.

[anandc1, 2016/02/22 20:58:08]

Done.

+    # In order to ensure the host is online with the expected JID, pass in the
+    # jid obtained from the host-log as a command-line parameter.
+    command = command.replace('\n', '') + ' --hostjid=%s' % host_jid
 
-  results = RunCommandInSubProcess(command)
+    results = RunCommandInSubProcess(command)
 
-  tear_down_index = results.find(TEST_ENVIRONMENT_TEAR_DOWN_INDICATOR)
-  if tear_down_index == -1:
-    # The test environment did not tear down. Something went horribly wrong.
-    return '[Command failed]: ' + command, host_log_file_names
+    tear_down_index = results.find(TEST_ENVIRONMENT_TEAR_DOWN_INDICATOR)
+    if tear_down_index == -1:
+      # The test environment did not tear down. Something went horribly wrong.
+      return '[Command failed]: ' + command, host_log_file_names
 
-  end_results_list = results[tear_down_index:].split('\n')
-  failed_tests_list = []
-  for result in end_results_list:
-    if result.startswith(FAILED_INDICATOR):
-      failed_tests_list.append(result)
+    end_results_list = results[tear_down_index:].split('\n')
+    test_failed = False
+    for result in end_results_list:
+      if result.startswith(FAILED_INDICATOR):
+        test_failed = True
+        if retries == MAX_RETRIES:
+          # Test failed and we have no more retries left.
+          failed_tests_list.append(result)
+
+    if test_failed:
+      retries += 1
+    else:
+      break
 
   if failed_tests_list:
     test_result = '[Command]: ' + command
     # Note: Skipping the first one is intentional.
     for i in range(1, len(failed_tests_list)):
       test_result += '    ' + failed_tests_list[i]
     return test_result, host_log_file_names
 
   # All tests passed!
   return '', host_log_file_names
@@ skipping 41 matching lines @@
   try:
     failing_tests, host_logs = main(command_line_args)
     if failing_tests:
       print '++++++++++FAILED TESTS++++++++++'
       print failing_tests.rstrip('\n')
       print '++++++++++++++++++++++++++++++++'
       raise Exception('At least one test failed.')
   finally:
     # Stop host and cleanup user-profile-dir.
     TestMachineCleanup(command_line_args.user_profile_dir, host_logs)