Add a retry to Chromoting tests on Linux builders.

testing/chromoting/chromoting_test_driver_launcher.py

Index: testing/chromoting/chromoting_test_driver_launcher.py
diff --git a/testing/chromoting/chromoting_test_driver_launcher.py b/testing/chromoting/chromoting_test_driver_launcher.py
index 186adb9341d20cd3f19d567f0a9e165f35f28db0..f8e84305ec6d7fbef49d038df18cd53c2257dee0 100644
--- a/testing/chromoting/chromoting_test_driver_launcher.py
+++ b/testing/chromoting/chromoting_test_driver_launcher.py
@@ -8,6 +8,7 @@ import argparse
 
 from chromoting_test_utilities import GetJidFromHostLog
 from chromoting_test_utilities import InitialiseTestMachineForLinux
+from chromoting_test_utilities import MAX_RETRIES
 from chromoting_test_utilities import PrintHostLogContents
 from chromoting_test_utilities import PROD_DIR_ID
 from chromoting_test_utilities import RunCommandInSubProcess
@@ -40,22 +41,33 @@ def LaunchCTDCommand(args, command):
     print 'Host-JID not found in log %s.' % host_log_file_names[-1]
     return '[Command failed]: %s, %s' % (command, host_log_file_names)
 
-  # In order to ensure the host is online with the expected JID, pass in the
-  # jid obtained from the host-log as a command-line parameter.
-  command = command.replace('\n', '') + ' --hostjid=%s' % host_jid
-
-  results = RunCommandInSubProcess(command)
-
-  tear_down_index = results.find(TEST_ENVIRONMENT_TEAR_DOWN_INDICATOR)
-  if tear_down_index == -1:
-    # The test environment did not tear down. Something went horribly wrong.
-    return '[Command failed]: ' + command, host_log_file_names
-
-  end_results_list = results[tear_down_index:].split('\n')
+  retries = 0
   failed_tests_list = []
-  for result in end_results_list:
-    if result.startswith(FAILED_INDICATOR):
-      failed_tests_list.append(result)
+  while retries <= MAX_RETRIES:

[joedow, 2016/02/20 17:08:38]

Can you add a TODO/bug to track the removal of the retry logic?  I don't think
we want to keep this for more than a few months as we stabilize the waterfall.

[anandc1, 2016/02/22 20:58:08]

Done.

+    # In order to ensure the host is online with the expected JID, pass in the
+    # jid obtained from the host-log as a command-line parameter.
+    command = command.replace('\n', '') + ' --hostjid=%s' % host_jid
+
+    results = RunCommandInSubProcess(command)
+
+    tear_down_index = results.find(TEST_ENVIRONMENT_TEAR_DOWN_INDICATOR)
+    if tear_down_index == -1:
+      # The test environment did not tear down. Something went horribly wrong.
+      return '[Command failed]: ' + command, host_log_file_names
+
+    end_results_list = results[tear_down_index:].split('\n')
+    test_failed = False
+    for result in end_results_list:
+      if result.startswith(FAILED_INDICATOR):
+        test_failed = True
+        if retries == MAX_RETRIES:
+          # Test failed and we have no more retries left.
+          failed_tests_list.append(result)
+
+    if test_failed:
+      retries += 1
+    else:
+      break
 
   if failed_tests_list:
     test_result = '[Command]: ' + command