[android] Retry data pushes that fail due to device busy.

build/android/pylib/android_commands.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Provides an interface to communicate with the device via the adb command.
 
 Assumes adb binary is currently on system path.
 """
 
 import collections
@@ skipping 765 matching lines @@
     self._pushed_files.append(device_path)
     self._potential_push_size += size
 
     if self.CheckMd5Sum(local_path, device_path):
       return
 
     self._actual_push_size += size
     # They don't match, so remove everything first and then create it.
     if os.path.isdir(local_path):
       self.RunShellCommand('rm -r %s' % device_path, timeout_time=2 * 60)
       self.RunShellCommand('mkdir -p %s' % device_path)

[frankf, 2013/07/18 17:54:07]

Is this necessary? Doesn't push overwrite? From the log, it seems the flakiness
is casued by rm -r /sdcard/thirdparty which is around 100MB.

 
     # NOTE: We can't use adb_interface.Push() because it hardcodes a timeout of
     # 60 seconds which isn't sufficient for a lot of users of this method.
     push_command = 'push %s %s' % (local_path, device_path)
     self._LogShell(push_command)
-    output = self._adb.SendCommand(push_command, timeout_time=30 * 60)
-    assert _HasAdbPushSucceeded(output)
+
+    # Retry push with increasing backoff if the device is busy.
+    retry = 0
+    while True:
+      output = self._adb.SendCommand(push_command, timeout_time=30 * 60)
+      if _HasAdbPushSucceeded(output):
+        return
+      if 'resource busy' in output and retry < 3:

[frankf, 2013/07/18 17:54:07]

If rm is required, then shouldn't you also being do it here?

+        retry += 1
+        wait_time = 5 * retry
+        logging.error('Push failed, retrying in %d seconds: %s' %
+                      (wait_time, output))
+        time.sleep(wait_time)
+      else:
+        raise Exception('Push failed: %s' % output)
 
   def GetPushSizeInfo(self):
     """Get total size of pushes to the device done via PushIfNeeded()
 
     Returns:
       A tuple:
         1. Total size of push requests to PushIfNeeded (MB)
         2. Total size that was actually pushed (MB)
     """
     return (self._potential_push_size, self._actual_push_size)
@@ skipping 587 matching lines @@
   """
   def __init__(self, output):
     self._output = output
 
   def write(self, data):
     data = data.replace('\r\r\n', '\n')
     self._output.write(data)
 
   def flush(self):
     self._output.flush()