Add a RevGitWithRetry method to retry repo sync and git push

prebuilt.py

 #!/usr/bin/python
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import datetime
 import multiprocessing
 import optparse
 import os
 import re
 import sys
 import tempfile
+import time
 
 from chromite.lib import cros_build_lib
 """
 This script is used to upload host prebuilts as well as board BINHOSTS.
 
 If the URL starts with 'gs://', we upload using gsutil to Google Storage.
 Otherwise, rsync is used.
 
 After a build is successfully uploaded a file is updated with the proper
 BINHOST version as well as the target board. This file is defined in GIT_FILE
@@ skipping 43 matching lines @@
 
 
 class UploadFailed(Exception):
   """Raised when one of the files uploaded failed."""
   pass
 
 class UnknownBoardFormat(Exception):
   """Raised when a function finds an unknown board format."""
   pass
 
+class GitPushFailed(Exception):
+  """Raised when a git push failed after retry."""
+
 
 def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'):
   """Update the key in file with the value passed.
   File format:
     key="value"
   Note quotes are added automatically
 
   Args:
     filename: Name of file to modify.
     value: Value to write with the key.
@@ skipping 24 matching lines @@
   if not found:
     file_lines.append(keyval_str % {'key': key, 'value': value})
 
   file_fh.close()
   # write out new file
   new_file_fh = open(filename, 'w')
   new_file_fh.write('\n'.join(file_lines))
   new_file_fh.close()
 
 
-def RevGitFile(filename, value):
+def RevGitPushWithRetry(retries=5):
+  """Repo sync and then push git changes in flight.
+
+    Args:
+      retries: The number of times to retry before giving up, default: 5
+
+    Raises:
+      GitPushFailed if push was unsuccessful after retries
+  """
+  for retry in range(1, retries+1):
+    try:
+      cros_build_lib.RunCommand('repo sync .', shell=True)
+      cros_build_lib.RunCommand('git push', shell=True)
+      break
+    except cros_build_lib.RunCommandError:
+      if retry < retries:
+        print 'Error pushing changes trying again (%s/%s)' % (retry, retries)
+        time.sleep(5*retry)
+  else:
+    raise GitPushFailed('Failed to push change after %s retries' % retries)
+
+
+def RevGitFile(filename, value, retries=5):
   """Update and push the git file.
 
-  Args:
-    filename: file to modify that is in a git repo already
-    key: board or host package type e.g. x86-dogfood
-    value: string representing the version of the prebuilt that has been
-      uploaded.
+    Args:
+      filename: file to modify that is in a git repo already
+      value: string representing the version of the prebuilt that has been
+        uploaded.
+      retries: The number of times to retry before giving up, default: 5
   """
   prebuilt_branch = 'prebuilt_branch'
   old_cwd = os.getcwd()
   os.chdir(os.path.dirname(filename))
 
-  cros_build_lib.RunCommand('repo sync', shell=True)
-  cros_build_lib.RunCommand('repo start %s  .' % prebuilt_branch, shell=True)
+  cros_build_lib.RunCommand('repo sync .', shell=True)
+  cros_build_lib.RunCommand('repo start %s .' % prebuilt_branch, shell=True)
   git_ssh_config_cmd = (
     'git config url.ssh://git@gitrw.chromium.org:9222.pushinsteadof '
     'http://git.chromium.org/git')
   cros_build_lib.RunCommand(git_ssh_config_cmd, shell=True)
   description = 'Update PORTAGE_BINHOST="%s" in %s' % (value, filename)
   print description
   try:
     UpdateLocalFile(filename, value)
     cros_build_lib.RunCommand('git config push.default tracking', shell=True)
     cros_build_lib.RunCommand('git commit -am "%s"' % description, shell=True)
-    cros_build_lib.RunCommand('repo sync', shell=True)
-    cros_build_lib.RunCommand('git push', shell=True)
+    RevGitPushWithRetry(retries)
   finally:
     cros_build_lib.RunCommand('repo abandon %s .' % prebuilt_branch, shell=True)
     os.chdir(old_cwd)
 
 
 def GetVersion():
   """Get the version to put in LATEST and update the git version with."""
   return datetime.datetime.now().strftime('%d.%m.%y.%H%M%S')
 
 
@@ skipping 217 matching lines @@
   elif re.match('.*?-\w+', target):
     overlay_str = 'overlay-%s' % target
     make_path = os.path.join(_BINHOST_BASE_DIR, overlay_str, 'make.conf')
   else:
     raise UnknownBoardFormat('Unknown format: %s' % target)
 
   return os.path.join(make_path)
 
 
 def UploadPrebuilt(build_path, upload_location, version, binhost_base_url,
-                   board=None, git_sync=False):
+                   board=None, git_sync=False, git_sync_retries=5):
   """Upload Host prebuilt files to Google Storage space.
 
   Args:
     build_path: The path to the root of the chroot.
     upload_location: The upload location.
     board: The board to upload to Google Storage, if this is None upload
       host packages.
     git_sync: If set, update make.conf of target to reference the latest
       prebuilt packages genereated here.
+    git_sync_retries: How many times to retry pushing when updating git files.
+      This helps avoid failures when multiple bots are modifying the same Repo.
+      default: 5
   """
 
   if not board:
     # We are uploading host packages
     # TODO(scottz): eventually add support for different host_targets
     package_path = os.path.join(build_path, _HOST_PACKAGES_PATH)
     url_suffix = _REL_HOST_PATH % {'version': version, 'target': _HOST_TARGET}
     package_string = _HOST_TARGET
     git_file = os.path.join(build_path, _PREBUILT_MAKE_CONF[_HOST_TARGET])
   else:
@@ skipping 15 matching lines @@
   else:
     ssh_server, remote_path = remote_location.split(':', 1)
     cmds = ['ssh %s mkdir -p %s' % (ssh_server, remote_path),
             'rsync -av %s/ %s/' % (package_path, remote_location)]
     for cmd in cmds:
       if not _RetryRun(cmd, shell=True):
         raise UploadFailed('Could not run %s' % cmd)
 
   if git_sync:
     url_value = '%s/%s/' % (binhost_base_url, url_suffix)
-    RevGitFile(git_file, url_value)
+    RevGitFile(git_file, url_value, retries=git_sync_retries)
 
 
 def usage(parser, msg):
   """Display usage message and parser help then exit with 1."""
   print >> sys.stderr, msg
   parser.print_help()
   sys.exit(1)
 
 
 def main():
@@ skipping 46 matching lines @@
                    options.binhost_base_url, git_sync=options.git_sync)
 
   if options.board:
     UploadPrebuilt(options.build_path, options.upload, version,
                    options.binhost_base_url, board=options.board,
                    git_sync=options.git_sync)
 
 
 if __name__ == '__main__':
   main()