Retry errors setting executable headers.

upload_to_google_storage.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Uploads files to Google Storage content addressed."""
 
 import hashlib
 import optparse
 import os
@@ skipping 58 matching lines @@
 def _upload_worker(
     thread_num, upload_queue, base_url, gsutil, md5_lock, force,
     use_md5, stdout_queue, ret_codes, gzip):
   while True:
     filename, sha1_sum = upload_queue.get()
     if not filename:
       break
     file_url = '%s/%s' % (base_url, sha1_sum)
     if gsutil.check_call('ls', file_url)[0] == 0 and not force:
       # File exists, check MD5 hash.
-      _, out, _ = gsutil.check_call('ls', '-L', file_url)
+      _, out, _ = gsutil.check_call_with_retries('ls', '-L', file_url)
       etag_match = re.search('ETag:\s+([a-z0-9]{32})', out)
       if etag_match:
         remote_md5 = etag_match.group(1)
         # Calculate the MD5 checksum to match it to Google Storage's ETag.
         with md5_lock:
           if use_md5:
             local_md5 = get_md5_cached(filename)
           else:
             local_md5 = get_md5(filename)
         if local_md5 == remote_md5:
           stdout_queue.put(
               '%d> File %s already exists and MD5 matches, upload skipped' %
               (thread_num, filename))
           continue
     stdout_queue.put('%d> Uploading %s...' % (
         thread_num, filename))
     gsutil_args = ['cp']
     if gzip:
       gsutil_args.extend(['-z', gzip])
     gsutil_args.extend([filename, file_url])
-    code, _, err = gsutil.check_call(*gsutil_args)
+    code, _, err = gsutil.check_call_with_retries(*gsutil_args)
     if code != 0:
       ret_codes.put(
           (code,
            'Encountered error on uploading %s to %s\n%s' %
            (filename, file_url, err)))
       continue
 
     # Mark executable files with the header "x-goog-meta-executable: 1" which
     # the download script will check for to preserve the executable bit.
     if not sys.platform.startswith('win'):
       if os.stat(filename).st_mode & stat.S_IEXEC:
-        code, _, err = gsutil.check_call('setmeta', '-h',
-                                         'x-goog-meta-executable:1', file_url)
-        if code:
+        code, _, err = gsutil.check_call_with_retries(
+            'setmeta', '-h', 'x-goog-meta-executable:1', file_url)
+        if not code:
           ret_codes.put(
               (code,
                'Encountered error on setting metadata on %s\n%s' %
                (file_url, err)))
 
 
 def get_targets(args, parser, use_null_terminator):
   if not args:
     parser.error('Missing target.')
 
@@ skipping 165 matching lines @@
       input_filenames, base_url, gsutil, options.force, options.use_md5,
       options.num_threads, options.skip_hashing, options.gzip)
 
 
 if __name__ == '__main__':
   try:
     sys.exit(main())
   except KeyboardInterrupt:
     sys.stderr.write('interrupted\n')
     sys.exit(1)