Retry errors setting executable headers.

upload_to_google_storage.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Uploads files to Google Storage content addressed."""
 
 import hashlib
 import optparse
 import os
@@ skipping 21 matching lines @@
 DEPS file to call download_from_google_storage.py.
 
 Example usages
 --------------
 
 Scan the current directory and upload all files larger than 1MB:
 find . -name .svn -prune -o -size +1000k -type f -print0 | %prog -0 -b bkt -
 (Replace "bkt" with the name of a writable bucket.)
 """
 
+SET_EXECUTABLE_RETRY_DELAY = 5.0
+SET_EXECUTABLE_MAX_TRIES = 6

[tandrii(chromium), 2016/03/24 09:49:24]

i think these are no longer neede.d

[dsansome, 2016/03/29 03:24:54]

Oops thanks

+
 
 def get_md5(filename):
   md5_calculator = hashlib.md5()
   with open(filename, 'rb') as f:
     while True:
       chunk = f.read(1024*1024)
       if not chunk:
         break
       md5_calculator.update(chunk)
   return md5_calculator.hexdigest()
@@ skipping 17 matching lines @@
 def _upload_worker(
     thread_num, upload_queue, base_url, gsutil, md5_lock, force,
     use_md5, stdout_queue, ret_codes, gzip):
   while True:
     filename, sha1_sum = upload_queue.get()
     if not filename:
       break
     file_url = '%s/%s' % (base_url, sha1_sum)
     if gsutil.check_call('ls', file_url)[0] == 0 and not force:
       # File exists, check MD5 hash.
-      _, out, _ = gsutil.check_call('ls', '-L', file_url)
+      _, out, _ = gsutil.check_call_with_retries('ls', '-L', file_url)
       etag_match = re.search('ETag:\s+([a-z0-9]{32})', out)
       if etag_match:
         remote_md5 = etag_match.group(1)
         # Calculate the MD5 checksum to match it to Google Storage's ETag.
         with md5_lock:
           if use_md5:
             local_md5 = get_md5_cached(filename)
           else:
             local_md5 = get_md5(filename)
         if local_md5 == remote_md5:
           stdout_queue.put(
               '%d> File %s already exists and MD5 matches, upload skipped' %
               (thread_num, filename))
           continue
     stdout_queue.put('%d> Uploading %s...' % (
         thread_num, filename))
     gsutil_args = ['cp']
     if gzip:
       gsutil_args.extend(['-z', gzip])
     gsutil_args.extend([filename, file_url])
-    code, _, err = gsutil.check_call(*gsutil_args)
+    code, _, err = gsutil.check_call_with_retries(*gsutil_args)

[tandrii(chromium), 2016/03/24 09:49:24]

+hinoka@ I'm not sure retrying any non-0 code is a good idea. Could there be
some uploading error code that really shouldn't be retried?

[hinoka, 2016/03/24 11:28:54]

In general I think if this one fails once, it might be permanent. But I could be
wrong and I don't see a huge detriment to retrying it anyways.

     if code != 0:
       ret_codes.put(
           (code,
            'Encountered error on uploading %s to %s\n%s' %
            (filename, file_url, err)))
       continue
 
     # Mark executable files with the header "x-goog-meta-executable: 1" which
     # the download script will check for to preserve the executable bit.
     if not sys.platform.startswith('win'):
       if os.stat(filename).st_mode & stat.S_IEXEC:
-        code, _, err = gsutil.check_call('setmeta', '-h',
-                                         'x-goog-meta-executable:1', file_url)
-        if code:
+        code, _, err = gsutil.check_call_with_retries(
+            'setmeta', '-h', 'x-goog-meta-executable:1', file_url)
+        if not code:
           ret_codes.put(
               (code,
-               'Encountered error on setting metadata on %s\n%s' %
-               (file_url, err)))
+               'Encountered error on setting metadata on %s after %d '
+               'tries\n%s' % (file_url, SET_EXECUTABLE_MAX_TRIES, err)))
 
 
 def get_targets(args, parser, use_null_terminator):
   if not args:
     parser.error('Missing target.')
 
   if len(args) == 1 and args[0] == '-':
     # Take stdin as a newline or null seperated list of files.
     if use_null_terminator:
       return sys.stdin.read().split('\0')
@@ skipping 161 matching lines @@
       input_filenames, base_url, gsutil, options.force, options.use_md5,
       options.num_threads, options.skip_hashing, options.gzip)
 
 
 if __name__ == '__main__':
   try:
     sys.exit(main())
   except KeyboardInterrupt:
     sys.stderr.write('interrupted\n')
     sys.exit(1)