Silence common noisiness in download_from_google_storage

download_from_google_storage.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Download files from Google Storage based on SHA1 sums."""
 
 
 import hashlib
 import optparse
@@ skipping 145 matching lines @@
               (sha1_match.groups(1)[0], full_path.replace('.sha1', '')))
           work_queue_size += 1
         else:
           if not ignore_errors:
             raise InvalidFileError('No sha1 sum found in %s.' % filename)
           print >> sys.stderr, 'No sha1 sum found in %s.' % filename
   return work_queue_size
 
 
 def _downloader_worker_thread(thread_num, q, force, base_url,
-                              gsutil, out_q, ret_codes):
+                              gsutil, out_q, ret_codes, verbose):
   while True:
     input_sha1_sum, output_filename = q.get()
     if input_sha1_sum is None:
       return
     if os.path.exists(output_filename) and not force:
       if get_sha1(output_filename) == input_sha1_sum:
-        out_q.put(
-            '%d> File %s exists and SHA1 matches. Skipping.' % (
-                thread_num, output_filename))
+        if verbose:
+          out_q.put(
+              '%d> File %s exists and SHA1 matches. Skipping.' % (
+                  thread_num, output_filename))
         continue
     # Check if file exists.
     file_url = '%s/%s' % (base_url, input_sha1_sum)
     if gsutil.check_call('ls', file_url)[0] != 0:
       out_q.put('%d> File %s for %s does not exist, skipping.' % (
           thread_num, file_url, output_filename))
       ret_codes.put((1, 'File %s for %s does not exist.' % (
           file_url, output_filename)))
       continue
     # Fetch the file.
@@ skipping 23 matching lines @@
   while True:
     line = output_queue.get()
     # Its plausible we want to print empty lines.
     if line is None:
       break
     print line
 
 
 def download_from_google_storage(
     input_filename, base_url, gsutil, num_threads, directory, recursive,
-    force, output, ignore_errors, sha1_file):
+    force, output, ignore_errors, sha1_file, verbose):
   # Start up all the worker threads.
   all_threads = []
   download_start = time.time()
   stdout_queue = Queue.Queue()
   work_queue = Queue.Queue()
   ret_codes = Queue.Queue()
   ret_codes.put((0, None))
   for thread_num in range(num_threads):
     t = threading.Thread(
         target=_downloader_worker_thread,
         args=[thread_num, work_queue, force, base_url,
-              gsutil, stdout_queue, ret_codes])
+              gsutil, stdout_queue, ret_codes, verbose])
     t.daemon = True
     t.start()
     all_threads.append(t)
   printer_thread = threading.Thread(target=printer_worker, args=[stdout_queue])
   printer_thread.daemon = True
   printer_thread.start()
 
   # Enumerate our work queue.
   work_queue_size = enumerate_work_queue(
       input_filename, work_queue, directory, recursive,
       ignore_errors, output, sha1_file)
   for _ in all_threads:
     work_queue.put((None, None))  # Used to tell worker threads to stop.
 
   # Wait for all downloads to finish.
   for t in all_threads:
     t.join()
   stdout_queue.put(None)
   printer_thread.join()
 
   # See if we ran into any errors.
   max_ret_code = 0
   for ret_code, message in ret_codes.queue:
     max_ret_code = max(ret_code, max_ret_code)
     if message:
       print >> sys.stderr, message
-  if not max_ret_code:
+  if verbose and not max_ret_code:
     print 'Success!'
 
-  print 'Downloading %d files took %1f second(s)' % (
-      work_queue_size, time.time() - download_start)
+  if verbose:
+    print 'Downloading %d files took %1f second(s)' % (
+        work_queue_size, time.time() - download_start)
   return max_ret_code
 
 
 def main(args):
   usage = ('usage: %prog [options] target\n'
            'Target must be:\n'
            '  (default) a sha1 sum ([A-Za-z0-9]{40}).\n'
            '  (-s or --sha1_file) a .sha1 file, containing a sha1 sum on '
            'the first line.\n'
            '  (-d or --directory) A directory to scan for .sha1 files.')
@@ skipping 28 matching lines @@
                     help='Alias for "gsutil config".  Run this if you want '
                          'to initialize your saved Google Storage '
                          'credentials.')
   parser.add_option('-n', '--no_auth', action='store_true',
                     help='Skip auth checking.  Use if it\'s known that the '
                          'target bucket is a public bucket.')
   parser.add_option('-p', '--platform',
                     help='A regular expression that is compared against '
                          'Python\'s sys.platform. If this option is specified, '
                          'the download will happen only if there is a match.')
+  parser.add_option('-v', '--verbose', default=False, action='store_true',

[M-A Ruel, 2013/11/25 19:13:44]

default=False is not needed, please remove.

[scottmg, 2013/11/25 19:23:40]

Done.

+                    help='Output extra diagnostic and progress information.')
 
   (options, args) = parser.parse_args()
 
   # Make sure we should run at all based on platform matching.
   if options.platform:
     if not re.match(options.platform, sys.platform):
-      print('The current platform doesn\'t match "%s", skipping.' %
-            options.platform)
+      if options.verbose:
+        print('The current platform doesn\'t match "%s", skipping.' %
+              options.platform)
       return 0
 
   # Set the boto file to /dev/null if we don't need auth.
   if options.no_auth:
     options.boto = os.devnull
 
   # Make sure we can find a working instance of gsutil.
   if os.path.exists(GSUTIL_DEFAULT_PATH):
     gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
   else:
@@ skipping 47 matching lines @@
                    % options.output)
 
   # Check we have a valid bucket with valid permissions.
   base_url, code = check_bucket_permissions(options.bucket, gsutil)
   if code:
     return code
 
   return download_from_google_storage(
       input_filename, base_url, gsutil, options.num_threads, options.directory,
       options.recursive, options.force, options.output, options.ignore_errors,
-      options.sha1_file)
+      options.sha1_file, options.verbose)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))