Add verification to downloaded files

download_from_google_storage.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Download files from Google Storage based on SHA1 sums."""
 
 
 import hashlib
 import optparse
@@ skipping 208 matching lines @@
     out_q.put('%d> Downloading %s...' % (thread_num, output_filename))
     try:
       os.remove(output_filename)  # Delete the file if it exists already.
     except OSError:
       if os.path.exists(output_filename):
         out_q.put('%d> Warning: deleting %s failed.' % (
             thread_num, output_filename))
     code, _, err = gsutil.check_call('cp', file_url, output_filename)
     if code != 0:
       out_q.put('%d> %s' % (thread_num, err))
       ret_codes.put((code, err))

[Vadim Sh., 2015/07/29 20:43:49]

is it ok that there's no "continue" after this line? IIUC, its a failure to
download the file.

[Ryan Tseng, 2015/07/29 20:51:24]

Good catch there should be a continue here.  I think the lack thereof is due to
this being the final point in the loop before the executable bit stuff was
added.

+    remote_sha1 = get_sha1(output_filename)
+    if remote_sha1 != input_sha1_sum:
+      msg = ('%d> ERROR remote sha1 (%s) does not match expected sha1 (%s).' %
+             (thread_num, remote_sha1, input_sha1_sum))
+      out_q.put(msg)
+      ret_codes.put((20, msg))
+      continue
 
     # Set executable bit.
     if sys.platform == 'cygwin':
       # Under cygwin, mark all files as executable. The executable flag in
       # Google Storage will not be set when uploading from Windows, so if
       # this script is running under cygwin and we're downloading an
       # executable, it will be unrunnable from inside cygwin without this.
       st = os.stat(output_filename)
       os.chmod(output_filename, st.st_mode | stat.S_IEXEC)
     elif sys.platform != 'win32':
@@ skipping 209 matching lines @@
   base_url = 'gs://%s' % options.bucket
 
   return download_from_google_storage(
       input_filename, base_url, gsutil, options.num_threads, options.directory,
       options.recursive, options.force, options.output, options.ignore_errors,
       options.sha1_file, options.verbose, options.auto_platform)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))