add only_if_modified param to GSUtils.upload_file()

py/utils/gs_utils.py

 #!/usr/bin/python
 
 # pylint: disable=C0301
 """
 Copyright 2014 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 Utilities for accessing Google Cloud Storage, using the boto library (wrapper
 for the XML API).
 
 API/library references:
 - https://developers.google.com/storage/docs/reference-guide
 - http://googlecloudstorage.blogspot.com/2012/09/google-cloud-storage-tutorial-using-boto.html
 """
 # pylint: enable=C0301
 
 # System-level imports
 import errno
+import hashlib
 import os
 import posixpath
 import re
 import sys
 
 # Imports from third-party code
 TRUNK_DIRECTORY = os.path.abspath(os.path.join(
     os.path.dirname(__file__), os.pardir, os.pardir))
 for import_subdir in ['boto']:
   import_dirpath = os.path.join(
@@ skipping 101 matching lines @@
     """Delete a single file within a GS bucket.
 
     TODO(epoger): what if bucket or path does not exist?  Should probably raise
     an exception.  Implement, and add a test to exercise this.
 
     Params:
       bucket: GS bucket to delete a file from
       path: full path (Posix-style) of the file within the bucket to delete
     """
     b = self._connect_to_bucket(bucket_name=bucket)
-    item = Key(b)
-    item.key = path
+    key = Key(b)
+    key.name = path
     try:
-      item.delete()
+      key.delete()
     except BotoServerError, e:
       e.body = (repr(e.body) +
                 ' while deleting bucket=%s, path=%s' % (bucket, path))
       raise
 
+  def get_last_modified_time(self, bucket, path):
+    """Gets the timestamp of when this file was last modified.
+
+    Params:
+      bucket: GS bucket in which to look for the file
+      path: full path (Posix-style) of the file within the bucket to check
+
+    Returns the last modified time, as a freeform string.  If the file was not
+    found, returns None.
+    """
+    b = self._connect_to_bucket(bucket_name=bucket)
+    try:
+      key = b.get_key(key_name=path)
+      if not key:
+        return None
+      return key.last_modified
+    except BotoServerError, e:
+      e.body = (repr(e.body) +
+                ' while getting attributes of bucket=%s, path=%s' % (
+                    bucket, path))
+      raise
+
   def upload_file(self, source_path, dest_bucket, dest_path,
-                  predefined_acl=None, fine_grained_acl_list=None):
+                  only_if_modified=False, predefined_acl=None,
+                  fine_grained_acl_list=None):
     """Upload contents of a local file to Google Storage.
 
-    TODO(epoger): Add the only_if_modified param provided by upload_file() in
-    https://github.com/google/skia-buildbot/blob/master/slave/skia_slave_scripts/utils/old_gs_utils.py ,
-    so we can replace that function with this one.
-
     params:
       source_path: full path (local-OS-style) on local disk to read from
       dest_bucket: GCS bucket to copy the file to
       dest_path: full path (Posix-style) within that bucket
+      only_if_modified: if True, only upload the file if it would actually
+          change the content on Google Storage (uploads the file if dest_path
+          does not exist, or if it exists but has different contents than
+          source_path).  Note that this may take longer than just uploading the
+          file without checking first, due to extra round-trips!
       predefined_acl: which predefined ACL to apply to the file on Google
           Storage; must be one of the PredefinedACL values defined above.
           If None, inherits dest_bucket's default object ACL.
           TODO(epoger): add unittests for this param, although it seems to work
           in my manual testing
       fine_grained_acl_list: list of (id_type, id_value, permission) tuples
           to apply to the uploaded file (on top of the predefined_acl),
           or None if predefined_acl is sufficient
     """
     b = self._connect_to_bucket(bucket_name=dest_bucket)
-    item = Key(b)
-    item.key = dest_path
+
+    if only_if_modified:
+      old_key = b.get_key(key_name=dest_path)
+      if old_key:
+        local_md5 = '"%s"' % _get_local_md5(path=source_path)
+        if local_md5 == old_key.etag:
+          print 'Skipping upload of unmodified file %s : %s' % (
+              source_path, local_md5)
+          return
+
+    key = Key(b)
+    key.name = dest_path
     try:
-      item.set_contents_from_filename(filename=source_path,
-                                      policy=predefined_acl)
+      key.set_contents_from_filename(filename=source_path,
+                                     policy=predefined_acl)
     except BotoServerError, e:
       e.body = (repr(e.body) +
                 ' while uploading source_path=%s to bucket=%s, path=%s' % (
-                    source_path, dest_bucket, item.key))
+                    source_path, dest_bucket, key.name))
       raise
     # TODO(epoger): This may be inefficient, because it calls
     # _connect_to_bucket() again.  Depending on how expensive that
     # call is, we may want to optimize this.
     for (id_type, id_value, permission) in fine_grained_acl_list or []:
       self.set_acl(
-          bucket=dest_bucket, path=item.key,
+          bucket=dest_bucket, path=key.name,
           id_type=id_type, id_value=id_value, permission=permission)
 
   def upload_dir_contents(self, source_dir, dest_bucket, dest_dir,
                           predefined_acl=None, fine_grained_acl_list=None):
     """Recursively upload contents of a local directory to Google Storage.
 
     params:
       source_dir: full path (local-OS-style) on local disk of directory to copy
           contents of
       dest_bucket: GCS bucket to copy the files into
@@ skipping 31 matching lines @@
       else:
         remote_path = filename
 
       if os.path.isdir(local_path):
         self.upload_dir_contents(  # recurse
             source_dir=local_path, dest_bucket=dest_bucket,
             dest_dir=remote_path,
             predefined_acl=predefined_acl,
             fine_grained_acl_list=fine_grained_acl_list)
       else:
-        item = Key(b)
-        item.key = remote_path
+        key = Key(b)
+        key.name = remote_path
         try:
-          item.set_contents_from_filename(
+          key.set_contents_from_filename(
               filename=local_path, policy=predefined_acl)
         except BotoServerError, e:
           e.body = (repr(e.body) +
                     ' while uploading local_path=%s to bucket=%s, path=%s' % (
                         local_path, dest_bucket, remote_path))
           raise
         # TODO(epoger): This may be inefficient, because it calls
         # _connect_to_bucket() for every file.  Depending on how expensive that
         # call is, we may want to optimize this.
         for (id_type, id_value, permission) in fine_grained_acl_list or []:
           self.set_acl(
               bucket=dest_bucket, path=remote_path,
               id_type=id_type, id_value=id_value, permission=permission)
 
   def download_file(self, source_bucket, source_path, dest_path,
                     create_subdirs_if_needed=False):
     """Downloads a single file from Google Cloud Storage to local disk.
 
     Args:
       source_bucket: GCS bucket to download the file from
       source_path: full path (Posix-style) within that bucket
       dest_path: full path (local-OS-style) on local disk to copy the file to
       create_subdirs_if_needed: boolean; whether to create subdirectories as
           needed to create dest_path
     """
     b = self._connect_to_bucket(bucket_name=source_bucket)
-    item = Key(b)
-    item.key = source_path
+    key = Key(b)
+    key.name = source_path
     if create_subdirs_if_needed:
       _makedirs_if_needed(os.path.dirname(dest_path))
     with open(dest_path, 'w') as f:
       try:
-        item.get_contents_to_file(fp=f)
+        key.get_contents_to_file(fp=f)
       except BotoServerError, e:
         e.body = (repr(e.body) +
                   ' while downloading bucket=%s, path=%s to local_path=%s' % (
                       source_bucket, source_path, dest_path))
         raise
 
   def download_dir_contents(self, source_bucket, source_dir, dest_dir):
     """Recursively download contents of a Google Storage directory to local disk
 
     params:
       source_bucket: GCS bucket to copy the files from
       source_dir: full path (Posix-style) within that bucket; read the files
           from this directory
       dest_dir: full path (local-OS-style) on local disk of directory to copy
           the files into
 
     The copy operates as a "merge with overwrite": any files in source_dir will
     be "overlaid" on top of the existing content in dest_dir.  Existing files
     with the same names will be overwritten.
 
     TODO(epoger): Download multiple files simultaneously to reduce latency.
     """
     _makedirs_if_needed(dest_dir)
     b = self._connect_to_bucket(bucket_name=source_bucket)
     (dirs, files) = self.list_bucket_contents(
         bucket=source_bucket, subdir=source_dir)
 
     for filename in files:
-      item = Key(b)
-      item.key = posixpath.join(source_dir, filename)
+      key = Key(b)
+      key.name = posixpath.join(source_dir, filename)
       dest_path = os.path.join(dest_dir, filename)
       with open(dest_path, 'w') as f:
         try:
-          item.get_contents_to_file(fp=f)
+          key.get_contents_to_file(fp=f)
         except BotoServerError, e:
           e.body = (repr(e.body) +
                     ' while downloading bucket=%s, path=%s to local_path=%s' % (
-                        source_bucket, item.key, dest_path))
+                        source_bucket, key.name, dest_path))
           raise
 
     for dirname in dirs:
       self.download_dir_contents(  # recurse
           source_bucket=source_bucket,
           source_dir=posixpath.join(source_dir, dirname),
           dest_dir=os.path.join(dest_dir, dirname))
 
   def get_acl(self, bucket, path, id_type, id_value):
     """Retrieve partial access permissions on a single file in Google Storage.
@@ skipping 99 matching lines @@
       bucket: name of the Google Storage bucket
       subdir: directory within the bucket to list, or None for root directory
     """
     # The GS command relies on the prefix (if any) ending with a slash.
     prefix = subdir or ''
     if prefix and not prefix.endswith('/'):
       prefix += '/'
     prefix_length = len(prefix) if prefix else 0
 
     b = self._connect_to_bucket(bucket_name=bucket)
-    lister = BucketListResultSet(bucket=b, prefix=prefix, delimiter='/')
+    items = BucketListResultSet(bucket=b, prefix=prefix, delimiter='/')
     dirs = []
     files = []
-    for item in lister:
+    for item in items:
       t = type(item)
       if t is Key:
-        files.append(item.key[prefix_length:])
+        files.append(item.name[prefix_length:])
       elif t is Prefix:
         dirs.append(item.name[prefix_length:-1])
     return (dirs, files)
 
   def _connect_to_bucket(self, bucket_name):
     """Returns a Bucket object we can use to access a particular bucket in GS.
 
     Params:
       bucket_name: name of the bucket (e.g., 'chromium-skia-gm')
     """
@@ skipping 42 matching lines @@
   exist yet.
 
   Args:
     path: full path of directory to create
   """
   try:
     os.makedirs(path)
   except OSError as e:
     if e.errno != errno.EEXIST:
       raise
+
+
+def _get_local_md5(path):
+  """Returns the MD5 hash of a file on local disk."""
+  hasher = hashlib.md5()
+  with open(path, 'rb') as f:
+    while True:
+      data = f.read(64*1024)
+      if not data:
+        return hasher.hexdigest()
+      hasher.update(data)