add only_if_modified param to GSUtils.upload_file()

py/utils/gs_utils.py

Index: py/utils/gs_utils.py
diff --git a/py/utils/gs_utils.py b/py/utils/gs_utils.py
index d49b3fe4f615da04ff9c05a05a7fa55d3e03ebda..e3b6630d22d28eed912bac895aa1eaae2670c2fe 100644
--- a/py/utils/gs_utils.py
+++ b/py/utils/gs_utils.py
@@ -18,6 +18,7 @@ API/library references:
 
 # System-level imports
 import errno
+import hashlib
 import os
 import posixpath
 import re
@@ -148,18 +149,42 @@ class GSUtils(object):
                 ' while deleting bucket=%s, path=%s' % (bucket, path))
       raise
 
+  def get_last_modified_time(self, bucket, path):
+    """Gets the timestamp of when this file was last modified.
+
+    Params:
+      bucket: GS bucket in which to look for the file
+      path: full path (Posix-style) of the file within the bucket to check
+
+    Returns the last modified time, as a freeform string.  If the file was not
+    found, returns None.
+    """
+    b = self._connect_to_bucket(bucket_name=bucket)
+    try:
+      key = b.get_key(key_name=path)
+      if not key:
+        return None
+      return key.last_modified
+    except BotoServerError, e:
+      e.body = (repr(e.body) +
+                ' while getting attributes of bucket=%s, path=%s' % (
+                    bucket, path))
+      raise
+
   def upload_file(self, source_path, dest_bucket, dest_path,
-                  predefined_acl=None, fine_grained_acl_list=None):
+                  only_if_modified=False, predefined_acl=None,
+                  fine_grained_acl_list=None):
     """Upload contents of a local file to Google Storage.
 
-    TODO(epoger): Add the only_if_modified param provided by upload_file() in
-    https://github.com/google/skia-buildbot/blob/master/slave/skia_slave_scripts/utils/old_gs_utils.py ,
-    so we can replace that function with this one.
-
     params:
       source_path: full path (local-OS-style) on local disk to read from
       dest_bucket: GCS bucket to copy the file to
       dest_path: full path (Posix-style) within that bucket
+      only_if_modified: if True, only upload the file if it would actually
+          change the content on Google Storage (uploads the file if dest_path
+          does not exist, or if it exists but has different contents than
+          source_path).  Note that this may take longer than just uploading the
+          file without checking first, due to extra round-trips!
       predefined_acl: which predefined ACL to apply to the file on Google
           Storage; must be one of the PredefinedACL values defined above.
           If None, inherits dest_bucket's default object ACL.
@@ -170,6 +195,16 @@ class GSUtils(object):
           or None if predefined_acl is sufficient
     """
     b = self._connect_to_bucket(bucket_name=dest_bucket)
+
+    if only_if_modified:
+      old_key = b.get_key(key_name=dest_path)
+      if old_key:
+        local_md5 = '"%s"' % _get_local_md5(path=source_path)
+        if local_md5 == old_key.etag:
+          print 'Skipping upload of unmodified file %s : %s' % (
+              source_path, local_md5)
+          return
+
     item = Key(b)
     item.key = dest_path
     try:
@@ -500,3 +535,14 @@ def _makedirs_if_needed(path):
   except OSError as e:
     if e.errno != errno.EEXIST:
       raise
+
+
+def _get_local_md5(path):
+  """Returns the MD5 hash of a file on local disk."""
+  hasher = hashlib.md5()
+  with open(path, 'rb') as f:
+    while True:
+      data = f.read(64*1024)
+      if not data:
+        return hasher.hexdigest()
+      hasher.update(data)