Make building of resource zips and srcjars hermetic

build/android/gyp/util/build_utils.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import ast
 import contextlib
 import fnmatch
 import json
 import os
 import pipes
 import re
 import shlex
 import shutil
 import subprocess
 import sys
 import tempfile
 import zipfile
 
 
 CHROMIUM_SRC = os.path.normpath(
     os.path.join(os.path.dirname(__file__),
                  os.pardir, os.pardir, os.pardir, os.pardir))
 COLORAMA_ROOT = os.path.join(CHROMIUM_SRC,
                              'third_party', 'colorama', 'src')
 # aapt should ignore OWNERS files in addition the default ignore pattern.
 AAPT_IGNORE_PATTERN = ('!OWNERS:!.svn:!.git:!.ds_store:!*.scc:.*:<dir>_*:' +
                        '!CVS:!thumbs.db:!picasa.ini:!*~:!*.d.stamp')
+HERMETIC_TIMESTAMP = (2001, 1, 1, 0, 0, 0)
 
 
 @contextlib.contextmanager
 def TempDir():
   dirname = tempfile.mkdtemp()
   try:
     yield dirname
   finally:
     shutil.rmtree(dirname)
 
@@ skipping 173 matching lines @@
       if no_clobber:
         output_path = os.path.join(path, name)
         if os.path.exists(output_path):
           raise Exception(
               'Path already exists from zip: %s %s %s'
               % (zip_path, name, output_path))
 
     z.extractall(path=path)
 
 
-def DoZip(inputs, output, base_dir):
+def DoZip(inputs, output, base_dir=None):
+  """Creates a zip file from a list of files.
+
+  Args:
+    inputs: A list of paths to zip, or a list of (zip_path, fs_path) tuples.
+    output: Destination .zip file.
+    base_dir: Prefix to strip from inputs.
+  """
+  input_tuples = []
+  for tup in inputs:
+    if isinstance(tup, basestring):
+      tup = (os.path.relpath(tup, base_dir), tup)
+    input_tuples.append(tup)
+
+  # Sort by zip path to ensure stable zip ordering.
+  input_tuples.sort(key=lambda tup: tup[0])
   with zipfile.ZipFile(output, 'w') as outfile:
-    for f in inputs:
-      CheckZipPath(os.path.relpath(f, base_dir))
-      outfile.write(f, os.path.relpath(f, base_dir))
+    for zip_path, fs_path in input_tuples:
+      CheckZipPath(zip_path)
+      zipinfo = zipfile.ZipInfo(filename=zip_path, date_time=HERMETIC_TIMESTAMP)

[jbudorick, 2015/09/01 18:11:33]

Maybe I'm not following ... if the inputs are the same, won't the timestamp be
the same? And if the timestamp changes, don't we want to update the zip?

[agrieve, 2015/09/01 19:06:16]

The most relevant example is in our process_resources.py step. The doesn't have
a good idea of when it needs to re-run, so it re-runs fairly aggressively,
(often) generating the same assets as before, but with new timestamps. Rules
that depend on process_resources use md5_check to see if the results have
changed, and if so re-run themselves. Without this change, the outputs looks
like they always change because md5_check is foiled by timestamps embedded in
the file.

+      with file(fs_path) as f:
+        contents = f.read()
+      outfile.writestr(zipinfo, contents)
 
 
 def ZipDir(output, base_dir):
-  with zipfile.ZipFile(output, 'w') as outfile:
-    for root, _, files in os.walk(base_dir):
-      for f in files:
-        path = os.path.join(root, f)
-        archive_path = os.path.relpath(path, base_dir)
-        CheckZipPath(archive_path)
-        outfile.write(path, archive_path)
+  """Creates a zip file from a directory."""
+  inputs = []
+  for root, _, files in os.walk(base_dir):
+    for f in files:
+      inputs.append(os.path.join(root, f))
+  DoZip(inputs, output, base_dir)
 
 
-def MergeZips(output, inputs, exclude_patterns=None):
+def MergeZips(output, inputs, exclude_patterns=None, path_transform=None):
+  path_transform = path_transform or (lambda p, z: p)
   added_names = set()
   def Allow(name):
     if exclude_patterns is not None:
       for p in exclude_patterns:
         if fnmatch.fnmatch(name, p):
           return False
     return True
 
   with zipfile.ZipFile(output, 'w') as out_zip:
     for in_file in inputs:
       with zipfile.ZipFile(in_file, 'r') as in_zip:
         for name in in_zip.namelist():
-          if name not in added_names and Allow(name):
-            out_zip.writestr(name, in_zip.read(name))
-            added_names.add(name)
+          dst_name = path_transform(name, in_file)
+          if dst_name not in added_names and Allow(name):
+            zipinfo = zipfile.ZipInfo(filename=dst_name,
+                                      date_time=HERMETIC_TIMESTAMP)
+            out_zip.writestr(zipinfo, in_zip.read(name))
+            added_names.add(dst_name)
 
 
 def PrintWarning(message):
   print 'WARNING: ' + message
 
 
 def PrintBigWarning(message):
   print '*****     ' * 8
   PrintWarning(message)
   print '*****     ' * 8
@@ skipping 103 matching lines @@
       file_jsons[file_path] = ReadJson(file_path)
 
     expansion = file_jsons[file_path]
     for k in lookup_path[1:]:
       expansion = expansion[k]
 
     new_args[i] = arg[:match.start()] + str(expansion)
 
   return new_args