Compressing .pak resources with new option: "type=GZIPPABLE_BINDATA"

tools/grit/grit/node/include.py

Index: tools/grit/grit/node/include.py
diff --git a/tools/grit/grit/node/include.py b/tools/grit/grit/node/include.py
index ebcd53108f503005dadae9ae67b192245fb0a66d..88aa09a5dffc70fa60dd9865068ba64e6e6033ec 100755
--- a/tools/grit/grit/node/include.py
+++ b/tools/grit/grit/node/include.py
@@ -7,16 +7,20 @@
 """
 
 import os
+import sys
 
+from grit import util
+import grit.format.gzip_string
 import grit.format.html_inline
-import grit.format.rc_header
 import grit.format.rc
-
+import grit.format.rc_header
 from grit.node import base
-from grit import util
 
 class IncludeNode(base.Node):
   """An <include> element."""
+
+  RESERVED_HEADER = '\xff\x1f\x8b'
+
   def __init__(self):
     super(IncludeNode, self).__init__()
 
@@ -37,7 +41,6 @@ class IncludeNode(base.Node):
           grit.format.html_inline.InlineToString(filename, self,
               allow_external_script=allow_external_script))
     return self._flattened_data
-
   def MandatoryAttributes(self):
     return ['name', 'type', 'file']
 
@@ -47,6 +50,7 @@ class IncludeNode(base.Node):
             'filenameonly': 'false',
             'mkoutput': 'false',
             'flattenhtml': 'false',
+            'compress': 'false',

[flackr, 2016/05/27 19:42:14]

We should still have a test of generating a pak with the compress flag set in
the grd file.

[smaier, 2016/05/27 21:20:26]

Done.

             'allowexternalscript': 'false',
             'relativepath': 'false',
             'use_base_dir': 'true',
@@ -90,6 +94,24 @@ class IncludeNode(base.Node):
       filename = self.ToRealPath(self.GetInputPath())
       data = util.ReadFile(filename, util.BINARY)
 
+    if 'compress' in self.attrs and self.attrs['compress'] == 'gzip':

[flackr, 2016/05/27 19:42:14]

For now we should assert that self.attrs['compress'] == 'gzip' since we
shouldn't see anything else.

[smaier, 2016/05/27 21:20:26]

By default, self.attrs['compress'] == 'false'. This if statement will only catch
if it is 'gzip', which is when we want it to compress.

I didn't see any other entries in self.attrs which are forced to only a subset
of strings, although they typically are "true" or "false". Not sure if this is
the property to start that enforcement on.

[flackr, 2016/05/30 18:55:33]

Fair enough, there are only a few instances where we require specific string
values.

+      # We only use rsyncable compression on Linux.
+      if sys.platform == 'linux2':
+        data = grit.format.gzip_string.GzipStringRsyncable(data)
+      else:
+        data = grit.format.gzip_string.GzipString(data)
+      data = self.RESERVED_HEADER[0] + data
+    elif data[:3] == self.RESERVED_HEADER:
+      # We are reserving these 3 bytes as the header for gzipped files in the
+      # data pack. 1f:8b is the first two bytes of a gzipped header, and ff is
+      # a custom byte we throw in front of the gzip header so that we prevent
+      # accidentally throwing this error on a resource we gzipped beforehand and
+      # don't wish to compress again. If this exception is hit, change the first
+      # byte of RESERVED_HEADER, and then mirror that update in
+      # ui/base/resource/resource_bundle.h
+      raise Exception('Resource included with first 3 bytes matching reserved'

[flackr, 2016/05/27 19:42:15]

You should define a new exception type. See
https://code.google.com/p/chromium/codesearch#chromium/src/tools/grit/grit/ex...

[smaier, 2016/05/27 21:20:26]

Done.

+                      'header');
+
     # Include does not care about the encoding, because it only returns binary
     # data.
     return id, data