update_nacl_manifest improvements

native_client_sdk/src/build_tools/manifest_util.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import copy
 import hashlib
 import json
+import string
 import sys
+import urllib2
 
 MANIFEST_VERSION = 2
 
 # Some commonly-used key names.
 ARCHIVES_KEY = 'archives'
 BUNDLES_KEY = 'bundles'
 NAME_KEY = 'name'
 REVISION_KEY = 'revision'
 VERSION_KEY = 'version'
 
@@ skipping 31 matching lines @@
 def DictToJSON(pydict):
   """Convert a dict to a JSON-formatted string."""
   pretty_string = json.dumps(pydict, sort_keys=False, indent=2)
   # json.dumps sometimes returns trailing whitespace and does not put
   # a newline at the end.  This code fixes these problems.
   pretty_lines = pretty_string.split('\n')
   return '\n'.join([line.rstrip() for line in pretty_lines]) + '\n'
 
 
 def DownloadAndComputeHash(from_stream, to_stream=None, progress_func=None):
-  ''' Download the archive data from from-stream and generate sha1 and
-      size info.
+  '''Download the archive data from from-stream and generate sha1 and
+  size info.
 
   Args:
     from_stream:   An input stream that supports read.
     to_stream:     [optional] the data is written to to_stream if it is
                    provided.
     progress_func: [optional] A function used to report download progress. If
                    provided, progress_func is called with progress=0 at the
                    beginning of the download, periodically with progress=1
                    during the download, and progress=100 at the end.
 
@@ skipping 67 matching lines @@
       raise Error('Archive "%s" has no checksum' % host_os)
     elif not isinstance(checksum, dict):
       raise Error('Archive "%s" has a checksum, but it is not a dict' % host_os)
     elif not len(checksum):
       raise Error('Archive "%s" has an empty checksum dict' % host_os)
     # Verify that all key names are valid.
     for key in self:
       if key not in VALID_ARCHIVE_KEYS:
         raise Error('Archive "%s" has invalid attribute "%s"' % (host_os, key))
 
+  def UpdateVitals(self, revision):
+    """Update the size and checksum information for this archive
+    based on the content currently at the URL.
+
+    This allows the template mandifest to be maintained without

[binji, 2012/10/16 23:09:10]

manifest

+    the need to size and checksums to be present.
+    """
+    template = string.Template(self['url'])
+    self['url'] = template.substitute({'revision': revision})
+    from_stream = urllib2.urlopen(self['url'])
+    sha1_hash, size = DownloadAndComputeHash(from_stream)
+    self['size'] = size
+    self['checksum'] = { 'sha1': sha1_hash }
+
   def __getattr__(self, name):
     """Retrieve values from this dict using attributes.
 
     This allows for foo.bar instead of foo['bar'].
 
     Args:
       name: the name of the key, 'bar' in the example above.
     Returns:
       The value associated with that key."""
     if name not in VALID_ARCHIVE_KEYS:
@@ skipping 49 matching lines @@
       bundle: The other bundle.  Must be a dict.
     """
     for k, v in bundle.iteritems():
       if k == ARCHIVES_KEY:
         for archive in v:
           self.RemoveArchive(archive['host_os'])
           self.get(k, []).append(archive)
       else:
         self[k] = v
 
+  def __str__(self):
+    return self.GetDataAsString()
+
   def GetDataAsString(self):
     """Returns the JSON bundle object, pretty-printed"""
     return DictToJSON(self)
 
   def LoadDataFromString(self, json_string):
     """Load a JSON bundle string. Raises an exception if json_string
        is not well-formed JSON.
 
     Args:
       json_string: a JSON-formatted string containing the bundle
@@ skipping 10 matching lines @@
       if key == ARCHIVES_KEY:
         archives = []
         for a in value:
           new_archive = Archive(a['host_os'])
           new_archive.CopyFrom(a)
           archives.append(new_archive)
         self[ARCHIVES_KEY] = archives
       else:
         self[key] = value
 
-  def Validate(self):
+  def Validate(self, add_missing_info=False):
     """Validate the content of the bundle. Raise an Error if an invalid or
        missing field is found. """
     # Check required fields.
-    if not self.get(NAME_KEY, None):
+    if not self.get(NAME_KEY):
       raise Error('Bundle has no name')
-    if self.get(REVISION_KEY, None) == None:
+    if self.get(REVISION_KEY) == None:
       raise Error('Bundle "%s" is missing a revision number' % self[NAME_KEY])
-    if self.get(VERSION_KEY, None) == None:
+    if self.get(VERSION_KEY) == None:
       raise Error('Bundle "%s" is missing a version number' % self[NAME_KEY])
-    if not self.get('description', None):
+    if not self.get('description'):
       raise Error('Bundle "%s" is missing a description' % self[NAME_KEY])
-    if not self.get('stability', None):
+    if not self.get('stability'):
       raise Error('Bundle "%s" is missing stability info' % self[NAME_KEY])
-    if self.get('recommended', None) == None:
+    if self.get('recommended') == None:
       raise Error('Bundle "%s" is missing the recommended field' %
                   self[NAME_KEY])
     # Check specific values
     if self['stability'] not in STABILITY_LITERALS:
       raise Error('Bundle "%s" has invalid stability field: "%s"' %
                   (self[NAME_KEY], self['stability']))
     if self['recommended'] not in YES_NO_LITERALS:
       raise Error(
           'Bundle "%s" has invalid recommended field: "%s"' %
           (self[NAME_KEY], self['recommended']))
     # Verify that all key names are valid.
     for key in self:
       if key not in VALID_BUNDLES_KEYS:
         raise Error('Bundle "%s" has invalid attribute "%s"' %
                     (self[NAME_KEY], key))
     # Validate the archives
     for archive in self[ARCHIVES_KEY]:
+      if add_missing_info and 'size' not in archive:
+        archive.UpdateVitals(self[REVISION_KEY])
       archive.Validate()
 
+

[binji, 2012/10/16 23:09:10]

no newline necessary here

   def GetArchive(self, host_os_name):
     """Retrieve the archive for the given host os.
 
     Args:
       host_os_name: name of host os whose archive must be retrieved.
     Return:
       An Archive instance or None if it doesn't exist."""
     for archive in self[ARCHIVES_KEY]:
       if archive.host_os == host_os_name or archive.host_os == 'all':
         return archive
@@ skipping 88 matching lines @@
   For ease of unit-testing, this class should not contain any file I/O.
   """
 
   def __init__(self):
     """Create a new SDKManifest object with default contents"""
     self._manifest_data = {
         "manifest_version": MANIFEST_VERSION,
         "bundles": [],
         }
 
-  def Validate(self):
+  def Validate(self, add_missing_info=False):
     """Validate the Manifest file and raises an exception for problems"""
     # Validate the manifest top level
     if self._manifest_data["manifest_version"] > MANIFEST_VERSION:
       raise Error("Manifest version too high: %s" %
                   self._manifest_data["manifest_version"])
     # Verify that all key names are valid.
     for key in self._manifest_data:
       if key not in VALID_MANIFEST_KEYS:
         raise Error('Manifest has invalid attribute "%s"' % key)
     # Validate each bundle
     for bundle in self._manifest_data[BUNDLES_KEY]:
-      bundle.Validate()
+      bundle.Validate(add_missing_info)
 
   def GetBundle(self, name):
     """Get a bundle from the array of bundles.
 
     Args:
       name: the name of the bundle to return.
     Return:
       The first bundle with the given name, or None if it is not found."""
     if not BUNDLES_KEY in self._manifest_data:
       return None
@@ skipping 37 matching lines @@
     Returns:
       True if Bundle needs to be updated.
     """
     if NAME_KEY not in bundle:
       raise KeyError("Bundle must have a 'name' key.")
     local_bundle = self.GetBundle(bundle[NAME_KEY])
     return (local_bundle == None) or (
            (local_bundle[VERSION_KEY], local_bundle[REVISION_KEY]) <
            (bundle[VERSION_KEY], bundle[REVISION_KEY]))
 
-  def MergeBundle(self, bundle, allow_existing = True):
+  def MergeBundle(self, bundle, allow_existing=True):
     """Merge a Bundle into this manifest.
 
     The new bundle is added if not present, or merged into the existing bundle.
 
     Args:
       bundle: The bundle to merge.
     """
     if NAME_KEY not in bundle:
       raise KeyError("Bundle must have a 'name' key.")
     local_bundle = self.GetBundle(bundle.name)
     if not local_bundle:
       self.SetBundle(bundle)
     else:
       if not allow_existing:
         raise Error('cannot merge manifest bundle \'%s\', it already exists'
                     % bundle.name)
       local_bundle.MergeWithBundle(bundle)
 
   def MergeManifest(self, manifest):
     '''Merge another manifest into this manifest, disallowing overiding.
 
     Args
       manifest: The manifest to merge.
     '''
     for bundle in manifest.GetBundles():
-      self.MergeBundle(bundle, allow_existing = False)
+      self.MergeBundle(bundle, allow_existing=False)
 
   def FilterBundles(self, predicate):
     """Filter the list of bundles by |predicate|.
 
     For all bundles in this manifest, if predicate(bundle) is False, the bundle
     is removed from the manifest.
 
     Args:
       predicate: a function that take a bundle and returns whether True to keep
       it or False to remove it.
     """
     self._manifest_data[BUNDLES_KEY] = filter(predicate, self.GetBundles())
 
-  def LoadDataFromString(self, json_string):
+  def LoadDataFromString(self, json_string, add_missing_info=False):
     """Load a JSON manifest string. Raises an exception if json_string
        is not well-formed JSON.
 
     Args:
       json_string: a JSON-formatted string containing the previous manifest
       all_hosts: True indicates that we should load bundles for all hosts.
           False (default) says to only load bundles for the current host"""
     new_manifest = json.loads(json_string)
     for key, value in new_manifest.items():
       if key == BUNDLES_KEY:
         # Remap each bundle in |value| to a Bundle instance
         bundles = []
         for b in value:
           new_bundle = Bundle(b[NAME_KEY])
           new_bundle.CopyFrom(b)
           bundles.append(new_bundle)
         self._manifest_data[key] = bundles
       else:
         self._manifest_data[key] = value
-    self.Validate()
+    self.Validate(add_missing_info)
+
+  def __str__(self):
+    return self.GetDataAsString()
 
   def GetDataAsString(self):
     """Returns the current JSON manifest object, pretty-printed"""
     return DictToJSON(self._manifest_data)