Add firmware image packing tool

utility/pack_image

+#!/usr/bin/env python2.6

[gauravsh, 2011/01/04 21:14:39]

make this version independent

[Hung-Te, 2011/01/05 03:33:54]

That reminds me something. Che-Liang used "with" which is supported since 2.6
(in 2.5, some special import is required).

Do we have some policy or coding style suggestion for such situation?

[Che-Liang Chiou, 2011/01/05 08:01:55]

The coding style asks for the shebang line with a specific version. 
http://google-styleguide.googlecode.com/svn/trunk/pyguide.html?showone=Python...

[gauravsh, 2011/01/06 02:06:45]

Chromium OS follows PEP-8, not Google python style. But since PEP-8 doesn't say
anything about this, and you use 2.6 features, this is fine.

+
+# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.

[gauravsh, 2011/01/04 21:14:39]

2011

[Che-Liang Chiou, 2011/01/05 08:01:55]

Done.

+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import mmap
+import os
+import re
+import struct
+import subprocess
+import sys
+import tempfile
+
+# TODO(clchiou): Rewrite this part by importing fmap after David's fmap

[Hung-Te, 2011/01/04 10:05:23]

Please use "official" instead of "David's"

[Che-Liang Chiou, 2011/01/05 08:01:55]

Done.

+# implementation is pulled into Chromium OS code base
+
+FMAP_SIGNATURE = "__FMAP__"
+
+FMAP_AREA_STATIC = 1 << 0
+FMAP_AREA_COMPRESSED = 1 << 1
+FMAP_AREA_RO = 1 << 2
+
+FMAP_STRLEN = 32
+FMAP_HEADER_FORMAT = "<8sBBQI%dsH" % FMAP_STRLEN
+FMAP_AREA_FORMAT = "<II%dsH" % FMAP_STRLEN
+
+FMAP_HEADER_NAMES = (
+    'signature',
+    'ver_major',
+    'ver_minor',
+    'base',
+    'size',
+    'name',
+    'nareas',
+)
+
+FMAP_AREA_NAMES = (
+    'offset',
+    'size',
+    'name',
+    'flags',
+)
+
+RE_ASSIGNMENT = re.compile(r'^(\w+)=(.*)$')
+
+VERBOSE = False
+
+
+class ConfigError(Exception):
+  pass
+
+
+class PackError(Exception):
+  pass
+
+
+class Entry(dict):

[gauravsh, 2011/01/04 21:14:39]

general comment our python style - Chromium OS follows PEP-8 but with Mixed case
method and function names and 2 space instead of 4 space indent. Your indent is
fine, but the method names are not.

Please make the change for the entire file.

[Che-Liang Chiou, 2011/01/05 08:01:55]

Done.

+
+  @staticmethod
+  def _check_fields(kwargs, fields):
+    for f in fields:
+      if f not in kwargs:
+        raise ConfigError('Entry: missing required field: %s' % f)
+
+  def __init__(self, **kwargs):
+    Entry._check_fields(kwargs, ('offset', 'length', 'name'))
+    super(Entry, self).__init__(kwargs)
+
+  def __getattr__(self, name):
+    return self[name]
+
+  def is_overlapped(self, entry):
+    return entry.offset <= self.offset < entry.offset + entry.length or \

[gauravsh, 2011/01/04 21:14:39]

you can get rid of the \ and use implicit line breaking by parenthesizing it (as
Hung-Te suggested elsewhere). This will make this easier to read too.

[Che-Liang Chiou, 2011/01/05 08:01:55]

Done.

+        self.offset <= entry.offset < self.offset + self.length
+
+  def pack(self, fw_image, entries):
+    raise PackError('class Entry does not implement pack()')
+
+
+class EntryFmap(Entry):
+
+  def __init__(self, **kwargs):
+    Entry._check_fields(kwargs, ('ver_major', 'ver_minor', 'base', 'size'))
+    super(EntryFmap, self).__init__(**kwargs)
+
+  def pack(self, fw_image, entries):
+    areas = [e for e in entries
+        if isinstance(e, EntryBlob) or isinstance(e, EntryKeyBlock)]
+    areas.sort(key=lambda a: a.offset)
+
+    offset = 0
+
+    # pack header
+    values = []
+    for name in FMAP_HEADER_NAMES:
+      if name == 'nareas':
+        v = len(areas)
+      elif name == 'signature':
+        v = FMAP_SIGNATURE
+      else:
+        v = self[name]
+      values.append(v)
+    offset = self._append(fw_image, offset,
+        struct.pack(FMAP_HEADER_FORMAT, *values))
+
+    # pack areas
+    for a in areas:
+      values = [a.length if name == 'size' else a[name]
+          for name in FMAP_AREA_NAMES]
+      offset = self._append(fw_image, offset,
+          struct.pack(FMAP_AREA_FORMAT, *values))
+
+    if offset > self.length:
+        raise PackError('fmap too large: %d > %d' % (offset, self.length))
+
+  def _append(self, fw_image, offset, blob):
+    size = len(blob)
+    fw_image[self.offset+offset:self.offset+offset+size] = blob
+    return offset + size
+
+
+class EntryBlob(Entry):
+
+  def __init__(self, **kwargs):
+    Entry._check_fields(kwargs, ('flags', 'path'))
+    super(EntryBlob, self).__init__(**kwargs)
+
+  def pack(self, fw_image, entries):
+    size = os.stat(self.path).st_size
+    if size > 0:
+      size = min(size, self.length)
+    else:
+      size = self.length
+    with open(self.path, 'r+b') as f:
+      fvimg = mmap.mmap(f.fileno(), size, mmap.MAP_PRIVATE, mmap.PROT_READ)

[gauravsh, 2011/01/04 21:14:39]

i am not sure the performance improvement from mmap is really needed here.

[Che-Liang Chiou, 2011/01/05 08:01:55]

Done.

+      fw_image.seek(self.offset)
+      fw_image.write(fvimg[0:size])
+      fvimg.close()
+
+
+class EntryKeyBlock(Entry):
+
+  def __init__(self, **kwargs):
+    Entry._check_fields(kwargs,
+        ('flags', 'keyblock', 'signprivate', 'version', 'fv', 'kernelkey'))
+    super(EntryKeyBlock, self).__init__(**kwargs)
+
+  def pack(self, fw_image, entries):
+    fd, path = tempfile.mkstemp()
+    try:
+      args = [
+          'vbutil_firmware',
+          '--vblock', path,
+          '--keyblock', self.keyblock,
+          '--signprivate', self.signprivate,
+          '--version', '%d' % self.version,
+          '--fv', self.fv,
+          '--kernelkey', self.kernelkey,
+      ]
+      if VERBOSE:
+        stdout = sys.stdout
+        stderr = sys.stderr
+      else:
+        stdout = None

[gauravsh, 2011/01/04 21:14:39]

this could be declared as a class variable right at the beginning of class
definition, you won't need the else block then.

[Che-Liang Chiou, 2011/01/05 08:01:55]

Done.

+        stderr = None
+      info('run: %s' % ' '.join(args))
+      proc = subprocess.Popen(args, stdout=stdout, stderr=stderr)
+      proc.wait()
+      if proc.returncode != 0:
+        raise PackError('cannot make key block: vbutil_firmware returns %d' %
+                        proc.returncode)
+
+      img = mmap.mmap(fd, 0, mmap.MAP_PRIVATE, mmap.PROT_READ)

[gauravsh, 2011/01/04 21:14:39]

in general, Google style recommends not using short forms for variable names
(i.e. image is better than img)

[Che-Liang Chiou, 2011/01/05 08:01:55]

Done.

+      size = img.size()
+      if size > self.length:
+        img.close()
+        raise PackError('key block too large: %d > %d' % (size, self.length))
+      fw_image.seek(self.offset)
+      fw_image.write(img)
+      img.close()
+    finally:
+      os.unlink(path)
+
+
+def parse_assignment(stmt):
+  m = RE_ASSIGNMENT.match(stmt)
+  if m is None:
+    raise ConfigError('illegal statement: %s' % repr(stmt))
+  return (m.group(1), parse_value(m.group(2)))
+
+
+def parse_value(expr):
+  if (expr.startswith('"') and expr.endswith('"')) or \

[Hung-Te, 2011/01/04 10:05:23]

You can use implicit multi-line expression, like
 ((a and b) or
  (b and c))

[Che-Liang Chiou, 2011/01/05 08:01:55]

Done.

+     (expr.startswith("'") and expr.endswith("'")):
+    expr = expr[1:-1]

[Hung-Te, 2011/01/04 10:05:23]

return expr[1:-1] ?
Otherwise it'll be converted to integer agian.

[Che-Liang Chiou, 2011/01/05 08:01:55]

Done.

+  try:
+    return int(expr, 0)
+  except ValueError:
+    return expr # if not a number, interpret as string literals
+
+
+def pack_image(entries, output_path, image_size):
+  entries = sorted(entries, key=lambda e: e.offset)
+  for e1, e2 in zip(entries, entries[1:]):
+    if e1.is_overlapped(e2):
+      raise PackError('overlapped entries: [%08x:%08x], [%08x:%08x]' %
+          (e1.offset, e1.offset + e1.length, e2.offset, e2.offset + e2.length))
+
+  # create image so that mmap will succeed
+  with open(output_path, 'w+b') as f:

[gauravsh, 2011/01/04 21:14:39]

again don't use 'f' here. Use 'file'. The only place where you short forms var
names are recommended are standard loop index variables like 'i'. 

[Hung-Te, 2011/01/05 03:33:54]

P.S: 'file' is a special keyword in python -- some other things like image_file
would be even better.

[Che-Liang Chiou, 2011/01/05 08:01:55]

Done; avoid use 'file'.

+    f.write('\0')
+
+  with open(output_path, 'rw+b') as f:
+    fw_image = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_WRITE)
+    fw_image.resize(image_size)
+    for entry in entries:
+      entry.pack(fw_image, entries)
+    fw_image.flush()
+    fw_image.close()
+
+
+def info(msg):
+  if VERBOSE:
+    print >>sys.stderr, 'INFO: %s' % msg
+
+
+def main():
+  global VERBOSE
+
+  if len(sys.argv) < 2:
+    print 'Usage: %s [-v] CONFIG_FILE [NAME=VALUE...]' % sys.argv[0]
+    sys.exit(1)
+
+  if sys.argv[1] == '-v':
+    VERBOSE = True
+    argv = sys.argv[0:1] + sys.argv[2:]
+  else:
+    argv = sys.argv
+
+  if len(argv) > 2:
+    env = dict(parse_assignment(stmt) for stmt in argv[2:])
+  else:
+    env = {}
+
+  execfile(argv[1], globals(), env)
+
+  for varname in ('ENTRIES', 'OUTPUT', 'SIZE'):
+    if varname not in env:
+      raise ConfigError('undefined variable: %s' % varname)
+    info('%s = %s' % (varname, repr(env[varname])))
+
+  pack_image(env['ENTRIES'], env['OUTPUT'], env['SIZE'])
+
+  sys.exit(0)
+
+
+if __name__ == '__main__':
+  main()