create_nmf can now determine arch of nexe files

native_client_sdk/src/tools/create_nmf.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from __future__ import with_statement
 
 import errno
 import optparse
 import os
 import re
 import shutil
 import subprocess
 import sys
 import urllib
+import struct

[binji, 2012/09/24 19:37:04]

alphabetize

 
 try:
   import json
 except ImportError:
   import simplejson as json
 
 NeededMatcher = re.compile('^ *NEEDED *([^ ]+)\n$')
 FormatMatcher = re.compile('^(.+):\\s*file format (.+)\n$')
 
-FORMAT_ARCH_MAP = {
+OBJDUMP_ARCH_MAP = {
     # Names returned by Linux's objdump:
     'elf64-x86-64': 'x86-64',
     'elf32-i386': 'x86-32',
+    'elf32-little': 'arm',
+    'elf32-littlearm': 'arm',
     # Names returned by x86_64-nacl-objdump:
     'elf64-nacl': 'x86-64',
     'elf32-nacl': 'x86-32',
 }
 
 ARCH_LOCATION = {
     'x86-32': 'lib32',
     'x86-64': 'lib64',
+    'arm': 'lib',
 }
 
-NAME_ARCH_MAP = {
-    '32.nexe': 'x86-32',
-    '64.nexe': 'x86-64',
-    'arm.nexe': 'arm'
-}
 
 # These constants are used within nmf files.
 RUNNABLE_LD = 'runnable-ld.so'  # Name of the dynamic loader
 MAIN_NEXE = 'main.nexe'  # Name of entry point for execution
 PROGRAM_KEY = 'program'  # Key of the program section in an nmf file
 URL_KEY = 'url'  # Key of the url field for a particular file in an nmf file
 FILES_KEY = 'files'  # Key of the files section in an nmf file
 
 # The proper name of the dynamic linker, as kept in the IRT.  This is
 # excluded from the nmf file by convention.
 LD_NACL_MAP = {
     'x86-32': 'ld-nacl-x86-32.so.1',
     'x86-64': 'ld-nacl-x86-64.so.1',
+    'arm': None,
 }
 
 
 def DebugPrint(message):
   if DebugPrint.debug_mode:
     sys.stderr.write('%s\n' % message)
 
 
 DebugPrint.debug_mode = False  # Set to True to enable extra debug prints
 
@@ skipping 15 matching lines @@
 
 class Error(Exception):
   '''Local Error class for this file.'''
   pass
 
 
 class ArchFile(object):
   '''Simple structure containing information about
 
   Attributes:
+    name: Name of this file
+    path: Full path to this file on the build system
     arch: Architecture of this file (e.g., x86-32)
-    filename: name of this file
-    path: Full path to this file on the build system
     url: Relative path to file in the staged web directory.
         Used for specifying the "url" attribute in the nmf file.'''
-  def __init__(self, arch, name, path='', url=None):
-    self.arch = arch
+
+  def __init__(self, name, path, url):
     self.name = name
     self.path = path
-    self.url = url or '/'.join([arch, name])
+    self.url = url
+    self.arch = None
+    self.ReadElfHeader()
+
+  def ReadElfHeader(self):

[binji, 2012/09/24 19:37:04]

make this private.

+    """Determine architecture of nexe by reading elf header."""
+    # From elf.h:
+    # typedef struct
+    # {
+    #   unsigned char   e_ident[EI_NIDENT]; /* Magic number and other info */

[binji, 2012/09/24 19:37:04]

kill tab

+    #   Elf64_Half e_type; /* Object file type */
+    #   Elf64_Half e_machine; /* Architecture */
+    #   ...
+    # } Elf32_Ehdr;
+    elf_header_format = '16s2H'
+    elf_header_size = struct.calcsize(elf_header_format)
+
+    with open(self.path, 'rb') as f:
+      header = f.read(elf_header_size)
+
+    header = struct.unpack(elf_header_format, header)
+    e_ident, _, e_machine = header
+
+    elf_magic = '\x7fELF'
+    if e_ident[:4] != elf_magic:
+      raise Error("Not a valid NaCL executable: %s" % self.path)
+
+    e_machine_mapping = {
+      3 : 'x86-32',
+      40 : 'arm',
+      62 : 'x86-64'
+    }
+    if e_machine not in e_machine_mapping:
+      raise Error("Unknown machine type: %s" % e_machine)
+
+    # set arch based on the machine type in the elf header
+    self.arch = e_machine_mapping[e_machine]
 
   def __repr__(self):
     return "<ArchFile %s>" % self.path
 
   def __str__(self):
     '''Return the file path when invoked with the str() function'''
     return self.path
 
 
 class NmfUtils(object):
   '''Helper class for creating and managing nmf files
 
   Attributes:
     manifest: A JSON-structured dict containing the nmf structure
     needed: A dict with key=filename and value=ArchFile (see GetNeeded)
   '''
 
-  def __init__(self, main_files=None, objdump='x86_64-nacl-objdump',
+  def __init__(self, main_files=None, objdump=None,
                lib_path=None, extra_files=None, lib_prefix=None,
                toolchain=None, remap=None):
     '''Constructor
 
     Args:
       main_files: List of main entry program files.  These will be named
           files->main.nexe for dynamic nexes, and program for static nexes
       objdump: path to x86_64-nacl-objdump tool (or Linux equivalent)
       lib_path: List of paths to library directories
       extra_files: List of extra files to include in the nmf
       lib_prefix: A list of path components to prepend to the library paths,
           both for staging the libraries and for inclusion into the nmf file.
           Examples:  ['..'], ['lib_dir']
       toolchain: Specify which toolchain newlib|glibc|pnacl which can require
           different forms of the NMF.
       remap: Remaps the library name in the manifest.
       '''
     self.objdump = objdump
     self.main_files = main_files or []
     self.extra_files = extra_files or []
     self.lib_path = lib_path or []
     self.manifest = None
-    self.needed = None
+    self.needed = {}
     self.lib_prefix = lib_prefix or []
     self.toolchain = toolchain
     self.remap = remap or {}
 
 
   def GleanFromObjdump(self, files):
     '''Get architecture and dependency information for given files
 
     Args:
       files: A dict with key=filename and value=list or set of archs.  E.g.:
           { '/path/to/my.nexe': ['x86-32']
             '/path/to/lib64/libmy.so': ['x86-64'],
             '/path/to/mydata.so': ['x86-32', 'x86-64'],
             '/path/to/my.data': None }  # Indicates all architectures
 
     Returns: A tuple with the following members:
       input_info: A dict with key=filename and value=ArchFile of input files.
           Includes the input files as well, with arch filled in if absent.
           Example: { '/path/to/my.nexe': ArchFile(my.nexe),
                      '/path/to/libfoo.so': ArchFile(libfoo.so) }
       needed: A set of strings formatted as "arch/name".  Example:
           set(['x86-32/libc.so', 'x86-64/libgcc.so'])
     '''
+    if not self.objdump:
+      raise Error("No objdump executable specified (see --help for more info)")
     DebugPrint("GleanFromObjdump(%s)" % ([self.objdump, '-p'] + files.keys()))
     proc = subprocess.Popen([self.objdump, '-p'] + files.keys(),
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, bufsize=-1)
     input_info = {}
     needed = set()
     output, err_output = proc.communicate()
     for line in output.splitlines(True):
       # Objdump should display the architecture first and then the dependencies
       # second for each file in the list.
       matched = FormatMatcher.match(line)
       if matched is not None:
         filename = matched.group(1)
-        arch = FORMAT_ARCH_MAP[matched.group(2)]
+        arch = OBJDUMP_ARCH_MAP[matched.group(2)]
         if files[filename] is None or arch in files[filename]:
           name = os.path.basename(filename)
           input_info[filename] = ArchFile(
-              arch=arch,
               name=name,
               path=filename,
               url='/'.join(self.lib_prefix + [ARCH_LOCATION[arch], name]))
       matched = NeededMatcher.match(line)
       if matched is not None:
         if files[filename] is None or arch in files[filename]:
           needed.add('/'.join([arch, matched.group(1)]))
     status = proc.poll()
     if status != 0:
       raise Error('%s\nStdError=%s\nobjdump failed with error code: %d' %
@@ skipping 23 matching lines @@
     Returns:
       A dict with key=filename and value=ArchFile of input files.
           Includes the input files as well, with arch filled in if absent.
           Example: { '/path/to/my.nexe': ArchFile(my.nexe),
                      '/path/to/libfoo.so': ArchFile(libfoo.so) }'''
     if self.needed:
       return self.needed
 
     runnable = (self.toolchain != 'newlib' and self.toolchain != 'pnacl')
     DebugPrint('GetNeeded(%s)' % self.main_files)
+
     if runnable:
       examined = set()
       all_files, unexamined = self.GleanFromObjdump(
           dict([(f, None) for f in self.main_files]))
       for name, arch_file in all_files.items():
         arch_file.url = name
         if unexamined:
           unexamined.add('/'.join([arch_file.arch, RUNNABLE_LD]))
       while unexamined:
         files_to_examine = {}
         for arch_name in unexamined:
           arch, name = arch_name.split('/')
           for path in self.FindLibsInPath(name):
             files_to_examine.setdefault(path, set()).add(arch)
         new_files, needed = self.GleanFromObjdump(files_to_examine)
         all_files.update(new_files)
         examined |= unexamined
         unexamined = needed - examined
+
       # With the runnable-ld.so scheme we have today, the proper name of
       # the dynamic linker should be excluded from the list of files.
-      ldso = [LD_NACL_MAP[arch] for arch in set(FORMAT_ARCH_MAP.values())]
+      ldso = [LD_NACL_MAP[arch] for arch in set(OBJDUMP_ARCH_MAP.values())]
       for name, arch_map in all_files.items():
         if arch_map.name in ldso:
           del all_files[name]
+
       self.needed = all_files
     else:
-      need = {}
       for filename in self.main_files:
-        arch = filename.split('_')[-1]
-        arch = NAME_ARCH_MAP[arch]
         url = os.path.split(filename)[1]
-        need[filename] = ArchFile(arch=arch, name=os.path.basename(filename),
-                                  path=filename, url=url)
-      self.needed = need
+        archfile = ArchFile(name=os.path.basename(filename),
+                            path=filename, url=url)
+        self.needed[filename] = archfile
 
     return self.needed
 
   def StageDependencies(self, destination_dir):
     '''Copies over the dependencies into a given destination directory
 
     Each library will be put into a subdirectory that corresponds to the arch.
 
     Args:
       destination_dir: The destination directory for staging the dependencies
@@ skipping 92 matching lines @@
 
 Trace.verbose = False
 
 
 def Main(argv):
   parser = optparse.OptionParser(
       usage='Usage: %prog [options] nexe [extra_libs...]')
   parser.add_option('-o', '--output', dest='output',
                     help='Write manifest file to FILE (default is stdout)',
                     metavar='FILE')
-  parser.add_option('-D', '--objdump', dest='objdump', default='objdump',
+  parser.add_option('-D', '--objdump', dest='objdump',
                     help='Use TOOL as the "objdump" tool to run',
                     metavar='TOOL')
   parser.add_option('-L', '--library-path', dest='lib_path',
                     action='append', default=[],
                     help='Add DIRECTORY to library search path',
                     metavar='DIRECTORY')
   parser.add_option('-P', '--path-prefix', dest='path_prefix', default='',
                     help='A path to prepend to shared libraries in the .nmf',
                     metavar='DIRECTORY')
   parser.add_option('-s', '--stage-dependencies', dest='stage_dependencies',
@@ skipping 61 matching lines @@
 
 
 # Invoke this file directly for simple testing.
 if __name__ == '__main__':
   try:
     rtn = Main(sys.argv[1:])
   except Error, e:
     print "%s: %s" % (os.path.basename(__file__), e)
     rtn = 1
   sys.exit(rtn)