Subzero: Make the szbuild.py script more automatic.

pydir/szbuild.py

 #!/usr/bin/env python2
 
 import argparse
 import os
 import pipes
 import re
 import sys
 
 from utils import shellcmd
 from utils import FindBaseNaCl
 
+def NewerThanOrNotThere(old_path, new_path):
+    """Returns whether old_path is newer than new_path.
+
+    Also returns true if either path doesn't exist.
+    """
+    if not (os.path.exists(old_path) and os.path.exists(new_path)):
+        return True
+    return os.path.getmtime(old_path) > os.path.getmtime(new_path)
+
 def BuildRegex(patterns, syms):
     """Build a regular expression string for inclusion or exclusion.
 
     Creates a regex string from an array of patterns and an array
     of symbol names.  Each element in the patterns array is either a
     regex, or a range of entries in the symbol name array, e.g. '2:9'.
     """
     pattern_list = []
     for pattern in patterns:
         if pattern[0].isdigit() or pattern[0] == ':':
@@ skipping 50 matching lines @@
     'accepted' symbol, and the llc version is used for a 'rejected'
     symbol.
 
     Each --include and --exclude argument can be a regular expression
     or a range of lines in the symbol file.  Each regular expression
     is wrapped inside '^$', so if you want a substring match on 'foo',
     use '.*foo.*' instead.  Ranges use python-style 'first:last'
     notation, so e.g. use '0:10' or ':10' for the first 10 lines of
     the file, or '1' for the second line of the file.
 
-    The --init argument does first-time initialization for the pexe,
-    including creation of the Subzero symbol file that is implicitly
-    used in the --include and --exclude arguments.  It can be removed
-    from the command line for subsequent executions if the pexe
-    doesn't change.
+    This script uses file modification timestamps to determine whether
+    llc and Subzero re-translation are needed.  It checks timestamps
+    of llc, llvm2ice, and the pexe against the translated object files
+    to determine the minimal work necessary.  The --init option
+    suppresses those checks and re-translates everything.
 
-    This scripts augments PATH so that various PNaCl and LLVM tools
-    can be run.  These extra paths are within the native_client tree.
+    This script augments PATH so that various PNaCl and LLVM tools can
+    be run.  These extra paths are within the native_client tree.
     When changes are made to these tools, copy them this way:
       cd native_client
       toolchain_build/toolchain_build_pnacl.py llvm_i686_linux \\
       --install=toolchain/linux_x86/pnacl_newlib
     """
     argparser = argparse.ArgumentParser(
         description='    ' + main.__doc__,
         formatter_class=argparse.RawTextHelpFormatter)
     argparser.add_argument('pexe', help='Finalized pexe to translate')
     argparser.add_argument('--init', dest='init', action='store_true',
-                           help='Perform first-time setup for the pexe')
+                           help='Force all re-translation of the pexe')

[jvoung (off chromium), 2014/09/16 00:43:58]

Would it make sense to call the flag --force, or something, now that its
behavior is a bit different?

[Jim Stichnoth, 2014/09/16 03:09:48]

You're right.  I changed it to --force.

     argparser.add_argument('--include', '-i', default=[], dest='include',
                            action='append',
                            help='Subzero symbols to include ' +
                                 '(regex or line range)')
     argparser.add_argument('--exclude', '-e', default=[], dest='exclude',
                            action='append',
                            help='Subzero symbols to exclude ' +
                                 '(regex or line range)')
     argparser.add_argument('--output', '-o', default='a.out', dest='output',
                            action='store',
@@ skipping 27 matching lines @@
     obj_sz = pexe_base + '.sz.o'
     asm_sz = pexe_base + '.sz.s'
     obj_llc_weak = pexe_base + '.weak.llc.o'
     obj_sz_weak = pexe_base + '.weak.sz.o'
     obj_partial = pexe_base + '.o'
     sym_llc = pexe_base + '.sym.llc.txt'
     sym_sz = pexe_base + '.sym.sz.txt'
     sym_sz_unescaped = pexe_base_unescaped + '.sym.sz.txt'
     whitelist_sz = pexe_base + '.wl.sz.txt'
     whitelist_sz_unescaped = pexe_base_unescaped + '.wl.sz.txt'
+    llvm2ice = (
+        '{root}/toolchain_build/src/subzero/llvm2ice'
+        ).format(root=nacl_root)
+    llcbin = (
+        '{root}/toolchain/linux_x86/pnacl_newlib/host_x86_32/bin/llc'
+        ).format(root=nacl_root)
+    opt_level = args.optlevel
 
-    if args.init:
-        opt_level = args.optlevel
+    if args.init or NewerThanOrNotThere(pexe, obj_llc) or \
+            NewerThanOrNotThere(llcbin, obj_llc):
         opt_level_map = { 'm1':'0', '-1':'0', '0':'0', '1':'1', '2':'2' }
         shellcmd((
             'pnacl-translate -ffunction-sections -c -arch x86-32-linux ' +
             '-O{level} --pnacl-driver-append-LLC_FLAGS_EXTRA=-externalize ' +
             '-o {obj} {pexe}'
             ).format(level=opt_level_map[opt_level], obj=obj_llc, pexe=pexe),
                  echo=args.verbose)
         shellcmd((
             'objcopy --redefine-sym _start=_user_start {obj}'
             ).format(obj=obj_llc), echo=args.verbose)
         shellcmd((
-            '{root}/toolchain_build/src/subzero/llvm2ice ' +
-            '-O{level} -bitcode-format=pnacl -disable-globals ' +
+            'nm {obj} | sed -n "s/.* [a-zA-Z] //p" > {sym}'
+            ).format(obj=obj_llc, sym=sym_llc), echo=args.verbose)
+    if args.init or NewerThanOrNotThere(pexe, obj_sz) or \
+            NewerThanOrNotThere(llvm2ice, obj_sz):
+        shellcmd((
+            '{l2i} -O{level} -bitcode-format=pnacl -disable-globals ' +
             '-externalize -ffunction-sections {pexe} -o {asm}'
-            ).format(root=nacl_root,level=opt_level, pexe=pexe, asm=asm_sz),
+            ).format(l2i=llvm2ice, level=opt_level, pexe=pexe, asm=asm_sz),
                  echo=args.verbose)
         shellcmd((
             'llvm-mc -arch=x86 -x86-asm-syntax=intel -filetype=obj -o {obj} ' +
             '{asm}'
             ).format(asm=asm_sz, obj=obj_sz), echo=args.verbose)
         shellcmd((
             'objcopy --redefine-sym _start=_user_start {obj}'
             ).format(obj=obj_sz), echo=args.verbose)
         shellcmd((
             'nm {obj} | sed -n "s/.* [a-zA-Z] //p" > {sym}'
             ).format(obj=obj_sz, sym=sym_sz), echo=args.verbose)
-        shellcmd((
-            'nm {obj} | sed -n "s/.* [a-zA-Z] //p" > {sym}'
-            ).format(obj=obj_llc, sym=sym_llc), echo=args.verbose)
 
     with open(sym_sz_unescaped) as f:
         sz_syms = f.read().splitlines()
     re_include_str = BuildRegex(args.include, sz_syms)
     re_exclude_str = BuildRegex(args.exclude, sz_syms)
     re_include = re.compile(re_include_str)
     re_exclude = re.compile(re_exclude_str)
     # If a symbol doesn't explicitly match re_include or re_exclude,
     # the default MatchSymbol() result is False, unless some --exclude
     # args are provided and no --include args are provided.
@@ skipping 41 matching lines @@
     # Put the extra verbose printing at the end.
     if args.verbose:
         print 'PATH={path}'.format(path=os.environ['PATH'])
         print 'include={regex}'.format(regex=re_include_str)
         print 'exclude={regex}'.format(regex=re_exclude_str)
         print 'default_match={dm}'.format(dm=default_match)
         print 'Number of Subzero syms = {num}'.format(num=len(sz_syms))
 
 if __name__ == '__main__':
     main()