Add a scons pnacl finalize step for pnacl_generate_pexe tests.

SConstruct

 #! -*- python -*-
 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import atexit
 import os
 import platform
 import re
 import subprocess
@@ skipping 334 matching lines @@
   BitFromArgument(env, 'x86_64_zero_based_sandbox', default=False,
     desc='Use the zero-address-based x86-64 sandbox model instead of '
       'the r15-based model.')
 
   BitFromArgument(env, 'android', default=False,
                   desc='Build for Android target')
 
   BitFromArgument(env, 'arm_hard_float', default=False,
                   desc='Build for hard float ARM ABI')
 
+  BitFromArgument(env, 'skip_nonstable_bitcode', default=False,
+                  desc='Skip tests involving non-stable bitcode')
+
   #########################################################################
   # EXPERIMENTAL
   # This is for generating a testing library for use within private test
   # enuminsts, where we want to compare and test different validators.
   #
   BitFromArgument(env, 'ncval_testing', default=False,
     desc='EXPERIMENTAL: Compile validator code for testing within enuminsts')
 
   # PNaCl sanity checks
   if ((env.Bit('pnacl_generate_pexe') or env.Bit('use_sandboxed_translator'))
@@ skipping 1202 matching lines @@
   # Make sure the pexe doesn't get removed by the fake builders when
   # built_elsewhere=1
   env.Precious(pexe)
   node = env.Command(target=nexe_name, source=[pexe_name],
                      action=[Action('${TRANSLATECOM}', '${TRANSLATECOMSTR}')])
   assert len(node) == 1, node
   return node[0]
 
 pre_base_env.AddMethod(GetTranslatedNexe)
 
-

[Mark Seaborn, 2013/03/28 01:31:34]

Nit: Can you leave 2 empty lines between top-level defs please?  That's the
usual Python style.

[jvoung - send to chromium..., 2013/03/28 15:50:29]

Done.

 def ShouldTranslateToNexe(env, pexe):
   """ Determine when we need to translate a Pexe to a Nexe.
   """
   # Check if we started with a pexe, so there is actually a translation step.
   if not env.Bit('pnacl_generate_pexe'):
     return False
 
   # There is no bitcode for trusted code.
   if env['NACL_BUILD_FAMILY'] == 'TRUSTED':
     return False
 
   # Often there is a build step (do_not_run_tests=1) and a test step
   # (which is run with -j1). Normally we want to translate in the build step
   # so we can translate in parallel. However when we do sandboxed translation
   # on arm hw, we do the build step on x86 and translation on arm, so we have
   # to force the translation to be done in the test step. Hence,
   # we check the bit 'translate_in_build_step' / check if we are
   # in the test step.
   return (
     env.Bit('translate_in_build_step') or not env.Bit('do_not_run_tests'))
 
 pre_base_env.AddMethod(ShouldTranslateToNexe)
 
+def GetFinalizedPexe(env, pexe):
+  """ Prep and finalize the ABI for a given pexe.
+  """
+  pexe_name = pexe.abspath
+  final_name = pexe_name[:pexe_name.index('.pexe')] + '.final.pexe'

[Mark Seaborn, 2013/03/28 01:31:34]

You should assert that pexe_name ends with ".pexe" so that this doesn't mangle
the name if it doesn't.

Could we use a slightly different naming convention?  e.g. ".final-pexe", so
that globbing for "*.pexe" and "*.final-pexe" match disjoint sets of files (for
when copying around sets of these manually).

I'd suggest reversing the naming actually: how about ".nonfinal-pexe" or
".unstripped-pexe" and ".pexe"?  We'd want to use a briefer name for the file
that's intended to be stable and published.

+  # Make sure the pexe doesn't get removed by the fake builders when
+  # built_elsewhere=1
+  env.Precious(pexe)
+  node = env.Command(target=final_name, source=[pexe_name],
+                     action=[Action('${PNACLFINALIZECOM}',
+                                    '${PNACLFINALIZECOMSTR}')])
+  assert len(node) == 1, node
+  return node[0]
+
+pre_base_env.AddMethod(GetFinalizedPexe)
+
+# Bit to be set by individual test/nacl.scons files that need to opt-out.

[Mark Seaborn, 2013/03/28 01:31:34]

Nit: "opt out", no hyphen :-)

[jvoung - send to chromium..., 2013/03/28 15:50:29]

Done.

+DeclareBit('nonstable_bitcode', 'Tests use non-stable bitcode features', False)
+
+def ShouldFinalizePexe(env, pexe):
+  """ Determine when we need to prep a pexe to finalize it for ABI stability.
+  """
+  if not env.Bit('pnacl_generate_pexe') or env.Bit('nonstable_bitcode'):
+    return False
+
+  # There is no bitcode for trusted code.
+  if env['NACL_BUILD_FAMILY'] == 'TRUSTED':
+    return False
+
+  # Unlike ShouldTranslateToNexe, we always prep during the build step,
+  # since there is no prep tool that can run on triggered bots such as the
+  # ARM HW bots.
+  return True
+
+pre_base_env.AddMethod(ShouldFinalizePexe)
+
 
 def CommandTestFileDumpCheck(env,
                              name,
                              target,
                              check_file,
                              objdump_flags):
   """Create a test that disassembles a binary (|target|) and checks for
   patterns in the |check_file|.  Disassembly is done using |objdump_flags|.
   """
 
   # Do not try to run OBJDUMP if 'built_elsewhere', since that *might* mean
   # that a toolchain is not even present.  E.g., the arm hw buildbots do
   # not have the pnacl toolchain. We should be able to look for the host
   # ARM objdump though... a TODO(jvoung) for when there is time.
   if env.Bit('built_elsewhere'):
     return []
+  if env.ShouldFinalizePexe(target):
+    target = env.GetFinalizedPexe(target)

[Mark Seaborn, 2013/03/28 01:31:34]

GetFinalizedPexe() is always called after ShouldFinalizePexe(), so why not merge
the two functions?

(The same goes for GetTranslatedNexe() but I'm not asking you to refactor that
now. :-) )

Is there a reason not to have GetTranslatedNexe() call ShouldFinalizePexe() for
you?  There are a few other calls to GetTranslatedNexe() in nacl.scons files --
should these be calling ShouldFinalizePexe()?

[jvoung - send to chromium..., 2013/03/28 15:50:29]

Yes, I think we can merge these.
I'd kept them separate because of the ShouldTranslateToNexe() condition is
different, so I didn't want to accidentally leave out a call to
GetFinalizedPexe, just because ShouldTranslateToNexe() is false.

We probably do want to have the finalized pexe for those other nacl.scons files
too, for consistency, so it might make sense to fuse all the Should* with the
Get*, and have one Get...() that does both.

With this file renaming, I'm also a bit worried now about nacl integration tests
making an URL for the non-finalized pexe in the autogenerated NMF files... but
that can probably be fixed in a later CL.

   if env.ShouldTranslateToNexe(target):
-    target_obj = env.GetTranslatedNexe(target)
-  else:
-    target_obj = target
+    target = env.GetTranslatedNexe(target)
   return env.CommandTestFileCheck(name,
-                                  ['${OBJDUMP}', objdump_flags, target_obj],
+                                  ['${OBJDUMP}', objdump_flags, target],
                                   check_file)
 
 pre_base_env.AddMethod(CommandTestFileDumpCheck)
 
 
 def CommandTestFileCheck(env, name, cmd, check_file):
   """Create a test that runs a |cmd| (array of strings),
   which is expected to print to stdout.  The results
   of stdout will then be piped to the file_check.py tool which
   will search for the regexes specified in |check_file|. """
@@ skipping 20 matching lines @@
                           # e.g., [ 'python', 'time_check.py', '--' ]
                           **extra):
   # Disable all sel_ldr tests for windows under coverage.
   # Currently several .S files block sel_ldr from being instrumented.
   # See http://code.google.com/p/nativeclient/issues/detail?id=831
   if ('TRUSTED_ENV' in env and
       env['TRUSTED_ENV'].Bit('coverage_enabled') and
       env['TRUSTED_ENV'].Bit('windows')):
     return []
 
+  if env.ShouldFinalizePexe(nexe):
+    # The nexe is actually a pexe and we need to prep it for
+    # ABI stability first.
+    nexe = env.GetFinalizedPexe(nexe)
+
   if env.ShouldTranslateToNexe(nexe):
     # The nexe is actually a pexe.  Translate it before we run it.
     nexe = env.GetTranslatedNexe(nexe)
   command = [nexe]
   if args is not None:
     command += args
 
   if loader is None:
     loader = env.GetSelLdr()
     if loader is None:
@@ skipping 2111 matching lines @@
   nacl_env.ValidateSdk()
 
 if BROKEN_TEST_COUNT > 0:
   msg = "There are %d broken tests." % BROKEN_TEST_COUNT
   if GetOption('brief_comstr'):
     msg += " Add --verbose to the command line for more information."
   print msg
 
 # separate warnings from actual build output
 Banner('B U I L D - O U T P U T:')