Dropping in software construction toolkit.

site_scons/site_tools/component_builders.py

+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Software construction toolkit builders for SCons."""
+
+
+import SCons
+
+
+__component_list = {}
+
+
+def _InitializeComponentBuilders(self):
+  """Re-initializes component builders module.
+
+  Args:
+    self: Parent environment.
+  """
+  self = self     # Silence gpylint
+
+  __component_list.clear()
+
+
+def _RetrieveComponents(component_name, filter_components=None):
+  """Get the list of all components required by the specified component.
+
+  Args:
+    component_name: Name of the base component.
+    filter_components: List of components NOT to include.
+
+  Returns:
+    A list of the transitive closure of all components required by the base
+    component.  That is, if A requires B and B requires C, this returns [B, C].
+
+  """
+  if filter_components:
+    filter_components = set(filter_components)
+  else:
+    filter_components = set()
+
+  components = set([component_name])    # Components always require themselves
+  new_components = set(components)
+  while new_components:
+    # Take next new component and add it to the list we've already scanned.
+    c = new_components.pop()
+    components.add(c)
+    # Add to the list of new components any of c's components that we haven't
+    # seen before.
+    new_components.update(__component_list.get(c, set())
+                          - components - filter_components)
+
+  return list(components)
+
+
+def _StoreComponents(self, component_name):
+  """Stores the list of child components for the specified component.
+
+  Args:
+    self: Environment containing component.
+    component_name: Name of the component.
+
+  Adds component references based on the LIBS and COMPONENTS variables in the
+  current environment.  Should be called at primary SConscript execution time;
+  use _RetrieveComponents() to get the final components lists in a Defer()'d
+  function.
+  """
+
+  components = set()
+  for clist in ('LIBS', 'COMPONENTS'):
+    components.update(map(self.subst, self.Flatten(self[clist])))
+
+  if component_name not in __component_list:
+    __component_list[component_name] = set()
+  __component_list[component_name].update(components)
+
+
+def _ComponentPlatformSetup(env, builder_name, **kwargs):
+  """Modify an environment to work with a component builder.
+
+  Args:
+    env: Environment to clone.
+    builder_name: Name of the builder.
+    kwargs: Keyword arguments.
+
+  Returns:
+    A modified clone of the environment.
+  """
+  # Clone environment so we can modify it
+  env = env.Clone()
+
+  # Add all keyword arguments to the environment
+  for k, v in kwargs.items():
+    env[k] = v
+
+  # Call platform-specific component setup function, if any
+  if env.get('COMPONENT_PLATFORM_SETUP'):
+    env['COMPONENT_PLATFORM_SETUP'](env, builder_name)
+
+  # Return the modified environment
+  return env
+
+#------------------------------------------------------------------------------
+
+# TODO(rspangler): Should be possible to refactor programs, test programs,
+# libs to all publish as packages, for simplicity and code reuse.
+
+
+def ComponentPackageDeferred(env):
+  """Deferred build steps for component package.
+
+  Args:
+    env: Environment from ComponentPackage().
+
+  Sets up the aliases to build the package.
+  """
+  package_name = env['PACKAGE_NAME']
+
+  # Install program and resources
+  all_outputs = []
+  components = _RetrieveComponents(package_name,
+                                   env.get('COMPONENT_PACKAGE_FILTER'))
+  for resource, dest_dir in env.get('COMPONENT_PACKAGE_RESOURCES').items():
+    all_outputs += env.ReplicatePublished(dest_dir, components, resource)
+
+  # Add installed program and resources to the alias
+  env.Alias(package_name, all_outputs)
+
+
+def ComponentPackage(self, package_name, dest_dir, **kwargs):
+  """Pseudo-builder for package containing other components.
+
+  Args:
+    self: Environment in which we were called.
+    package_name: Name of package.
+    dest_dir: Destination directory for package.
+    args: Positional arguments.
+    kwargs: Keyword arguments.
+
+  Returns:
+    The alias node for the package.
+  """
+  # Clone and modify environment
+  env = _ComponentPlatformSetup(self, 'ComponentPackage', **kwargs)
+
+  env.Replace(
+      PACKAGE_NAME=package_name,
+      PACKAGE_DIR=dest_dir,
+  )
+
+  # Add an empty alias for the package and add it to the right groups
+  a = env.Alias(package_name, [])
+  for group in env['COMPONENT_PACKAGE_GROUPS']:
+    SCons.Script.Alias(group, a)
+
+  # Store list of components for this program
+  env._StoreComponents(package_name)
+
+  # Set up deferred call to replicate resources
+  env.Defer(ComponentPackageDeferred)
+
+  # Return the alias, since it's the only node we have
+  return a
+
+#------------------------------------------------------------------------------
+
+
+def ComponentObject(self, *args, **kwargs):
+  """Pseudo-builder for object to handle platform-dependent type.
+
+  Args:
+    self: Environment in which we were called.
+    args: Positional arguments.
+    kwargs: Keyword arguments.
+
+  Returns:
+    Passthrough return code from env.StaticLibrary() or env.SharedLibrary().
+
+  TODO(rspangler): Perhaps this should be a generator builder, so it can take
+  a list of inputs and return a list of outputs?
+  """
+  # Clone and modify environment
+  env = _ComponentPlatformSetup(self, 'ComponentObject', **kwargs)
+
+  # Make appropriate object type
+  if env.get('COMPONENT_STATIC'):
+    return env.StaticObject(*args, **kwargs)
+  else:
+    return env.SharedObject(*args, **kwargs)
+
+#------------------------------------------------------------------------------
+
+
+def ComponentLibrary(self, lib_name, *args, **kwargs):
+  """Pseudo-builder for library to handle platform-dependent type.
+
+  Args:
+    self: Environment in which we were called.
+    lib_name: Library name.
+    args: Positional arguments.
+    kwargs: Keyword arguments.
+
+  Returns:
+    Passthrough return code from env.StaticLibrary() or env.SharedLibrary().
+  """
+  # Clone and modify environment
+  env = _ComponentPlatformSetup(self, 'ComponentLibrary', **kwargs)
+
+  # Make appropriate library type
+  if env.get('COMPONENT_STATIC'):
+    lib_outputs = env.StaticLibrary(lib_name, *args, **kwargs)
+  else:
+    lib_outputs = env.SharedLibrary(lib_name, *args, **kwargs)
+
+  # Scan library outputs for files we need to link against this library, and
+  # files we need to run executables linked against this library.
+  need_for_link = []
+  need_for_debug = []
+  need_for_run = []
+  for o in lib_outputs:
+    if o.suffix in env['COMPONENT_LIBRARY_LINK_SUFFIXES']:
+      need_for_link.append(o)
+    if o.suffix in env['COMPONENT_LIBRARY_DEBUG_SUFFIXES']:
+      need_for_debug.append(o)
+    if o.suffix == env['SHLIBSUFFIX']:
+      need_for_run.append(o)
+  all_outputs = lib_outputs
+
+  # Install library in intermediate directory, so other libs and programs can
+  # link against it
+  all_outputs += env.Replicate('$COMPONENT_LIBRARY_DIR', need_for_link)
+
+  # Publish output
+  env.Publish(lib_name, 'run', need_for_run)
+  env.Publish(lib_name, 'debug', need_for_debug)
+
+  # Add an alias to build and copy the library, and add it to the right groups
+  a = self.Alias(lib_name, all_outputs)
+  for group in env['COMPONENT_LIBRARY_GROUPS']:
+    SCons.Script.Alias(group, a)
+
+  # Store list of components for this library
+  env._StoreComponents(lib_name)
+
+  # If library should publish itself, publish as if it was a program
+  if env.get('COMPONENT_LIBRARY_PUBLISH'):
+    env['PROGRAM_BASENAME'] = lib_name
+    env.Defer(ComponentProgramDeferred)
+
+  # Return the library outputs
+  return lib_outputs
+
+#------------------------------------------------------------------------------
+
+
+def ComponentTestProgramDeferred(env):
+  """Deferred build steps for test program.
+
+  Args:
+    env: Environment from ComponentTestProgram().
+
+  Sets up the aliases to compile and run the test program.
+  """
+  prog_name = env['PROGRAM_BASENAME']
+
+  # Install program and resources
+  all_outputs = []
+  components = _RetrieveComponents(prog_name)
+  for resource, dest_dir in env.get('COMPONENT_TEST_RESOURCES').items():
+    all_outputs += env.ReplicatePublished(dest_dir, components, resource)
+
+  # Add installed program and resources to the alias
+  env.Alias(prog_name, all_outputs)
+
+  # Add an alias for running the test in the test directory, if there's a test
+  # command line.
+  if env.get('COMPONENT_TEST_CMDLINE'):
+    # Test program is the first run resource we replicated.
+    test_program = env.ReplicatePublished('$TESTS_DIR', prog_name, 'run')
+    env.Replace(
+        COMMAND_OUTPUT_CMDLINE=env['COMPONENT_TEST_CMDLINE'],
+        COMMAND_OUTPUT_RUN_DIR='$TESTS_DIR',
+    )
+    test_out = env.CommandOutput(
+        '$TEST_OUTPUT_DIR/${PROGRAM_BASENAME}.out.txt', test_program)
+    # Running the test requires the test and its libs copied to the tests dir
+    env.Depends(test_out, all_outputs)
+    env.ComponentTestOutput('run_' + prog_name, test_out)
+
+
+def ComponentTestProgram(self, prog_name, *args, **kwargs):
+  """Pseudo-builder for test program to handle platform-dependent type.
+
+  Args:
+    self: Environment in which we were called.
+    prog_name: Test program name.
+    args: Positional arguments.
+    kwargs: Keyword arguments.
+
+  Returns:
+    Output node list from env.Program().
+
+  TODO(rspangler): Should have some sort of support for S/M/L categorization
+  """
+  # Clone and modify environment
+  env = _ComponentPlatformSetup(self, 'ComponentTestProgram', **kwargs)
+
+  env['PROGRAM_BASENAME'] = prog_name
+  env['PROGRAM_NAME'] = '$PROGPREFIX$PROGRAM_BASENAME$PROGSUFFIX'
+
+  # Call env.Program()
+  out_nodes = env.Program(prog_name, *args, **kwargs)
+
+  # Publish output
+  env.Publish(prog_name, 'run', out_nodes[0])
+  env.Publish(prog_name, 'debug', out_nodes[1:])
+
+  # Add an alias to build the program to the right groups
+  a = env.Alias(prog_name, out_nodes)
+  for group in env['COMPONENT_TEST_PROGRAM_GROUPS']:
+    SCons.Script.Alias(group, a)
+
+  # Store list of components for this program
+  env._StoreComponents(prog_name)
+
+  # Set up deferred call to replicate resources and run test
+  env.Defer(ComponentTestProgramDeferred)
+
+  # Return the output node
+  return out_nodes
+
+#------------------------------------------------------------------------------
+
+
+def ComponentProgramDeferred(env):
+  """Deferred build steps for program.
+
+  Args:
+    env: Environment from ComponentProgram().
+
+  Sets up the aliases to compile the program.
+  """
+  prog_name = env['PROGRAM_BASENAME']
+
+  # Install program and resources
+  all_outputs = []
+  components = _RetrieveComponents(prog_name)
+  for resource, dest_dir in env.get('COMPONENT_PROGRAM_RESOURCES').items():
+    all_outputs += env.ReplicatePublished(dest_dir, components, resource)
+
+  # Add installed program and resources to the alias
+  env.Alias(prog_name, all_outputs)
+
+
+def ComponentProgram(self, prog_name, *args, **kwargs):
+  """Pseudo-builder for program to handle platform-dependent type.
+
+  Args:
+    self: Environment in which we were called.
+    prog_name: Test program name.
+    args: Positional arguments.
+    kwargs: Keyword arguments.
+
+  Returns:
+    Output node list from env.Program().
+  """
+  # Clone and modify environment
+  env = _ComponentPlatformSetup(self, 'ComponentProgram', **kwargs)
+
+  env['PROGRAM_BASENAME'] = prog_name
+
+  # Call env.Program()
+  out_nodes = env.Program(prog_name, *args, **kwargs)
+
+  # Publish output
+  env.Publish(prog_name, 'run', out_nodes[0])
+  env.Publish(prog_name, 'debug', out_nodes[1:])
+
+  # Add an alias to build the program to the right groups
+  a = env.Alias(prog_name, out_nodes)
+  for group in env['COMPONENT_PROGRAM_GROUPS']:
+    SCons.Script.Alias(group, a)
+
+  # Store list of components for this program
+  env._StoreComponents(prog_name)
+
+  # Set up deferred call to replicate resources
+  env.Defer(ComponentProgramDeferred)
+
+  # Return the output nodes
+  return out_nodes
+
+#------------------------------------------------------------------------------
+
+
+def ComponentTestOutput(self, test_name, nodes):
+  """Pseudo-builder for test output.
+
+  Args:
+    self: Environment in which we were called.
+    test_name: Test name.
+    nodes: List of files/Nodes output by the test.
+
+  Returns:
+    Passthrough return code from env.Alias().
+  """
+
+  # Add an alias for the test outputs, and add it to the right groups
+  a = self.Alias(test_name, nodes)
+  for group in self['COMPONENT_TEST_OUTPUT_GROUPS']:
+    SCons.Script.Alias(group, a)
+
+  # Return the output node
+  return a
+
+#------------------------------------------------------------------------------
+
+
+def generate(env):
+  # NOTE: SCons requires the use of this name, which fails gpylint.
+  """SCons entry point for this tool."""
+
+  env.Replace(
+      COMPONENT_LIBRARY_DIR='$TARGET_ROOT/lib',
+      STAGING_DIR='$TARGET_ROOT/staging',
+      TESTS_DIR='$TARGET_ROOT/tests',
+      TEST_OUTPUT_DIR='$TARGET_ROOT/test_output',
+      # Default command line for a test is just the name of the file.
+      # TODO(rspangler): Why doesn't the following work:
+      # COMPONENT_TEST_CMDLINE='${SOURCE.abspath}',
+      # (it generates a SCons error)
+      COMPONENT_TEST_CMDLINE='${PROGRAM_NAME}',
+      COMPONENT_STATIC=True,  # Static linking is a sensible default.
+      # Don't publish libraries to the staging dir by themselves by default.
+      COMPONENT_LIBRARY_PUBLISH=False,
+  )
+  env.Append(
+      LIBPATH=['$COMPONENT_LIBRARY_DIR'],
+      RPATH=['$COMPONENT_LIBRARY_DIR'],
+
+      # Default alias groups for component builders
+      COMPONENT_PACKAGE_GROUPS=['all_packages'],
+      COMPONENT_LIBRARY_GROUPS=['all_libraries'],
+      COMPONENT_PROGRAM_GROUPS=['all_programs'],
+      COMPONENT_TEST_PROGRAM_GROUPS=['all_test_programs'],
+      COMPONENT_TEST_OUTPUT_GROUPS=['run_all_tests'],
+
+      # Additional components whose resources should be copied into program
+      # directories, in addition to those from LIBS and the program itself.
+      LIBS=[],
+      COMPONENTS=[],
+
+      # Dicts of what resources should go in each destination directory for
+      # programs and test programs.
+      COMPONENT_PACKAGE_RESOURCES={
+          'run': '$PACKAGE_DIR',
+          'debug': '$PACKAGE_DIR',
+      },
+      COMPONENT_PROGRAM_RESOURCES={
+          'run': '$STAGING_DIR',
+          'debug': '$STAGING_DIR',
+      },
+      COMPONENT_TEST_RESOURCES={
+          'run': '$TESTS_DIR',
+          'debug': '$TESTS_DIR',
+          'test_input': '$TESTS_DIR',
+      },
+  )
+
+  # Add our pseudo-builder methods
+  env.AddMethod(_InitializeComponentBuilders)
+  env.AddMethod(_StoreComponents)
+  env.AddMethod(ComponentPackage)
+  env.AddMethod(ComponentObject)
+  env.AddMethod(ComponentLibrary)
+  env.AddMethod(ComponentProgram)
+  env.AddMethod(ComponentTestProgram)
+  env.AddMethod(ComponentTestOutput)
+
+  # Add our target groups
+  AddTargetGroup('all_libraries', 'libraries can be built')
+  AddTargetGroup('all_programs', 'programs can be built')
+  AddTargetGroup('all_test_programs', 'tests can be built')
+  AddTargetGroup('all_packages', 'packages can be built')
+  AddTargetGroup('run_all_tests', 'tests can be run')