Teach gyp about configurable filter commands, add "pymod_do_main" filter

pylib/gyp/input.py

 #!/usr/bin/python
 
 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from compiler.ast import Const
 from compiler.ast import Dict
 from compiler.ast import Discard
 from compiler.ast import List
 from compiler.ast import Module
 from compiler.ast import Node
 from compiler.ast import Stmt
 import compiler
 import copy
 import gyp.common
 import optparse
 import os.path
 import re
 import shlex

[Mark Mentovai, 2011/05/25 02:12:34]

shlex was already in here. LGTM for a quick follow-up to remove the duplicate.

 import subprocess
+import shlex

[tony, 2011/05/24 23:31:39]

Nit: alphabetize

 import sys
 
 
 # A list of types that are treated as linkable.
 linkable_types = ['executable', 'shared_library', 'loadable_module']
 
 # A list of sections that contain links to other targets.
 dependency_sections = ['dependencies', 'export_dependent_settings']
 
 # base_path_sections is a list of sections defined by GYP that contain
@@ skipping 432 matching lines @@
   """Returns True if |string| is in its canonical integer form.
 
   The canonical form is such that str(int(string)) == string.
   """
   if not isinstance(string, str) or not canonical_int_re.match(string):
     return False
 
   return True
 
 
-early_variable_re = re.compile('(?P<replace>(?P<type><((!?@?)|\|)?)'
-                               '\((?P<is_array>\s*\[?)'
-                               '(?P<content>.*?)(\]?)\))')
-late_variable_re = re.compile('(?P<replace>(?P<type>>((!?@?)|\|)?)'
-                              '\((?P<is_array>\s*\[?)'
-                              '(?P<content>.*?)(\]?)\))')
+# This matches things like "<(asdf)", "<!(cmd)", "<!@(cmd)", "<|(list)",
+# "<!interpreter(arguments)", "<([list])", and even "<([)" and "<(<())".
+# In the last case, the inner "<()" is captured in match['content'].
+early_variable_re = re.compile(
+    '(?P<replace>(?P<type><(?:(?:!?@?)|\|)?)'
+    '(?P<command_string>[-a-zA-Z0-9_.]+)?'
+    '\((?P<is_array>\s*\[?)'
+    '(?P<content>.*?)(\]?)\))')
+
+# This matches the same as early_variable_re, but with '>' instead of '<'.
+late_variable_re = re.compile(
+    '(?P<replace>(?P<type>>(?:(?:!?@?)|\|)?)'
+    '(?P<command_string>[-a-zA-Z0-9_.]+)?'
+    '\((?P<is_array>\s*\[?)'
+    '(?P<content>.*?)(\]?)\))')
 
 # Global cache of results from running commands so they don't have to be run
 # more then once.
 cached_command_results = {}
 
 
 def FixupPlatformCommand(cmd):
   if sys.platform == 'win32':
     if type(cmd) == list:
       cmd = [re.sub('^cat ', 'type ', cmd[0])] + cmd[1:]
@@ skipping 28 matching lines @@
     # of what's intended for replacement.
     matches.reverse()
     for match_group in matches:
       match = match_group.groupdict()
       gyp.DebugOutput(gyp.DEBUG_VARIABLES,
                       "Matches: %s" % repr(match))
       # match['replace'] is the substring to look for, match['type']
       # is the character code for the replacement type (< > <! >! <| >| <@
       # >@ <!@ >!@), match['is_array'] contains a '[' for command
       # arrays, and match['content'] is the name of the variable (< >)
-      # or command to run (<! >!).
+      # or command to run (<! >!). match['command_string'] is an optional
+      # command string. Currently, only 'pymod_do_main' is supported.
 
       # run_command is true if a ! variant is used.
       run_command = '!' in match['type']
+      command_string = match['command_string']
 
       # file_list is true if a | variant is used.
       file_list = '|' in match['type']
 
       # Capture these now so we can adjust them later.
       replace_start = match_group.start('replace')
       replace_end = match_group.end('replace')
 
       # Find the ending paren, and re-evaluate the contained string.
       (c_start, c_end) = FindEnclosingBracketGroup(input_str[replace_start:])
@@ skipping 85 matching lines @@
         # is invoked it produces different output by design. When the need
         # arises, the syntax should be extended to support no caching off a
         # command's output so it is run every time.
         cache_key = str(contents)
         cached_value = cached_command_results.get(cache_key, None)
         if cached_value is None:
           gyp.DebugOutput(gyp.DEBUG_VARIABLES,
                           "Executing command '%s' in directory '%s'" %
                           (contents,build_file_dir))
 
-          # Fix up command with platform specific workarounds.
-          contents = FixupPlatformCommand(contents)
-          p = subprocess.Popen(contents, shell=use_shell,
-                               stdout=subprocess.PIPE,
-                               stderr=subprocess.PIPE,
-                               stdin=subprocess.PIPE,
-                               cwd=build_file_dir)
+          replacement = ''
 
-          (p_stdout, p_stderr) = p.communicate('')
+          if command_string == 'pymod_do_main':

[TVL, 2011/05/24 23:44:03]

drive by - should we also support pymod_main for the things that just call it
Main()?

[Nico, 2011/05/25 00:41:26]

We can add this once we need it I think.

+            # <!pymod_do_main(modulename param eters) loads |modulename| as a
+            # python module and then calls that module's DoMain() function,
+            # passing ["param", "eters"] as a single list argument. For modules
+            # that don't load quickly, this can be faster than
+            # <!(python modulename param eters). Do this in |build_file_dir|.
+            oldwd = os.getcwd()  # Python doesn't like os.open('.'): no fchdir.
+            os.chdir(build_file_dir)
 
-          if p.wait() != 0 or p_stderr:
-            sys.stderr.write(p_stderr)
-            # Simulate check_call behavior, since check_call only exists
-            # in python 2.5 and later.
-            raise Exception("Call to '%s' returned exit status %d." %
-                            (contents, p.returncode))
-          replacement = p_stdout.rstrip()
+            parsed_contents = shlex.split(contents)
+            py_module = __import__(parsed_contents[0])
+            replacement = str(py_module.DoMain(parsed_contents[1:])).rstrip()
+
+            os.chdir(oldwd)
+            assert replacement != None
+          elif command_string:
+            raise Exception("Unknown command string '%s' in '%s'." %
+                            (command_string, contents))
+          else:
+            # Fix up command with platform specific workarounds.
+            contents = FixupPlatformCommand(contents)
+            p = subprocess.Popen(contents, shell=use_shell,
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE,
+                                 stdin=subprocess.PIPE,
+                                 cwd=build_file_dir)
+
+            (p_stdout, p_stderr) = p.communicate('')

[tony, 2011/05/24 23:31:39]

Nit: no () on the left side

+
+            if p.wait() != 0 or p_stderr:
+              sys.stderr.write(p_stderr)
+              # Simulate check_call behavior, since check_call only exists
+              # in python 2.5 and later.
+              raise Exception("Call to '%s' returned exit status %d." %
+                              (contents, p.returncode))
+            replacement = p_stdout.rstrip()
 
           cached_command_results[cache_key] = replacement
         else:
           gyp.DebugOutput(gyp.DEBUG_VARIABLES,
                           "Had cache value for command '%s' in directory '%s'" %
                           (contents,build_file_dir))
           replacement = cached_value
 
       else:
         if not contents in variables:
@@ skipping 1577 matching lines @@
     ValidateRunAsInTarget(target, target_dict, build_file)
     ValidateActionsInTarget(target, target_dict, build_file)
 
   # Generators might not expect ints.  Turn them into strs.
   TurnIntIntoStrInDict(data)
 
   # TODO(mark): Return |data| for now because the generator needs a list of
   # build files that came in.  In the future, maybe it should just accept
   # a list, and not the whole data dict.
   return [flat_list, targets, data]