Teach gyp about configurable filter commands, add "pymod_do_main" filter

pylib/gyp/input.py

Index: pylib/gyp/input.py
===================================================================
--- pylib/gyp/input.py	(revision 917)
+++ pylib/gyp/input.py	(working copy)
@@ -19,6 +19,7 @@
 import re
 import shlex
 import subprocess
+import shlex
 import sys
 
 
@@ -471,13 +472,22 @@
   return True
 
 
-early_variable_re = re.compile('(?P<replace>(?P<type><((!?@?)|\|)?)'
-                               '\((?P<is_array>\s*\[?)'
-                               '(?P<content>.*?)(\]?)\))')
-late_variable_re = re.compile('(?P<replace>(?P<type>>((!?@?)|\|)?)'
-                              '\((?P<is_array>\s*\[?)'
-                              '(?P<content>.*?)(\]?)\))')
+# This matches things like "<(asdf)", "<!(cmd)", "<!@(cmd)", "<|(list)",
+# "<!interpreter(arguments)", "<([list])", and even "<([)" and "<(<())".
+# In the last case, the inner "<()" is captured in match['content'].
+early_variable_re = re.compile(
+    '(?P<replace>(?P<type><(?:(?:!?@?)|\|)?)'
+    '(?P<command_string>[-a-zA-Z0-9_.]+)?'
+    '\((?P<is_array>\s*\[?)'
+    '(?P<content>.*?)(\]?)\))')
 
+# This matches the same as early_variable_re, but with '>' instead of '<'.
+late_variable_re = re.compile(
+    '(?P<replace>(?P<type>>(?:(?:!?@?)|\|)?)'
+    '(?P<command_string>[-a-zA-Z0-9_.]+)?'
+    '\((?P<is_array>\s*\[?)'
+    '(?P<content>.*?)(\]?)\))')
+
 # Global cache of results from running commands so they don't have to be run
 # more then once.
 cached_command_results = {}
@@ -525,10 +535,12 @@
       # is the character code for the replacement type (< > <! >! <| >| <@
       # >@ <!@ >!@), match['is_array'] contains a '[' for command
       # arrays, and match['content'] is the name of the variable (< >)
-      # or command to run (<! >!).
+      # or command to run (<! >!). match['command_string'] is an optional
+      # command string. Currently, only 'python' is supported.

[Mark Mentovai, 2011/05/24 21:54:57]

Except you’ve called it pymod_do_main in the code below.

[Nico, 2011/05/24 22:02:19]

Done.

 
       # run_command is true if a ! variant is used.
       run_command = '!' in match['type']
+      command_string = match['command_string']
 
       # file_list is true if a | variant is used.
       file_list = '|' in match['type']
@@ -634,24 +646,42 @@
                           "Executing command '%s' in directory '%s'" %
                           (contents,build_file_dir))
 
-          # Fix up command with platform specific workarounds.
-          contents = FixupPlatformCommand(contents)
-          p = subprocess.Popen(contents, shell=use_shell,
-                               stdout=subprocess.PIPE,
-                               stderr=subprocess.PIPE,
-                               stdin=subprocess.PIPE,
-                               cwd=build_file_dir)
+          replacement = ''
 
-          (p_stdout, p_stderr) = p.communicate('')
+          if command_string == 'pymod_do_main':
+            # <!pymod_do_main(modulename param eters) loads |modulename| as a
+            # python module and then calls that module's DoMain() function,
+            # passing "param eters" as a single string argument. For modules

[Mark Mentovai, 2011/05/24 22:04:16]

Sorry, I read this comment saying |passing "param eters" as a single string
argument| and my mind put an implicit |' '.join()| around your list.

Update the comment, then!

[Nico, 2011/05/24 22:09:52]

Done.

+            # that don't load quickly, this can be faster than
+            # <!(python modulename param eters). Do this in |build_file_dir|.
+            oldwd = os.getcwd()  # Python doesn't like os.open('.'): no fchdir.
+            os.chdir(build_file_dir)
 
-          if p.wait() != 0 or p_stderr:
-            sys.stderr.write(p_stderr)
-            # Simulate check_call behavior, since check_call only exists
-            # in python 2.5 and later.
-            raise Exception("Call to '%s' returned exit status %d." %
-                            (contents, p.returncode))
-          replacement = p_stdout.rstrip()
+            parsed_contents = shlex.split(contents)
+            py_module = __import__(parsed_contents[0])
+            replacement = str(py_module.DoMain(parsed_contents[1:])).rstrip()

[Mark Mentovai, 2011/05/24 21:54:57]

This is hokey. Any reason you don’t just pass parsed_contents[1:] to DoMain? Is
there some existing API that you’re trying to maintain conformity with?

If you need to keep a string:

Since you split the string up with shlex.split, you should really join it back
together with something that makes strings following the same rules shlex uses.

[Nico, 2011/05/24 22:02:19]

I think that's what I'm doing.
DoMain returns a unicode, but gyp asserts that replacement is a str somewhere
below. And the rstrip() just gets rid of eventual trailing newlines.

 
+            os.chdir(oldwd)
+            assert replacement != None
+          else:

[Mark Mentovai, 2011/05/24 21:54:57]

assert on unknown command_strings?

[Nico, 2011/05/24 22:02:19]

Done.

+            # Fix up command with platform specific workarounds.
+            contents = FixupPlatformCommand(contents)
+            p = subprocess.Popen(contents, shell=use_shell,
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE,
+                                 stdin=subprocess.PIPE,
+                                 cwd=build_file_dir)
+
+            (p_stdout, p_stderr) = p.communicate('')
+
+            if p.wait() != 0 or p_stderr:
+              sys.stderr.write(p_stderr)
+              # Simulate check_call behavior, since check_call only exists
+              # in python 2.5 and later.
+              raise Exception("Call to '%s' returned exit status %d." %
+                              (contents, p.returncode))
+            replacement = p_stdout.rstrip()
+
           cached_command_results[cache_key] = replacement
         else:
           gyp.DebugOutput(gyp.DEBUG_VARIABLES,