Add simple inline macros to js2c and use that for typed array constructors.

tools/js2c.py

Index: tools/js2c.py
diff --git a/tools/js2c.py b/tools/js2c.py
old mode 100644
new mode 100755
index 9492b0030c0dbf56f91f78208230e3aa89db300e..f67d053ad26360c07cf190fefea4c7cb73ce6ab6
--- a/tools/js2c.py
+++ b/tools/js2c.py
@@ -116,41 +116,47 @@ def ExpandConstants(lines, constants):
   return lines
 
 
+def ExpandMacroDefinition(lines, pos, name_pattern, macro, expander):
+  pattern_match = name_pattern.search(lines, pos)
+  while pattern_match is not None:
+    # Scan over the arguments
+    height = 1
+    start = pattern_match.start()
+    end = pattern_match.end()
+    assert lines[end - 1] == '('
+    last_match = end
+    arg_index = [0]  # Wrap state into array, to work around Python "scoping"
+    mapping = { }
+    def add_arg(str):
+      # Remember to expand recursively in the arguments
+      replacement = expander(str.strip())
+      mapping[macro.args[arg_index[0]]] = replacement
+      arg_index[0] += 1
+    while end < len(lines) and height > 0:
+      # We don't count commas at higher nesting levels.
+      if lines[end] == ',' and height == 1:
+        add_arg(lines[last_match:end])
+        last_match = end + 1
+      elif lines[end] in ['(', '{', '[']:
+        height = height + 1
+      elif lines[end] in [')', '}', ']']:
+        height = height - 1
+      end = end + 1
+    # Remember to add the last match.
+    add_arg(lines[last_match:end-1])
+    result = macro.expand(mapping)
+    # Replace the occurrence of the macro with the expansion
+    lines = lines[:start] + result + lines[end:]
+    pattern_match = name_pattern.search(lines, start + len(result))
+  return lines
+
 def ExpandMacros(lines, macros):
   # We allow macros to depend on the previously declared macros, but
   # we don't allow self-dependecies or recursion.
   for name_pattern, macro in reversed(macros):
-    pattern_match = name_pattern.search(lines, 0)
-    while pattern_match is not None:
-      # Scan over the arguments
-      height = 1
-      start = pattern_match.start()
-      end = pattern_match.end()
-      assert lines[end - 1] == '('
-      last_match = end
-      arg_index = [0]  # Wrap state into array, to work around Python "scoping"
-      mapping = { }
-      def add_arg(str):
-        # Remember to expand recursively in the arguments
-        replacement = ExpandMacros(str.strip(), macros)
-        mapping[macro.args[arg_index[0]]] = replacement
-        arg_index[0] += 1
-      while end < len(lines) and height > 0:
-        # We don't count commas at higher nesting levels.
-        if lines[end] == ',' and height == 1:
-          add_arg(lines[last_match:end])
-          last_match = end + 1
-        elif lines[end] in ['(', '{', '[']:
-          height = height + 1
-        elif lines[end] in [')', '}', ']']:
-          height = height - 1
-        end = end + 1
-      # Remember to add the last match.
-      add_arg(lines[last_match:end-1])
-      result = macro.expand(mapping)
-      # Replace the occurrence of the macro with the expansion
-      lines = lines[:start] + result + lines[end:]
-      pattern_match = name_pattern.search(lines, start + len(result))
+    def expander(s):
+      return ExpandMacros(s, macros)
+    lines = ExpandMacroDefinition(lines, 0, name_pattern, macro, expander)
   return lines
 
 class TextMacro:
@@ -210,6 +216,34 @@ def ReadMacros(lines):
           raise ("Illegal line: " + line)
   return (constants, macros)
 
+INLINE_MACRO_PATTERN = re.compile(r'macro\s+([a-zA-Z0-9_]+)\s*\(([^)]*)\)\s*\n')
+INLINE_MACRO_END_PATTERN = re.compile(r'endmacro\s*\n')
+
+def ExpandInlineMacros(lines, filename):
+  pos = 0
+  while True:

[rossberg, 2013/10/28 10:40:55]

Wait, you iterate over the whole file N times, once for each macro? Shouldn't
you first collect the macros, and then expand them simultaneously, like in the
non-inline case? In fact, shouldn't you all expansions together?

[Dmitry Lomov (no reviews), 2013/11/02 12:56:16]

Macros in macros + the macro scope is from definition to the end of file. Perf
impact is negligible (note that we iterate the whole file once for each macro in
the "predefined macros" case too).

I am not sure what is your concern here.

+    macro_match = INLINE_MACRO_PATTERN.search(lines, pos)
+    if macro_match is None:
+      # no more macros
+      return lines
+    name = macro_match.group(1)
+    args = [match.strip() for match in macro_match.group(2).split(',')]
+    end_macro_match = INLINE_MACRO_END_PATTERN.search(lines, macro_match.end());
+    if end_macro_match is None:
+      raise ("Macro %s unclosed in %s" % (name, filename))
+    body = lines[macro_match.end():end_macro_match.start()]
+
+    # remove macro definition
+    lines = lines[:macro_match.start()] + lines[end_macro_match.end():]
+    name_pattern = re.compile("\\b%s\\(" % name)
+    macro = TextMacro(args, body)
+
+    # advance position to where the macro defintion was
+    pos = macro_match.start()
+
+    def non_expander(s):
+      return s
+    lines = ExpandMacroDefinition(lines, pos, name_pattern, macro, non_expander)
 
 HEADER_TEMPLATE = """\
 // Copyright 2011 Google Inc. All Rights Reserved.
@@ -325,6 +359,8 @@ def JS2C(source, target, env):
     lines = ReadFile(filename)
     lines = ExpandConstants(lines, consts)
     lines = ExpandMacros(lines, macros)
+    lines = RemoveCommentsAndTrailingWhitespace(lines)

[rossberg, 2013/10/28 10:40:55]

Why is this needed?

[Dmitry Lomov (no reviews), 2013/11/02 12:56:16]

To handle cases like 
  macro abc(f,g,h) // this is a comment.

+    lines = ExpandInlineMacros(lines, filename)
     Validate(lines, filename)
     lines = minifier.JSMinify(lines)
     id = (os.path.split(filename)[1])[:-3]