Expand C++ macros in tools/generate-runtime-tests.py to increase coverage

tools/generate-runtime-tests.py

 #!/usr/bin/env python
 # Copyright 2014 the V8 project authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import itertools
 import multiprocessing
 import optparse
 import os
 import random
 import re
 import shutil
 import signal
 import string
 import subprocess
 import sys
 import time
 
-# TODO(jkummerow): Support DATA_VIEW_{G,S}ETTER in runtime.cc
-
 FILENAME = "src/runtime.cc"
 HEADERFILENAME = "src/runtime.h"
 FUNCTION = re.compile("^RUNTIME_FUNCTION\(Runtime_(\w+)")
 ARGSLENGTH = re.compile(".*ASSERT\(.*args\.length\(\) == (\d+)\);")
 FUNCTIONEND = "}\n"
+MACRO = re.compile(r"^#define ([^ ]+)\(([^)]*)\) *([^\\]*)\\?\n$")
+FIRST_WORD = re.compile("^\s*(.*?)[\s({\[]")
 
 WORKSPACE = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), ".."))
 BASEPATH = os.path.join(WORKSPACE, "test", "mjsunit", "runtime-gen")
 THIS_SCRIPT = os.path.relpath(sys.argv[0])
 
+# Expand these macros, they define further runtime functions.
+EXPAND_MACROS = [
+  "BUFFER_VIEW_GETTER",
+  "DATA_VIEW_GETTER",
+  "DATA_VIEW_SETTER",
+  "RUNTIME_UNARY_MATH",
+]
+# TODO(jkummerow): We could also whitelist the following macros, but the
+# functions they define are so trivial that it's unclear how much benefit
+# that would provide:
+# ELEMENTS_KIND_CHECK_RUNTIME_FUNCTION
+# FIXED_TYPED_ARRAYS_CHECK_RUNTIME_FUNCTION
+# TYPED_ARRAYS_CHECK_RUNTIME_FUNCTION
+
 # Counts of functions in each detection state. These are used to assert
 # that the parser doesn't bit-rot. Change the values as needed when you add,
 # remove or change runtime functions, but make sure we don't lose our ability
 # to parse them!
-EXPECTED_FUNCTION_COUNT = 338
-EXPECTED_FUZZABLE_COUNT = 305
+EXPECTED_FUNCTION_COUNT = 362
+EXPECTED_FUZZABLE_COUNT = 329
 EXPECTED_CCTEST_COUNT = 6
 EXPECTED_UNKNOWN_COUNT = 5
 
 
 # Don't call these at all.
 BLACKLISTED = [
   "Abort",  # Kills the process.
   "AbortJS",  # Kills the process.
   "CompileForOnStackReplacement",  # Riddled with ASSERTs.
   "IS_VAR",  # Not implemented in the runtime.
@@ skipping 526 matching lines @@
         result += self._Int32(offset_name)
       if die < 0.33:
         length_name = name + "_length"
         args.append(length_name)
         result += self._Int32(length_name)
       return (result +
               self._Variable(name,
                              "new %sArray(%s)" % (arraytype, ", ".join(args)),
                              fallback="new %sArray(8)" % arraytype))
 
+  def _JSArrayBufferView(self, name, recursion_budget):
+    if random.random() < 0.4:
+      return self._JSDataView(name, recursion_budget)
+    else:
+      return self._JSTypedArray(name, recursion_budget)
+
   def _JSWeakCollection(self, name, recursion_budget):
     ctor = random.choice([self._JSMap, self._JSSet])
     return ctor(name, recursion_budget, weak="Weak")
 
   def _PropertyDetails(self, name, recursion_budget):
     # TODO(jkummerow): Be more clever here?
     return self._Int32(name)
 
   def _JSObject(self, name, recursion_budget):
     die = random.random()
@@ skipping 34 matching lines @@
     if random.random() < 0.2:
       return self._Smi(name, recursion_budget)
     return self._HeapObject(name, recursion_budget)
 
   GENERATORS = {
     "Boolean": ["true", _Boolean],
     "HeapObject": ["new Object()", _HeapObject],
     "Int32": ["32", _Int32],
     "JSArray": ["new Array()", _JSArray],
     "JSArrayBuffer": ["new ArrayBuffer(8)", _JSArrayBuffer],
-    "JSDataView": ["new DataView(new ArrayBuffer(8))", _JSDataView],
+    "JSArrayBufferView": ["new Int32Array(2)", _JSArrayBufferView],
+    "JSDataView": ["new DataView(new ArrayBuffer(24))", _JSDataView],
     "JSDate": ["new Date()", _JSDate],
     "JSFunction": ["function() {}", _JSFunction],
     "JSFunctionProxy": ["Proxy.createFunction({}, function() {})",
                         _JSFunctionProxy],
     "JSGeneratorObject": ["(function*(){ yield 1; })()", _JSGeneratorObject],
     "JSMap": ["new Map()", _JSMap],
     "JSMapIterator": ["%MapCreateIterator(new Map(), 3)", _JSMapIterator],
     "JSObject": ["new Object()", _JSObject],
     "JSProxy": ["Proxy.create({})", _JSProxy],
     "JSReceiver": ["new Object()", _JSReceiver],
@@ skipping 102 matching lines @@
         argcount = max([self.args[i].index + 1 for i in self.args])
       else:
         argcount = 0
     for i in range(argcount):
       if i > 0: s.append(", ")
       s.append(self.args[i].type if i in self.args else "<unknown>")
     s.append(")")
     return "".join(s)
 
 
+class Macro(object):
+  def __init__(self, match):
+    self.name = match.group(1)
+    self.args = [s.strip() for s in match.group(2).split(",")]
+    self.lines = []
+    self.indentation = 0
+    self.AddLine(match.group(3))
+
+  def AddLine(self, line):
+    if not line: return
+    if not self.lines:
+      # This is the first line, detect indentation.
+      self.indentation = len(line) - len(line.lstrip())
+    line = line.rstrip("\\\n ")
+    if not line: return
+    assert len(line[:self.indentation].strip()) == 0, \
+        ("expected whitespace: '%s', full line: '%s'" %
+         (line[:self.indentation], line))
+    line = line[self.indentation:]
+    if not line: return
+    self.lines.append(line + "\n")
+
+  def Finalize(self):
+    for arg in self.args:
+      pattern = re.compile(r"(##|\b)%s(##|\b)" % arg)
+      for i in range(len(self.lines)):
+        self.lines[i] = re.sub(pattern, "%%(%s)s" % arg, self.lines[i])
+
+  def FillIn(self, arg_values):
+    filler = {}
+    assert len(arg_values) == len(self.args)
+    for i in range(len(self.args)):
+      filler[self.args[i]] = arg_values[i]
+    result = []
+    for line in self.lines:
+      result.append(line % filler)
+    return result
+
+
 # Parses HEADERFILENAME to find out which runtime functions are "inline".
 def FindInlineRuntimeFunctions():
   inline_functions = []
   with open(HEADERFILENAME, "r") as f:
     inline_list = "#define INLINE_FUNCTION_LIST(F) \\\n"
     inline_opt_list = "#define INLINE_OPTIMIZED_FUNCTION_LIST(F) \\\n"
     inline_function = re.compile(r"^\s*F\((\w+), \d+, \d+\)\s*\\?")
     mode = "SEARCHING"
     for line in f:
       if mode == "ACTIVE":
         match = inline_function.match(line)
         if match:
           inline_functions.append(match.group(1))
         if not line.endswith("\\\n"):
           mode = "SEARCHING"
       elif mode == "SEARCHING":
         if line == inline_list or line == inline_opt_list:
           mode = "ACTIVE"
   return inline_functions
 
 
+def ReadFileAndExpandMacros(filename):
+  found_macros = {}
+  expanded_lines = []
+  with open(filename, "r") as f:
+    found_macro = None
+    for line in f:
+      if found_macro is not None:
+        found_macro.AddLine(line)
+        if not line.endswith("\\\n"):
+          found_macro.Finalize()
+          found_macro = None
+        continue
+
+      match = MACRO.match(line)
+      if match:
+        found_macro = Macro(match)
+        if found_macro.name in EXPAND_MACROS:
+          found_macros[found_macro.name] = found_macro
+        else:
+          found_macro = None
+        continue
+
+      match = FIRST_WORD.match(line)
+      if match:
+        first_word = match.group(1)
+        if first_word in found_macros:
+          MACRO_CALL = re.compile("%s\(([^)]*)\)" % first_word)
+          match = MACRO_CALL.match(line)
+          assert match
+          args = [s.strip() for s in match.group(1).split(",")]
+          expanded_lines += found_macros[first_word].FillIn(args)
+          continue
+
+      expanded_lines.append(line)
+  return expanded_lines
+
+
 # Detects runtime functions by parsing FILENAME.
 def FindRuntimeFunctions():
   inline_functions = FindInlineRuntimeFunctions()
   functions = []
-  with open(FILENAME, "r") as f:
-    function = None
-    partial_line = ""
-    for line in f:
-      # Multi-line definition support, ignoring macros.
-      if line.startswith("RUNTIME_FUNCTION") and not line.endswith("{\n"):
-        if line.endswith("\\\n"): continue
-        partial_line = line.rstrip()
-        continue
-      if partial_line:
-        partial_line += " " + line.strip()
-        if partial_line.endswith("{"):
-          line = partial_line
-          partial_line = ""
-        else:
-          continue
-
-      match = FUNCTION.match(line)
-      if match:
-        function = Function(match)
-        if function.name in inline_functions:
-          function.inline = "_"
-        continue
-      if function is None: continue
-
-      match = ARGSLENGTH.match(line)
-      if match:
-        function.SetArgsLength(match)
+  expanded_lines = ReadFileAndExpandMacros(FILENAME)
+  function = None
+  partial_line = ""
+  for line in expanded_lines:
+    # Multi-line definition support, ignoring macros.
+    if line.startswith("RUNTIME_FUNCTION") and not line.endswith("{\n"):
+      if line.endswith("\\\n"): continue
+      partial_line = line.rstrip()
+      continue
+    if partial_line:
+      partial_line += " " + line.strip()
+      if partial_line.endswith("{"):
+        line = partial_line
+        partial_line = ""
+      else:
         continue
 
-      if function.TryParseArg(line):
-        continue
+    match = FUNCTION.match(line)
+    if match:
+      function = Function(match)
+      if function.name in inline_functions:
+        function.inline = "_"
+      continue
+    if function is None: continue
 
-      if line == FUNCTIONEND:
-        if function is not None:
-          functions.append(function)
-          function = None
+    match = ARGSLENGTH.match(line)
+    if match:
+      function.SetArgsLength(match)
+      continue
+
+    if function.TryParseArg(line):
+      continue
+
+    if line == FUNCTIONEND:
+      if function is not None:
+        functions.append(function)
+        function = None
   return functions
 
 # Classifies runtime functions.
 def ClassifyFunctions(functions):
   # Can be fuzzed with a JavaScript testcase.
   js_fuzzable_functions = []
   # We have enough information to fuzz these, but they need inputs that
   # cannot be created or passed around in JavaScript.
   cctest_fuzzable_functions = []
   # This script does not have enough information about these.
@@ skipping 290 matching lines @@
       for i in range(len(processes)):
         processes[i].join()
     except KeyboardInterrupt:
       stop_running.set()
       for i in range(len(processes)):
         processes[i].join()
     return 0
 
 if __name__ == "__main__":
   sys.exit(Main())