Add builtin detector to generate-runtime-tests.py

tools/generate-runtime-tests.py

Index: tools/generate-runtime-tests.py
diff --git a/tools/generate-runtime-tests.py b/tools/generate-runtime-tests.py
index 8770aa2916b51c76e49ec2e241fe782dac4714bf..429271a1d193ed0453692f88309a93546c51adf4 100755
--- a/tools/generate-runtime-tests.py
+++ b/tools/generate-runtime-tests.py
@@ -4,6 +4,7 @@
 # found in the LICENSE file.
 
 import itertools
+import js2c
 import multiprocessing
 import optparse
 import os
@@ -50,6 +51,7 @@ EXPECTED_FUNCTION_COUNT = 362
 EXPECTED_FUZZABLE_COUNT = 329
 EXPECTED_CCTEST_COUNT = 6
 EXPECTED_UNKNOWN_COUNT = 5
+EXPECTED_BUILTINS_COUNT = 827
 
 
 # Don't call these at all.
@@ -298,10 +300,6 @@ class Generator(object):
     return self._Variable(name,
         "\"%s\" + (function() { return \"%s\";})()" % (s1, s2))
 
-  def _ExternalString(self, name):
-    # Needs --expose-externalize-string.
-    return None
-
   def _InternalizedString(self, name):
     return self._Variable(name, "\"%s\"" % self._RawRandomString(0, 20))
 
@@ -918,6 +916,60 @@ def FindRuntimeFunctions():
         function = None
   return functions
 
+
+# Hack: This must have the same fields as class Function above, because the
+# two are used polymorphically in RunFuzzer(). We could use inheritance...
+class Builtin(object):
+  def __init__(self, match):
+    self.name = match.group(1)
+    args = match.group(2)
+    self.argslength = 0 if args == "" else args.count(",") + 1
+    self.inline = ""
+    self.args = {}
+    if self.argslength > 0:
+      args = args.split(",")
+      for i in range(len(args)):
+        # a = args[i].strip()  # TODO: filter out /* comments */ first.
+        a = ""
+        self.args[i] = Arg("Object", a, i)
+
+  def __str__(self):
+    return "%s(%d)" % (self.name, self.argslength)
+
+
+def FindJSBuiltins():
+  PATH = "src"
+  fileslist = []
+  for (root, dirs, files) in os.walk(PATH):
+    for f in files:
+      if f.endswith(".js"):
+        fileslist.append(os.path.join(root, f))
+  builtins = []
+  regexp = re.compile("^function (\w+)\s*\((.*?)\) {")
+  matches = 0
+  for filename in fileslist:
+    with open(filename, "r") as f:
+      file_contents = f.read()
+    file_contents = js2c.ExpandInlineMacros(file_contents)
+    lines = file_contents.split("\n")
+    partial_line = ""
+    for line in lines:
+      if line.startswith("function") and not '{' in line:
+        partial_line += line.rstrip()
+        continue
+      if partial_line:
+        partial_line += " " + line.strip()
+        if '{' in line:
+          line = partial_line
+          partial_line = ""
+        else:
+          continue
+      match = regexp.match(line)
+      if match:
+        builtins.append(Builtin(match))
+  return builtins
+
+
 # Classifies runtime functions.
 def ClassifyFunctions(functions):
   # Can be fuzzed with a JavaScript testcase.
@@ -1018,7 +1070,23 @@ def _SaveFileName(save_path, process_id, save_file_index):
   return "%s/fuzz_%d_%d.js" % (save_path, process_id, save_file_index)
 
 
+def _GetFuzzableRuntimeFunctions():
+  functions = FindRuntimeFunctions()
+  (js_fuzzable_functions, cctest_fuzzable_functions, unknown_functions) = \
+      ClassifyFunctions(functions)
+  return js_fuzzable_functions
+
+
+FUZZ_TARGET_LISTS = {
+  "runtime": _GetFuzzableRuntimeFunctions,
+  "builtins": FindJSBuiltins,
+}
+
+
 def RunFuzzer(process_id, options, stop_running):
+  MAX_SLEEP_TIME = 0.1
+  INITIAL_SLEEP_TIME = 0.001
+  SLEEP_TIME_FACTOR = 1.25
   base_file_name = "/dev/shm/runtime_fuzz_%d" % process_id
   test_file_name = "%s.js" % base_file_name
   stderr_file_name = "%s.out" % base_file_name
@@ -1026,19 +1094,14 @@ def RunFuzzer(process_id, options, stop_running):
   while os.path.exists(_SaveFileName(options.save_path, process_id,
                                      save_file_index)):
     save_file_index += 1
-  MAX_SLEEP_TIME = 0.1
-  INITIAL_SLEEP_TIME = 0.001
-  SLEEP_TIME_FACTOR = 1.5
-
-  functions = FindRuntimeFunctions()
-  (js_fuzzable_functions, cctest_fuzzable_functions, unknown_functions) = \
-      ClassifyFunctions(functions)
 
+  targets = FUZZ_TARGET_LISTS[options.fuzz_target]()
   try:
     for i in range(options.num_tests):
       if stop_running.is_set(): break
-      function = random.choice(js_fuzzable_functions)  # TODO: others too
-      if function.argslength == 0: continue
+      function = None
+      while function is None or function.argslength == 0:
+        function = random.choice(targets)
       args = []
       definitions = []
       gen = Generator()
@@ -1086,11 +1149,11 @@ def RunFuzzer(process_id, options, stop_running):
         save_file_index += 1
   except KeyboardInterrupt:
     stop_running.set()
-  except Exception, e:
-    print e
   finally:
-    os.remove(test_file_name)
-    os.remove(stderr_file_name)
+    if os.path.exists(test_file_name):
+      os.remove(test_file_name)
+    if os.path.exists(stderr_file_name):
+      os.remove(stderr_file_name)
 
 
 def BuildOptionParser():
@@ -1110,6 +1173,9 @@ fuzz      Generate fuzz tests, run them, save those that crashed (see options).
   o = optparse.OptionParser(usage=usage)
   o.add_option("--binary", default="out/x64.debug/d8",
                help="d8 binary used for running fuzz tests (default: %default)")
+  o.add_option("--fuzz-target", default="runtime",
+               help="Set of functions targeted by fuzzing. Allowed values: "
+                    "%s (default: %%default)" % ", ".join(FUZZ_TARGET_LISTS))
   o.add_option("-n", "--num-tests", default=1000, type="int",
                help="Number of fuzz tests to generate per worker process"
                     " (default: %default)")
@@ -1122,12 +1188,21 @@ fuzz      Generate fuzz tests, run them, save those that crashed (see options).
   return o
 
 
+def ProcessOptions(options, args):
+  options.save_path = os.path.expanduser(options.save_path)
+  if options.fuzz_target not in FUZZ_TARGET_LISTS:
+    print("Invalid fuzz target: %s" % options.fuzz_target)
+    return False
+  if len(args) != 1 or args[0] == "help":
+    return False
+  return True
+
+
 def Main():
   parser = BuildOptionParser()
   (options, args) = parser.parse_args()
-  options.save_path = os.path.expanduser(options.save_path)
 
-  if len(args) != 1 or args[0] == "help":
+  if not ProcessOptions(options, args):
     parser.print_help()
     return 1
   action = args[0]
@@ -1135,14 +1210,10 @@ def Main():
   functions = FindRuntimeFunctions()
   (js_fuzzable_functions, cctest_fuzzable_functions, unknown_functions) = \
       ClassifyFunctions(functions)
+  builtins = FindJSBuiltins()
 
   if action == "test":
-    gen = Generator()
-    vartype = "JSTypedArray"
-    print("simple: %s" % gen.RandomVariable("x", vartype, True))
-    for i in range(10):
-      print("----")
-      print("%s" % "\n".join(gen.RandomVariable("x", vartype, False)))
+    print("put your temporary debugging code here")
     return 0
 
   if action == "info":
@@ -1150,6 +1221,7 @@ def Main():
           "cctest_fuzzable_functions: %d, unknown_functions: %d"
           % (len(functions), len(js_fuzzable_functions),
              len(cctest_fuzzable_functions), len(unknown_functions)))
+    print("%d JavaScript builtins" % len(builtins))
     print("unknown functions:")
     for f in unknown_functions:
       print(f)
@@ -1175,6 +1247,8 @@ def Main():
                          "cctest-fuzzable functions")
     errors += CheckCount(unknown_functions, EXPECTED_UNKNOWN_COUNT,
                          "functions with incomplete type information")
+    errors += CheckCount(builtins, EXPECTED_BUILTINS_COUNT,
+                         "JavaScript builtins")
 
     def CheckTestcasesExisting(functions):
       errors = 0
@@ -1196,6 +1270,19 @@ def Main():
 
     errors += CheckTestcasesExisting(js_fuzzable_functions)
 
+    def CheckNameClashes(runtime_functions, builtins):
+      errors = 0
+      runtime_map = {}
+      for f in runtime_functions:
+        runtime_map[f.name] = 1
+      for b in builtins:
+        if b.name in runtime_map:
+          print("Builtin/Runtime_Function name clash: %s" % b.name)
+          errors += 1
+      return errors
+
+    errors += CheckNameClashes(functions, builtins)
+
     if errors > 0:
       return 1
     print("Generated runtime tests: all good.")