Add builtin detector to generate-runtime-tests.py

tools/generate-runtime-tests.py

 #!/usr/bin/env python
 # Copyright 2014 the V8 project authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import itertools
+import js2c
 import multiprocessing
 import optparse
 import os
 import random
 import re
 import shutil
 import signal
 import string
 import subprocess
 import sys
@@ skipping 26 matching lines @@
 # TYPED_ARRAYS_CHECK_RUNTIME_FUNCTION
 
 # Counts of functions in each detection state. These are used to assert
 # that the parser doesn't bit-rot. Change the values as needed when you add,
 # remove or change runtime functions, but make sure we don't lose our ability
 # to parse them!
 EXPECTED_FUNCTION_COUNT = 362
 EXPECTED_FUZZABLE_COUNT = 329
 EXPECTED_CCTEST_COUNT = 6
 EXPECTED_UNKNOWN_COUNT = 5
+EXPECTED_BUILTINS_COUNT = 827
 
 
 # Don't call these at all.
 BLACKLISTED = [
   "Abort",  # Kills the process.
   "AbortJS",  # Kills the process.
   "CompileForOnStackReplacement",  # Riddled with ASSERTs.
   "IS_VAR",  # Not implemented in the runtime.
   "ListNatives",  # Not available in Release mode.
   "SetAllocationTimeout",  # Too slow for fuzzing.
@@ skipping 228 matching lines @@
     # 'ffoo12345678901234567890'.substr(1)
     return self._Variable(name, "\"%s\".substr(1)" % s)
 
   def _ConsString(self, name):
     s1 = self._RawRandomString(8, 15)
     s2 = self._RawRandomString(8, 15)
     # 'foo12345' + (function() { return 'bar12345';})()
     return self._Variable(name,
         "\"%s\" + (function() { return \"%s\";})()" % (s1, s2))
 
-  def _ExternalString(self, name):
-    # Needs --expose-externalize-string.
-    return None
-
   def _InternalizedString(self, name):
     return self._Variable(name, "\"%s\"" % self._RawRandomString(0, 20))
 
   def _String(self, name, recursion_budget):
     die = random.random()
     if die < 0.5:
       string = random.choice(self.USUAL_SUSPECT_PROPERTIES)
       return self._Variable(name, "\"%s\"" % string)
     elif die < 0.6:
       number_name = name + "_number"
@@ skipping 596 matching lines @@
 
     if function.TryParseArg(line):
       continue
 
     if line == FUNCTIONEND:
       if function is not None:
         functions.append(function)
         function = None
   return functions
 
+
+# Hack: This must have the same fields as class Function above, because the
+# two are used polymorphically in RunFuzzer(). We could use inheritance...
+class Builtin(object):
+  def __init__(self, match):
+    self.name = match.group(1)
+    args = match.group(2)
+    self.argslength = 0 if args == "" else args.count(",") + 1
+    self.inline = ""
+    self.args = {}
+    if self.argslength > 0:
+      args = args.split(",")
+      for i in range(len(args)):
+        # a = args[i].strip()  # TODO: filter out /* comments */ first.
+        a = ""
+        self.args[i] = Arg("Object", a, i)
+
+  def __str__(self):
+    return "%s(%d)" % (self.name, self.argslength)
+
+
+def FindJSBuiltins():
+  PATH = "src"
+  fileslist = []
+  for (root, dirs, files) in os.walk(PATH):
+    for f in files:
+      if f.endswith(".js"):
+        fileslist.append(os.path.join(root, f))
+  builtins = []
+  regexp = re.compile("^function (\w+)\s*\((.*?)\) {")
+  matches = 0
+  for filename in fileslist:
+    with open(filename, "r") as f:
+      file_contents = f.read()
+    file_contents = js2c.ExpandInlineMacros(file_contents)
+    lines = file_contents.split("\n")
+    partial_line = ""
+    for line in lines:
+      if line.startswith("function") and not '{' in line:
+        partial_line += line.rstrip()
+        continue
+      if partial_line:
+        partial_line += " " + line.strip()
+        if '{' in line:
+          line = partial_line
+          partial_line = ""
+        else:
+          continue
+      match = regexp.match(line)
+      if match:
+        builtins.append(Builtin(match))
+  return builtins
+
+
 # Classifies runtime functions.
 def ClassifyFunctions(functions):
   # Can be fuzzed with a JavaScript testcase.
   js_fuzzable_functions = []
   # We have enough information to fuzz these, but they need inputs that
   # cannot be created or passed around in JavaScript.
   cctest_fuzzable_functions = []
   # This script does not have enough information about these.
   unknown_functions = []
 
@@ skipping 80 matching lines @@
   shutil.rmtree(BASEPATH)  # Re-generate everything.
   os.makedirs(BASEPATH)
   for f in functions:
     GenerateJSTestcaseForFunction(f)
 
 
 def _SaveFileName(save_path, process_id, save_file_index):
   return "%s/fuzz_%d_%d.js" % (save_path, process_id, save_file_index)
 
 
+def _GetFuzzableRuntimeFunctions():
+  functions = FindRuntimeFunctions()
+  (js_fuzzable_functions, cctest_fuzzable_functions, unknown_functions) = \
+      ClassifyFunctions(functions)
+  return js_fuzzable_functions
+
+
+FUZZ_TARGET_LISTS = {
+  "runtime": _GetFuzzableRuntimeFunctions,
+  "builtins": FindJSBuiltins,
+}
+
+
 def RunFuzzer(process_id, options, stop_running):
+  MAX_SLEEP_TIME = 0.1
+  INITIAL_SLEEP_TIME = 0.001
+  SLEEP_TIME_FACTOR = 1.25
   base_file_name = "/dev/shm/runtime_fuzz_%d" % process_id
   test_file_name = "%s.js" % base_file_name
   stderr_file_name = "%s.out" % base_file_name
   save_file_index = 0
   while os.path.exists(_SaveFileName(options.save_path, process_id,
                                      save_file_index)):
     save_file_index += 1
-  MAX_SLEEP_TIME = 0.1
-  INITIAL_SLEEP_TIME = 0.001
-  SLEEP_TIME_FACTOR = 1.5
 
-  functions = FindRuntimeFunctions()
-  (js_fuzzable_functions, cctest_fuzzable_functions, unknown_functions) = \
-      ClassifyFunctions(functions)
-
+  targets = FUZZ_TARGET_LISTS[options.fuzz_target]()
   try:
     for i in range(options.num_tests):
       if stop_running.is_set(): break
-      function = random.choice(js_fuzzable_functions)  # TODO: others too
-      if function.argslength == 0: continue
+      function = None
+      while function is None or function.argslength == 0:
+        function = random.choice(targets)
       args = []
       definitions = []
       gen = Generator()
       for i in range(function.argslength):
         arg = function.args[i]
         argname = "arg%d%s" % (i, arg.name)
         args.append(argname)
         definitions += gen.RandomVariable(argname, arg.type, simple=False)
       testcase = _GenerateTestcase(function, definitions, args, True)
       with open(test_file_name, "w") as f:
@@ skipping 27 matching lines @@
             if line.strip() == "# Allocation failed - process out of memory":
               oom = True
               break
         if oom: continue
         save_name = _SaveFileName(options.save_path, process_id,
                                   save_file_index)
         shutil.copyfile(test_file_name, save_name)
         save_file_index += 1
   except KeyboardInterrupt:
     stop_running.set()
-  except Exception, e:
-    print e
   finally:
-    os.remove(test_file_name)
-    os.remove(stderr_file_name)
+    if os.path.exists(test_file_name):
+      os.remove(test_file_name)
+    if os.path.exists(stderr_file_name):
+      os.remove(stderr_file_name)
 
 
 def BuildOptionParser():
   usage = """Usage: %%prog [options] ACTION
 
 where ACTION can be:
 
 info      Print diagnostic info.
 check     Check that runtime functions can be parsed as expected, and that
           test cases exist.
 generate  Parse source code for runtime functions, and auto-generate
           test cases for them. Warning: this will nuke and re-create
           %(path)s.
 fuzz      Generate fuzz tests, run them, save those that crashed (see options).
 """ % {"path": os.path.relpath(BASEPATH)}
 
   o = optparse.OptionParser(usage=usage)
   o.add_option("--binary", default="out/x64.debug/d8",
                help="d8 binary used for running fuzz tests (default: %default)")
+  o.add_option("--fuzz-target", default="runtime",
+               help="Set of functions targeted by fuzzing. Allowed values: "
+                    "%s (default: %%default)" % ", ".join(FUZZ_TARGET_LISTS))
   o.add_option("-n", "--num-tests", default=1000, type="int",
                help="Number of fuzz tests to generate per worker process"
                     " (default: %default)")
   o.add_option("--save-path", default="~/runtime_fuzz_output",
                help="Path to directory where failing tests will be stored"
                     " (default: %default)")
   o.add_option("--timeout", default=20, type="int",
                help="Timeout for each fuzz test (in seconds, default:"
                     "%default)")
   return o
 
 
+def ProcessOptions(options, args):
+  options.save_path = os.path.expanduser(options.save_path)
+  if options.fuzz_target not in FUZZ_TARGET_LISTS:
+    print("Invalid fuzz target: %s" % options.fuzz_target)
+    return False
+  if len(args) != 1 or args[0] == "help":
+    return False
+  return True
+
+
 def Main():
   parser = BuildOptionParser()
   (options, args) = parser.parse_args()
-  options.save_path = os.path.expanduser(options.save_path)
 
-  if len(args) != 1 or args[0] == "help":
+  if not ProcessOptions(options, args):
     parser.print_help()
     return 1
   action = args[0]
 
   functions = FindRuntimeFunctions()
   (js_fuzzable_functions, cctest_fuzzable_functions, unknown_functions) = \
       ClassifyFunctions(functions)
+  builtins = FindJSBuiltins()
 
   if action == "test":
-    gen = Generator()
-    vartype = "JSTypedArray"
-    print("simple: %s" % gen.RandomVariable("x", vartype, True))
-    for i in range(10):
-      print("----")
-      print("%s" % "\n".join(gen.RandomVariable("x", vartype, False)))
+    print("put your temporary debugging code here")
     return 0
 
   if action == "info":
     print("%d functions total; js_fuzzable_functions: %d, "
           "cctest_fuzzable_functions: %d, unknown_functions: %d"
           % (len(functions), len(js_fuzzable_functions),
              len(cctest_fuzzable_functions), len(unknown_functions)))
+    print("%d JavaScript builtins" % len(builtins))
     print("unknown functions:")
     for f in unknown_functions:
       print(f)
     return 0
 
   if action == "check":
     errors = 0
 
     def CheckCount(actual, expected, description):
       if len(actual) != expected:
         print("Expected to detect %d %s, but found %d." % (
               expected, description, len(actual)))
         print("If this change is intentional, please update the expectations"
               " at the top of %s." % THIS_SCRIPT)
         return 1
       return 0
 
     errors += CheckCount(functions, EXPECTED_FUNCTION_COUNT,
                          "functions in total")
     errors += CheckCount(js_fuzzable_functions, EXPECTED_FUZZABLE_COUNT,
                          "JavaScript-fuzzable functions")
     errors += CheckCount(cctest_fuzzable_functions, EXPECTED_CCTEST_COUNT,
                          "cctest-fuzzable functions")
     errors += CheckCount(unknown_functions, EXPECTED_UNKNOWN_COUNT,
                          "functions with incomplete type information")
+    errors += CheckCount(builtins, EXPECTED_BUILTINS_COUNT,
+                         "JavaScript builtins")
 
     def CheckTestcasesExisting(functions):
       errors = 0
       for f in functions:
         if not os.path.isfile(os.path.join(BASEPATH, f.Filename())):
           print("Missing testcase for %s, please run '%s generate'" %
                 (f.name, THIS_SCRIPT))
           errors += 1
       files = filter(lambda filename: not filename.startswith("."),
                      os.listdir(BASEPATH))
       if (len(files) != len(functions)):
         unexpected_files = set(files) - set([f.Filename() for f in functions])
         for f in unexpected_files:
           print("Unexpected testcase: %s" % os.path.join(BASEPATH, f))
           errors += 1
         print("Run '%s generate' to automatically clean these up."
               % THIS_SCRIPT)
       return errors
 
     errors += CheckTestcasesExisting(js_fuzzable_functions)
 
+    def CheckNameClashes(runtime_functions, builtins):
+      errors = 0
+      runtime_map = {}
+      for f in runtime_functions:
+        runtime_map[f.name] = 1
+      for b in builtins:
+        if b.name in runtime_map:
+          print("Builtin/Runtime_Function name clash: %s" % b.name)
+          errors += 1
+      return errors
+
+    errors += CheckNameClashes(functions, builtins)
+
     if errors > 0:
       return 1
     print("Generated runtime tests: all good.")
     return 0
 
   if action == "generate":
     GenerateTestcases(js_fuzzable_functions)
     return 0
 
   if action == "fuzz":
@@ skipping 10 matching lines @@
       for i in range(len(processes)):
         processes[i].join()
     except KeyboardInterrupt:
       stop_running.set()
       for i in range(len(processes)):
         processes[i].join()
     return 0
 
 if __name__ == "__main__":
   sys.exit(Main())