Refactor the tools/test.py script and related testcfg.py files....

tools/test.py

 #!/usr/bin/env python
 #
 # Copyright 2008 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 313 matching lines @@
   def __init__(self, exit_code, timed_out, stdout, stderr):
     self.exit_code = exit_code
     self.timed_out = timed_out
     self.stdout = stdout
     self.stderr = stderr
     self.failed = None
 
 
 class TestCase(object):
 
-  def __init__(self, context, path):
+  def __init__(self, context, path, mode):
     self.path = path
     self.context = context
     self.duration = None
+    self.mode = mode
 
   def IsNegative(self):
     return False
 
   def CompareTime(self, other):
     return cmp(other.duration, self.duration)
 
   def DidFail(self, output):
     if output.failed is None:
       output.failed = self.IsFailureOutput(output)
     return output.failed
 
   def IsFailureOutput(self, output):
     return output.exit_code != 0
 
   def GetSource(self):
     return "(no source available)"
 
   def RunCommand(self, command):
     full_command = self.context.processor(command)
-    output = Execute(full_command, self.context, self.context.timeout)
+    output = Execute(full_command,
+                     self.context,
+                     self.context.GetTimeout(self.mode))
     self.Cleanup()
-    return TestOutput(self, full_command, output)
+    return TestOutput(self,
+                      full_command,
+                      output,
+                      self.context.store_unexpected_output)
 
   def BeforeRun(self):
     pass
 
-  def AfterRun(self):
+  def AfterRun(self, result):
     pass
 
   def Run(self):
     self.BeforeRun()
     try:
       result = self.RunCommand(self.GetCommand())
     finally:
-      self.AfterRun()
+      self.AfterRun(result)
     return result
 
   def Cleanup(self):
     return
 
 
 class TestOutput(object):
 
-  def __init__(self, test, command, output):
+  def __init__(self, test, command, output, store_unexpected_output):
     self.test = test
     self.command = command
     self.output = output
+    self.store_unexpected_output = store_unexpected_output
 
   def UnexpectedOutput(self):
     if self.HasCrashed():
       outcome = CRASH
     elif self.HasTimedOut():
       outcome = TIMEOUT
     elif self.HasFailed():
       outcome = FAIL
     else:
       outcome = PASS
     return not outcome in self.test.outcomes
 
+  def HasPreciousOutput(self):
+    return self.UnexpectedOutput() and self.store_unexpected_output
+
   def HasCrashed(self):
     if utils.IsWindows():
       return 0x80000000 & self.output.exit_code and not (0x3FFFFF00 & self.output.exit_code)
     else:
       # Timed out tests will have exit_code -signal.SIGTERM.
       if self.output.timed_out:
         return False
       return self.output.exit_code < 0 and \
              self.output.exit_code != -signal.SIGABRT
 
@@ skipping 142 matching lines @@
 
 class TestSuite(object):
 
   def __init__(self, name):
     self.name = name
 
   def GetName(self):
     return self.name
 
 
+# Use this to run several variants of the tests, e.g.:
+# VARIANT_FLAGS = [[], ['--always_compact', '--noflush_code']]
+VARIANT_FLAGS = [[]]
+
+
 class TestRepository(TestSuite):
 
   def __init__(self, path):
     normalized_path = abspath(path)
     super(TestRepository, self).__init__(basename(normalized_path))
     self.path = normalized_path
     self.is_loaded = False
     self.config = None
 
   def GetConfiguration(self, context):
     if self.is_loaded:
       return self.config
     self.is_loaded = True
     file = None
     try:
       (file, pathname, description) = imp.find_module('testcfg', [ self.path ])
       module = imp.load_module('testcfg', file, pathname, description)
       self.config = module.GetConfiguration(context, self.path)
     finally:
       if file:
         file.close()
     return self.config
 
   def GetBuildRequirements(self, path, context):
     return self.GetConfiguration(context).GetBuildRequirements()
 
-  def ListTests(self, current_path, path, context, mode):
-    return self.GetConfiguration(context).ListTests(current_path, path, mode)
+  def AddTestsToList(self, result, current_path, path, context, mode):
+    for v in VARIANT_FLAGS:
+      tests = self.GetConfiguration(context).ListTests(current_path, path, mode)
+      for t in tests: t.variant_flags = v
+      result += tests
+
 
   def GetTestStatus(self, context, sections, defs):
     self.GetConfiguration(context).GetTestStatus(sections, defs)
 
 
 class LiteralTestSuite(TestSuite):
 
   def __init__(self, tests):
     super(LiteralTestSuite, self).__init__('root')
     self.tests = tests
 
   def GetBuildRequirements(self, path, context):
     (name, rest) = CarCdr(path)
     result = [ ]
     for test in self.tests:
       if not name or name.match(test.GetName()):
         result += test.GetBuildRequirements(rest, context)
     return result
 
   def ListTests(self, current_path, path, context, mode):
     (name, rest) = CarCdr(path)
     result = [ ]
     for test in self.tests:
       test_name = test.GetName()
       if not name or name.match(test_name):
         full_path = current_path + [test_name]
-        result += test.ListTests(full_path, path, context, mode)
+        test.AddTestsToList(result, full_path, path, context, mode)
     return result
 
   def GetTestStatus(self, context, sections, defs):
     for test in self.tests:
       test.GetTestStatus(context, sections, defs)
 
 
-SUFFIX = {'debug': '_g', 'release': ''}
+SUFFIX = {
+    'debug'   : '_g',
+    'release' : '' }
+FLAGS = {
+    'debug'   : ['--enable-slow-asserts', '--debug-code', '--verify-heap'],
+    'release' : []}
+TIMEOUT_SCALEFACTOR = {
+    'debug'   : 4,
+    'release' : 1 }
 
 
 class Context(object):
 
-  def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, suppress_dialogs):
+  def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, suppress_dialogs, store_unexpected_output):
     self.workspace = workspace
     self.buildspace = buildspace
     self.verbose = verbose
     self.vm_root = vm
     self.timeout = timeout
     self.processor = processor
     self.suppress_dialogs = suppress_dialogs
+    self.store_unexpected_output = store_unexpected_output
 
   def GetVm(self, mode):
     name = self.vm_root + SUFFIX[mode]
     if utils.IsWindows() and not name.endswith('.exe'):
       name = name + '.exe'
     return name
 
+  def GetVmCommand(self, testcase, mode):
+    return [self.GetVm(mode)] + self.GetVmFlags(testcase, mode)
+
+  def GetVmFlags(self, testcase, mode):
+    return testcase.variant_flags + FLAGS[mode]
+
+  def GetTimeout(self, mode):
+    return self.timeout * TIMEOUT_SCALEFACTOR[mode]
+
 def RunTestCases(cases_to_run, progress, tasks):
   progress = PROGRESS_INDICATORS[progress](cases_to_run)
   return progress.Run(tasks)
 
 
 def BuildRequirements(context, requirements, mode, scons_flags):
   command_line = (['scons', '-Y', context.workspace, 'mode=' + ",".join(mode)]
                   + requirements
                   + scons_flags)
   output = ExecuteNoCapture(command_line, context)
@@ skipping 462 matching lines @@
   result.add_option("--warn-unused", help="Report unused rules",
       default=False, action="store_true")
   result.add_option("-j", help="The number of parallel tasks to run",
       default=1, type="int")
   result.add_option("--time", help="Print timing information after running",
       default=False, action="store_true")
   result.add_option("--suppress-dialogs", help="Suppress Windows dialogs for crashing tests",
         dest="suppress_dialogs", default=True, action="store_true")
   result.add_option("--no-suppress-dialogs", help="Display Windows dialogs for crashing tests",
         dest="suppress_dialogs", action="store_false")
-  result.add_option("--shell", help="Path to V8 shell", default="shell");
+  result.add_option("--shell", help="Path to V8 shell", default="shell")
+  result.add_option("--store-unexpected-output", 
+      help="Store the temporary JS files from tests that fails",
+      dest="store_unexpected_output", default=True, action="store_true")
+  result.add_option("--no-store-unexpected-output", 
+      help="Deletes the temporary JS files from tests that fails",
+      dest="store_unexpected_output", action="store_false")
   return result
 
 
 def ProcessOptions(options):
   global VERBOSE
   VERBOSE = options.verbose
   options.mode = options.mode.split(',')
   for mode in options.mode:
     if not mode in ['debug', 'release']:
       print "Unknown mode %s" % mode
@@ skipping 116 matching lines @@
       paths.append(path)
 
   # Check for --valgrind option. If enabled, we overwrite the special
   # command flag with a command that uses the run-valgrind.py script.
   if options.valgrind:
     run_valgrind = join(workspace, "tools", "run-valgrind.py")
     options.special_command = "python -u " + run_valgrind + " @"
 
   shell = abspath(options.shell)
   buildspace = dirname(shell)
+
   context = Context(workspace, buildspace, VERBOSE,
                     shell,
                     options.timeout,
                     GetSpecialCommandProcessor(options.special_command),
-                    options.suppress_dialogs)
+                    options.suppress_dialogs,
+                    options.store_unexpected_output)
   # First build the required targets
   if not options.no_build:
     reqs = [ ]
     for path in paths:
       reqs += root.GetBuildRequirements(path, context)
     reqs = list(set(reqs))
     if len(reqs) > 0:
       if options.j != 1:
         options.scons_flags += ['-j', str(options.j)]
       if not BuildRequirements(context, reqs, options.mode, options.scons_flags):
         return 1
 
   # Just return if we are only building the targets for running the tests.
   if options.build_only:
     return 0
-  
+
   # Get status for tests
   sections = [ ]
   defs = { }
   root.GetTestStatus(context, sections, defs)
   config = Configuration(sections, defs)
 
   # List the tests
   all_cases = [ ]
   all_unused = [ ]
   unclassified_tests = [ ]
@@ skipping 69 matching lines @@
     for entry in timed_tests[:20]:
       t = FormatTime(entry.duration)
       sys.stderr.write("%4i (%s) %s\n" % (index, t, entry.GetLabel()))
       index += 1
 
   return result
 
 
 if __name__ == '__main__':
   sys.exit(Main())