Various test-related changes.

tools/test.py

 #!/usr/bin/env python
 #
 # Copyright 2008 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 12 matching lines @@
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import imp
 import optparse
 import os
-from os.path import join, dirname, abspath, basename
+from os.path import join, dirname, abspath, basename, isdir
 import platform
 import re
 import signal
 import subprocess
 import sys
 import tempfile
 import time
 import utils
 
 
 VERBOSE = False
 
 
 # ---------------------------------------------
 # --- P r o g r e s s   I n d i c a t o r s ---
 # ---------------------------------------------
 
 
 class ProgressIndicator(object):
 
   def __init__(self, cases):
     self.cases = cases
     self.succeeded = 0
     self.failed = 0
     self.remaining = len(self.cases)
     self.total = len(self.cases)
     self.failed_tests = [ ]
 
+  def PrintFailureHeader(self, test):
+    if test.IsNegative():
+      negative_marker = '[negative] '
+    else:
+      negative_marker = ''
+    print "=== %(label)s %(negative)s===" % {
+      'label': test.GetLabel(),
+      'negative': negative_marker
+    }
+    print "Path: %s" % "/".join(test.path)
+
   def Run(self):
     self.Starting()
     for test in self.cases:
       case = test.case
       self.AboutToRun(case)
       output = case.Run()
       if output.UnexpectedOutput():
         self.failed += 1
         self.failed_tests.append(output)
       else:
@@ skipping 17 matching lines @@
 
 
 class SimpleProgressIndicator(ProgressIndicator):
 
   def Starting(self):
     print 'Running %i tests' % len(self.cases)
 
   def Done(self):
     print
     for failed in self.failed_tests:
-      print "=== %s (%s) ===" % (failed.test.GetLabel(), "/".join(failed.test.path))
+      self.PrintFailureHeader(failed.test)
       if failed.output.stderr:
         print "--- stderr ---"
         print failed.output.stderr.strip()
       if failed.output.stdout:
         print "--- stdout ---"
         print failed.output.stdout.strip()
       print "Command: %s" % EscapeCommand(failed.command)
     if len(self.failed_tests) == 0:
       print "==="
       print "=== All tests succeeded"
@@ skipping 47 matching lines @@
     pass
 
   def Done(self):
     self.PrintProgress('Done')
 
   def AboutToRun(self, case):
     self.PrintProgress(case.GetLabel())
 
   def HasRun(self, output):
     if output.UnexpectedOutput():
-      print "=== %s (%s) ===" % (output.test.GetLabel(), "/".join(output.test.path))
+      self.ClearLine(self.last_status_length)
+      self.PrintFailureHeader(output.test)
       print "Command: %s" % EscapeCommand(output.command)
       stdout = output.output.stdout.strip()
       if len(stdout):
         print self.templates['stdout'] % stdout
       stderr = output.output.stderr.strip()
       if len(stderr):
         print self.templates['stderr'] % stderr
 
   def Truncate(self, str, length):
     if length and (len(str) > (length - 3)):
@@ skipping 643 matching lines @@
 
   def __init__(self, sections, defs):
     self.sections = sections
     self.defs = defs
 
   def ClassifyTests(self, cases, env):
     sections = [s for s in self.sections if s.condition.Evaluate(env, self.defs)]
     all_rules = reduce(list.__add__, [s.rules for s in sections], [])
     unused_rules = set(all_rules)
     result = [ ]
+    all_outcomes = set([])
     for case in cases:
       matches = [ r for r in all_rules if r.Contains(case.path) ]
       outcomes = set([])
       for rule in matches:
         outcomes = outcomes.union(rule.GetOutcomes(env, self.defs))
         unused_rules.discard(rule)
       if not outcomes:
         outcomes = [PASS]
       case.outcomes = outcomes
+      all_outcomes = all_outcomes.union(outcomes)
       result.append(ClassifiedTest(case, outcomes))
-    return (result, list(unused_rules))
+    return (result, list(unused_rules), all_outcomes)
 
 
 class Section(object):
   """A section of the configuration file.  Sections are enabled or
   disabled prior to running the tests, based on their conditions"""
 
   def __init__(self, condition):
     self.condition = condition
     self.rules = [ ]
 
@@ skipping 96 matching lines @@
       default=[], action="append")
   result.add_option("-t", "--timeout", help="Timeout in seconds",
       default=60, type="int")
   result.add_option("--arch", help='The architecture to run tests for',
       default='none')
   result.add_option("--simulator", help="Run tests with architecture simulator",
       default='none')
   result.add_option("--special-command", default=None)
   result.add_option("--cat", help="Print the source of the tests",
       default=False, action="store_true")
+  result.add_option("--warn-unused", help="Report unused rules",
+      default=False, action="store_true")
   return result
 
 
 def ProcessOptions(options):
   global VERBOSE
   VERBOSE = options.verbose
   options.mode = options.mode.split(',')
   for mode in options.mode:
     if not mode in ['debug', 'release']:
       print "Unknown mode %s" % mode
@@ skipping 72 matching lines @@
     prefix = value[:pos].split()
     suffix = value[pos+1:].split()
     def ExpandCommand(args):
       return prefix + args + suffix
     return ExpandCommand
 
 
 BUILT_IN_TESTS = ['mjsunit', 'cctest']
 
 
+def GetSuites(test_root):
+  return [ f for f in os.listdir(test_root) if isdir(join(test_root, f)) ]
+
+
 def Main():
   parser = BuildOptions()
   (options, args) = parser.parse_args()
   if not ProcessOptions(options):
     parser.print_help()
     return 1
 
   workspace = abspath(join(dirname(sys.argv[0]), '..'))
-  repositories = [TestRepository(join(workspace, 'test', name)) for name in BUILT_IN_TESTS]
+  suites = GetSuites(join(workspace, 'test'))
+  repositories = [TestRepository(join(workspace, 'test', name)) for name in suites]
   repositories += [TestRepository(a) for a in options.suite]
 
   root = LiteralTestSuite(repositories)
   if len(args) == 0:
     paths = [SplitPath(t) for t in BUILT_IN_TESTS]
   else:
     paths = [ ]
     for arg in args:
       path = SplitPath(arg)
       paths.append(path)
@@ skipping 16 matching lines @@
   # Get status for tests
   sections = [ ]
   defs = { }
   root.GetTestStatus(context, sections, defs)
   config = Configuration(sections, defs)
 
   # List the tests
   all_cases = [ ]
   all_unused = [ ]
   unclassified_tests = [ ]
+  globally_unused_rules = None
   for path in paths:
     for mode in options.mode:
       env = {
         'mode': mode,
         'system': platform.system().lower(),
         'arch': options.arch
       }
       test_list = root.ListTests([], path, context, mode)
       unclassified_tests += test_list
-      (cases, unused_rules) = config.ClassifyTests(test_list, env)
+      (cases, unused_rules, all_outcomes) = config.ClassifyTests(test_list, env)
+      if globally_unused_rules is None:
+        globally_unused_rules = set(unused_rules)
+      else:
+        globally_unused_rules = globally_unused_rules.intersection(unused_rules)
       all_cases += cases
       all_unused.append(unused_rules)
 
   if options.cat:
     visited = set()
     for test in unclassified_tests:
       key = tuple(test.path)
       if key in visited:
         continue
       visited.add(key)
       print "--- begin source: %s ---" % test.GetLabel()
       source = test.GetSource().strip()
       print source
       print "--- end source: %s ---" % test.GetLabel()
     return 0
 
-#  for rule in unused_rules:
-#    print "Rule for '%s' was not used." % '/'.join([str(s) for s in rule.path])
+  if options.warn_unused:
+    for rule in globally_unused_rules:
+      print "Rule for '%s' was not used." % '/'.join([str(s) for s in rule.path])
 
   if options.report:
     PrintReport(all_cases)
 
   if len(all_cases) == 0:
     print "No tests to run."
     return 0
   else:
     try:
       if RunTestCases(all_cases, options.progress):
         return 0
       else:
         return 1
     except KeyboardInterrupt:
       print "Interrupted"
       return 1
 
 
 if __name__ == '__main__':
   sys.exit(Main())