Add concept of slow tests.

webkit/tools/layout_tests/layout_package/test_expectations.py

 # Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A helper class for reading in and dealing with tests expectations
 for layout tests.
 """
 
 import logging
 import os
 import re
 import sys
 import path_utils
 import compare_failures
 
 
 # Test expectation and modifier constants.
-(PASS, FAIL, TIMEOUT, CRASH, SKIP, WONTFIX, DEFER, NONE) = range(8)
+(PASS, FAIL, TIMEOUT, CRASH, SKIP, WONTFIX, DEFER, SLOW, NONE) = range(9)
 
 class TestExpectations:
   TEST_LIST = "test_expectations.txt"
 
   def __init__(self, tests, directory, platform, is_debug_mode):
     """Reads the test expectations files from the given directory."""
     path = os.path.join(directory, self.TEST_LIST)
     self._expected_failures = TestExpectationsFile(path, tests, platform,
         is_debug_mode)
 
@@ skipping 76 matching lines @@
 
   def IsDeferred(self, test):
     return self._expected_failures.HasModifier(test, DEFER)
 
   def IsFixable(self, test):
     return self._expected_failures.HasModifier(test, NONE)
 
   def IsIgnored(self, test):
     return self._expected_failures.HasModifier(test, WONTFIX)
 
+  def HasModifier(self, test, modifier):
+    return self._expected_failures.HasModifier(test, modifier)
+
 def StripComments(line):
   """Strips comments from a line and return None if the line is empty
   or else the contents of line with leading and trailing spaces removed
   and all other whitespace collapsed"""
 
   commentIndex = line.find('//')
   if commentIndex is -1:
     commentIndex = len(line)
 
   line = re.sub(r'\s+', ' ', line[:commentIndex].strip())
@@ skipping 13 matching lines @@
     ...
 
   To add other options:
     SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS
     DEBUG : LayoutTests/fast/js/no-good.js = TIMEOUT PASS
     DEBUG SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS
     LINUX DEBUG SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS
     DEFER LINUX WIN : LayoutTests/fast/js/no-good.js = TIMEOUT PASS
 
   SKIP: Doesn't run the test.
+  SLOW: The test takes a long time to run, but does not timeout indefinitely.
   WONTFIX: For tests that we never intend to pass on a given platform.
   DEFER: Test does not count in our statistics for the current release.
   DEBUG: Expectations apply only to the debug build.
   RELEASE: Expectations apply only to release build.
   LINUX/WIN/MAC: Expectations apply only to these platforms.
 
-  A test can be included twice, but not via the same path. If a test is included
-  twice, then the more precise path wins.
+  Notes:
+    -A test cannot be both SLOW and TIMEOUT
+    -A test cannot be both DEFER and WONTFIX
+    -A test can be included twice, but not via the same path.
+    -If a test is included twice, then the more precise path wins.
+    -CRASH tests cannot be DEFER or WONTFIX
   """
 
   EXPECTATIONS = { 'pass': PASS,
                    'fail': FAIL,
                    'timeout': TIMEOUT,
                    'crash': CRASH }
 
   PLATFORMS = [ 'mac', 'linux', 'win' ]
   
   BUILD_TYPES = [ 'debug', 'release' ]
 
   MODIFIERS = { 'skip': SKIP,
                 'wontfix': WONTFIX,
                 'defer': DEFER,
+                'slow': SLOW,
                 'none': NONE }
 
   def __init__(self, path, full_test_list, platform, is_debug_mode):
     """
     path: The path to the expectation file. An error is thrown if a test is
         listed more than once.
     full_test_list: The list of all tests to be run pending processing of the
         expections for those tests.
     platform: Which platform from self.PLATFORMS to filter tests for.
     is_debug_mode: Whether we testing a test_shell built debug mode.
@@ skipping 111 matching lines @@
 
       tests_and_expecation_parts = test_and_expectations.split('=')
       if (len(tests_and_expecation_parts) is not 2):
         self._AddError(lineno, 'Missing expectations.', test_and_expectations)
         continue
 
       test_list_path = tests_and_expecation_parts[0].strip()
       expectations = self._ParseExpectations(tests_and_expecation_parts[1],
           lineno, test_list_path)
 
+      if 'slow' in options and TIMEOUT in expectations:
+        self._AddError(lineno, 'A test cannot be both slow and timeout. If the '
+            'test times out indefinitely, the it should be listed as timeout.',
+            test_and_expectations)
+
       full_path = os.path.join(path_utils.LayoutDataDir(), test_list_path)
       full_path = os.path.normpath(full_path)
       # WebKit's way of skipping tests is to add a -disabled suffix.
       # So we should consider the path existing if the path or the -disabled
       # version exists.
       if not os.path.exists(full_path) and not \
         os.path.exists(full_path + '-disabled'):
         # Log a non fatal error here since you hit this case any time you
         # update test_expectations.txt without syncing the LayoutTests
         # directory
@@ skipping 101 matching lines @@
   def _AddError(self, lineno, msg, path):
     """Reports an error that will prevent running the tests. Does not
     immediately raise an exception because we'd like to aggregate all the
     errors so they can all be printed out."""
     self._errors.append('\nLine:%s %s %s' % (lineno, msg, path))
 
   def _LogNonFatalError(self, lineno, msg, path):
     """Reports an error that will not prevent running the tests. These are
     still errors, but not bad enough to warrant breaking test running."""
     self._non_fatal_errors.append('Line:%s %s %s' % (lineno, msg, path))