Fix sorting of test list so Purify test slices are alphabetically consecutive...

webkit/tools/layout_tests/run_webkit_tests.py

 #!/bin/env python
 # Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Run layout tests using the test_shell.
 
 This is a port of the existing webkit test script run-webkit-tests.
 
 The TestRunner class runs a series of tests (TestType interface) against a set
@@ skipping 59 matching lines @@
     Args:
       options: a dictionary of command line options
       paths: a list of paths to crawl looking for test files
     """
     self._options = options
 
     self._http_server = http_server.Lighttpd(options.results_directory)
     # a list of TestType objects
     self._test_types = []
 
-    # a set of test files
+    # a set of test files, and the same tests as a list
     self._test_files = set()
+    self._test_files_list = None
     self._file_dir = os.path.join(os.path.dirname(sys.argv[0]), TEST_FILE_DIR)
     self._file_dir = path_utils.GetAbsolutePath(self._file_dir)
 
     if options.lint_test_files:
       # Creating the expecations for each platform/target pair does all the
       # test list parsing and ensures it's correct syntax(e.g. no dupes).
       self._ParseExpectations('win', is_debug_mode=True)
       self._ParseExpectations('win', is_debug_mode=False)
       self._ParseExpectations('mac', is_debug_mode=True)
       self._ParseExpectations('mac', is_debug_mode=False)
       self._ParseExpectations('linux', is_debug_mode=True)
       self._ParseExpectations('linux', is_debug_mode=False)
     else:
       self._GatherTestFiles(paths)
       self._expectations = self._ParseExpectations(
           platform_utils.GetTestListPlatformName().lower(),
           options.target == 'Debug')
-      self._GenerateExpecationsAndPrintOutput()
+      self._PrepareListsAndPrintOutput()
 
   def __del__(self):
     sys.stdout.flush()
     sys.stderr.flush()
     # Stop the http server.
     self._http_server.Stop()
 
   def _GatherTestFiles(self, paths):
     """Generate a set of test files and place them in self._test_files
 
@@ skipping 45 matching lines @@
       return test_expectations.TestExpectations(test_files,
                                                 self._file_dir,
                                                 platform,
                                                 is_debug_mode)
     except SyntaxError, err:
       if self._options.lint_test_files:
         print str(err)
       else:
         raise err
 
-  def _GenerateExpecationsAndPrintOutput(self):
-    """Create appropriate subsets of self._tests_files in 
+  def _PrepareListsAndPrintOutput(self):
+    """Create appropriate subsets of test lists and print test counts.
+
+    Create appropriate subsets of self._tests_files in
     self._ignored_failures, self._fixable_failures, and self._fixable_crashes.
+    Also remove skipped files from self._test_files, extract a subset of tests
+    if desired, and create the sorted self._test_files_list.
     """
     # Filter and sort out files from the skipped, ignored, and fixable file
     # lists.
     saved_test_files = set()
     if len(self._test_files) == 1:
       # If there's only one test file, we don't want to skip it, but we do want
       # to sort it.  So we save it to add back to the list later.
       saved_test_files = self._test_files
 
     # Remove skipped - both fixable and ignored - files from the
     # top-level list of files to test.
     skipped = (self._expectations.GetFixableSkipped() |
                self._expectations.GetIgnoredSkipped())
 
     self._test_files -= skipped
 
     # If there was only one test file, run the test even if it was skipped.
     if len(saved_test_files):
       self._test_files = saved_test_files
 
     logging.info('Skipped: %d tests' % len(skipped))
     logging.info('Skipped tests do not appear in any of the below numbers\n')
 
+    # Create a sorted list of test files so the subset chunk, if used, contains
+    # alphabetically consecutive tests.
+    self._test_files_list = list(self._test_files)
+    if self._options.randomize_order:
+      random.shuffle(self._test_files_list)
+    else:
+      self._test_files_list.sort(self.TestFilesSort)
+
     # If the user specifies they just want to run a subset chunk of the tests,
     # just grab a subset of the non-skipped tests.
     if self._options.run_chunk:
-      test_files = list(self._test_files)
+      test_files = self._test_files_list
       try:
         (chunk_num, chunk_len) = self._options.run_chunk.split(":")
         chunk_num = int(chunk_num)
         assert(chunk_num >= 0)
         chunk_len = int(chunk_len)
         assert(chunk_len > 0)
       except:
         logging.critical("invalid chunk '%s'" % self._options.run_chunk)
         sys.exit(1)
       num_tests = len(test_files)
       slice_start = (chunk_num * chunk_len) % num_tests
       slice_end = min(num_tests + 1, slice_start + chunk_len)
       files = test_files[slice_start:slice_end]
       logging.info('Run: %d tests (chunk slice [%d:%d] of %d)' % (
           chunk_len, slice_start, slice_end, num_tests))
       if slice_end - slice_start < chunk_len:
         extra = 1 + chunk_len - (slice_end - slice_start)
         logging.info('   last chunk is partial, appending [0:%d]' % extra)
         files.extend(test_files[0:extra])
+      self._test_files_list = files
       self._test_files = set(files)
       # update expectations so that the stats are calculated correctly
       self._expectations = self._ParseExpectations(
           platform_utils.GetTestListPlatformName().lower(),
           options.target == 'Debug')
     else:
       logging.info('Run: %d tests' % len(self._test_files))
 
     logging.info('Deferred: %d tests' % 
                  len(self._expectations.GetFixableDeferred()))
@@ skipping 64 matching lines @@
       if proc.wait() != 0:
         logging.info("Aborting because system dependencies check failed.\n"
                      "To override, invoke with --nocheck-sys-deps")
         sys.exit(1)
 
     logging.info("Starting tests")
 
     # Create the output directory if it doesn't already exist.
     google.path_utils.MaybeMakeDirectory(self._options.results_directory)
 
-    test_files = list(self._test_files)
-    if self._options.randomize_order:
-      random.shuffle(test_files)
-    else:
-      test_files.sort(self.TestFilesSort)
+    test_files = self._test_files_list
 
     # Create the thread safe queue of (test filenames, test URIs) tuples. Each
     # TestShellThread pulls values from this queue.
     filename_queue = Queue.Queue()
     for test_file in test_files:
       filename_queue.put((test_file, path_utils.FilenameToUri(test_file)))
 
     # If we have http tests, the first one will be an http test.
     if test_files and test_files[0].find(self.HTTP_SUBDIR) >= 0:
       self._http_server.Start()
@@ skipping 456 matching lines @@
   option_parser.add_option("", "--randomize-order", action="store_true",
                            default=False,
                            help=("Run tests in random order (useful for "
                                  "tracking down corruption)"))
   option_parser.add_option("", "--run-chunk",
                            default=None,
                            help=("Run a specified chunk (n:l), the nth of len l"
                                  ", of the layout tests"))
   options, args = option_parser.parse_args()
   main(options, args)