Revise run_webkit_test.PrepareListsAndPrintOutput() to keep accurate counts...

webkit/tools/layout_tests/run_webkit_tests.py

 #!/usr/bin/env python
 # Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Run layout tests using the test_shell.
 
 This is a port of the existing webkit test script run-webkit-tests.
 
 The TestRunner class runs a series of tests (TestType interface) against a set
@@ skipping 186 matching lines @@
     except Exception, err:
       if self._options.lint_test_files:
         print str(err)
       else:
         raise err
 
   def PrepareListsAndPrintOutput(self):
     """Create appropriate subsets of test lists and returns a ResultSummary
     object. Also prints expected test counts."""
 
-    result_summary = ResultSummary(self._expectations, self._test_files)
-
     # Remove skipped - both fixable and ignored - files from the
     # top-level list of files to test.
-    skipped = set()
-    if len(self._test_files) > 1 and not self._options.force:
+    num_all_test_files = len(self._test_files)
+    skipped = ()
+    if num_all_test_files > 1 and not self._options.force:
       skipped = self._expectations.GetTestsWithResultType(
-          test_expectations.SKIP)
-      for test in skipped:
-        result_summary.Add(test, [], test_expectations.SKIP)
+                     test_expectations.SKIP)

[ojan, 2009/12/04 00:58:30]

nit: indent should be 4 spaces here.

       self._test_files -= skipped
 
-    if self._options.force:
-      logging.info('Skipped: 0 tests (--force)')
-    else:
-      logging.info('Skipped: %d tests' % len(skipped))
-    logging.info('Skipped tests do not appear in any of the below numbers\n')
-
     # Create a sorted list of test files so the subset chunk, if used, contains
     # alphabetically consecutive tests.
     self._test_files_list = list(self._test_files)
     if self._options.randomize_order:
       random.shuffle(self._test_files_list)
     else:
       self._test_files_list.sort(self.TestFilesSort)
 
-    # Chunking replaces self._expectations, which loses all the skipped test
-    # information. Keep the prechunk expectations for tracking number of
-    # skipped tests.
-    self.prechunk_expectations = self._expectations;
-
     # If the user specifies they just want to run a subset of the tests,
     # just grab a subset of the non-skipped tests.
     if self._options.run_chunk or self._options.run_part:
       chunk_value = self._options.run_chunk or self._options.run_part
       test_files = self._test_files_list
       try:
         (chunk_num, chunk_len) = chunk_value.split(":")
         chunk_num = int(chunk_num)
         assert(chunk_num >= 0)
         test_size = int(chunk_len)
@@ skipping 23 matching lines @@
           rounded_tests = num_tests + test_size - (num_tests % test_size)
 
         chunk_len = rounded_tests / test_size
         slice_start = chunk_len * (chunk_num - 1)
         # It does not mind if we go over test_size.
 
       # Get the end offset of the slice.
       slice_end = min(num_tests, slice_start + chunk_len)
 
       files = test_files[slice_start:slice_end]
+
       tests_run_msg = 'Run: %d tests (chunk slice [%d:%d] of %d)' % (
           (slice_end - slice_start), slice_start, slice_end, num_tests)
       logging.info(tests_run_msg)
 
       # If we reached the end and we don't have enough tests, we run some
       # from the beginning.
       if self._options.run_chunk and (slice_end - slice_start < chunk_len):
         extra = 1 + chunk_len - (slice_end - slice_start)
         extra_msg = '   last chunk is partial, appending [0:%d]' % extra
         logging.info(extra_msg)
         tests_run_msg += "\n" + extra_msg
         files.extend(test_files[0:extra])
-      self._test_files_list = files
-      self._test_files = set(files)
-
       tests_run_filename = os.path.join(self._options.results_directory,
                                         "tests_run.txt")
       tests_run_file = open(tests_run_filename, "w")
       tests_run_file.write(tests_run_msg + "\n")
       tests_run_file.close()
 
-      # update expectations so that the stats are calculated correctly
+      # Update expectations so that the stats are calculated correctly.
+      len_skip_chunk = int(len(files) * len(skipped) /
+                           float(len(self._test_files)))
+      skip_chunk_list = list(skipped)[0:len_skip_chunk]
+      skip_chunk = set(skip_chunk_list)
+      self._test_files_list = files + skip_chunk_list
+      self._test_files = set(self._test_files_list)
+
       self._expectations = self.ParseExpectations(
           path_utils.PlatformName(), options.target == 'Debug')
+
+      self._test_files = set(files)

[ojan, 2009/12/04 00:58:30]

Maybe add some comments here? It's confusing that we set _test_files, then set
it to something different after ParseExpectations. I get it that the point is to
calculate statistics correctly, but I had to stare at this code for a while to
figure that out (and to get that this wasn't actually running skipped tests).

+      self._test_files_list = files
     else:
+      skip_chunk = skipped
       logging.info('Run: %d tests' % len(self._test_files))
 
+    result_summary = ResultSummary(self._expectations,
+                                   self._test_files | skip_chunk)
     self._PrintExpectedResultsOfType(result_summary, test_expectations.PASS,
         "passes")
     self._PrintExpectedResultsOfType(result_summary, test_expectations.FAIL,
         "failures")
     self._PrintExpectedResultsOfType(result_summary, test_expectations.FLAKY,
         "flaky")
     self._PrintExpectedResultsOfType(result_summary, test_expectations.SKIP,
         "skipped")
+
+    if self._options.force:
+      logging.info('Running all tests, including skips (--force)')
+    else:
+      for test in skip_chunk:
+        result_summary.Add(test, [], test_expectations.SKIP)
+
     return result_summary
 
   def AddTestType(self, test_type):
     """Add a TestType to the TestRunner."""
     self._test_types.append(test_type)
 
   # We sort the tests so that tests using the http server will run first.  We
   # are seeing some flakiness, maybe related to apache getting swapped out,
   # slow, or stuck after not serving requests for a while.
   def TestFilesSort(self, x, y):
@@ skipping 1112 matching lines @@
                                  "this script."))
   option_parser.add_option("", "--find-baselines", action="store_true",
                            default=False,
                            help="Prints a table mapping tests to their "
                                 "expected results")
   option_parser.add_option("", "--experimental-fully-parallel",
                            action="store_true", default=False,
                            help="run all tests in parallel")
   options, args = option_parser.parse_args()
   main(options, args)