In update-test-expectation, skip lines with directories.

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/update_test_expectations.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Updates TestExpectations based on results in builder bots.
 
 Scans the TestExpectations file and uses results from actual builder bots runs
 to remove tests that are marked as flaky but don't fail in the specified way.
 
 E.g. If a test has this expectation:
@@ skipping 86 matching lines @@
             return False
 
         # Don't check lines that have expectations like NeedsRebaseline or Skip.
         if self._has_unstrippable_expectations(expectations):
             return False
 
         # Don't check lines unless they're flaky. i.e. At least one expectation is a PASS.
         if not self._has_pass_expectation(expectations):
             return False
 
+        # Don't check lines that have expectations for directories, since
+        # the flakiness of all sub-tests isn't as easy to check.
+        if self._is_directory(test_expectation_line.name):
+            return False
+
         # The line can be deleted if the only expectation on the line that appears in the actual
         # results is the PASS expectation.
         builders_checked = []
         for config in test_expectation_line.matching_configurations:
             builder_name = self._host.builders.builder_name_for_specifiers(config.version, config.build_type)
 
             if not builder_name:
                 _log.debug('No builder with config %s', config)
                 # For many configurations, there is no matching builder in
                 # webkitpy/common/config/builders.py. We ignore these
@@ skipping 67 matching lines @@
 
         Returns:
             True if at least one of the expectations is unstrippable. False
             otherwise.
         """
         unstrippable_expectations = ('REBASELINE', 'NEEDSREBASELINE',
                                      'NEEDSMANUALREBASELINE', 'SLOW',
                                      'SKIP')
         return any(s in expectations for s in unstrippable_expectations)
 
+    def _is_directory(self, name):
+        return [name] != self._port.tests([name])

[bokan, 2017/02/07 20:06:28]

This will return true for tests files that don't exist or are typo'd. Maybe we
should also check that the path exists on the filesystem?

[qyearsley, 2017/02/07 21:22:42]

That sounds reasonable. In the latest patch I've now changed this to just
directly check whether the path on the test expectation line is a directory;
does the latest patch look OK?

+
     def _get_builder_results_by_path(self):
         """Returns a dictionary of results for each builder.
 
         Returns a dictionary where each key is a builder and value is a dictionary containing
         the distinct results for each test. E.g.
 
         {
             'WebKit Linux Precise': {
                   'test1.html': ['PASS', 'IMAGE'],
                   'test2.html': ['PASS'],
@@ skipping 126 matching lines @@
         """Writes the given TestExpectations object to the filesystem.
 
         Args:
             test_expectations: The TestExpectations object to write.
             test_expectations_file: The full file path of the Blink
                 TestExpectations file. This file will be overwritten.
         """
         self._host.filesystem.write_text_file(
             test_expectations_file,
             TestExpectations.list_to_string(test_expectations, reconstitute_only_these=[]))