Change update_test_expectations to not require builders for all configurations.

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/update_test_expectations.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Updates TestExpectations based on results in builder bots.
 
 Scans the TestExpectations file and uses results from actual builder bots runs
 to remove tests that are marked as flaky but don't fail in the specified way.
 
 E.g. If a test has this expectation:
@@ skipping 15 matching lines @@
 import argparse
 import logging
 
 from webkitpy.layout_tests.models.test_expectations import TestExpectations
 
 _log = logging.getLogger(__name__)
 
 
 def main(host, bot_test_expectations_factory, argv):
     parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter)
-    parser.parse_args(argv)
+    parser.add_argument('--verbose', '-v', action='store_true', default=False, help='enable more verbose logging')
+    args = parser.parse_args(argv)
+
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(levelname)s: %(message)s")
 
     port = host.port_factory.get()
-
-    logging.basicConfig(level=logging.INFO, format="%(message)s")
-
     expectations_file = port.path_to_generic_test_expectations_file()
     if not host.filesystem.isfile(expectations_file):
         _log.warning("Didn't find generic expectations file at: " + expectations_file)
         return 1
 
     remove_flakes_o_matic = RemoveFlakesOMatic(host,
                                                port,
                                                bot_test_expectations_factory)
 
     test_expectations = remove_flakes_o_matic.get_updated_test_expectations()
 
     remove_flakes_o_matic.write_test_expectations(test_expectations,
                                                   expectations_file)
     return 0
 
 
 class RemoveFlakesOMatic(object):
+
     def __init__(self, host, port, bot_test_expectations_factory):
         self._host = host
         self._port = port
         self._expectations_factory = bot_test_expectations_factory
         self._builder_results_by_path = {}
 
     def _can_delete_line(self, test_expectation_line):
         """Returns whether a given line in the expectations can be removed.
 
         Uses results from builder bots to determine if a given line is stale and
@@ skipping 16 matching lines @@
         # Don't check lines that have expectations like NeedsRebaseline or Skip.
         if self._has_unstrippable_expectations(expectations):
             return False
 
         # Don't check lines unless they're flaky. i.e. At least one expectation is a PASS.
         if not self._has_pass_expectation(expectations):
             return False
 
         # The line can be deleted if the only expectation on the line that appears in the actual
         # results is the PASS expectation.
+        builders_checked = []
         for config in test_expectation_line.matching_configurations:
             builder_name = self._host.builders.builder_name_for_specifiers(config.version, config.build_type)
 
             if not builder_name:
-                _log.error('Failed to get builder for config [%s, %s, %s]',
-                           config.version, config.architecture, config.build_type)
+                _log.debug('No builder with config %s', config)
                 # TODO(bokan): Matching configurations often give us bots that don't have a
                 # builder in builders.py's exact_matches. Should we ignore those or be conservative
                 # and assume we need these expectations to make a decision?
-                return False
+                continue
+
+            builders_checked.append(builder_name)
 
             if builder_name not in self._builder_results_by_path.keys():
                 _log.error('Failed to find results for builder "%s"', builder_name)
                 return False
 
             results_by_path = self._builder_results_by_path[builder_name]
 
             # No results means the tests were all skipped or all results are passing.
             if test_expectation_line.path not in results_by_path.keys():
                 continue
 
             results_for_single_test = results_by_path[test_expectation_line.path]
 
             if self._expectations_that_were_met(test_expectation_line, results_for_single_test) != set(['PASS']):
                 return False
 
+        if builders_checked:
+            _log.debug('Checked builders:\n  %s', '\n  '.join(builders_checked))
+        else:
+            _log.warning('No matching builders for line, deleting line.')
+        _log.info('Deleting line "%s"', test_expectation_line.original_string.strip())
         return True
 
     def _has_pass_expectation(self, expectations):
         return 'PASS' in expectations
 
     def _expectations_that_were_met(self, test_expectation_line, results_for_single_test):
         """Returns the set of expectations that appear in the given results.
 
         e.g. If the test expectations is "bug(test) fast/test.html [Crash Failure Pass]"
         and the results are ['TEXT', 'PASS', 'PASS', 'TIMEOUT'], then this method would
@@ skipping 151 matching lines @@
         """Writes the given TestExpectations object to the filesystem.
 
         Args:
             test_expectations: The TestExpectations object to write.
             test_expectations_file: The full file path of the Blink
                 TestExpectations file. This file will be overwritten.
         """
         self._host.filesystem.write_text_file(
             test_expectations_file,
             TestExpectations.list_to_string(test_expectations, reconstitute_only_these=[]))