Added update_test_expectations script to remove non-flaky test expectations.

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/layout_package/bot_test_expectations.py

 # Copyright (C) 2013 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 # copyright notice, this list of conditions and the following disclaimer
@@ skipping 122 matching lines @@
         if not results_json:
             return None
         return BotTestExpectations(results_json)
 
     def expectations_for_builder(self, builder):
         results_json = self._results_json_for_builder(builder)
         if not results_json:
             return None
         return BotTestExpectations(results_json)
 
 class BotTestExpectations(object):

[qyearsley, 2016/03/07 19:09:53]

Does a BotTestExpectations object represent the set of layout test expectations
for a given single bot and test combination?

[bokan, 2016/03/07 22:59:15]

No, it's the results of running the layout tests on the bots. I admit the name
is a bit misleading.

     # FIXME: Get this from the json instead of hard-coding it.
     RESULT_TYPES_TO_IGNORE = ['N', 'X', 'Y']  # NO_DATA, SKIP, NOTRUN
 
     # TODO(ojan): Remove this once crbug.com/514378 is fixed.
     # The JSON can contain results for expectations, not just actual result types.
     NON_RESULT_TYPES = ['S', 'X']  # SLOW, SKIP
 
     # specifiers arg is used in unittests to avoid the static dependency on builders.
     def __init__(self, results_json, specifiers=None):
         self.results_json = results_json
@@ skipping 59 matching lines @@
             additional_expectations = set(e for e in result_exp if not expected(e))
 
             # Test did not have unexpected results.
             if not additional_expectations:
                 continue
 
             expectations.update(additional_expectations)
             unexpected_results_by_path[test_path] = sorted(map(exp_to_string, expectations))
         return unexpected_results_by_path
 
+    def all_results_by_path(self):
+        results_by_path = {}
+        for test_path, entry in self.results_json.walk_results():
+            results_dict = entry.get(self.results_json.RESULTS_KEY, {})
+
+            result_types = self._all_types_in_results(results_dict)
+
+            if not result_types:
+                continue
+
+            # Distinct results as non-encoded strings.
+            result_strings = map(self.results_json.expectation_for_type, result_types)

[qyearsley, 2016/03/07 19:09:53]

What are the possible result strings that are returned here? What would it mean
for the result strings to be encoded?

[bokan, 2016/03/07 22:59:15]

The result codes come from the bot so I'm not sure these are exhaustive and if
they vary by bot, this came off the Nexus 4 bot:

u'A': u'AUDIO',
 u'C': u'CRASH',
 u'F': u'TEXT',
 u'I': u'IMAGE',
 u'K': u'LEAK',
 u'L': u'FLAKY',
 u'N': u'NO DATA',
 u'O': u'MISSING',
 u'P': u'PASS',
 u'Q': u'FAIL',
 u'S': u'SLOW',
 u'T': u'TIMEOUT',
 u'U': u'UNKNOWN',
 u'V': u'VERYFLAKY',
 u'X': u'SKIP',
 u'Y': u'NOTRUN',
 u'Z': u'IMAGE+TEXT'}

The encoded version is the one letter version which is what's stored in the
JSON. the expectation_for_type method converts to the human readable string.

+
+            results_by_path[test_path] = sorted(result_strings)
+        return results_by_path
+
     def expectation_lines(self, only_ignore_very_flaky):
         lines = []
         for test_path, entry in self.results_json.walk_results():
             flaky_types = self._flaky_types_in_results(entry, only_ignore_very_flaky)
             if len(flaky_types) > 1:
                 line = self._line_from_test_and_flaky_types(test_path, flaky_types)
                 lines.append(line)
         return lines
 
     def _all_types_in_results(self, run_length_encoded_results):
@@ skipping 56 matching lines @@
                 # The only thing we lose by not fixing this is that a test that was flaky
                 # and got fixed will still get printed out until 100 runs have passed.
                 if not TestExpectations.result_was_expected(result_enum, latest_expectations, test_needs_rebaselining=False):
                     has_unexpected_results = True
                     break
 
             if has_unexpected_results:
                 flaky_results = flaky_results.union(set(result_types))
 
         return flaky_results