rebaseline_server: apply ignored-tests.txt within rebaseline_server, not just on the bots

gm/rebaseline_server/compare_to_expectations.py

 #!/usr/bin/python
 
 """
 Copyright 2013 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
 """
@@ skipping 30 matching lines @@
 import imagepair
 import imagepairset
 import results
 
 EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [
     results.KEY__EXPECTATIONS__BUGS,
     results.KEY__EXPECTATIONS__IGNOREFAILURE,
     results.KEY__EXPECTATIONS__REVIEWED,
 ]
 DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
+DEFAULT_IGNORE_FAILURES_FILE = 'ignored-tests.txt'
 
 IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image')
 
 
 class ExpectationComparisons(results.BaseComparisons):
   """Loads actual and expected GM results into an ImagePairSet.
 
   Loads actual and expected results from all builders, except for those skipped
   by _ignore_builder().
 
   Once this object has been constructed, the results (in self._results[])
   are immutable.  If you want to update the results based on updated JSON
   file contents, you will need to create a new ExpectationComparisons object."""
 
   def __init__(self, actuals_root=results.DEFAULT_ACTUALS_DIR,
                expected_root=DEFAULT_EXPECTATIONS_DIR,
+               ignore_failures_file=DEFAULT_IGNORE_FAILURES_FILE,
                generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
                diff_base_url=None, builder_regex_list=None):
     """
     Args:
       actuals_root: root directory containing all actual-results.json files
       expected_root: root directory containing all expected-results.json files
+      ignore_failures_file: if a file with this name is found within
+          expected_root, ignore failures for any tests listed in the file
       generated_images_root: directory within which to create all pixel diffs;
           if this directory does not yet exist, it will be created
       diff_base_url: base URL within which the client should look for diff
           images; if not specified, defaults to a "file:///" URL representation
           of generated_images_root
       builder_regex_list: List of regular expressions specifying which builders
           we will process. If None, process all builders.
     """
     time_start = int(time.time())
     if builder_regex_list != None:
       self.set_match_builders_pattern_list(builder_regex_list)
     self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
     self._diff_base_url = (
         diff_base_url or
         download_actuals.create_filepath_url(generated_images_root))
     self._actuals_root = actuals_root
     self._expected_root = expected_root
+    self._ignore_failures_on_these_tests = []
+    if ignore_failures_file:
+      self._ignore_failures_on_these_tests = (
+          ExpectationComparisons._read_noncomment_lines(
+              os.path.join(expected_root, ignore_failures_file)))
     self._load_actual_and_expected()
     self._timestamp = int(time.time())
     logging.info('Results complete; took %d seconds.' %
                  (self._timestamp - time_start))
 
   def edit_expectations(self, modifications):
     """Edit the expectations stored within this object and write them back
     to disk.
 
     Note that this will NOT update the results stored in self._results[] ;
@@ skipping 209 matching lines @@
           # from the most recent actuals).  Treat these as successes
           # instead of failures.
           #
           # TODO(epoger): Do we need to do something similar in
           # other cases, such as when we have recently marked a test
           # as ignoreFailure but it still shows up in the 'failed'
           # category?  Maybe we should not rely on the result_type
           # categories recorded within the gm_actuals AT ALL, and
           # instead evaluate the result_type ourselves based on what
           # we see in expectations vs actual checksum?
-          # See related http://skbug.com/2514 ('rebaseline_server: apply
-          # ignored-tests.txt within rebaseline_server, not just on the bots')
           if expected_image_relative_url == actual_image_relative_url:
             updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED
+          elif ((result_type == results.KEY__RESULT_TYPE__FAILED) and
+                (test in self._ignore_failures_on_these_tests)):
+            updated_result_type = results.KEY__RESULT_TYPE__FAILUREIGNORED
           else:
             updated_result_type = result_type
           extra_columns_dict = {
               results.KEY__EXTRACOLUMNS__RESULT_TYPE: updated_result_type,
               results.KEY__EXTRACOLUMNS__BUILDER: builder,
               results.KEY__EXTRACOLUMNS__TEST: test,
               results.KEY__EXTRACOLUMNS__CONFIG: config,
           }
           try:
             image_pair = imagepair.ImagePair(
@@ skipping 22 matching lines @@
   parser = argparse.ArgumentParser()
   parser.add_argument(
       '--actuals', default=results.DEFAULT_ACTUALS_DIR,
       help='Directory containing all actual-result JSON files; defaults to '
       '\'%(default)s\' .')
   parser.add_argument(
       '--expectations', default=DEFAULT_EXPECTATIONS_DIR,
       help='Directory containing all expected-result JSON files; defaults to '
       '\'%(default)s\' .')
   parser.add_argument(
+      '--ignore-failures-file', default=DEFAULT_IGNORE_FAILURES_FILE,
+      help='If a file with this name is found within the EXPECTATIONS dir, '
+      'ignore failures for any tests listed in the file; defaults to '
+      '\'%(default)s\' .')
+  parser.add_argument(
       '--outfile', required=True,
       help='File to write result summary into, in JSON format.')
   parser.add_argument(
       '--results', default=results.KEY__HEADER__RESULTS_FAILURES,
       help='Which result types to include. Defaults to \'%(default)s\'; '
       'must be one of ' +
       str([results.KEY__HEADER__RESULTS_FAILURES,
            results.KEY__HEADER__RESULTS_ALL]))
   parser.add_argument(
       '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT,
       help='Directory within which to download images and generate diffs; '
       'defaults to \'%(default)s\' .')
   args = parser.parse_args()
-  results_obj = ExpectationComparisons(actuals_root=args.actuals,
-                                       expected_root=args.expectations,
-                                       generated_images_root=args.workdir)
+  results_obj = ExpectationComparisons(
+      actuals_root=args.actuals,
+      expected_root=args.expectations,
+      ignore_failures_file=args.ignore_failures_file,
+      generated_images_root=args.workdir)
   gm_json.WriteToFile(
       results_obj.get_packaged_results_of_type(results_type=args.results),
       args.outfile)
 
 
 if __name__ == '__main__':
   main()