Truly ignore failures in skimage.

tools/skimage_main.cpp

 /*
  * Copyright 2011 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 #include "gm_expectations.h"
 #include "SkBitmap.h"
 #include "SkColorPriv.h"
@@ skipping 75 matching lines @@
         if (dot != NULL) {
             int32_t index = SkToS32(dot - cstyleDst);
             dst->remove(index, dst->size() - index);
         }
         dst->append(suffix);
     }
 }
 
 // Store the names of the filenames to report later which ones failed, succeeded, and were
 // invalid.
+// FIXME: Add more arrays, for more specific types of errors, and make the output simpler.
+// If each array holds one type of error, the output can change from:
+//
+// Failures:
+//      <image> failed for such and such reason
+//      <image> failed for some different reason
+//
+// to:
+//
+// Such and such failures:
+//      <image>
+//
+// Different reason failures:
+//      <image>
+//
 static SkTArray<SkString, false> gInvalidStreams;
 static SkTArray<SkString, false> gMissingCodecs;
 static SkTArray<SkString, false> gDecodeFailures;
 static SkTArray<SkString, false> gEncodeFailures;
 static SkTArray<SkString, false> gSuccessfulDecodes;
 static SkTArray<SkString, false> gSuccessfulSubsetDecodes;
 static SkTArray<SkString, false> gFailedSubsetDecodes;
 // Files/subsets that do not have expectations. Not reported as a failure of the test so
 // the bots will not turn red with each new image test.
 static SkTArray<SkString, false> gMissingExpectations;
 static SkTArray<SkString, false> gMissingSubsetExpectations;
+// For files that are expected to fail.
+static SkTArray<SkString, false> gKnownFailures;
+static SkTArray<SkString, false> gKnownSubsetFailures;
 
 static SkBitmap::Config gPrefConfig(SkBitmap::kNo_Config);
 
 // Expections read from a file specified by readExpectationsPath. The expectations must have been
 // previously written using createExpectationsPath.
 SkAutoTUnref<skiagm::JsonExpectationsSource> gJsonExpectations;
 
 static bool write_bitmap(const char outName[], const SkBitmap& bm) {
     if (SkImageEncoder::EncodeFile(outName, bm, SkImageEncoder::kPNG_Type, 100)) {
         return true;
@@ skipping 64 matching lines @@
 static void write_expectations(const SkBitmap& bitmap, const char* filename) {
     if (!FLAGS_createExpectationsPath.isEmpty()) {
         // Creates an Expectations object, and add it to the list to write.
         skiagm::Expectations expectation(bitmap);
         Json::Value value = expectation.asJsonValue();
         gExpectationsToWrite[filename] = value;
     }
 }
 
 /**
- *  Return true if this filename is a known failure, and therefore a failure
- *  to decode should be ignored.
- */
-static bool expect_to_fail(const char* filename) {
-    if (NULL == gJsonExpectations.get()) {
-        return false;
-    }
-    skiagm::Expectations jsExpectations = gJsonExpectations->get(filename);
-    return jsExpectations.ignoreFailure();
-}
-
-/**
  *  Compare against an expectation for this filename, if there is one.
  *  @param digest GmResultDigest, computed from the decoded bitmap, to compare to the
- *           expectation.
+ *          expectation.
  *  @param filename String used to find the expected value.
  *  @param failureArray Array to add a failure message to on failure.
- *  @param missingArray Array to add missing expectation to on failure.
+ *  @param missingArray Array to add failure message to when missing image
+ *          expectation.
+ *  @param ignoreArray Array to add failure message to when the image does not match
+ *          the expectation, but this is a failure we can ignore.
  *  @return bool True in any of these cases:
  *                  - the bitmap matches the expectation.
  *               False in any of these cases:
  *                  - there is no expectations file.
  *                  - there is an expectations file, but no expectation for this bitmap.
  *                  - there is an expectation for this bitmap, but it did not match.
  *                  - expectation could not be computed from the bitmap.
  */
 static bool compare_to_expectations_if_necessary(const skiagm::GmResultDigest& digest,
                                                  const char* filename,
                                                  SkTArray<SkString, false>* failureArray,
-                                                 SkTArray<SkString, false>* missingArray) {
+                                                 SkTArray<SkString, false>* missingArray,
+                                                 SkTArray<SkString, false>* ignoreArray) {
     if (!digest.isValid()) {
         if (failureArray != NULL) {
             failureArray->push_back().printf("decoded %s, but could not create a GmResultDigest.",
                                              filename);
         }
         return false;
     }
 
     if (NULL == gJsonExpectations.get()) {
         return false;
     }
 
     skiagm::Expectations jsExpectation = gJsonExpectations->get(filename);
     if (jsExpectation.empty()) {
         if (missingArray != NULL) {
             missingArray->push_back().printf("decoded %s, but could not find expectation.",
                                              filename);
         }
         return false;
     }
 
     if (jsExpectation.match(digest)) {
         return true;
     }
 
-    if (failureArray != NULL) {
+    if (jsExpectation.ignoreFailure()) {
+        ignoreArray->push_back().printf("%s does not match expectation, but this is known.",
+                                        filename);
+    } else if (failureArray != NULL) {
         failureArray->push_back().printf("decoded %s, but the result does not match "
                                          "expectations.",
                                          filename);
     }
     return false;
 }
 
 /**
  *  Helper function to write a bitmap subset to a file. Only called if subsets were created
  *  and a writePath was provided. Creates a subdirectory called 'subsets' and writes a PNG to
@@ skipping 146 matching lines @@
     codec->setSkipWritingZeroes(FLAGS_skip);
     codec->setSampleSize(FLAGS_sampleSize);
     stream.rewind();
 
     // Create a string representing just the filename itself, for use in json expectations.
     SkString basename = SkOSPath::SkBasename(srcPath);
     const char* filename = basename.c_str();
 
     if (!codec->decode(&stream, &bitmap, gPrefConfig,
                        SkImageDecoder::kDecodePixels_Mode)) {
-        if (expect_to_fail(filename)) {
-            gSuccessfulDecodes.push_back().appendf(
-                "failed to decode %s, which is a known failure.", srcPath);
-        } else {
-            gDecodeFailures.push_back().set(srcPath);
+        if (NULL != gJsonExpectations.get()) {
+            skiagm::Expectations jsExpectations = gJsonExpectations->get(filename);
+            if (jsExpectations.ignoreFailure()) {
+                // This is a known failure.
+                gKnownFailures.push_back().appendf(
+                    "failed to decode %s, which is a known failure.", srcPath);
+                return;
+            }
+            if (jsExpectations.empty()) {
+                // This is a failure, but it is a new file. Mark it as missing, with
+                // a note that it should be marked failing.
+                gMissingExpectations.push_back().appendf(
+                    "new file %s (with no expectations) FAILED to decode.", srcPath);
+                return;
+            }
         }
+
+        // If there was a failure, and either there was no expectations file, or
+        // the expectations file listed a valid expectation, report the failure.
+        gDecodeFailures.push_back().set(srcPath);
         return;
     }
 
     // Test decoding just the bounds. The bounds should always match.
     {
         stream.rewind();
         SkBitmap dim;
         if (!codec->decode(&stream, &dim, SkImageDecoder::kDecodeBounds_Mode)) {
             SkString failure = SkStringPrintf("failed to decode bounds for %s", srcPath);
             gDecodeFailures.push_back() = failure;
         } else {
             // Now check that the bounds match:
             if (dim.width() != bitmap.width() || dim.height() != bitmap.height()) {
                 SkString failure = SkStringPrintf("bounds do not match for %s", srcPath);
                 gDecodeFailures.push_back() = failure;
             }
         }
     }
 
     skiagm::GmResultDigest digest(bitmap);
     if (compare_to_expectations_if_necessary(digest, filename,
                                              &gDecodeFailures,
-                                             &gMissingExpectations)) {
+                                             &gMissingExpectations,
+                                             &gKnownFailures)) {
         gSuccessfulDecodes.push_back().printf("%s [%d %d]", srcPath, bitmap.width(),
                                               bitmap.height());
     } else if (!FLAGS_mismatchPath.isEmpty()) {
         SkString outPath;
         make_outname(&outPath, FLAGS_mismatchPath[0], srcPath, ".png");
         if (write_bitmap(outPath.c_str(), bitmap)) {
             gSuccessfulDecodes.push_back().appendf("\twrote %s", outPath.c_str());
         } else {
             gEncodeFailures.push_back().set(outPath);
         }
@@ skipping 32 matching lines @@
                 // FIXME: Come up with a more representative set of rectangles.
                 SkIRect rect = generate_random_rect(&rand, width, height);
                 SkString subsetDim = SkStringPrintf("[%d,%d,%d,%d]", rect.fLeft, rect.fTop,
                                                     rect.fRight, rect.fBottom);
                 if (codec->decodeSubset(&bitmapFromDecodeSubset, rect, gPrefConfig)) {
                     SkString subsetName = SkStringPrintf("%s_%s", filename, subsetDim.c_str());
                     skiagm::GmResultDigest subsetDigest(bitmapFromDecodeSubset);
                     if (compare_to_expectations_if_necessary(subsetDigest,
                                                              subsetName.c_str(),
                                                              &gFailedSubsetDecodes,
-                                                             &gMissingSubsetExpectations)) {
+                                                             &gMissingSubsetExpectations,
+                                                             &gKnownSubsetFailures)) {
                         gSuccessfulSubsetDecodes.push_back().printf("Decoded subset %s from %s",
                                                               subsetDim.c_str(), srcPath);
                     } else if (!FLAGS_mismatchPath.isEmpty()) {
                         write_subset(FLAGS_mismatchPath[0], filename, subsetDim.c_str(),
                                      &bitmapFromDecodeSubset, rect, bitmap);
                     }
 
                     write_expectations(bitmapFromDecodeSubset, subsetName.c_str());
                     if (writePath != NULL) {
                         write_subset(writePath->c_str(), filename, subsetDim.c_str(),
@@ skipping 202 matching lines @@
     failed |= print_strings("Missing codec", gMissingCodecs);
     failed |= print_strings("Failed to decode", gDecodeFailures);
     failed |= print_strings("Failed to encode", gEncodeFailures);
     print_strings("Decoded", gSuccessfulDecodes);
     print_strings("Missing expectations", gMissingExpectations);
 
     if (FLAGS_testSubsetDecoding) {
         failed |= print_strings("Failed subset decodes", gFailedSubsetDecodes);
         print_strings("Decoded subsets", gSuccessfulSubsetDecodes);
         print_strings("Missing subset expectations", gMissingSubsetExpectations);
+        print_strings("Known subset failures", gKnownSubsetFailures);
     }
 
+    print_strings("Known failures", gKnownFailures);
+
     return failed ? -1 : 0;
 }
 
 #if !defined SK_BUILD_FOR_IOS
 int main(int argc, char * const argv[]) {
     return tool_main(argc, (char**) argv);
 }
 #endif