gm: report ExpectationsMismatch and RenderModeMismatch results separately

gm/gmmain.cpp

 /*
  * Copyright 2011 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 /*
  * Code for the "gm" (Golden Master) rendering comparison tool.
  *
@@ skipping 254 matching lines @@
         int renderModeCount = 0;
         this->fRenderModesEncountered.find(renderModeDescriptor, &renderModeCount);
         renderModeCount++;
         this->fRenderModesEncountered.set(renderModeDescriptor, renderModeCount);
 
         if (errorCombination.isEmpty()) {
             return;
         }
 
         // Things to do only if there is some error condition.
-        SkString fullName = make_name(name.c_str(), renderModeDescriptor);
+        SkString fullName = name;
+        fullName.append(renderModeDescriptor);
         for (int typeInt = 0; typeInt <= kLast_ErrorType; typeInt++) {
             ErrorType type = static_cast<ErrorType>(typeInt);
             if (errorCombination.includes(type)) {
                 fFailedTests[type].push_back(fullName);
             }
         }
     }
 
     /**
      * Return the number of significant (non-ignorable) errors we have
@@ skipping 390 matching lines @@
      * (if any) that we saw along the way.
      *
      * If fMismatchPath has been set, and there are pixel diffs, then the
      * actual bitmap will be written out to a file within fMismatchPath.
      *
      * @param expectations what expectations to compare actualBitmap against
      * @param actualBitmap the image we actually generated
      * @param baseNameString name of test without renderModeDescriptor added
      * @param renderModeDescriptor e.g., "-rtree", "-deferred"
      * @param addToJsonSummary whether to add these results (both actual and
-     *        expected) to the JSON summary
-     *
-     * TODO: For now, addToJsonSummary is only set to true within
-     * compare_test_results_to_stored_expectations(), so results of our
-     * in-memory comparisons (Rtree vs regular, etc.) are not written to the
-     * JSON summary.  We may wish to change that.
+     *        expected) to the JSON summary. Regardless of this setting, if
+     *        we find an image mismatch in this test, we will write these
+     *        results to the JSON summary.  (This is so that we will always
+     *        report errors across rendering modes, such as pipe vs tiled.
+     *        See https://codereview.chromium.org/13650002/ )
      */
     ErrorCombination compare_to_expectations(Expectations expectations,
                                              const SkBitmap& actualBitmap,
                                              const SkString& baseNameString,
                                              const char renderModeDescriptor[],
-                                             bool addToJsonSummary=false) {
+                                             bool addToJsonSummary) {
         ErrorCombination errors;
         Checksum actualChecksum = SkBitmapChecksummer::Compute64(actualBitmap);
         SkString completeNameString = baseNameString;
         completeNameString.append(renderModeDescriptor);
         const char* completeName = completeNameString.c_str();
 
         if (expectations.empty()) {
             errors.add(kMissingExpectations_ErrorType);
         } else if (!expectations.match(actualChecksum)) {
-            errors.add(kImageMismatch_ErrorType);
+            addToJsonSummary = true;
+            // The error mode we record depends on whether this was running
+            // in a non-standard renderMode.
+            if ('\0' == *renderModeDescriptor) {
+                errors.add(kExpectationsMismatch_ErrorType);
+            } else {
+                errors.add(kRenderModeMismatch_ErrorType);

[epoger, 2013/04/04 17:29:44]

patchset 4 records the error as RenderModeMismatch if it was running a
renderMode with a non-blank renderModeDescriptor!

+            }
 
             // Write out the "actuals" for any mismatches, if we have
             // been directed to do so.
             if (fMismatchPath) {
                 SkString path =
                     make_filename(fMismatchPath, renderModeDescriptor,
                                   baseNameString.c_str(), "png");
                 write_bitmap(path, actualBitmap);
             }
 
@@ skipping 28 matching lines @@
             asJsonValue(actualChecksum);
         if (result.isEmpty()) {
             this->fJsonActualResults_Succeeded[testName] = actualResults;
         } else {
             if (ignoreFailure) {
                 // TODO: Once we have added the ability to compare
                 // actual results against expectations in a JSON file
                 // (where we can set ignore-failure to either true or
                 // false), add test cases that exercise ignored
                 // failures (both for kMissingExpectations_ErrorType
-                // and kImageMismatch_ErrorType).
+                // and kExpectationsMismatch_ErrorType).
                 this->fJsonActualResults_FailureIgnored[testName] =
                     actualResults;
             } else {
                 if (result.includes(kMissingExpectations_ErrorType)) {
                     // TODO: What about the case where there IS an
                     // expected image checksum, but that gm test
                     // doesn't actually run?  For now, those cases
                     // will always be ignored, because gm only looks
                     // at expectations that correspond to gm tests
                     // that were actually run.
                     //
                     // Once we have the ability to express
                     // expectations as a JSON file, we should fix this
                     // (and add a test case for which an expectation
                     // is given but the test is never run).
                     this->fJsonActualResults_NoComparison[testName] =
                         actualResults;
                 }
-                if (result.includes(kImageMismatch_ErrorType)) {
+                if (result.includes(kExpectationsMismatch_ErrorType) ||
+                    result.includes(kRenderModeMismatch_ErrorType)) {
                     this->fJsonActualResults_Failed[testName] = actualResults;
                 }
             }
         }
     }
 
     /**
      * Add this test to the JSON collection of expected results.
      */
     void add_expected_results_to_json_summary(const char testName[],
@@ skipping 73 matching lines @@
      * @param gm which test generated the bitmap
      * @param gRec
      * @param renderModeDescriptor
      * @param actualBitmap actual bitmap generated by this run
      * @param referenceBitmap bitmap we expected to be generated
      */
     ErrorCombination compare_test_results_to_reference_bitmap(
         GM* gm, const ConfigData& gRec, const char renderModeDescriptor [],
         SkBitmap& actualBitmap, const SkBitmap* referenceBitmap) {
 
-        // TODO(epoger): This method is run to compare results across
-        // different rendering modes (as opposed to
-        // compare_test_results_to_stored_expectations(), which
-        // compares results against expectations stored on disk).  If
-        // we would like the GenerateGMs step to distinguish between
-        // those two types of mismatches, we should report image
-        // mismatches in here with a different ErrorType.
         SkASSERT(referenceBitmap);
         SkString name = make_name(gm->shortName(), gRec.fName);
         Expectations expectations(*referenceBitmap);
         return compare_to_expectations(expectations, actualBitmap,
-                                       name, renderModeDescriptor);
+                                       name, renderModeDescriptor, false);
     }
 
     static SkPicture* generate_new_picture(GM* gm, BbhType bbhType, uint32_t recordFlags,
                                            SkScalar scale = SK_Scalar1) {
         // Pictures are refcounted so must be on heap
         SkPicture* pict;
         int width = SkScalarCeilToInt(SkScalarMul(SkIntToScalar(gm->getISize().width()), scale));
         int height = SkScalarCeilToInt(SkScalarMul(SkIntToScalar(gm->getISize().height()), scale));
 
         if (kTileGrid_BbhType == bbhType) {
@@ skipping 98 matching lines @@
                 // TODO(epoger): Report this as a new ErrorType,
                 // something like kImageGeneration_ErrorType?
                 return kEmpty_ErrorCombination;
             }
             return compare_test_results_to_reference_bitmap(
                 gm, gRec, "-deferred", bitmap, &referenceBitmap);
         }
         return kEmpty_ErrorCombination;
     }
 
-    ErrorCombination test_pipe_playback(GM* gm,
-                                        const ConfigData& gRec,
-                                        const SkBitmap& referenceBitmap) {
+    ErrorCombination test_pipe_playback(GM* gm, const ConfigData& gRec,
+                                        const SkBitmap& referenceBitmap, bool simulateFailure) {

[borenet, 2013/04/04 18:05:45]

I guess we had this discussion before: neither of us likes this, but it's the
cleanest way to self-test this.

[epoger, 2013/04/04 18:06:48]

Yup.  It's the least bad option we could come up with. :-)

         ErrorCombination errors;
         for (size_t i = 0; i < SK_ARRAY_COUNT(gPipeWritingFlagCombos); ++i) {
             SkBitmap bitmap;
             SkISize size = gm->getISize();
             setup_bitmap(gRec, size, &bitmap);
             SkCanvas canvas(bitmap);
             installFilter(&canvas);
             PipeController pipeController(&canvas);
             SkGPipeWriter writer;
             SkCanvas* pipeCanvas = writer.startRecording(
               &pipeController, gPipeWritingFlagCombos[i].flags);
-            invokeGM(gm, pipeCanvas, false, false);
+            if (!simulateFailure) {
+                invokeGM(gm, pipeCanvas, false, false);
+            }
             complete_bitmap(&bitmap);
             writer.endRecording();
             SkString string("-pipe");
             string.append(gPipeWritingFlagCombos[i].name);
             errors.add(compare_test_results_to_reference_bitmap(
                 gm, gRec, string.c_str(), bitmap, &referenceBitmap));
             if (!errors.isEmpty()) {
                 break;
             }
         }
@@ skipping 146 matching lines @@
 DEFINE_string(modulo, "", "[--modulo <remainder> <divisor>]: only run tests for which "
               "testIndex %% divisor == remainder.");
 DEFINE_bool(pdf, true, "Exercise the pdf rendering test pass.");
 DEFINE_bool(pipe, true, "Exercise the SkGPipe replay test pass.");
 DEFINE_string2(readPath, r, "", "Read reference images from this dir, and report "
                "any differences between those and the newly generated ones.");
 DEFINE_bool(replay, true, "Exercise the SkPicture replay test pass.");
 DEFINE_string2(resourcePath, i, "", "Directory that stores image resources.");
 DEFINE_bool(rtree, true, "Exercise the R-Tree variant of SkPicture test pass.");
 DEFINE_bool(serialize, true, "Exercise the SkPicture serialization & deserialization test pass.");
+DEFINE_bool(simulatePipePlaybackFailure, false, "Simulate a rendering failure in pipe mode only.");
 DEFINE_bool(tiledPipe, false, "Exercise tiled SkGPipe replay.");
 DEFINE_bool(tileGrid, true, "Exercise the tile grid variant of SkPicture.");
 DEFINE_string(tileGridReplayScales, "", "Space separated list of floating-point scale "
               "factors to be used for tileGrid playback testing. Default value: 1.0");
 DEFINE_string(writeJsonSummaryPath, "", "Write a JSON-formatted result summary to this file.");
 DEFINE_bool2(verbose, v, false, "Print diagnostics (e.g. list each config to be tested).");
 DEFINE_string2(writePath, w, "",  "Write rendered images into this directory.");
 DEFINE_string2(writePicturePath, wp, "", "Write .skp files into this directory.");
 
 static int findConfig(const char config[]) {
@@ skipping 214 matching lines @@
             const char* pictureSuffix = "skp";
             SkString path = make_filename(FLAGS_writePicturePath[0], "",
                                           gm->shortName(), pictureSuffix);
             SkFILEWStream stream(path.c_str());
             pict->serialize(&stream);
         }
 
         errorsForAllModes.add(pictErrors);
     }
 
-    // TODO: add a test in which the RTree rendering results in a
-    // different bitmap than the standard rendering.  It should
-    // show up as failed in the JSON summary, and should be listed
-    // in the stdout also.
     if (!(gmFlags & GM::kSkipPicture_Flag) && FLAGS_rtree) {
         SkPicture* pict = gmmain.generate_new_picture(
             gm, kRTree_BbhType, SkPicture::kUsePathBoundsForClip_RecordingFlag);
         SkAutoUnref aur(pict);
         SkBitmap bitmap;
         gmmain.generate_image_from_picture(gm, compareConfig, pict, &bitmap);
         errorsForAllModes.add(gmmain.compare_test_results_to_reference_bitmap(
             gm, compareConfig, "-rtree", bitmap, &comparisonBitmap));
     }
 
@@ skipping 24 matching lines @@
                 gm, compareConfig, suffix.c_str(), bitmap, &comparisonBitmap));
         }
     }
 
     // run the pipe centric GM steps
     if (!(gmFlags & GM::kSkipPipe_Flag)) {
 
         ErrorCombination pipeErrors;
 
         if (FLAGS_pipe) {
-            pipeErrors.add(gmmain.test_pipe_playback(gm, compareConfig, comparisonBitmap));
+            pipeErrors.add(gmmain.test_pipe_playback(gm, compareConfig, comparisonBitmap,
+                                                     FLAGS_simulatePipePlaybackFailure));
         }
 
         if ((pipeErrors.isEmpty()) &&
             FLAGS_tiledPipe && !(gmFlags & GM::kSkipTiled_Flag)) {
             pipeErrors.add(gmmain.test_tiled_pipe_playback(gm, compareConfig, comparisonBitmap));
         }
 
         errorsForAllModes.add(pipeErrors);
     }
     return errorsForAllModes;
@@ skipping 343 matching lines @@
     if (FLAGS_forceBWtext) {
         canvas->setDrawFilter(SkNEW(BWTextDrawFilter))->unref();
     }
 }
 
 #if !defined(SK_BUILD_FOR_IOS) && !defined(SK_BUILD_FOR_NACL)
 int main(int argc, char * const argv[]) {
     return tool_main(argc, (char**) argv);
 }
 #endif