gm: report drawing mode discrepancies (e.g. pipe vs tiled) explicitly

gm/gmmain.cpp

 /*

[epoger, 2013/03/20 19:06:12]

patchset 1: same as patchset 4 from https://codereview.chromium.org/12825005 ,
but synced to r8270 to pick up lots of conflicting changes to gmmain.cpp

  * Copyright 2011 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 /*
  * Code for the "gm" (Golden Master) rendering comparison tool.
  *
  * If you make changes to this, re-run the self-tests at gm/tests/run.sh
@@ skipping 242 matching lines @@
         // The common case: no error means nothing to record.
         if (kEmptyErrorBitfield == errorType) {
             return;
         }
 
         // If only certain error type(s) were reported, we know we can ignore them.
         if (errorType == (errorType & kIgnorable_ErrorBitmask)) {
             return;
         }
 
-        FailRec& rec = fFailedTests.push_back(make_name(
-            name.c_str(), renderModeDescriptor));
+        SkString completeName = name;
+        completeName.append(renderModeDescriptor);
+        FailRec& rec = fFailedTests.push_back(completeName);
         rec.fIsPixelError =
             (kEmptyErrorBitfield != (errorType & kImageMismatch_ErrorBitmask));
     }
 
     // List contents of fFailedTests via SkDebug.
     void ListErrors() {
         for (int i = 0; i < fFailedTests.count(); ++i) {
             if (fFailedTests[i].fIsPixelError) {
                 gm_fprintf(stderr, "\t\t%s pixel_error\n", fFailedTests[i].fName.c_str());
             } else {
@@ skipping 307 matching lines @@
      * _ErrorBitmask values otherwise.
      *
      * If fMismatchPath has been set, and there are pixel diffs, then the
      * actual bitmap will be written out to a file within fMismatchPath.
      *
      * @param expectations what expectations to compare actualBitmap against
      * @param actualBitmap the image we actually generated
      * @param baseNameString name of test without renderModeDescriptor added
      * @param renderModeDescriptor e.g., "-rtree", "-deferred"
      * @param addToJsonSummary whether to add these results (both actual and
-     *        expected) to the JSON summary
-     *
-     * TODO: For now, addToJsonSummary is only set to true within
-     * compare_test_results_to_stored_expectations(), so results of our
-     * in-memory comparisons (Rtree vs regular, etc.) are not written to the
-     * JSON summary.  We may wish to change that.
+     *        expected) to the JSON summary. Regardless of this setting, if
+     *        we find an image mismatch in this test, we will write these
+     *        results to the JSON summary.  (This is so that we will always
+     *        report errors across rendering modes, such as pipe vs tiled.
+     *        See https://codereview.chromium.org/12825005/ )
      */
     ErrorBitfield compare_to_expectations(Expectations expectations,
                                           const SkBitmap& actualBitmap,
                                           const SkString& baseNameString,
                                           const char renderModeDescriptor[],
-                                          bool addToJsonSummary=false) {
+                                          bool addToJsonSummary) {
         ErrorBitfield retval;
         Checksum actualChecksum = SkBitmapChecksummer::Compute64(actualBitmap);
         SkString completeNameString = baseNameString;
         completeNameString.append(renderModeDescriptor);
         const char* completeName = completeNameString.c_str();
 
         if (expectations.empty()) {
             retval = kMissingExpectations_ErrorBitmask;
         } else if (expectations.match(actualChecksum)) {
             retval = kEmptyErrorBitfield;
         } else {
+            addToJsonSummary = true;
             retval = kImageMismatch_ErrorBitmask;
 
             // Write out the "actuals" for any mismatches, if we have
             // been directed to do so.
             if (fMismatchPath) {
                 SkString path =
                     make_filename(fMismatchPath, renderModeDescriptor,
                                   baseNameString.c_str(), "png");
                 write_bitmap(path, actualBitmap);
             }
@@ skipping 147 matching lines @@
      * @param referenceBitmap bitmap we expected to be generated
      */
     ErrorBitfield compare_test_results_to_reference_bitmap(
         GM* gm, const ConfigData& gRec, const char renderModeDescriptor [],
         SkBitmap& actualBitmap, const SkBitmap* referenceBitmap) {
 
         SkASSERT(referenceBitmap);
         SkString name = make_name(gm->shortName(), gRec.fName);
         Expectations expectations(*referenceBitmap);
         return compare_to_expectations(expectations, actualBitmap,
-                                       name, renderModeDescriptor);
+                                       name, renderModeDescriptor, false);
     }
 
     static SkPicture* generate_new_picture(GM* gm, BbhType bbhType, uint32_t recordFlags,
                                            SkScalar scale = SK_Scalar1) {
         // Pictures are refcounted so must be on heap
         SkPicture* pict;
         int width = SkScalarCeilToInt(SkScalarMul(SkIntToScalar(gm->getISize().width()), scale));
         int height = SkScalarCeilToInt(SkScalarMul(SkIntToScalar(gm->getISize().height()), scale));
 
         if (kTileGrid_BbhType == bbhType) {
@@ skipping 87 matching lines @@
             // expected in some cases, so don't report a test failure.
             if (!generate_image(gm, gRec, context, rt, &bitmap, true)) {
                 return kEmptyErrorBitfield;
             }
             return compare_test_results_to_reference_bitmap(
                 gm, gRec, "-deferred", bitmap, &referenceBitmap);
         }
         return kEmptyErrorBitfield;
     }
 
-    ErrorBitfield test_pipe_playback(GM* gm,
-                                     const ConfigData& gRec,
-                                     const SkBitmap& referenceBitmap) {
+    ErrorBitfield test_pipe_playback(GM* gm, const ConfigData& gRec,
+                                     const SkBitmap& referenceBitmap, bool simulateFailure) {
         ErrorBitfield errors = kEmptyErrorBitfield;
         for (size_t i = 0; i < SK_ARRAY_COUNT(gPipeWritingFlagCombos); ++i) {
             SkBitmap bitmap;
             SkISize size = gm->getISize();
             setup_bitmap(gRec, size, &bitmap);
             SkCanvas canvas(bitmap);
             installFilter(&canvas);
             PipeController pipeController(&canvas);
             SkGPipeWriter writer;
             SkCanvas* pipeCanvas = writer.startRecording(
               &pipeController, gPipeWritingFlagCombos[i].flags);
-            invokeGM(gm, pipeCanvas, false, false);
+            if (!simulateFailure) {
+                invokeGM(gm, pipeCanvas, false, false);
+            }
             complete_bitmap(&bitmap);
             writer.endRecording();
             SkString string("-pipe");
             string.append(gPipeWritingFlagCombos[i].name);
             errors |= compare_test_results_to_reference_bitmap(
                 gm, gRec, string.c_str(), bitmap, &referenceBitmap);
             if (errors != kEmptyErrorBitfield) {
                 break;
             }
         }
@@ skipping 137 matching lines @@
 DEFINE_string(modulo, "", "[--modulo <remainder> <divisor>]: only run tests for which "
               "testIndex %% divisor == remainder.");
 DEFINE_bool(pdf, true, "Exercise the pdf rendering test pass.");
 DEFINE_bool(pipe, true, "Exercise the SkGPipe replay test pass.");
 DEFINE_string2(readPath, r, "", "Read reference images from this dir, and report "
                "any differences between those and the newly generated ones.");
 DEFINE_bool(replay, true, "Exercise the SkPicture replay test pass.");
 DEFINE_string2(resourcePath, i, "", "Directory that stores image resources.");
 DEFINE_bool(rtree, true, "Exercise the R-Tree variant of SkPicture test pass.");
 DEFINE_bool(serialize, true, "Exercise the SkPicture serialization & deserialization test pass.");
+DEFINE_bool(simulatePipePlaybackFailure, false, "Simulate a rendering failure in pipe mode only.");
 DEFINE_bool(tiledPipe, false, "Exercise tiled SkGPipe replay.");
 DEFINE_bool(tileGrid, true, "Exercise the tile grid variant of SkPicture.");
 DEFINE_string(tileGridReplayScales, "", "Space separated list of floating-point scale "
               "factors to be used for tileGrid playback testing. Default value: 1.0");
 DEFINE_string(writeJsonSummaryPath, "", "Write a JSON-formatted result summary to this file.");
 DEFINE_bool2(verbose, v, false, "Print diagnostics (e.g. list each config to be tested).");
 DEFINE_string2(writePath, w, "",  "Write rendered images into this directory.");
 DEFINE_string2(writePicturePath, wp, "", "Write .skp files into this directory.");
 
 static int findConfig(const char config[]) {
@@ skipping 286 matching lines @@
                 gm, compareConfig, suffix.c_str(), bitmap, &comparisonBitmap);
         }
     }
 
     // run the pipe centric GM steps
     if (!(gmFlags & GM::kSkipPipe_Flag)) {
 
         ErrorBitfield pipeErrors = kEmptyErrorBitfield;
 
         if (FLAGS_pipe) {
-            pipeErrors |= gmmain.test_pipe_playback(gm, compareConfig, comparisonBitmap);
+            pipeErrors |= gmmain.test_pipe_playback(gm, compareConfig, comparisonBitmap,
+                                                    FLAGS_simulatePipePlaybackFailure);
         }
 
         if ((kEmptyErrorBitfield == pipeErrors) &&
             FLAGS_tiledPipe && !(gmFlags & GM::kSkipTiled_Flag)) {
             pipeErrors |= gmmain.test_tiled_pipe_playback(gm, compareConfig, comparisonBitmap);
         }
 
         errorsForAllModes |= pipeErrors;
     }
     return errorsForAllModes;
@@ skipping 144 matching lines @@
         moduloRemainder = -1;
     }
     if (moduloRemainder < 0 || moduloRemainder >= moduloDivisor) {
         moduloRemainder = -1;
     }
 
     // Accumulate success of all tests.
     int testsRun = 0;
     int testsPassed = 0;
     int testsFailed = 0;
+    int testsWithDrawingModeDiscrepancies = 0;
     int testsMissingReferenceImages = 0;
 
     int gmIndex = -1;
     SkString moduloStr;
 
     // If we will be writing out files, prepare subdirectories.
     if (FLAGS_writePath.count() == 1) {
         if (!sk_mkdir(FLAGS_writePath[0])) {
             return -1;
         }
@@ skipping 25 matching lines @@
         const char* shortName = gm->shortName();
         if (skip_name(FLAGS_match, shortName)) {
             SkDELETE(gm);
             continue;
         }
 
         SkISize size = gm->getISize();
         gm_fprintf(stdout, "%sdrawing... %s [%d %d]\n", moduloStr.c_str(), shortName,
                    size.width(), size.height());
 
-        ErrorBitfield testErrors = kEmptyErrorBitfield;
-        testErrors |= run_multiple_configs(gmmain, gm, configs, grFactory);
+        ErrorBitfield compositeErrors = kEmptyErrorBitfield;
+        ErrorBitfield multipleConfigErrors = run_multiple_configs(gmmain, gm, configs, grFactory);
+        compositeErrors |= multipleConfigErrors;
 
         SkBitmap comparisonBitmap;
         const ConfigData compareConfig =
-            { SkBitmap::kARGB_8888_Config, kRaster_Backend, kDontCare_GLContextType, 0, kRW_ConfigFlag, "comparison", false };
-        testErrors |= gmmain.generate_image(gm, compareConfig, NULL, NULL, &comparisonBitmap, false);
+            { SkBitmap::kARGB_8888_Config, kRaster_Backend, kDontCare_GLContextType, 0,
+              kRW_ConfigFlag, "comparison", false };
+        ErrorBitfield generateModeBaselineErrors = gmmain.generate_image(
+            gm, compareConfig, NULL, NULL, &comparisonBitmap, false);
+        compositeErrors |= generateModeBaselineErrors;
 
-        // TODO(epoger): only run this if gmmain.generate_image() succeeded?
+        // TODO(epoger): only run this if generateModeBaselineErrors is kEmptyErrorBitfield?
         // Otherwise, what are we comparing against?
-        testErrors |= run_multiple_modes(gmmain, gm, compareConfig, comparisonBitmap);
+        ErrorBitfield multipleModeErrors = run_multiple_modes(gmmain, gm, compareConfig,
+                                                              comparisonBitmap);
+        compositeErrors |= multipleModeErrors;
 
-        // Update overall results.
-        // We only tabulate the particular error types that we currently
-        // care about (e.g., missing reference images). Later on, if we
-        // want to also tabulate other error types, we can do so.
+        // A non-ignorable error in run_multiple_configs(), or ANY error in
+        // generate_image()/run_multiple_modes(), counts as a failure.
         testsRun++;
-        if (!gmmain.fExpectationsSource.get() ||
-            (kEmptyErrorBitfield != (kMissingExpectations_ErrorBitmask & testErrors))) {
-            testsMissingReferenceImages++;
-        }
-        if (testErrors == (testErrors & kIgnorable_ErrorBitmask)) {
+        if (kEmptyErrorBitfield != multipleModeErrors) {
+            testsWithDrawingModeDiscrepancies++;
+            testsFailed++;
+        } else if (kEmptyErrorBitfield != generateModeBaselineErrors) {
+            testsFailed++;
+        } else if (compositeErrors == (compositeErrors & kIgnorable_ErrorBitmask)) {
             testsPassed++;
         } else {
             testsFailed++;
         }
+        // Any other result categories we care to report.
+        if (!gmmain.fExpectationsSource.get() ||
+            (kEmptyErrorBitfield != (kMissingExpectations_ErrorBitmask & compositeErrors))) {
+            testsMissingReferenceImages++;
+        }
 
         SkDELETE(gm);
     }
-    gm_fprintf(stdout, "Ran %d tests: %d passed, %d failed, %d missing reference images\n",
-               testsRun, testsPassed, testsFailed, testsMissingReferenceImages);
+    gm_fprintf(stdout, "Ran %d tests: %d passed, %d failed, %d with drawing mode discrepancies, "
+               "%d missing reference images\n",
+               testsRun, testsPassed, testsFailed, testsWithDrawingModeDiscrepancies,
+               testsMissingReferenceImages);
     gmmain.ListErrors();
 
     if (FLAGS_writeJsonSummaryPath.count() == 1) {
         Json::Value actualResults;
         actualResults[kJsonKey_ActualResults_Failed] =
             gmmain.fJsonActualResults_Failed;
         actualResults[kJsonKey_ActualResults_FailureIgnored] =
             gmmain.fJsonActualResults_FailureIgnored;
         actualResults[kJsonKey_ActualResults_NoComparison] =
             gmmain.fJsonActualResults_NoComparison;
@@ skipping 33 matching lines @@
     if (FLAGS_forceBWtext) {
         canvas->setDrawFilter(SkNEW(BWTextDrawFilter))->unref();
     }
 }
 
 #if !defined(SK_BUILD_FOR_IOS) && !defined(SK_BUILD_FOR_NACL)
 int main(int argc, char * const argv[]) {
     return tool_main(argc, (char**) argv);
 }
 #endif