gm self-test: make sure we report failures in individual rendering modes

gm/gmmain.cpp

 /*

[epoger, 2013/03/14 22:36:27]

As of patchset 3+4, we now report any differences across drawing modes as
testsWithDrawingModeDiscrepancies.

  * Copyright 2011 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 /*
  * Code for the "gm" (Golden Master) rendering comparison tool.
  *
  * If you make changes to this, re-run the self-tests at gm/tests/run.sh
@@ skipping 175 matching lines @@
     { " cross-process, shared address", SkGPipeWriter::kCrossProcess_Flag
         | SkGPipeWriter::kSharedAddressSpace_Flag }
 };
 
 class GMMain {
 public:
     GMMain() {
         // Set default values of member variables, which tool_main()
         // may override.
         fUseFileHierarchy = false;
+        fSimulatePipePlaybackFailure = false;
         fMismatchPath = NULL;
     }
 
     SkString make_name(const char shortName[], const char configName[]) {
         SkString name;
         if (0 == strlen(configName)) {
             name.append(shortName);
         } else if (fUseFileHierarchy) {
             name.appendf("%s%c%s", configName, SkPATH_SEPARATOR, shortName);
         } else {
@@ skipping 47 matching lines @@
         // The common case: no error means nothing to record.
         if (kEmptyErrorBitfield == errorType) {
             return;
         }
 
         // If only certain error type(s) were reported, we know we can ignore them.
         if (errorType == (errorType & kIgnorable_ErrorBitmask)) {
             return;
         }
 
-        FailRec& rec = fFailedTests.push_back(make_name(
-            name.c_str(), renderModeDescriptor));
+        SkString completeName = name;
+        completeName.append(renderModeDescriptor);
+        FailRec& rec = fFailedTests.push_back(completeName);
         rec.fIsPixelError =
             (kEmptyErrorBitfield != (errorType & kImageMismatch_ErrorBitmask));
     }
 
     // List contents of fFailedTests via SkDebug.
     void ListErrors() {
         for (int i = 0; i < fFailedTests.count(); ++i) {
             if (fFailedTests[i].fIsPixelError) {
                 gm_fprintf(stderr, "\t\t%s pixel_error\n", fFailedTests[i].fName.c_str());
             } else {
@@ skipping 311 matching lines @@
      * _ErrorBitmask values otherwise.
      *
      * If fMismatchPath has been set, and there are pixel diffs, then the
      * actual bitmap will be written out to a file within fMismatchPath.
      *
      * @param expectations what expectations to compare actualBitmap against
      * @param actualBitmap the image we actually generated
      * @param baseNameString name of test without renderModeDescriptor added
      * @param renderModeDescriptor e.g., "-rtree", "-deferred"
      * @param addToJsonSummary whether to add these results (both actual and
-     *        expected) to the JSON summary
-     *
-     * TODO: For now, addToJsonSummary is only set to true within
-     * compare_test_results_to_stored_expectations(), so results of our
-     * in-memory comparisons (Rtree vs regular, etc.) are not written to the
-     * JSON summary.  We may wish to change that.
+     *        expected) to the JSON summary. Regardless of this setting, if
+     *        we find an image mismatch in this test, we will write these
+     *        results to the JSON summary.  (This is so that we will always
+     *        report errors across rendering modes, such as pipe vs tiled.
+     *        See https://codereview.chromium.org/12825005/
      */
     ErrorBitfield compare_to_expectations(Expectations expectations,
                                           const SkBitmap& actualBitmap,
                                           const SkString& baseNameString,
                                           const char renderModeDescriptor[],
-                                          bool addToJsonSummary=false) {
+                                          bool addToJsonSummary) {
         ErrorBitfield retval;
         Checksum actualChecksum = SkBitmapChecksummer::Compute64(actualBitmap);
         SkString completeNameString = baseNameString;
         completeNameString.append(renderModeDescriptor);
         const char* completeName = completeNameString.c_str();
 
         if (expectations.empty()) {
             retval = kMissingExpectations_ErrorBitmask;
         } else if (expectations.match(actualChecksum)) {
             retval = kEmptyErrorBitfield;
         } else {
+            addToJsonSummary = true;
             retval = kImageMismatch_ErrorBitmask;
 
             // Write out the "actuals" for any mismatches, if we have
             // been directed to do so.
             if (fMismatchPath) {
                 SkString path =
                     make_filename(fMismatchPath, renderModeDescriptor,
                                   baseNameString.c_str(), "png");
                 write_bitmap(path, actualBitmap);
             }
@@ skipping 147 matching lines @@
      * @param referenceBitmap bitmap we expected to be generated
      */
     ErrorBitfield compare_test_results_to_reference_bitmap(
         GM* gm, const ConfigData& gRec, const char renderModeDescriptor [],
         SkBitmap& actualBitmap, const SkBitmap* referenceBitmap) {
 
         SkASSERT(referenceBitmap);
         SkString name = make_name(gm->shortName(), gRec.fName);
         Expectations expectations(*referenceBitmap);
         return compare_to_expectations(expectations, actualBitmap,
-                                       name, renderModeDescriptor);
+                                       name, renderModeDescriptor, false);
     }
 
     static SkPicture* generate_new_picture(GM* gm, BbhType bbhType, uint32_t recordFlags,
                                            SkScalar scale = SK_Scalar1) {
         // Pictures are refcounted so must be on heap
         SkPicture* pict;
         int width = SkScalarCeilToInt(SkScalarMul(SkIntToScalar(gm->getISize().width()), scale));
         int height = SkScalarCeilToInt(SkScalarMul(SkIntToScalar(gm->getISize().height()), scale));
 
         if (kTileGrid_BbhType == bbhType) {
@@ skipping 100 matching lines @@
         ErrorBitfield errors = kEmptyErrorBitfield;
         for (size_t i = 0; i < SK_ARRAY_COUNT(gPipeWritingFlagCombos); ++i) {
             SkBitmap bitmap;
             SkISize size = gm->getISize();
             setup_bitmap(gRec, size, &bitmap);
             SkCanvas canvas(bitmap);
             PipeController pipeController(&canvas);
             SkGPipeWriter writer;
             SkCanvas* pipeCanvas = writer.startRecording(
               &pipeController, gPipeWritingFlagCombos[i].flags);
-            invokeGM(gm, pipeCanvas, false, false);
+            if (!this->fSimulatePipePlaybackFailure) {
+                invokeGM(gm, pipeCanvas, false, false);
+            }
             complete_bitmap(&bitmap);
             writer.endRecording();
             SkString string("-pipe");
             string.append(gPipeWritingFlagCombos[i].name);
             errors |= compare_test_results_to_reference_bitmap(
                 gm, gRec, string.c_str(), bitmap, &referenceBitmap);
             if (errors != kEmptyErrorBitfield) {
                 break;
             }
         }
@@ skipping 25 matching lines @@
         }
         return errors;
     }
 
     //
     // member variables.
     // They are public for now, to allow easier setting by tool_main().
     //
 
     bool fUseFileHierarchy;
+    bool fSimulatePipePlaybackFailure;
 
     const char* fMismatchPath;
 
     // information about all failed tests we have encountered so far
     SkTArray<FailRec> fFailedTests;
 
     // Where to read expectations (expected image checksums, etc.) from.
     // If unset, we don't do comparisons.
     SkAutoTUnref<ExpectationsSource> fExpectationsSource;
 
@@ skipping 83 matching lines @@
 "    [--modulo <remainder> <divisor>]: only run tests for which \n"
 "        testIndex %% divisor == remainder\n"
 "    [--nopdf]: skip the pdf rendering test pass\n"
 "    [--nopipe]: Skip SkGPipe replay\n"
 "    [--readPath|-r <path>]: read reference images from this dir, and report\n"
 "        any differences between those and the newly generated ones\n"
 "    [--noreplay]: do not exercise SkPicture replay\n"
 "    [--resourcePath|-i <path>]: directory that stores image resources\n"
 "    [--nortree]: Do not exercise the R-Tree variant of SkPicture\n"
 "    [--noserialize]: do not exercise SkPicture serialization & deserialization\n"
+"    [--simulatePipePlaybackFailure]: simulate a rendering failure in pipe mode only\n"
 "    [--tiledPipe]: Exercise tiled SkGPipe replay\n"
 "    [--notileGrid]: Do not exercise the tile grid variant of SkPicture\n"
 "    [--tileGridReplayScales <scales>]: Comma separated list of floating-point scale\n"
 "        factors to be used for tileGrid playback testing. Default value: 1.0\n"
 "    [--writeJsonSummary <path>]: write a JSON-formatted result summary to this file\n"
 "    [--verbose] print diagnostics (e.g. list each config to be tested)\n"
 "    [--writePath|-w <path>]: write rendered images into this directory\n"
 "    [--writePicturePath|-wp <path>]: write .skp files into this directory\n"
              );
 }
@@ skipping 62 matching lines @@
 #endif
 }
 
 template <typename T> void appendUnique(SkTDArray<T>* array, const T& value) {
     int index = array->find(value);
     if (index < 0) {
         *array->append() = value;
     }
 }
 
+/**
+ * Run this test in a number of different configs (8888, 565, PDF,
+ * etc.), confirming that the resulting bitmaps match expectations
+ * (which may be different for each config).
+ */
+ErrorBitfield run_multiple_configs(GMMain &gmmain, GM *gm, const SkTDArray<size_t> &configs,
+                                   GrContextFactory *grFactory, int gpuCacheSizeBytes,
+                                   int gpuCacheSizeCount, const char *writePath, bool doPDF,
+                                   bool doDeferred) {
+    uint32_t gmFlags = gm->getFlags();
+    ErrorBitfield testErrors = kEmptyErrorBitfield;
+    for (int i = 0; i < configs.count(); i++) {
+        ConfigData config = gRec[configs[i]];
+
+        // Skip any tests that we don't even need to try.
+        if ((kPDF_Backend == config.fBackend) &&
+            (!doPDF || (gmFlags & GM::kSkipPDF_Flag)))
+            {
+                continue;
+            }
+        if ((gmFlags & GM::kSkip565_Flag) &&
+            (kRaster_Backend == config.fBackend) &&
+            (SkBitmap::kRGB_565_Config == config.fConfig)) {
+            continue;
+        }
+        if ((gmFlags & GM::kSkipGPU_Flag) &&
+            kGPU_Backend == config.fBackend) {
+            continue;
+        }
+
+        // Now we know that we want to run this test and record its
+        // success or failure.
+        ErrorBitfield renderErrors = kEmptyErrorBitfield;
+        GrRenderTarget* renderTarget = NULL;
+#if SK_SUPPORT_GPU
+        SkAutoTUnref<GrRenderTarget> rt;
+        AutoResetGr autogr;
+        if ((kEmptyErrorBitfield == renderErrors) &&
+            kGPU_Backend == config.fBackend) {
+            GrContext* gr = grFactory->get(config.fGLContextType);
+            bool grSuccess = false;
+            if (gr) {
+                // create a render target to back the device
+                GrTextureDesc desc;
+                desc.fConfig = kSkia8888_GrPixelConfig;
+                desc.fFlags = kRenderTarget_GrTextureFlagBit;
+                desc.fWidth = gm->getISize().width();
+                desc.fHeight = gm->getISize().height();
+                desc.fSampleCnt = config.fSampleCnt;
+                GrTexture* tex = gr->createUncachedTexture(desc, NULL, 0);
+                if (tex) {
+                    rt.reset(tex->asRenderTarget());
+                    rt.get()->ref();
+                    tex->unref();
+                    autogr.set(gr);
+                    renderTarget = rt.get();
+                    grSuccess = NULL != renderTarget;
+                }
+                // Set the user specified cache limits if non-default.
+                size_t bytes;
+                int count;
+                gr->getTextureCacheLimits(&count, &bytes);
+                if (-1 != gpuCacheSizeBytes) {
+                    bytes = static_cast<size_t>(gpuCacheSizeBytes);
+                }
+                if (-1 != gpuCacheSizeCount) {
+                    count = gpuCacheSizeCount;
+                }
+                gr->setTextureCacheLimits(count, bytes);
+            }
+            if (!grSuccess) {
+                renderErrors |= kNoGpuContext_ErrorBitmask;
+            }
+        }
+#endif
+
+        SkBitmap comparisonBitmap;
+
+        if (kEmptyErrorBitfield == renderErrors) {
+            renderErrors |= gmmain.test_drawing(gm, config, writePath,
+                                                GetGr(),
+                                                renderTarget,
+                                                &comparisonBitmap);
+        }
+
+        if (doDeferred && !renderErrors &&
+            (kGPU_Backend == config.fBackend ||
+             kRaster_Backend == config.fBackend)) {
+            renderErrors |= gmmain.test_deferred_drawing(gm, config,
+                                                         comparisonBitmap,
+                                                         GetGr(),
+                                                         renderTarget);
+        }
+
+        testErrors |= renderErrors;
+    }
+    return testErrors;
+}
+
+/**
+ * Run this test in a number of different drawing modes (pipe,
+ * deferred, tiled, etc.), confirming that the resulting bitmaps are
+ * *exactly* the same in all drawing modes.
+ *
+ * TODO(epoger): Right now, we only run the different drawing modes
+ * with the 8888 config.  Would there be value in running all those
+ * different drawing modes in whatever configs (8888, 565, PDF) we are
+ * testing?
+ */
+ErrorBitfield run_multiple_drawing_modes(GMMain &gmmain, GM *gm,
+                                         const char *writePicturePath, bool doReplay,
+                                         bool doSerialize, bool doRTree, bool doTileGrid,
+                                         const SkTDArray<SkScalar> &tileGridReplayScales,
+                                         bool doPipe, bool doTiledPipe) {
+    uint32_t gmFlags = gm->getFlags();
+    SkBitmap comparisonBitmap;
+    const ConfigData compareConfig =
+        { SkBitmap::kARGB_8888_Config, kRaster_Backend, kDontCare_GLContextType, 0,
+          kRW_ConfigFlag, "comparison" };
+    ErrorBitfield testErrors = gmmain.generate_image(gm, compareConfig, NULL, NULL,
+                                                     &comparisonBitmap, false);
+
+    // run the picture centric GM steps
+    if (!(gmFlags & GM::kSkipPicture_Flag)) {
+
+        ErrorBitfield pictErrors = kEmptyErrorBitfield;
+
+        //SkAutoTUnref<SkPicture> pict(generate_new_picture(gm));
+        SkPicture* pict = gmmain.generate_new_picture(gm, kNone_BbhType, 0);
+        SkAutoUnref aur(pict);
+
+        if ((kEmptyErrorBitfield == testErrors) && doReplay) {
+            SkBitmap bitmap;
+            gmmain.generate_image_from_picture(gm, compareConfig, pict,
+                                               &bitmap);
+            pictErrors |= gmmain.compare_test_results_to_reference_bitmap(
+                gm, compareConfig, "-replay", bitmap, &comparisonBitmap);
+        }
+
+        if ((kEmptyErrorBitfield == testErrors) &&
+            (kEmptyErrorBitfield == pictErrors) &&
+            doSerialize) {
+            SkPicture* repict = gmmain.stream_to_new_picture(*pict);
+            SkAutoUnref aurr(repict);
+
+            SkBitmap bitmap;
+            gmmain.generate_image_from_picture(gm, compareConfig, repict,
+                                               &bitmap);
+            pictErrors |= gmmain.compare_test_results_to_reference_bitmap(
+                                                                          gm, compareConfig, "-serialize", bitmap, &comparisonBitmap);
+        }
+
+        if (writePicturePath) {
+            const char* pictureSuffix = "skp";
+            SkString path = make_filename(writePicturePath, "",
+                                          gm->shortName(),
+                                          pictureSuffix);
+            SkFILEWStream stream(path.c_str());
+            pict->serialize(&stream);
+        }
+
+        testErrors |= pictErrors;
+    }
+
+    // TODO: add a test in which the RTree rendering results in a
+    // different bitmap than the standard rendering.  It should
+    // show up as failed in the JSON summary, and should be listed
+    // in the stdout also.
+    if (!(gmFlags & GM::kSkipPicture_Flag) && doRTree) {
+        SkPicture* pict = gmmain.generate_new_picture(
+                                                      gm, kRTree_BbhType, SkPicture::kUsePathBoundsForClip_RecordingFlag);
+        SkAutoUnref aur(pict);
+        SkBitmap bitmap;
+        gmmain.generate_image_from_picture(gm, compareConfig, pict,
+                                           &bitmap);
+        testErrors |= gmmain.compare_test_results_to_reference_bitmap(
+                                                                      gm, compareConfig, "-rtree", bitmap, &comparisonBitmap);
+    }
+
+    if (!(gmFlags & GM::kSkipPicture_Flag) && doTileGrid) {
+        for(int scaleIndex = 0; scaleIndex < tileGridReplayScales.count(); ++scaleIndex) {
+            SkScalar replayScale = tileGridReplayScales[scaleIndex];
+            if ((gmFlags & GM::kSkipScaledReplay_Flag) && replayScale != 1)
+                continue;
+            // We record with the reciprocal scale to obtain a replay
+            // result that can be validated against comparisonBitmap.
+            SkScalar recordScale = SkScalarInvert(replayScale);
+            SkPicture* pict = gmmain.generate_new_picture(
+                                                          gm, kTileGrid_BbhType, SkPicture::kUsePathBoundsForClip_RecordingFlag,
+                                                          recordScale);
+            SkAutoUnref aur(pict);
+            SkBitmap bitmap;
+            gmmain.generate_image_from_picture(gm, compareConfig, pict,
+                                               &bitmap, replayScale);
+            SkString suffix("-tilegrid");
+            if (SK_Scalar1 != replayScale) {
+                suffix += "-scale-";
+                suffix.appendScalar(replayScale);
+            }
+            testErrors |= gmmain.compare_test_results_to_reference_bitmap(
+                                                                          gm, compareConfig, suffix.c_str(), bitmap,
+                                                                          &comparisonBitmap);
+        }
+    }
+
+    // run the pipe centric GM steps
+    if (!(gmFlags & GM::kSkipPipe_Flag)) {
+
+        ErrorBitfield pipeErrors = kEmptyErrorBitfield;
+
+        if ((kEmptyErrorBitfield == testErrors) && doPipe) {
+            pipeErrors |= gmmain.test_pipe_playback(gm, compareConfig,
+                                                    comparisonBitmap);
+        }
+
+        if ((kEmptyErrorBitfield == testErrors) &&
+            (kEmptyErrorBitfield == pipeErrors) &&
+            doTiledPipe && !(gmFlags & GM::kSkipTiled_Flag)) {
+            pipeErrors |= gmmain.test_tiled_pipe_playback(gm, compareConfig,
+                                                          comparisonBitmap);
+        }
+
+        testErrors |= pipeErrors;
+    }
+    return testErrors;
+}
+
 int tool_main(int argc, char** argv);
 int tool_main(int argc, char** argv) {
 
 #if SK_ENABLE_INST_COUNT
     gPrintInstCount = true;
 #endif
 
     SkGraphics::Init();
     // we don't need to see this during a run
     gSkSuppressFontCachePurgeSpew = true;
@@ skipping 24 matching lines @@
 
     SkTDArray<size_t> configs;
     SkTDArray<size_t> excludeConfigs;
     SkTDArray<SkScalar> tileGridReplayScales;
     *tileGridReplayScales.append() = SK_Scalar1; // By default only test at scale 1.0
     bool userConfig = false;
 
     int moduloRemainder = -1;
     int moduloDivisor = -1;
 
-#if SK_SUPPORT_GPU
-    struct {
-        int     fBytes;
-        int     fCount;
-    } gpuCacheSize = { -1, -1 }; // -1s mean use the default
-#endif
+    int gpuCacheSizeBytes = -1;
+    int gpuCacheSizeCount = -1;
+    // -1 means use the default
 
     const char* const commandName = argv[0];
     char* const* stop = argv + argc;
     for (++argv; argv < stop; ++argv) {
         if (strcmp(*argv, "--config") == 0) {
             argv++;
             if (argv < stop) {
                 int index = findConfig(*argv);
                 if (index >= 0) {
                     appendUnique<size_t>(&configs, index);
@@ skipping 55 matching lines @@
                 usage(commandName);
                 return -1;
             }
         } else if (strcmp(*argv, "--enable-missing-warning") == 0) {
             notifyMissingReadReference = true;
         } else if (strcmp(*argv, "--forceBWtext") == 0) {
             gForceBWtext = true;
 #if SK_SUPPORT_GPU
         } else if (strcmp(*argv, "--gpuCacheSize") == 0) {
             if (stop - argv > 2) {
-                gpuCacheSize.fBytes = atoi(*++argv);
-                gpuCacheSize.fCount = atoi(*++argv);
+                gpuCacheSizeBytes = atoi(*++argv);
+                gpuCacheSizeCount = atoi(*++argv);
             } else {
                 gm_fprintf(stderr, "missing arg for --gpuCacheSize\n");
                 usage(commandName);
                 return -1;
             }
 #endif
         } else if (strcmp(*argv, "--help") == 0 || strcmp(*argv, "-h") == 0) {
             usage(commandName);
             return -1;
         } else if (strcmp(*argv, "--hierarchy") == 0) {
@@ skipping 37 matching lines @@
         } else if ((0 == strcmp(*argv, "--resourcePath")) ||
                    (0 == strcmp(*argv, "-i"))) {
             argv++;
             if (argv < stop && **argv) {
                 resourcePath = *argv;
             }
         } else if (strcmp(*argv, "--serialize") == 0) {
             doSerialize = true;
         } else if (strcmp(*argv, "--noserialize") == 0) {
             doSerialize = false;
+        } else if (strcmp(*argv, "--simulatePipePlaybackFailure") == 0) {
+            gmmain.fSimulatePipePlaybackFailure = true;
         } else if (strcmp(*argv, "--tiledPipe") == 0) {
             doTiledPipe = true;
         } else if (!strcmp(*argv, "--verbose") || !strcmp(*argv, "-v")) {
             doVerbose = true;
         } else if ((0 == strcmp(*argv, "--writePath")) ||
             (0 == strcmp(*argv, "-w"))) {
             argv++;
             if (argv < stop && **argv) {
                 writePath = *argv;
             }
@@ skipping 75 matching lines @@
         moduloRemainder = -1;
     }
     if (moduloRemainder < 0 || moduloRemainder >= moduloDivisor) {
         moduloRemainder = -1;
     }
 
     // Accumulate success of all tests.
     int testsRun = 0;
     int testsPassed = 0;
     int testsFailed = 0;
+    int testsWithDrawingModeDiscrepancies = 0;
     int testsMissingReferenceImages = 0;
 
 #if SK_SUPPORT_GPU
     GrContextFactory* grFactory = new GrContextFactory;
 #endif
 
     int gmIndex = -1;
     SkString moduloStr;
 
     // If we will be writing out files, prepare subdirectories.
@@ skipping 29 matching lines @@
         const char* shortName = gm->shortName();
         if (skip_name(fMatches, shortName)) {
             SkDELETE(gm);
             continue;
         }
 
         SkISize size = gm->getISize();
         gm_fprintf(stdout, "%sdrawing... %s [%d %d]\n", moduloStr.c_str(), shortName,
                    size.width(), size.height());
 
-        ErrorBitfield testErrors = kEmptyErrorBitfield;
-        uint32_t gmFlags = gm->getFlags();
+        ErrorBitfield compositeErrors = kEmptyErrorBitfield;
+        ErrorBitfield multipleConfigErrors = run_multiple_configs(
+            gmmain, gm, configs, grFactory, gpuCacheSizeBytes, gpuCacheSizeCount, writePath,
+            doPDF, doDeferred);
+        compositeErrors |= multipleConfigErrors;
+        ErrorBitfield multipleModeErrors = run_multiple_drawing_modes(
+            gmmain, gm, writePicturePath, doReplay, doSerialize, doRTree, doTileGrid,
+            tileGridReplayScales, doPipe, doTiledPipe);
+        compositeErrors |= multipleModeErrors;
 
-        for (int i = 0; i < configs.count(); i++) {
-            ConfigData config = gRec[configs[i]];
-
-            // Skip any tests that we don't even need to try.
-            if ((kPDF_Backend == config.fBackend) &&
-                (!doPDF || (gmFlags & GM::kSkipPDF_Flag)))
-                {
-                    continue;
-                }
-            if ((gmFlags & GM::kSkip565_Flag) &&
-                (kRaster_Backend == config.fBackend) &&
-                (SkBitmap::kRGB_565_Config == config.fConfig)) {
-                continue;
-            }
-            if ((gmFlags & GM::kSkipGPU_Flag) &&
-                kGPU_Backend == config.fBackend) {
-                continue;
-            }
-
-            // Now we know that we want to run this test and record its
-            // success or failure.
-            ErrorBitfield renderErrors = kEmptyErrorBitfield;
-            GrRenderTarget* renderTarget = NULL;
-#if SK_SUPPORT_GPU
-            SkAutoTUnref<GrRenderTarget> rt;
-            AutoResetGr autogr;
-            if ((kEmptyErrorBitfield == renderErrors) &&
-                kGPU_Backend == config.fBackend) {
-                GrContext* gr = grFactory->get(config.fGLContextType);
-                bool grSuccess = false;
-                if (gr) {
-                    // create a render target to back the device
-                    GrTextureDesc desc;
-                    desc.fConfig = kSkia8888_GrPixelConfig;
-                    desc.fFlags = kRenderTarget_GrTextureFlagBit;
-                    desc.fWidth = gm->getISize().width();
-                    desc.fHeight = gm->getISize().height();
-                    desc.fSampleCnt = config.fSampleCnt;
-                    GrTexture* tex = gr->createUncachedTexture(desc, NULL, 0);
-                    if (tex) {
-                        rt.reset(tex->asRenderTarget());
-                        rt.get()->ref();
-                        tex->unref();
-                        autogr.set(gr);
-                        renderTarget = rt.get();
-                        grSuccess = NULL != renderTarget;
-                    }
-                    // Set the user specified cache limits if non-default.
-                    size_t bytes;
-                    int count;
-                    gr->getTextureCacheLimits(&count, &bytes);
-                    if (-1 != gpuCacheSize.fBytes) {
-                        bytes = static_cast<size_t>(gpuCacheSize.fBytes);
-                    }
-                    if (-1 != gpuCacheSize.fCount) {
-                        count = gpuCacheSize.fCount;
-                    }
-                    gr->setTextureCacheLimits(count, bytes);
-                }
-                if (!grSuccess) {
-                    renderErrors |= kNoGpuContext_ErrorBitmask;
-                }
-            }
-#endif
-
-            SkBitmap comparisonBitmap;
-
-            if (kEmptyErrorBitfield == renderErrors) {
-                renderErrors |= gmmain.test_drawing(gm, config, writePath,
-                                                    GetGr(),
-                                                    renderTarget,
-                                                    &comparisonBitmap);
-            }
-
-            if (doDeferred && !renderErrors &&
-                (kGPU_Backend == config.fBackend ||
-                 kRaster_Backend == config.fBackend)) {
-                renderErrors |= gmmain.test_deferred_drawing(gm, config,
-                                                             comparisonBitmap,
-                                                             GetGr(),
-                                                             renderTarget);
-            }
-
-            testErrors |= renderErrors;
-        }
-
-        SkBitmap comparisonBitmap;
-        const ConfigData compareConfig =
-            { SkBitmap::kARGB_8888_Config, kRaster_Backend, kDontCare_GLContextType, 0, kRW_ConfigFlag, "comparison" };
-        testErrors |= gmmain.generate_image(gm, compareConfig, NULL, NULL, &comparisonBitmap, false);
-
-        // run the picture centric GM steps
-        if (!(gmFlags & GM::kSkipPicture_Flag)) {
-
-            ErrorBitfield pictErrors = kEmptyErrorBitfield;
-
-            //SkAutoTUnref<SkPicture> pict(generate_new_picture(gm));
-            SkPicture* pict = gmmain.generate_new_picture(gm, kNone_BbhType, 0);
-            SkAutoUnref aur(pict);
-
-            if ((kEmptyErrorBitfield == testErrors) && doReplay) {
-                SkBitmap bitmap;
-                gmmain.generate_image_from_picture(gm, compareConfig, pict,
-                                                   &bitmap);
-                pictErrors |= gmmain.compare_test_results_to_reference_bitmap(
-                    gm, compareConfig, "-replay", bitmap, &comparisonBitmap);
-            }
-
-            if ((kEmptyErrorBitfield == testErrors) &&
-                (kEmptyErrorBitfield == pictErrors) &&
-                doSerialize) {
-                SkPicture* repict = gmmain.stream_to_new_picture(*pict);
-                SkAutoUnref aurr(repict);
-
-                SkBitmap bitmap;
-                gmmain.generate_image_from_picture(gm, compareConfig, repict,
-                                                   &bitmap);
-                pictErrors |= gmmain.compare_test_results_to_reference_bitmap(
-                    gm, compareConfig, "-serialize", bitmap, &comparisonBitmap);
-            }
-
-            if (writePicturePath) {
-                const char* pictureSuffix = "skp";
-                SkString path = make_filename(writePicturePath, "",
-                                              gm->shortName(),
-                                              pictureSuffix);
-                SkFILEWStream stream(path.c_str());
-                pict->serialize(&stream);
-            }
-
-            testErrors |= pictErrors;
-        }
-
-        // TODO: add a test in which the RTree rendering results in a
-        // different bitmap than the standard rendering.  It should
-        // show up as failed in the JSON summary, and should be listed
-        // in the stdout also.
-        if (!(gmFlags & GM::kSkipPicture_Flag) && doRTree) {
-            SkPicture* pict = gmmain.generate_new_picture(
-                gm, kRTree_BbhType, SkPicture::kUsePathBoundsForClip_RecordingFlag);
-            SkAutoUnref aur(pict);
-            SkBitmap bitmap;
-            gmmain.generate_image_from_picture(gm, compareConfig, pict,
-                                               &bitmap);
-            testErrors |= gmmain.compare_test_results_to_reference_bitmap(
-                gm, compareConfig, "-rtree", bitmap, &comparisonBitmap);
-        }
-
-        if (!(gmFlags & GM::kSkipPicture_Flag) && doTileGrid) {
-            for(int scaleIndex = 0; scaleIndex < tileGridReplayScales.count(); ++scaleIndex) {
-                SkScalar replayScale = tileGridReplayScales[scaleIndex];
-                if ((gmFlags & GM::kSkipScaledReplay_Flag) && replayScale != 1)
-                    continue;
-                // We record with the reciprocal scale to obtain a replay
-                // result that can be validated against comparisonBitmap.
-                SkScalar recordScale = SkScalarInvert(replayScale);
-                SkPicture* pict = gmmain.generate_new_picture(
-                    gm, kTileGrid_BbhType, SkPicture::kUsePathBoundsForClip_RecordingFlag,
-                    recordScale);
-                SkAutoUnref aur(pict);
-                SkBitmap bitmap;
-                gmmain.generate_image_from_picture(gm, compareConfig, pict,
-                                                   &bitmap, replayScale);
-                SkString suffix("-tilegrid");
-                if (SK_Scalar1 != replayScale) {
-                    suffix += "-scale-";
-                    suffix.appendScalar(replayScale);
-                }
-                testErrors |= gmmain.compare_test_results_to_reference_bitmap(
-                    gm, compareConfig, suffix.c_str(), bitmap,
-                    &comparisonBitmap);
-            }
-        }
-
-        // run the pipe centric GM steps
-        if (!(gmFlags & GM::kSkipPipe_Flag)) {
-
-            ErrorBitfield pipeErrors = kEmptyErrorBitfield;
-
-            if ((kEmptyErrorBitfield == testErrors) && doPipe) {
-                pipeErrors |= gmmain.test_pipe_playback(gm, compareConfig,
-                                                        comparisonBitmap);
-            }
-
-            if ((kEmptyErrorBitfield == testErrors) &&
-                (kEmptyErrorBitfield == pipeErrors) &&
-                doTiledPipe && !(gmFlags & GM::kSkipTiled_Flag)) {
-                pipeErrors |= gmmain.test_tiled_pipe_playback(gm, compareConfig,
-                                                              comparisonBitmap);
-            }
-
-            testErrors |= pipeErrors;
-        }
-
-        // Update overall results.
-        // We only tabulate the particular error types that we currently
-        // care about (e.g., missing reference images). Later on, if we
-        // want to also tabulate other error types, we can do so.
+        // A non-ignorable error in run_multiple_configs, or ANY error in
+        // run_multiple_drawing_modes, counts as a failure.
         testsRun++;
-        if (!gmmain.fExpectationsSource.get() ||
-            (kEmptyErrorBitfield != (kMissingExpectations_ErrorBitmask & testErrors))) {
-            testsMissingReferenceImages++;
-        }
-        if (testErrors == (testErrors & kIgnorable_ErrorBitmask)) {
+        if (kEmptyErrorBitfield != multipleModeErrors) {
+            testsWithDrawingModeDiscrepancies++;
+            testsFailed++;
+        } else if (compositeErrors == (compositeErrors & kIgnorable_ErrorBitmask)) {
             testsPassed++;
         } else {
             testsFailed++;
         }
+        // Any other result categories we care to report.
+        if (!gmmain.fExpectationsSource.get() ||
+            (kEmptyErrorBitfield != (kMissingExpectations_ErrorBitmask & compositeErrors))) {
+            testsMissingReferenceImages++;
+        }
 
         SkDELETE(gm);
     }
-    gm_fprintf(stdout, "Ran %d tests: %d passed, %d failed, %d missing reference images\n",
-               testsRun, testsPassed, testsFailed, testsMissingReferenceImages);
+    gm_fprintf(stdout, "Ran %d tests: %d passed, %d failed, %d with drawing mode discrepancies, "
+               "%d missing reference images\n",
+               testsRun, testsPassed, testsFailed, testsWithDrawingModeDiscrepancies,
+               testsMissingReferenceImages);
     gmmain.ListErrors();
 
     if (NULL != writeJsonSummaryPath) {
         Json::Value actualResults;
         actualResults[kJsonKey_ActualResults_Failed] =
             gmmain.fJsonActualResults_Failed;
         actualResults[kJsonKey_ActualResults_FailureIgnored] =
             gmmain.fJsonActualResults_FailureIgnored;
         actualResults[kJsonKey_ActualResults_NoComparison] =
             gmmain.fJsonActualResults_NoComparison;
@@ skipping 27 matching lines @@
     SkGraphics::Term();
 
     return (0 == testsFailed) ? 0 : -1;
 }
 
 #if !defined(SK_BUILD_FOR_IOS) && !defined(SK_BUILD_FOR_NACL)
 int main(int argc, char * const argv[]) {
     return tool_main(argc, (char**) argv);
 }
 #endif