GM: run mode comparisons for all raster configs.

gm/gmmain.cpp

 /*
  * Copyright 2011 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 /*
  * Code for the "gm" (Golden Master) rendering comparison tool.
  *
@@ skipping 318 matching lines @@
     bool ShouldIgnoreTest(const SkString &name) const {
         for (int i = 0; i < fIgnorableTestSubstrings.count(); i++) {
             if (name.contains(fIgnorableTestSubstrings[i].c_str())) {
                 return true;
             }
         }
         return false;
     }
 
     /**
+     * Calls RecordTestResults to record that we skipped a test for a single rendermode.
+     */
+    void RecordSkippedTest(const SkString& shortNamePlusConfig,
+                           const char renderModeDescriptor []) {
+        this->RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
+                                renderModeDescriptor);
+    }
+
+    /**
      * Records the results of this test in fTestsRun and fFailedTests.
      *
      * We even record successes, and errors that we regard as
      * "ignorable"; we can filter them out later.
      */
     void RecordTestResults(const ErrorCombination& errorCombination,
                            const SkString& shortNamePlusConfig,
                            const char renderModeDescriptor []) {
         // Things to do regardless of errorCombination.
         fTestsRun++;
@@ skipping 799 matching lines @@
         const SkString shortNamePlusConfig = make_shortname_plus_config(gm->shortName(),
                                                                         gRec.fName);
         ErrorCombination errors;
         for (size_t i = 0; i < SK_ARRAY_COUNT(gPipeWritingFlagCombos); ++i) {
             SkString renderModeDescriptor("-pipe");
             renderModeDescriptor.append(gPipeWritingFlagCombos[i].name);
 
             if (gm->getFlags() & GM::kSkipPipe_Flag
                 || (gPipeWritingFlagCombos[i].flags == SkGPipeWriter::kCrossProcess_Flag
                     && gm->getFlags() & GM::kSkipPipeCrossProcess_Flag)) {
-                RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
+                RecordSkippedTest(shortNamePlusConfig,

[epoger, 2014/01/10 19:36:39]

Patchset 5: created a new RecordSkippedTest() method, to record that we skipped
a test for a single renderMode.

In my next patchset, I will add a variant that records skipping a test for ALL
renderModes.

To be continued...

                                   renderModeDescriptor.c_str());
                 errors.add(kIntentionallySkipped_ErrorType);
             } else {
                 SkBitmap bitmap;
                 SkISize size = gm->getISize();
                 setup_bitmap(gRec, size, &bitmap);
                 SkCanvas canvas(bitmap);
                 installFilter(&canvas);
                 // Pass a decoding function so the factory GM (which has an SkBitmap
                 // with encoded data) will not fail playback.
@@ skipping 22 matching lines @@
                                               const SkBitmap& referenceBitmap) {
         const SkString shortNamePlusConfig = make_shortname_plus_config(gm->shortName(),
                                                                         gRec.fName);
         ErrorCombination errors;
         for (size_t i = 0; i < SK_ARRAY_COUNT(gPipeWritingFlagCombos); ++i) {
             SkString renderModeDescriptor("-tiled pipe");
             renderModeDescriptor.append(gPipeWritingFlagCombos[i].name);
 
             if ((gm->getFlags() & GM::kSkipPipe_Flag) ||
                 (gm->getFlags() & GM::kSkipTiled_Flag)) {
-                RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
+                RecordSkippedTest(shortNamePlusConfig,
                                   renderModeDescriptor.c_str());
                 errors.add(kIntentionallySkipped_ErrorType);
             } else {
                 SkBitmap bitmap;
                 SkISize size = gm->getISize();
                 setup_bitmap(gRec, size, &bitmap);
                 SkCanvas canvas(bitmap);
                 installFilter(&canvas);
                 TiledPipeController pipeController(bitmap, &SkImageDecoder::DecodeMemory);
                 SkGPipeWriter writer;
@@ skipping 296 matching lines @@
 }
 
 template <typename T> void appendUnique(SkTDArray<T>* array, const T& value) {
     int index = array->find(value);
     if (index < 0) {
         *array->append() = value;
     }
 }
 
 /**
- * Run this test in a number of different configs (8888, 565, PDF,
- * etc.), confirming that the resulting bitmaps match expectations
- * (which may be different for each config).
- *
- * Returns all errors encountered while doing so.
- */
-ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm,
-                                      const SkTDArray<size_t> &configs,
-                                      const SkTDArray<const PDFRasterizerData*> &pdfRasterizers,
-                                      GrContextFactory *grFactory);
-ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm,
-                                      const SkTDArray<size_t> &configs,
-                                      const SkTDArray<const PDFRasterizerData*> &pdfRasterizers,
-                                      GrContextFactory *grFactory) {
-    const char renderModeDescriptor[] = "";
-    ErrorCombination errorsForAllConfigs;
-    uint32_t gmFlags = gm->getFlags();
-
-    for (int i = 0; i < configs.count(); i++) {
-        ConfigData config = gRec[configs[i]];
-        const SkString shortNamePlusConfig = gmmain.make_shortname_plus_config(gm->shortName(),
-                                                                               config.fName);
-
-        // Skip any tests that we don't even need to try.
-        // If any of these were skipped on a per-GM basis, record them as
-        // kIntentionallySkipped.
-        if (kPDF_Backend == config.fBackend) {
-            if (gmFlags & GM::kSkipPDF_Flag) {
-                gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
-                                         renderModeDescriptor);
-                errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
-                continue;
-            }
-        }
-        if ((gmFlags & GM::kSkip565_Flag) &&
-            (kRaster_Backend == config.fBackend) &&
-            (SkBitmap::kRGB_565_Config == config.fConfig)) {
-            gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
-                                     renderModeDescriptor);
-            errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
-            continue;
-        }
-        if (((gmFlags & GM::kSkipGPU_Flag) && kGPU_Backend == config.fBackend) ||
-            ((gmFlags & GM::kGPUOnly_Flag) && kGPU_Backend != config.fBackend)) {
-            gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
-                                     renderModeDescriptor);
-            errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
-            continue;
-        }
-
-        // Now we know that we want to run this test and record its
-        // success or failure.
-        ErrorCombination errorsForThisConfig;
-        GrSurface* gpuTarget = NULL;
-#if SK_SUPPORT_GPU
-        SkAutoTUnref<GrSurface> auGpuTarget;
-        if ((errorsForThisConfig.isEmpty()) && (kGPU_Backend == config.fBackend)) {
-            if (FLAGS_resetGpuContext) {
-                grFactory->destroyContexts();
-            }
-            GrContext* gr = grFactory->get(config.fGLContextType);
-            bool grSuccess = false;
-            if (gr) {
-                // create a render target to back the device
-                GrTextureDesc desc;
-                desc.fConfig = kSkia8888_GrPixelConfig;
-                desc.fFlags = kRenderTarget_GrTextureFlagBit;
-                desc.fWidth = gm->getISize().width();
-                desc.fHeight = gm->getISize().height();
-                desc.fSampleCnt = config.fSampleCnt;
-                auGpuTarget.reset(gr->createUncachedTexture(desc, NULL, 0));
-                if (NULL != auGpuTarget) {
-                    gpuTarget = auGpuTarget;
-                    grSuccess = true;
-                    // Set the user specified cache limits if non-default.
-                    size_t bytes;
-                    int count;
-                    gr->getTextureCacheLimits(&count, &bytes);
-                    if (DEFAULT_CACHE_VALUE != gGpuCacheSizeBytes) {
-                        bytes = static_cast<size_t>(gGpuCacheSizeBytes);
-                    }
-                    if (DEFAULT_CACHE_VALUE != gGpuCacheSizeCount) {
-                        count = gGpuCacheSizeCount;
-                    }
-                    gr->setTextureCacheLimits(count, bytes);
-                }
-            }
-            if (!grSuccess) {
-                errorsForThisConfig.add(kNoGpuContext_ErrorType);
-            }
-        }
-#endif
-
-        SkBitmap comparisonBitmap;
-
-        const char* writePath;
-        if (FLAGS_writePath.count() == 1) {
-            writePath = FLAGS_writePath[0];
-        } else {
-            writePath = NULL;
-        }
-
-        if (errorsForThisConfig.isEmpty()) {
-            errorsForThisConfig.add(gmmain.test_drawing(gm, config, pdfRasterizers,
-                                                        writePath, gpuTarget,
-                                                        &comparisonBitmap));
-            gmmain.RecordTestResults(errorsForThisConfig, shortNamePlusConfig, "");
-        }
-
-        if (FLAGS_deferred && errorsForThisConfig.isEmpty() &&
-            (kGPU_Backend == config.fBackend || kRaster_Backend == config.fBackend)) {
-            errorsForThisConfig.add(gmmain.test_deferred_drawing(gm, config, comparisonBitmap,
-                                                                 gpuTarget));
-        }
-
-        errorsForAllConfigs.add(errorsForThisConfig);
-    }
-    return errorsForAllConfigs;
-}
-
-/**
  * Run this test in a number of different drawing modes (pipe,
  * deferred, tiled, etc.), confirming that the resulting bitmaps all
  * *exactly* match comparisonBitmap.
  *
  * Returns all errors encountered while doing so.
  */
 ErrorCombination run_multiple_modes(GMMain &gmmain, GM *gm, const ConfigData &compareConfig,
                                     const SkBitmap &comparisonBitmap,
                                     const SkTDArray<SkScalar> &tileGridReplayScales);
 ErrorCombination run_multiple_modes(GMMain &gmmain, GM *gm, const ConfigData &compareConfig,
                                     const SkBitmap &comparisonBitmap,
                                     const SkTDArray<SkScalar> &tileGridReplayScales) {
     ErrorCombination errorsForAllModes;
     uint32_t gmFlags = gm->getFlags();
     const SkString shortNamePlusConfig = gmmain.make_shortname_plus_config(gm->shortName(),
                                                                            compareConfig.fName);
 
     SkPicture* pict = gmmain.generate_new_picture(gm, kNone_BbhType, 0);
     SkAutoUnref aur(pict);
     if (FLAGS_replay) {
         const char renderModeDescriptor[] = "-replay";
         if (gmFlags & GM::kSkipPicture_Flag) {
-            gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
+            gmmain.RecordSkippedTest(shortNamePlusConfig,
                                      renderModeDescriptor);
             errorsForAllModes.add(kIntentionallySkipped_ErrorType);
         } else {
             SkBitmap bitmap;
             gmmain.generate_image_from_picture(gm, compareConfig, pict, &bitmap);
             errorsForAllModes.add(gmmain.compare_test_results_to_reference_bitmap(
                 gm->shortName(), compareConfig.fName, renderModeDescriptor, bitmap,
                 &comparisonBitmap));
         }
     }
 
     if (FLAGS_serialize) {
         const char renderModeDescriptor[] = "-serialize";
         if (gmFlags & GM::kSkipPicture_Flag) {
-            gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
+            gmmain.RecordSkippedTest(shortNamePlusConfig,
                                      renderModeDescriptor);
             errorsForAllModes.add(kIntentionallySkipped_ErrorType);
         } else {
             SkPicture* repict = gmmain.stream_to_new_picture(*pict);
             SkAutoUnref aurr(repict);
             SkBitmap bitmap;
             gmmain.generate_image_from_picture(gm, compareConfig, repict, &bitmap);
             errorsForAllModes.add(gmmain.compare_test_results_to_reference_bitmap(
                 gm->shortName(), compareConfig.fName, renderModeDescriptor, bitmap,
                 &comparisonBitmap));
@@ skipping 12 matching lines @@
         // break anybody.
         SkString path = gmmain.make_filename(FLAGS_writePicturePath[0], gm->shortName(),
                                              compareConfig.fName, "", pictureSuffix);
         SkFILEWStream stream(path.c_str());
         pict->serialize(&stream);
     }
 
     if (FLAGS_rtree) {
         const char renderModeDescriptor[] = "-rtree";
         if (gmFlags & GM::kSkipPicture_Flag) {
-            gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
+            gmmain.RecordSkippedTest(shortNamePlusConfig,
                                      renderModeDescriptor);
             errorsForAllModes.add(kIntentionallySkipped_ErrorType);
         } else {
             SkPicture* pict = gmmain.generate_new_picture(
                 gm, kRTree_BbhType, SkPicture::kOptimizeForClippedPlayback_RecordingFlag);
             SkAutoUnref aur(pict);
             SkBitmap bitmap;
             gmmain.generate_image_from_picture(gm, compareConfig, pict, &bitmap);
             errorsForAllModes.add(gmmain.compare_test_results_to_reference_bitmap(
                 gm->shortName(), compareConfig.fName, renderModeDescriptor, bitmap,
                 &comparisonBitmap));
         }
     }
 
     if (FLAGS_tileGrid) {
         for(int scaleIndex = 0; scaleIndex < tileGridReplayScales.count(); ++scaleIndex) {
             SkScalar replayScale = tileGridReplayScales[scaleIndex];
             SkString renderModeDescriptor("-tilegrid");
             if (SK_Scalar1 != replayScale) {
                 renderModeDescriptor += "-scale-";
                 renderModeDescriptor.appendScalar(replayScale);
             }
 
             if ((gmFlags & GM::kSkipPicture_Flag) ||
                 ((gmFlags & GM::kSkipScaledReplay_Flag) && replayScale != 1)) {
-                gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
+                gmmain.RecordSkippedTest(shortNamePlusConfig,
                                          renderModeDescriptor.c_str());
                 errorsForAllModes.add(kIntentionallySkipped_ErrorType);
             } else {
                 // We record with the reciprocal scale to obtain a replay
                 // result that can be validated against comparisonBitmap.
                 SkScalar recordScale = SkScalarInvert(replayScale);
                 SkPicture* pict = gmmain.generate_new_picture(
                     gm, kTileGrid_BbhType, SkPicture::kUsePathBoundsForClip_RecordingFlag,
                     recordScale);
                 SkAutoUnref aur(pict);
@@ skipping 15 matching lines @@
         errorsForAllModes.add(gmmain.test_pipe_playback(gm, compareConfig, comparisonBitmap,
                                                         FLAGS_simulatePipePlaybackFailure));
         if (FLAGS_tiledPipe) {
             errorsForAllModes.add(gmmain.test_tiled_pipe_playback(gm, compareConfig,
                                                                   comparisonBitmap));
         }
     }
     return errorsForAllModes;
 }
 
+
+/**
+ * Run this test in a number of different configs (8888, 565, PDF,
+ * etc.), confirming that the resulting bitmaps match expectations
+ * (which may be different for each config).
+ *
+ * Returns all errors encountered while doing so.
+ */
+ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm,
+                                      const SkTDArray<size_t> &configs,
+                                      const SkTDArray<const PDFRasterizerData*> &pdfRasterizers,
+                                      const SkTDArray<SkScalar> &tileGridReplayScales,
+                                      GrContextFactory *grFactory);
+ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm,
+                                      const SkTDArray<size_t> &configs,
+                                      const SkTDArray<const PDFRasterizerData*> &pdfRasterizers,
+                                      const SkTDArray<SkScalar> &tileGridReplayScales,
+                                      GrContextFactory *grFactory) {
+    const char renderModeDescriptor[] = "";
+    ErrorCombination errorsForAllConfigs;
+    uint32_t gmFlags = gm->getFlags();
+
+    for (int i = 0; i < configs.count(); i++) {
+        ConfigData config = gRec[configs[i]];
+        const SkString shortNamePlusConfig = gmmain.make_shortname_plus_config(gm->shortName(),
+                                                                               config.fName);
+
+        // Skip any tests that we don't even need to try.
+        // If any of these were skipped on a per-GM basis, record them as
+        // kIntentionallySkipped.
+        if (kPDF_Backend == config.fBackend) {
+            if (gmFlags & GM::kSkipPDF_Flag) {
+                gmmain.RecordSkippedTest(shortNamePlusConfig,
+                                         renderModeDescriptor);
+                errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
+                continue;
+            }
+        }
+        if ((gmFlags & GM::kSkip565_Flag) &&
+            (kRaster_Backend == config.fBackend) &&
+            (SkBitmap::kRGB_565_Config == config.fConfig)) {
+            gmmain.RecordSkippedTest(shortNamePlusConfig,
+                                     renderModeDescriptor);
+            errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
+            continue;
+        }
+        if (((gmFlags & GM::kSkipGPU_Flag) && kGPU_Backend == config.fBackend) ||
+            ((gmFlags & GM::kGPUOnly_Flag) && kGPU_Backend != config.fBackend)) {
+            gmmain.RecordSkippedTest(shortNamePlusConfig,
+                                     renderModeDescriptor);
+            errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
+            continue;
+        }
+
+        // Now we know that we want to run this test and record its
+        // success or failure.
+        ErrorCombination errorsForThisConfig;
+        GrSurface* gpuTarget = NULL;
+#if SK_SUPPORT_GPU
+        SkAutoTUnref<GrSurface> auGpuTarget;
+        if ((errorsForThisConfig.isEmpty()) && (kGPU_Backend == config.fBackend)) {
+            if (FLAGS_resetGpuContext) {
+                grFactory->destroyContexts();
+            }
+            GrContext* gr = grFactory->get(config.fGLContextType);
+            bool grSuccess = false;
+            if (gr) {
+                // create a render target to back the device
+                GrTextureDesc desc;
+                desc.fConfig = kSkia8888_GrPixelConfig;
+                desc.fFlags = kRenderTarget_GrTextureFlagBit;
+                desc.fWidth = gm->getISize().width();
+                desc.fHeight = gm->getISize().height();
+                desc.fSampleCnt = config.fSampleCnt;
+                auGpuTarget.reset(gr->createUncachedTexture(desc, NULL, 0));
+                if (NULL != auGpuTarget) {
+                    gpuTarget = auGpuTarget;
+                    grSuccess = true;
+                    // Set the user specified cache limits if non-default.
+                    size_t bytes;
+                    int count;
+                    gr->getTextureCacheLimits(&count, &bytes);
+                    if (DEFAULT_CACHE_VALUE != gGpuCacheSizeBytes) {
+                        bytes = static_cast<size_t>(gGpuCacheSizeBytes);
+                    }
+                    if (DEFAULT_CACHE_VALUE != gGpuCacheSizeCount) {
+                        count = gGpuCacheSizeCount;
+                    }
+                    gr->setTextureCacheLimits(count, bytes);
+                }
+            }
+            if (!grSuccess) {
+                errorsForThisConfig.add(kNoGpuContext_ErrorType);
+            }
+        }
+#endif
+
+        SkBitmap comparisonBitmap;
+
+        const char* writePath;
+        if (FLAGS_writePath.count() == 1) {
+            writePath = FLAGS_writePath[0];
+        } else {
+            writePath = NULL;
+        }
+
+        if (errorsForThisConfig.isEmpty()) {
+            errorsForThisConfig.add(gmmain.test_drawing(gm, config, pdfRasterizers,
+                                                        writePath, gpuTarget,
+                                                        &comparisonBitmap));
+            gmmain.RecordTestResults(errorsForThisConfig, shortNamePlusConfig, "");
+        }
+
+        // TODO: run only if gmmain.test_drawing succeeded.
+        if (kRaster_Backend == config.fBackend) {
+            run_multiple_modes(gmmain, gm, config, comparisonBitmap, tileGridReplayScales);
+        }
+
+        if (FLAGS_deferred && errorsForThisConfig.isEmpty() &&
+            (kGPU_Backend == config.fBackend || kRaster_Backend == config.fBackend)) {
+            errorsForThisConfig.add(gmmain.test_deferred_drawing(gm, config, comparisonBitmap,
+                                                                 gpuTarget));
+        }
+
+        errorsForAllConfigs.add(errorsForThisConfig);
+    }
+    return errorsForAllConfigs;
+}
+
+
 /**
  * Read individual lines from a file, pushing them into the given array.
  *
  * @param filename path to the file to read
  * @param lines array of strings to add the lines to
  * @returns true if able to read lines from the file
  */
 static bool read_lines_from_file(const char* filename, SkTArray<SkString> &lines) {
     SkAutoTUnref<SkStream> streamWrapper(SkStream::NewFromFile(filename));
     SkStream *stream = streamWrapper.get();
@@ skipping 497 matching lines @@
 
         if (SkCommandLineFlags::ShouldSkip(FLAGS_match, shortName)) {
             continue;
         }
 
         gmsRun++;
         SkISize size = gm->getISize();
         gm_fprintf(stdout, "%sdrawing... %s [%d %d]\n", moduloStr.c_str(), shortName,
                    size.width(), size.height());
 
-        run_multiple_configs(gmmain, gm, configs, pdfRasterizers, grFactory);
-
-        SkBitmap comparisonBitmap;
-        const ConfigData compareConfig =
-            { SkBitmap::kARGB_8888_Config, kRaster_Backend, kDontCare_GLContextType, 0, kRW_ConfigFlag, "comparison", false };
-        gmmain.generate_image(gm, compareConfig, NULL, &comparisonBitmap, false);
-
-        // TODO(epoger): only run this if gmmain.generate_image() succeeded?
-        // Otherwise, what are we comparing against?
-        run_multiple_modes(gmmain, gm, compareConfig, comparisonBitmap, tileGridReplayScales);
+        run_multiple_configs(gmmain, gm, configs, pdfRasterizers, tileGridReplayScales, grFactory);
     }
 
     SkTArray<SkString> modes;
     gmmain.GetRenderModesEncountered(modes);
     bool reportError = false;
     if (gmmain.NumSignificantErrors() > 0) {
         reportError = true;
     }
-    int expectedNumberOfTests = gmsRun * (configs.count() + modes.count());
+
+    // We test every GM against every config, and for every raster config also test every mode.
+    int rasterConfigs = 0;
+    for (int i = 0; i < configs.count(); i++) {
+        if (gRec[configs[i]].fBackend == kRaster_Backend) {
+            rasterConfigs++;
+        }
+    }
+    const int expectedNumberOfTests = gmsRun * configs.count()
+                                    + gmsRun * rasterConfigs * modes.count();
 
     // Output summary to stdout.
     if (FLAGS_verbose) {
         gm_fprintf(stdout, "Ran %d GMs\n", gmsRun);
         gm_fprintf(stdout, "... over %2d configs [%s]\n", configs.count(),
                    list_all_config_names(configs).c_str());
         gm_fprintf(stdout, "...  and %2d modes   [%s]\n", modes.count(), list_all(modes).c_str());
         gm_fprintf(stdout, "... so there should be a total of %d tests.\n", expectedNumberOfTests);
     }
     gmmain.ListErrors(FLAGS_verbose);
@@ skipping 45 matching lines @@
     if (FLAGS_forceBWtext) {
         canvas->setDrawFilter(SkNEW(BWTextDrawFilter))->unref();
     }
 }
 
 #if !defined(SK_BUILD_FOR_IOS) && !defined(SK_BUILD_FOR_NACL)
 int main(int argc, char * const argv[]) {
     return tool_main(argc, (char**) argv);
 }
 #endif