GM: run mode comparisons for all raster configs.

gm/gmmain.cpp

 /*
  * Copyright 2011 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 /*
  * Code for the "gm" (Golden Master) rendering comparison tool.
  *
@@ skipping 300 matching lines @@
         while ((mode = iter.next(NULL)) != NULL) {
             SkString modeAsString = SkString(mode);
             // TODO(epoger): It seems a bit silly that all of these modes were
             // recorded with a leading "-" which we have to remove here
             // (except for mode "", which means plain old original mode).
             // But that's how renderModeDescriptor has been passed into
             // compare_test_results_to_reference_bitmap() historically,
             // and changing that now may affect other parts of our code.
             if (modeAsString.startsWith("-")) {
                 modeAsString.remove(0, 1);
-                modes.push_back(modeAsString);
             }

[epoger, 2014/01/10 20:53:27]

There's one renderMode whose name doesn't start with a "-"; that's the default
renderMode.  Let's start reporting it along with the other renderModes, for
completeness' sake.

Besides, when we call RecordTestResults() for all of the
fTestsSkippedOnAllRenderModes entries, we need to report that we skipped the
default renderMode along with the others.  Rather than handle that as a special
case, I think it's better to include the default renderMode in the list.

+            modes.push_back(modeAsString);
         }
     }
 
     /**
      * Returns true if failures on this test should be ignored.
      */
     bool ShouldIgnoreTest(const SkString &name) const {
         for (int i = 0; i < fIgnorableTestSubstrings.count(); i++) {
             if (name.contains(fIgnorableTestSubstrings[i].c_str())) {
                 return true;
             }
         }
         return false;
     }
 
     /**
+     * Calls RecordTestResults to record that we skipped a test.
+     *
+     * Depending on the backend, this may mean that we skipped a single rendermode, or all
+     * rendermodes; see http://skbug.com/1994 and https://codereview.chromium.org/129203002/
+     */
+    void RecordSkippedTest(const SkString& shortNamePlusConfig,
+                           const char renderModeDescriptor [], Backend backend) {
+        if (kRaster_Backend == backend) {
+            // Skipping a test on kRaster_Backend means that we will skip ALL renderModes
+            // (as opposed to other backends, on which we only run the default renderMode).
+            //
+            // We cannot call RecordTestResults yet, because we won't know the full set of
+            // renderModes until we have run all tests.
+            fTestsSkippedOnAllRenderModes.push_back(shortNamePlusConfig);
+        } else {
+            this->RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
+                                    renderModeDescriptor);
+        }
+    }
+
+    /**
      * Records the results of this test in fTestsRun and fFailedTests.
      *
      * We even record successes, and errors that we regard as
      * "ignorable"; we can filter them out later.
      */
     void RecordTestResults(const ErrorCombination& errorCombination,
                            const SkString& shortNamePlusConfig,
                            const char renderModeDescriptor []) {
         // Things to do regardless of errorCombination.
         fTestsRun++;
@@ skipping 884 matching lines @@
 
     bool fUseFileHierarchy, fWriteChecksumBasedFilenames;
     ErrorCombination fIgnorableErrorTypes;
     SkTArray<SkString> fIgnorableTestSubstrings;
 
     const char* fMismatchPath;
     const char* fMissingExpectationsPath;
 
     // collection of tests that have failed with each ErrorType
     SkTArray<SkString> fFailedTests[kLast_ErrorType+1];
+    SkTArray<SkString> fTestsSkippedOnAllRenderModes;
     int fTestsRun;
     SkTDict<int> fRenderModesEncountered;
 
     // Where to read expectations (expected image hash digests, etc.) from.
     // If unset, we don't do comparisons.
     SkAutoTUnref<ExpectationsSource> fExpectationsSource;
 
     // JSON summaries that we generate as we go (just for output).
     Json::Value fJsonExpectedResults;
     Json::Value fJsonActualResults_Failed;
@@ skipping 255 matching lines @@
 }
 
 template <typename T> void appendUnique(SkTDArray<T>* array, const T& value) {
     int index = array->find(value);
     if (index < 0) {
         *array->append() = value;
     }
 }
 
 /**
- * Run this test in a number of different configs (8888, 565, PDF,
- * etc.), confirming that the resulting bitmaps match expectations
- * (which may be different for each config).
- *
- * Returns all errors encountered while doing so.
- */
-ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm,
-                                      const SkTDArray<size_t> &configs,
-                                      const SkTDArray<const PDFRasterizerData*> &pdfRasterizers,
-                                      GrContextFactory *grFactory);
-ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm,
-                                      const SkTDArray<size_t> &configs,
-                                      const SkTDArray<const PDFRasterizerData*> &pdfRasterizers,
-                                      GrContextFactory *grFactory) {
-    const char renderModeDescriptor[] = "";
-    ErrorCombination errorsForAllConfigs;
-    uint32_t gmFlags = gm->getFlags();
-
-    for (int i = 0; i < configs.count(); i++) {
-        ConfigData config = gRec[configs[i]];
-        const SkString shortNamePlusConfig = gmmain.make_shortname_plus_config(gm->shortName(),
-                                                                               config.fName);
-
-        // Skip any tests that we don't even need to try.
-        // If any of these were skipped on a per-GM basis, record them as
-        // kIntentionallySkipped.
-        if (kPDF_Backend == config.fBackend) {
-            if (gmFlags & GM::kSkipPDF_Flag) {
-                gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
-                                         renderModeDescriptor);
-                errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
-                continue;
-            }
-        }
-        if ((gmFlags & GM::kSkip565_Flag) &&
-            (kRaster_Backend == config.fBackend) &&
-            (SkBitmap::kRGB_565_Config == config.fConfig)) {
-            gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
-                                     renderModeDescriptor);
-            errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
-            continue;
-        }
-        if (((gmFlags & GM::kSkipGPU_Flag) && kGPU_Backend == config.fBackend) ||
-            ((gmFlags & GM::kGPUOnly_Flag) && kGPU_Backend != config.fBackend)) {
-            gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
-                                     renderModeDescriptor);
-            errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
-            continue;
-        }
-
-        // Now we know that we want to run this test and record its
-        // success or failure.
-        ErrorCombination errorsForThisConfig;
-        GrSurface* gpuTarget = NULL;
-#if SK_SUPPORT_GPU
-        SkAutoTUnref<GrSurface> auGpuTarget;
-        if ((errorsForThisConfig.isEmpty()) && (kGPU_Backend == config.fBackend)) {
-            if (FLAGS_resetGpuContext) {
-                grFactory->destroyContexts();
-            }
-            GrContext* gr = grFactory->get(config.fGLContextType);
-            bool grSuccess = false;
-            if (gr) {
-                // create a render target to back the device
-                GrTextureDesc desc;
-                desc.fConfig = kSkia8888_GrPixelConfig;
-                desc.fFlags = kRenderTarget_GrTextureFlagBit;
-                desc.fWidth = gm->getISize().width();
-                desc.fHeight = gm->getISize().height();
-                desc.fSampleCnt = config.fSampleCnt;
-                auGpuTarget.reset(gr->createUncachedTexture(desc, NULL, 0));
-                if (NULL != auGpuTarget) {
-                    gpuTarget = auGpuTarget;
-                    grSuccess = true;
-                    // Set the user specified cache limits if non-default.
-                    size_t bytes;
-                    int count;
-                    gr->getTextureCacheLimits(&count, &bytes);
-                    if (DEFAULT_CACHE_VALUE != gGpuCacheSizeBytes) {
-                        bytes = static_cast<size_t>(gGpuCacheSizeBytes);
-                    }
-                    if (DEFAULT_CACHE_VALUE != gGpuCacheSizeCount) {
-                        count = gGpuCacheSizeCount;
-                    }
-                    gr->setTextureCacheLimits(count, bytes);
-                }
-            }
-            if (!grSuccess) {
-                errorsForThisConfig.add(kNoGpuContext_ErrorType);
-            }
-        }
-#endif
-
-        SkBitmap comparisonBitmap;
-
-        const char* writePath;
-        if (FLAGS_writePath.count() == 1) {
-            writePath = FLAGS_writePath[0];
-        } else {
-            writePath = NULL;
-        }
-
-        if (errorsForThisConfig.isEmpty()) {
-            errorsForThisConfig.add(gmmain.test_drawing(gm, config, pdfRasterizers,
-                                                        writePath, gpuTarget,
-                                                        &comparisonBitmap));
-            gmmain.RecordTestResults(errorsForThisConfig, shortNamePlusConfig, "");
-        }
-
-        if (FLAGS_deferred && errorsForThisConfig.isEmpty() &&
-            (kGPU_Backend == config.fBackend || kRaster_Backend == config.fBackend)) {
-            errorsForThisConfig.add(gmmain.test_deferred_drawing(gm, config, comparisonBitmap,
-                                                                 gpuTarget));
-        }
-
-        errorsForAllConfigs.add(errorsForThisConfig);
-    }
-    return errorsForAllConfigs;
-}
-
-/**
  * Run this test in a number of different drawing modes (pipe,
  * deferred, tiled, etc.), confirming that the resulting bitmaps all
  * *exactly* match comparisonBitmap.
  *
  * Returns all errors encountered while doing so.
  */
 ErrorCombination run_multiple_modes(GMMain &gmmain, GM *gm, const ConfigData &compareConfig,
                                     const SkBitmap &comparisonBitmap,
                                     const SkTDArray<SkScalar> &tileGridReplayScales);
 ErrorCombination run_multiple_modes(GMMain &gmmain, GM *gm, const ConfigData &compareConfig,
@@ skipping 112 matching lines @@
         errorsForAllModes.add(gmmain.test_pipe_playback(gm, compareConfig, comparisonBitmap,
                                                         FLAGS_simulatePipePlaybackFailure));
         if (FLAGS_tiledPipe) {
             errorsForAllModes.add(gmmain.test_tiled_pipe_playback(gm, compareConfig,
                                                                   comparisonBitmap));
         }
     }
     return errorsForAllModes;
 }
 
+
+/**
+ * Run this test in a number of different configs (8888, 565, PDF,
+ * etc.), confirming that the resulting bitmaps match expectations
+ * (which may be different for each config).
+ *
+ * Returns all errors encountered while doing so.
+ */
+ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm,
+                                      const SkTDArray<size_t> &configs,
+                                      const SkTDArray<const PDFRasterizerData*> &pdfRasterizers,
+                                      const SkTDArray<SkScalar> &tileGridReplayScales,
+                                      GrContextFactory *grFactory);
+ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm,
+                                      const SkTDArray<size_t> &configs,
+                                      const SkTDArray<const PDFRasterizerData*> &pdfRasterizers,
+                                      const SkTDArray<SkScalar> &tileGridReplayScales,
+                                      GrContextFactory *grFactory) {
+    const char renderModeDescriptor[] = "";
+    ErrorCombination errorsForAllConfigs;
+    uint32_t gmFlags = gm->getFlags();
+
+    for (int i = 0; i < configs.count(); i++) {
+        ConfigData config = gRec[configs[i]];
+        const SkString shortNamePlusConfig = gmmain.make_shortname_plus_config(gm->shortName(),
+                                                                               config.fName);
+
+        // Skip any tests that we don't even need to try.
+        // If any of these were skipped on a per-GM basis, record them as
+        // kIntentionallySkipped.
+        if (kPDF_Backend == config.fBackend) {
+            if (gmFlags & GM::kSkipPDF_Flag) {
+                gmmain.RecordSkippedTest(shortNamePlusConfig,
+                                         renderModeDescriptor,
+                                         config.fBackend);
+                errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
+                continue;
+            }
+        }
+        if ((gmFlags & GM::kSkip565_Flag) &&
+            (kRaster_Backend == config.fBackend) &&
+            (SkBitmap::kRGB_565_Config == config.fConfig)) {
+            gmmain.RecordSkippedTest(shortNamePlusConfig,
+                                     renderModeDescriptor,
+                                     config.fBackend);
+            errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
+            continue;
+        }
+        if (((gmFlags & GM::kSkipGPU_Flag) && kGPU_Backend == config.fBackend) ||
+            ((gmFlags & GM::kGPUOnly_Flag) && kGPU_Backend != config.fBackend)) {
+            gmmain.RecordSkippedTest(shortNamePlusConfig,
+                                     renderModeDescriptor,
+                                     config.fBackend);
+            errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
+            continue;
+        }
+
+        // Now we know that we want to run this test and record its
+        // success or failure.
+        ErrorCombination errorsForThisConfig;
+        GrSurface* gpuTarget = NULL;
+#if SK_SUPPORT_GPU
+        SkAutoTUnref<GrSurface> auGpuTarget;
+        if ((errorsForThisConfig.isEmpty()) && (kGPU_Backend == config.fBackend)) {
+            if (FLAGS_resetGpuContext) {
+                grFactory->destroyContexts();
+            }
+            GrContext* gr = grFactory->get(config.fGLContextType);
+            bool grSuccess = false;
+            if (gr) {
+                // create a render target to back the device
+                GrTextureDesc desc;
+                desc.fConfig = kSkia8888_GrPixelConfig;
+                desc.fFlags = kRenderTarget_GrTextureFlagBit;
+                desc.fWidth = gm->getISize().width();
+                desc.fHeight = gm->getISize().height();
+                desc.fSampleCnt = config.fSampleCnt;
+                auGpuTarget.reset(gr->createUncachedTexture(desc, NULL, 0));
+                if (NULL != auGpuTarget) {
+                    gpuTarget = auGpuTarget;
+                    grSuccess = true;
+                    // Set the user specified cache limits if non-default.
+                    size_t bytes;
+                    int count;
+                    gr->getTextureCacheLimits(&count, &bytes);
+                    if (DEFAULT_CACHE_VALUE != gGpuCacheSizeBytes) {
+                        bytes = static_cast<size_t>(gGpuCacheSizeBytes);
+                    }
+                    if (DEFAULT_CACHE_VALUE != gGpuCacheSizeCount) {
+                        count = gGpuCacheSizeCount;
+                    }
+                    gr->setTextureCacheLimits(count, bytes);
+                }
+            }
+            if (!grSuccess) {
+                errorsForThisConfig.add(kNoGpuContext_ErrorType);
+            }
+        }
+#endif
+
+        SkBitmap comparisonBitmap;
+
+        const char* writePath;
+        if (FLAGS_writePath.count() == 1) {
+            writePath = FLAGS_writePath[0];
+        } else {
+            writePath = NULL;
+        }
+
+        if (errorsForThisConfig.isEmpty()) {
+            errorsForThisConfig.add(gmmain.test_drawing(gm, config, pdfRasterizers,
+                                                        writePath, gpuTarget,
+                                                        &comparisonBitmap));
+            gmmain.RecordTestResults(errorsForThisConfig, shortNamePlusConfig, "");
+        }
+
+        // TODO: run only if gmmain.test_drawing succeeded.
+        if (kRaster_Backend == config.fBackend) {
+            run_multiple_modes(gmmain, gm, config, comparisonBitmap, tileGridReplayScales);
+        }
+
+        if (FLAGS_deferred && errorsForThisConfig.isEmpty() &&
+            (kGPU_Backend == config.fBackend || kRaster_Backend == config.fBackend)) {
+            errorsForThisConfig.add(gmmain.test_deferred_drawing(gm, config, comparisonBitmap,
+                                                                 gpuTarget));
+        }
+
+        errorsForAllConfigs.add(errorsForThisConfig);
+    }
+    return errorsForAllConfigs;
+}
+
+
 /**
  * Read individual lines from a file, pushing them into the given array.
  *
  * @param filename path to the file to read
  * @param lines array of strings to add the lines to
  * @returns true if able to read lines from the file
  */
 static bool read_lines_from_file(const char* filename, SkTArray<SkString> &lines) {
     SkAutoTUnref<SkStream> streamWrapper(SkStream::NewFromFile(filename));
     SkStream *stream = streamWrapper.get();
@@ skipping 497 matching lines @@
 
         if (SkCommandLineFlags::ShouldSkip(FLAGS_match, shortName)) {
             continue;
         }
 
         gmsRun++;
         SkISize size = gm->getISize();
         gm_fprintf(stdout, "%sdrawing... %s [%d %d]\n", moduloStr.c_str(), shortName,
                    size.width(), size.height());
 
-        run_multiple_configs(gmmain, gm, configs, pdfRasterizers, grFactory);
-
-        SkBitmap comparisonBitmap;
-        const ConfigData compareConfig =
-            { SkBitmap::kARGB_8888_Config, kRaster_Backend, kDontCare_GLContextType, 0, kRW_ConfigFlag, "comparison", false };
-        gmmain.generate_image(gm, compareConfig, NULL, &comparisonBitmap, false);
-
-        // TODO(epoger): only run this if gmmain.generate_image() succeeded?
-        // Otherwise, what are we comparing against?
-        run_multiple_modes(gmmain, gm, compareConfig, comparisonBitmap, tileGridReplayScales);
+        run_multiple_configs(gmmain, gm, configs, pdfRasterizers, tileGridReplayScales, grFactory);
     }
 
     SkTArray<SkString> modes;
     gmmain.GetRenderModesEncountered(modes);
+    int modeCount = modes.count();
+
+    // Now that we have run all the tests and thus know the full set of renderModes that we
+    // tried to run, we can call RecordTestResults() to record the cases in which we skipped
+    // ALL renderModes.
+    // See http://skbug.com/1994 and https://codereview.chromium.org/129203002/
+    int testCount = gmmain.fTestsSkippedOnAllRenderModes.count();
+    for (int testNum = 0; testNum < testCount; ++testNum) {
+        SkString *shortNamePlusConfig = &gmmain.fTestsSkippedOnAllRenderModes[testNum];
+        for (int modeNum = 0; modeNum < modeCount; ++modeNum) {
+            gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, *shortNamePlusConfig,
+                                     modes[modeNum].c_str());
+        }
+    }
+
     bool reportError = false;
     if (gmmain.NumSignificantErrors() > 0) {
         reportError = true;
     }
-    int expectedNumberOfTests = gmsRun * (configs.count() + modes.count());
+
+    // We test every GM against every config, and for every raster config also test every mode.
+    int rasterConfigs = 0;
+    for (int i = 0; i < configs.count(); i++) {
+        if (gRec[configs[i]].fBackend == kRaster_Backend) {
+            rasterConfigs++;
+        }
+    }
+    // For raster configs, we run all renderModes; for non-raster configs, just default renderMode
+    const int expectedNumberOfTests = rasterConfigs * gmsRun * modeCount
+                                    + (configs.count() - rasterConfigs) * gmsRun;
 
     // Output summary to stdout.
     if (FLAGS_verbose) {
         gm_fprintf(stdout, "Ran %d GMs\n", gmsRun);
         gm_fprintf(stdout, "... over %2d configs [%s]\n", configs.count(),
                    list_all_config_names(configs).c_str());
-        gm_fprintf(stdout, "...  and %2d modes   [%s]\n", modes.count(), list_all(modes).c_str());
+        gm_fprintf(stdout, "...  and %2d modes   [%s]\n", modeCount, list_all(modes).c_str());
         gm_fprintf(stdout, "... so there should be a total of %d tests.\n", expectedNumberOfTests);
     }
     gmmain.ListErrors(FLAGS_verbose);
 
     // TODO(epoger): Enable this check for Android, too, once we resolve
     // https://code.google.com/p/skia/issues/detail?id=1222
     // ('GM is unexpectedly skipping tests on Android')
 #ifndef SK_BUILD_FOR_ANDROID
     if (expectedNumberOfTests != gmmain.fTestsRun) {
         gm_fprintf(stderr, "expected %d tests, but ran or skipped %d tests\n",
@@ skipping 38 matching lines @@
     if (FLAGS_forceBWtext) {
         canvas->setDrawFilter(SkNEW(BWTextDrawFilter))->unref();
     }
 }
 
 #if !defined(SK_BUILD_FOR_IOS) && !defined(SK_BUILD_FOR_NACL)
 int main(int argc, char * const argv[]) {
     return tool_main(argc, (char**) argv);
 }
 #endif