GM: add --missingExpectationsPath option, similar to --mismatchPath

gm/gmmain.cpp

 /*
  * Copyright 2011 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 /*
  * Code for the "gm" (Golden Master) rendering comparison tool.
  *
@@ skipping 175 matching lines @@
 static bool encode_to_dct_stream(SkWStream* stream, const SkBitmap& bitmap, const SkIRect& rect);
 
 const static ErrorCombination kDefaultIgnorableErrorTypes = ErrorCombination()
     .plus(kMissingExpectations_ErrorType)
     .plus(kIntentionallySkipped_ErrorType);
 
 class GMMain {
 public:
     GMMain() : fUseFileHierarchy(false), fWriteChecksumBasedFilenames(false),
                fIgnorableErrorTypes(kDefaultIgnorableErrorTypes),
-               fMismatchPath(NULL), fTestsRun(0), fRenderModesEncountered(1) {}
+               fMismatchPath(NULL), fMissingExpectationsPath(NULL), fTestsRun(0),

[epoger, 2013/06/21 17:46:44]

Patchset 2 adds the new missingExpectationsPath flag to GM

[borenet, 2013/06/21 18:01:14]

So, we put the actuals for GMs with no expectations in a different place from
those with mismatches.  Is this so that we can upload them to different places? 
I think we want to upload them in either case.

[epoger, 2013/06/21 18:06:46]

As you'll see in https://codereview.chromium.org/16907006/ ('buildbots: only
store/upload GM results we haven't seen before, to save time'), my plan is to
store both of them in the SAME directory.

Another way to achieve this would be to change the meaning of --mismatchPath to
be: write out any files that either mismatch expectations, OR DON'T HAVE ANY. 
But it seemed safer/cleaner to add a new option, and allow the caller to put the
images wherever he wants them.

[borenet, 2013/06/21 18:14:09]

I think that's fine, I just can't think of a case where we'd actually put them
in different places.

+               fRenderModesEncountered(1) {}
 
     /**
      * Assemble shortNamePlusConfig from (surprise!) shortName and configName.
      *
      * The method for doing so depends on whether we are using hierarchical naming.
      * For example, shortName "selftest1" and configName "8888" could be assembled into
      * either "selftest1_8888" or "8888/selftest1".
      */
     SkString make_shortname_plus_config(const char *shortName, const char *configName) {
         SkString name;
@@ skipping 553 matching lines @@
                    " max per-channel mismatch R=%d G=%d B=%d A=%d\n",
                    testName, differingPixels, width*height, errR, errG, errB, errA);
     }
 
     /**
      * Compares actual hash digest to expectations, returning the set of errors
      * (if any) that we saw along the way.
      *
      * If fMismatchPath has been set, and there are pixel diffs, then the
      * actual bitmap will be written out to a file within fMismatchPath.
+     * And similarly for fMissingExpectationsPath...
      *
      * @param expectations what expectations to compare actualBitmap against
      * @param actualBitmapAndDigest the SkBitmap we actually generated, and its GmResultDigest
      * @param shortName name of test, e.g. "selftest1"
      * @param configName name of config, e.g. "8888"
      * @param renderModeDescriptor e.g., "-rtree", "-deferred"
      * @param addToJsonSummary whether to add these results (both actual and
      *        expected) to the JSON summary. Regardless of this setting, if
      *        we find an image mismatch in this test, we will write these
      *        results to the JSON summary.  (This is so that we will always
      *        report errors across rendering modes, such as pipe vs tiled.
      *        See https://codereview.chromium.org/13650002/ )
      */
     ErrorCombination compare_to_expectations(Expectations expectations,
                                              const BitmapAndDigest& actualBitmapAndDigest,
                                              const char *shortName, const char *configName,
                                              const char *renderModeDescriptor,
                                              bool addToJsonSummary) {
         ErrorCombination errors;
         SkString shortNamePlusConfig = make_shortname_plus_config(shortName, configName);
         SkString completeNameString(shortNamePlusConfig);
         completeNameString.append(renderModeDescriptor);
         completeNameString.append(".");
         completeNameString.append(kPNG_FileExtension);
         const char* completeName = completeNameString.c_str();
 
         if (expectations.empty()) {
             errors.add(kMissingExpectations_ErrorType);
+
+            // Write out the "actuals" for any tests without expectations, if we have
+            // been directed to do so.
+            if (fMissingExpectationsPath) {
+                SkString path = make_bitmap_filename(fMissingExpectationsPath, shortName,
+                                                     configName, renderModeDescriptor,
+                                                     actualBitmapAndDigest.fDigest);
+                write_bitmap(path, actualBitmapAndDigest.fBitmap);
+            }
+
         } else if (!expectations.match(actualBitmapAndDigest.fDigest)) {
             addToJsonSummary = true;
             // The error mode we record depends on whether this was running
             // in a non-standard renderMode.
             if ('\0' == *renderModeDescriptor) {
                 errors.add(kExpectationsMismatch_ErrorType);
             } else {
                 errors.add(kRenderModeMismatch_ErrorType);
             }
 
@@ skipping 363 matching lines @@
 
     //
     // member variables.
     // They are public for now, to allow easier setting by tool_main().
     //
 
     bool fUseFileHierarchy, fWriteChecksumBasedFilenames;
     ErrorCombination fIgnorableErrorTypes;
 
     const char* fMismatchPath;
+    const char* fMissingExpectationsPath;
 
     // collection of tests that have failed with each ErrorType
     SkTArray<SkString> fFailedTests[kLast_ErrorType+1];
     int fTestsRun;
     SkTDict<int> fRenderModesEncountered;
 
     // Where to read expectations (expected image hash digests, etc.) from.
     // If unset, we don't do comparisons.
     SkAutoTUnref<ExpectationsSource> fExpectationsSource;
 
@@ skipping 119 matching lines @@
               "Space-separated list of ErrorTypes that should be ignored. If any *other* error "
               "types are encountered, the tool will exit with a nonzero return value.");
 DEFINE_string(match, "", "[~][^]substring[$] [...] of test name to run.\n"
               "Multiple matches may be separated by spaces.\n"
               "~ causes a matching test to always be skipped\n"
               "^ requires the start of the test to match\n"
               "$ requires the end of the test to match\n"
               "^ and $ requires an exact match\n"
               "If a test does not match any list entry,\n"
               "it is skipped unless some list entry starts with ~");
+DEFINE_string(missingExpectationsPath, "", "Write images for tests without expectations "
+              "into this directory.");
 DEFINE_string(mismatchPath, "", "Write images for tests that failed due to "
               "pixel mismatches into this directory.");
 DEFINE_string(modulo, "", "[--modulo <remainder> <divisor>]: only run tests for which "
               "testIndex %% divisor == remainder.");
 DEFINE_bool(pdf, true, "Exercise the pdf rendering test pass.");
 DEFINE_bool(pipe, true, "Exercise the SkGPipe replay test pass.");
 DEFINE_string2(readPath, r, "", "Read reference images from this dir, and report "
                "any differences between those and the newly generated ones.");
 DEFINE_bool(replay, true, "Exercise the SkPicture replay test pass.");
 DEFINE_string2(resourcePath, i, "", "Directory that stores image resources.");
@@ skipping 462 matching lines @@
     SkString usage;
     usage.printf("Run the golden master tests.\n");
     SkCommandLineFlags::SetUsage(usage.c_str());
     SkCommandLineFlags::Parse(argc, argv);
 
     gmmain.fUseFileHierarchy = FLAGS_hierarchy;
     gmmain.fWriteChecksumBasedFilenames = FLAGS_writeChecksumBasedFilenames;
     if (FLAGS_mismatchPath.count() == 1) {
         gmmain.fMismatchPath = FLAGS_mismatchPath[0];
     }
+    if (FLAGS_missingExpectationsPath.count() == 1) {
+        gmmain.fMissingExpectationsPath = FLAGS_missingExpectationsPath[0];
+    }
 
     for (int i = 0; i < FLAGS_config.count(); i++) {
         const char* config = FLAGS_config[i];
         userConfig = true;
         bool exclude = false;
         if (*config == kExcludeConfigChar) {
             exclude = true;
             config += 1;
         }
         int index = findConfig(config);
@@ skipping 168 matching lines @@
                 JsonExpectationsSource, (readPath)));
         }
     }
     if (FLAGS_verbose) {
         if (FLAGS_writePath.count() == 1) {
             gm_fprintf(stdout, "writing to %s\n", FLAGS_writePath[0]);
         }
         if (NULL != gmmain.fMismatchPath) {
             gm_fprintf(stdout, "writing mismatches to %s\n", gmmain.fMismatchPath);
         }
+        if (NULL != gmmain.fMissingExpectationsPath) {
+            gm_fprintf(stdout, "writing images without expectations to %s\n",
+                       gmmain.fMissingExpectationsPath);
+        }
         if (FLAGS_writePicturePath.count() == 1) {
             gm_fprintf(stdout, "writing pictures to %s\n", FLAGS_writePicturePath[0]);
         }
         if (FLAGS_resourcePath.count() == 1) {
             gm_fprintf(stdout, "reading resources from %s\n", FLAGS_resourcePath[0]);
         }
     }
 
     if (moduloDivisor <= 0) {
         moduloRemainder = -1;
@@ skipping 10 matching lines @@
     if (FLAGS_writePath.count() == 1) {
         if (!prepare_subdirectories(FLAGS_writePath[0], gmmain.fUseFileHierarchy, configs)) {
             return -1;
         }
     }
     if (NULL != gmmain.fMismatchPath) {
         if (!prepare_subdirectories(gmmain.fMismatchPath, gmmain.fUseFileHierarchy, configs)) {
             return -1;
         }
     }
+    if (NULL != gmmain.fMissingExpectationsPath) {
+        if (!prepare_subdirectories(gmmain.fMissingExpectationsPath, gmmain.fUseFileHierarchy,
+                                    configs)) {
+            return -1;
+        }
+    }
 
     if (FLAGS_pdfJpegQuality < -1 || FLAGS_pdfJpegQuality > 100) {
         gm_fprintf(stderr, "%s\n", "pdfJpegQuality must be in [-1 .. 100] range.");
     }
 
     Iter iter;
     GM* gm;
     while ((gm = iter.next()) != NULL) {
         SkAutoTDelete<GM> adgm(gm);
         ++gmIndex;
@@ skipping 91 matching lines @@
     if (FLAGS_forceBWtext) {
         canvas->setDrawFilter(SkNEW(BWTextDrawFilter))->unref();
     }
 }
 
 #if !defined(SK_BUILD_FOR_IOS) && !defined(SK_BUILD_FOR_NACL)
 int main(int argc, char * const argv[]) {
     return tool_main(argc, (char**) argv);
 }
 #endif