Add option to dump images from nanobench.

bench/nanobench.cpp

 /*
  * Copyright 2014 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 #include <ctype.h>
 
 #include "Benchmark.h"
@@ skipping 13 matching lines @@
 #include "SkSurface.h"
 
 #if SK_SUPPORT_GPU
     #include "gl/GrGLDefines.h"
     #include "GrContextFactory.h"
     GrContextFactory gGrFactory;
 #endif
 
 __SK_FORCE_IMAGE_DECODER_LINKING;
 
-#if SK_DEBUG
-    DEFINE_bool(runOnce, true, "Run each benchmark just once?");
+static const int kAutoTuneLoops = -1;
+
+static const int kDefaultLoops = 
+#ifdef SK_DEBUG
+    1;
 #else
-    DEFINE_bool(runOnce, false, "Run each benchmark just once?");
+    kAutoTuneLoops;
 #endif
 
+static SkString loops_help_txt() {
+    SkString help;
+    help.printf("Number of times to run each bench. Set this to %d to auto-"
+                "tune per-bench. Timings are only reported when auto-tuning.",
+                kAutoTuneLoops);
+    return help;
+}
+
+DEFINE_int32(loops, kDefaultLoops, loops_help_txt().c_str());

[mtklein, 2014/08/07 18:00:47]

This seems somewhat unsafe unless DEFINE_int32 immediate copies the help string.
 I think Valgrind's going to yell at us if we ever pass --help. :P

[bsalomon, 2014/08/07 20:08:10]

It does copy it into an SkString.

+
+DEFINE_string2(writePath, w, "", "If set, write benches here as .pngs.");
+
 DEFINE_int32(samples, 10, "Number of samples to measure for each bench.");
 DEFINE_int32(overheadLoops, 100000, "Loops to estimate timer overhead.");
 DEFINE_double(overheadGoal, 0.0001,
               "Loop until timer overhead is at most this fraction of our measurments.");
 DEFINE_double(gpuMs, 5, "Target bench time in millseconds for GPU.");
 DEFINE_int32(gpuFrameLag, 5, "Overestimate of maximum number of frames GPU allows to lag.");
 
 DEFINE_string(outResultsFile, "", "If given, write results here as JSON.");
 DEFINE_int32(maxCalibrationAttempts, 3,
              "Try up to this many times to guess loops for a bench, or skip the bench.");
@@ skipping 48 matching lines @@
         SkDebugf("ERROR: clamping loops from %d to 1.\n", loops);
         return 1;
     }
     if (loops > FLAGS_maxLoops) {
         SkDebugf("WARNING: clamping loops from %d to FLAGS_maxLoops, %d.\n", loops, FLAGS_maxLoops);
         return FLAGS_maxLoops;
     }
     return loops;
 }
 
+static bool write_canvas_png(SkCanvas* canvas, const SkString& filename) {
+    if (filename.isEmpty()) {
+        return false;
+    }
+    if (kUnknown_SkColorType == canvas->imageInfo().fColorType) {
+        return false;
+    }
+    SkBitmap bmp;
+    bmp.setInfo(canvas->imageInfo());
+    if (!canvas->readPixels(&bmp, 0, 0)) {
+        SkDebugf("Can't read canvas pixels.\n");
+        return false;
+    }
+    SkString dir = SkOSPath::Dirname(filename.c_str());
+    if (!sk_mkdir(dir.c_str())) {
+        SkDebugf("Can't make dir %s.\n", dir.c_str());
+        return false;
+    }
+    SkFILEWStream stream(filename.c_str());
+    if (!stream.isValid()) {
+        SkDebugf("Can't write %s.\n", filename.c_str());
+        return false;
+    }
+    if (!SkImageEncoder::EncodeStream(&stream, bmp, SkImageEncoder::kPNG_Type, 100)) {
+        SkDebugf("Can't encode a PNG.\n");
+        return false;
+    }
+    return true;
+}
+
+static int kFailedLoops = -2;
 static int cpu_bench(const double overhead, Benchmark* bench, SkCanvas* canvas, double* samples) {
     // First figure out approximately how many loops of bench it takes to make overhead negligible.
     double bench_plus_overhead = 0.0;
     int round = 0;
-    while (bench_plus_overhead < overhead) {
-        if (round++ == FLAGS_maxCalibrationAttempts) {
-            SkDebugf("WARNING: Can't estimate loops for %s (%s vs. %s); skipping.\n",
-                     bench->getName(), HUMANIZE(bench_plus_overhead), HUMANIZE(overhead));
-            return 0;
+    if (kAutoTuneLoops == FLAGS_loops) {
+        while (bench_plus_overhead < overhead) {
+            if (round++ == FLAGS_maxCalibrationAttempts) {
+                SkDebugf("WARNING: Can't estimate loops for %s (%s vs. %s); skipping.\n",
+                         bench->getName(), HUMANIZE(bench_plus_overhead), HUMANIZE(overhead));
+                return kFailedLoops;
+            }
+            bench_plus_overhead = time(1, bench, canvas, NULL);
         }
-        bench_plus_overhead = time(1, bench, canvas, NULL);
     }
 
     // Later we'll just start and stop the timer once but loop N times.
     // We'll pick N to make timer overhead negligible:
     //
     //          overhead
     //  -------------------------  < FLAGS_overheadGoal
     //  overhead + N * Bench Time
     //
     // where bench_plus_overhead ≈ overhead + Bench Time.
     //
     // Doing some math, we get:
     //
     //  (overhead / FLAGS_overheadGoal) - overhead
     //  ------------------------------------------  < N
     //       bench_plus_overhead - overhead)
     //
     // Luckily, this also works well in practice. :)
-    const double numer = overhead / FLAGS_overheadGoal - overhead;
-    const double denom = bench_plus_overhead - overhead;
-    const int loops = clamp_loops(FLAGS_runOnce ? 1 : (int)ceil(numer / denom));
+    int loops = FLAGS_loops;
+    if (kAutoTuneLoops == loops) {
+        const double numer = overhead / FLAGS_overheadGoal - overhead;
+        const double denom = bench_plus_overhead - overhead;
+        loops = (int)ceil(numer / denom);
+    }
+    loops = clamp_loops(loops);
 
     for (int i = 0; i < FLAGS_samples; i++) {
         samples[i] = time(loops, bench, canvas, NULL) / loops;
     }
     return loops;
 }
 
 #if SK_SUPPORT_GPU
 static int gpu_bench(SkGLContextHelper* gl,
                      Benchmark* bench,
                      SkCanvas* canvas,
                      double* samples) {
     gl->makeCurrent();
     // Make sure we're done with whatever came before.
     SK_GL(*gl, Finish());
 
     // First, figure out how many loops it'll take to get a frame up to FLAGS_gpuMs.
-    int loops = 1;
-    if (!FLAGS_runOnce) {
+    int loops = FLAGS_loops;
+    if (kAutoTuneLoops == loops) {
+        loops = 1;
         double elapsed = 0;
         do {
             loops *= 2;
             // If the GPU lets frames lag at all, we need to make sure we're timing
             // _this_ round, not still timing last round.  We force this by looping
             // more times than any reasonable GPU will allow frames to lag.
             for (int i = 0; i < FLAGS_gpuFrameLag; i++) {
                 elapsed = time(loops, bench, canvas, gl);
             }
         } while (elapsed < FLAGS_gpuMs);
@@ skipping 298 matching lines @@
     const char* fSourceType;
     int fCurrentScale;
     int fCurrentSKP;
 };
 
 int nanobench_main();
 int nanobench_main() {
     SetupCrashHandler();
     SkAutoGraphics ag;
 
-    if (FLAGS_runOnce) {
+    if (kAutoTuneLoops != FLAGS_loops) {
         FLAGS_samples     = 1;
         FLAGS_gpuFrameLag = 0;
     }
 
+    if (!FLAGS_writePath.isEmpty()) {
+        SkDebugf("Writing files to %s.", FLAGS_writePath[0]);
+        if (FLAGS_writePath.count() > 1) {
+            SkDebugf(" Ignoring: ", FLAGS_writePath[0]);

[mtklein, 2014/08/07 18:00:47]

bogus extra arg to SkDebugf?

Maybe it's just me being cavalier, but maybe you can just drop this whole
too-many-args section?

[bsalomon, 2014/08/07 20:08:10]

Done.

+            for (int i = 1; i < FLAGS_writePath.count(); ++i) {
+                SkDebugf("%s ", FLAGS_writePath[i]);
+            }
+        }
+        SkDebugf("\n");
+        if (!sk_mkdir(FLAGS_writePath[0])) {

[mtklein, 2014/08/07 18:00:47]

Don't we do this inside write_canvas_png?

[bsalomon, 2014/08/07 20:08:10]

It seems that sk_mkdir doesn't mkdirs recursively. So this attempts to make the
top level dir and that one makes the per-config dirs. It also prevents lots of
spew when the root dir can't be created.

+            SkDebugf("Could not create %s. Files won't be written.\n", FLAGS_writePath[0]);
+            FLAGS_writePath.set(0, NULL);
+        }
+    }
+
     MultiResultsWriter log;
     SkAutoTDelete<NanoJSONResultsWriter> json;
     if (!FLAGS_outResultsFile.isEmpty()) {
         const char* gitHash = FLAGS_gitHash.isEmpty() ? "unknown-revision" : FLAGS_gitHash[0];
         json.reset(SkNEW(NanoJSONResultsWriter(FLAGS_outResultsFile[0], gitHash)));
         log.add(json.get());
     }
     CallEnd<MultiResultsWriter> ender(log);
 
     if (1 == FLAGS_key.count() % 2) {
         SkDebugf("ERROR: --key must be passed with an even number of arguments.\n");
         return 1;
     }
     for (int i = 1; i < FLAGS_key.count(); i += 2) {
         log.key(FLAGS_key[i-1], FLAGS_key[i]);
     }
     fill_static_options(&log);
 
     const double overhead = estimate_timer_overhead();
     SkDebugf("Timer overhead: %s\n", HUMANIZE(overhead));
 
     SkAutoTMalloc<double> samples(FLAGS_samples);
 
-    if (FLAGS_runOnce) {
-        SkDebugf("--runOnce is true; times would only be misleading so we won't print them.\n");
+    if (kAutoTuneLoops != FLAGS_loops) {
+        SkDebugf("Fixed number of loops; times would only be misleading so we won't print them.\n");
     } else if (FLAGS_verbose) {
         // No header.
     } else if (FLAGS_quiet) {
         SkDebugf("median\tbench\tconfig\n");
     } else {
         SkDebugf("loops\tmin\tmedian\tmean\tmax\tstddev\tsamples\tconfig\tbench\n");
     }
 
     SkTDArray<Config> configs;
     create_configs(&configs);
@@ skipping 16 matching lines @@
             SkCanvas* canvas = targets[j]->surface.get() ? targets[j]->surface->getCanvas() : NULL;
             const char* config = targets[j]->config.name;
 
 #if SK_DEBUG
             // skia:2797  Some SKPs SkASSERT in debug mode.  Skip them for now.
             if (0 == strcmp("565", config) && SkStrContains(bench->getName(), ".skp")) {
                 SkDebugf("Skipping 565 %s.  See skia:2797\n", bench->getName());
                 continue;
             }
 #endif
-
+            SkString pngFilename;
+            if (canvas && !FLAGS_writePath.isEmpty() && NULL != FLAGS_writePath[0]) {
+                canvas->clear(SK_ColorWHITE);

[mtklein, 2014/08/07 18:00:47]

Perhaps we should always clear canvas to white?  and move the rest of this down
to where we call write_canvas_png?

[bsalomon, 2014/08/07 20:08:10]

Done.

+                pngFilename = SkOSPath::Join(FLAGS_writePath[0], config);
+                pngFilename = SkOSPath::Join(pngFilename.c_str(), bench->getName());
+                pngFilename.append(".png");
+            }
             const int loops =
 #if SK_SUPPORT_GPU
                 Benchmark::kGPU_Backend == targets[j]->config.backend
                 ? gpu_bench(targets[j]->gl, bench.get(), canvas, samples.get())
                 :
 #endif
                  cpu_bench(       overhead, bench.get(), canvas, samples.get());
 
-            if (loops == 0) {
+            if (!pngFilename.isEmpty()) {
+                write_canvas_png(canvas, pngFilename);
+            }
+
+            if (kFailedLoops == loops) {
                 // Can't be timed.  A warning note has already been printed.
                 continue;
             }
 
             Stats stats(samples.get(), FLAGS_samples);
             log.config(config);
             benchStream.fillCurrentOptions(&log);
 #if SK_SUPPORT_GPU
             if (Benchmark::kGPU_Backend == targets[j]->config.backend) {
                 fill_gpu_options(&log, targets[j]->gl);
             }
 #endif
             log.timer("min_ms",    stats.min);
             log.timer("median_ms", stats.median);
             log.timer("mean_ms",   stats.mean);
             log.timer("max_ms",    stats.max);
             log.timer("stddev_ms", sqrt(stats.var));
 
-            if (FLAGS_runOnce) {
+            if (kAutoTuneLoops != FLAGS_loops) {
                 if (targets.count() == 1) {
                     config = ""; // Only print the config if we run the same bench on more than one.
                 }
                 SkDebugf("%s\t%s\n", bench->getName(), config);
             } else if (FLAGS_verbose) {
                 for (int i = 0; i < FLAGS_samples; i++) {
                     SkDebugf("%s  ", HUMANIZE(samples[i]));
                 }
                 SkDebugf("%s\n", bench->getName());
             } else if (FLAGS_quiet) {
@@ skipping 30 matching lines @@
 
     return 0;
 }
 
 #if !defined SK_BUILD_FOR_IOS
 int main(int argc, char** argv) {
     SkCommandLineFlags::Parse(argc, argv);
     return nanobench_main();
 }
 #endif