Add samplingTime mode to nanobench

bench/nanobench.cpp

 /*
  * Copyright 2014 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 #include <ctype.h>
 
 #include "nanobench.h"
@@ skipping 37 matching lines @@
     #include "gl/GrGLDefines.h"
     #include "GrCaps.h"
     #include "GrContextFactory.h"
     SkAutoTDelete<GrContextFactory> gGrFactory;
 #endif
 
     struct GrContextOptions;
 
 __SK_FORCE_IMAGE_DECODER_LINKING;
 
+static const int kTimedSampling = 0;
+
 static const int kAutoTuneLoops = 0;
 
 static const int kDefaultLoops =
 #ifdef SK_DEBUG
     1;
 #else
     kAutoTuneLoops;
 #endif
 
 static SkString loops_help_txt() {
     SkString help;
     help.printf("Number of times to run each bench. Set this to %d to auto-"
                 "tune for each bench. Timings are only reported when auto-tuning.",
                 kAutoTuneLoops);
     return help;
 }
 
+static SkString to_string(int n) {
+    SkString str;
+    str.appendS32(n);
+    return str;
+}
+
 DEFINE_int32(loops, kDefaultLoops, loops_help_txt().c_str());
 
 DEFINE_int32(samples, 10, "Number of samples to measure for each bench.");
+DEFINE_string(samplingTime, "0", "Amount of time to run each bench. Takes precedence over samples."
+                                 "Must be \"0\", \"%%lfs\", or \"%%lfms\"");
 DEFINE_int32(overheadLoops, 100000, "Loops to estimate timer overhead.");
 DEFINE_double(overheadGoal, 0.0001,
               "Loop until timer overhead is at most this fraction of our measurments.");
 DEFINE_double(gpuMs, 5, "Target bench time in millseconds for GPU.");
 DEFINE_int32(gpuFrameLag, 5, "If unknown, estimated maximum number of frames GPU allows to lag.");
 DEFINE_bool(gpuCompressAlphaMasks, false, "Compress masks generated from falling back to "
                                           "software path rendering.");
 
 DEFINE_string(outResultsFile, "", "If given, write results here as JSON.");
 DEFINE_int32(maxCalibrationAttempts, 3,
@@ skipping 174 matching lines @@
         return false;
     }
     if (!SkImageEncoder::EncodeStream(&stream, bmp, SkImageEncoder::kPNG_Type, 100)) {
         SkDebugf("Can't encode a PNG.\n");
         return false;
     }
     return true;
 }
 
 static int kFailedLoops = -2;
-static int cpu_bench(const double overhead, Target* target, Benchmark* bench, double* samples) {
+static int setup_cpu_bench(const double overhead, Target* target, Benchmark* bench) {
     // First figure out approximately how many loops of bench it takes to make overhead negligible.
     double bench_plus_overhead = 0.0;
     int round = 0;
     int loops = bench->calculateLoops(FLAGS_loops);
     if (kAutoTuneLoops == loops) {
         while (bench_plus_overhead < overhead) {
             if (round++ == FLAGS_maxCalibrationAttempts) {
                 SkDebugf("WARNING: Can't estimate loops for %s (%s vs. %s); skipping.\n",
                          bench->getUniqueName(), HUMANIZE(bench_plus_overhead), HUMANIZE(overhead));
                 return kFailedLoops;
@@ skipping 20 matching lines @@
     // Luckily, this also works well in practice. :)
     if (kAutoTuneLoops == loops) {
         const double numer = overhead / FLAGS_overheadGoal - overhead;
         const double denom = bench_plus_overhead - overhead;
         loops = (int)ceil(numer / denom);
         loops = clamp_loops(loops);
     } else {
         loops = detect_forever_loops(loops);
     }
 
-    for (int i = 0; i < FLAGS_samples; i++) {
-        samples[i] = time(loops, bench, target) / loops;
-    }
     return loops;
 }
 
-static int gpu_bench(Target* target,
-                     Benchmark* bench,
-                     double* samples,
-                     int maxGpuFrameLag) {
+static int setup_gpu_bench(Target* target, Benchmark* bench, int maxGpuFrameLag) {
     // First, figure out how many loops it'll take to get a frame up to FLAGS_gpuMs.
     int loops = bench->calculateLoops(FLAGS_loops);
     if (kAutoTuneLoops == loops) {
         loops = 1;
         double elapsed = 0;
         do {
             if (1<<30 == loops) {
                 // We're about to wrap.  Something's wrong with the bench.
                 loops = 0;
                 break;
@@ skipping 15 matching lines @@
     } else {
         loops = detect_forever_loops(loops);
     }
 
     // Pretty much the same deal as the calibration: do some warmup to make
     // sure we're timing steady-state pipelined frames.
     for (int i = 0; i < maxGpuFrameLag - 1; i++) {
         time(loops, bench, target);
     }
 
-    // Now, actually do the timing!
-    for (int i = 0; i < FLAGS_samples; i++) {
-        samples[i] = time(loops, bench, target) / loops;
-    }
-
     return loops;
 }
 
 static SkString to_lower(const char* str) {
     SkString lower(str);
     for (size_t i = 0; i < lower.size(); i++) {
         lower[i] = tolower(lower[i]);
     }
     return lower;
 }
@@ skipping 566 matching lines @@
 #if SK_SUPPORT_GPU
     GrContextOptions grContextOpts;
     grContextOpts.fDrawPathToCompressedTexture = FLAGS_gpuCompressAlphaMasks;
     gGrFactory.reset(SkNEW_ARGS(GrContextFactory, (grContextOpts)));
 #endif
 
     if (FLAGS_veryVerbose) {
         FLAGS_verbose = true;
     }
 
+    double samplingTimeMs = 0;
+    if (0 != strcmp("0", FLAGS_samplingTime[0])) {
+        SkSTArray<8, char> timeUnit;
+        timeUnit.push_back_n(static_cast<int>(strlen(FLAGS_samplingTime[0])) + 1);
+        if (2 != sscanf(FLAGS_samplingTime[0], "%lf%s", &samplingTimeMs, timeUnit.begin()) ||
+            (0 != strcmp("s", timeUnit.begin()) && 0 != strcmp("ms", timeUnit.begin()))) {
+            SkDebugf("Invalid --samplingTime \"%s\". Must be \"0\", \"%%lfs\", or \"%%lfms\"\n",
+                     FLAGS_samplingTime[0]);
+            exit(0);
+        }
+        if (0 == strcmp("s", timeUnit.begin())) {
+            samplingTimeMs *= 1000;
+        }
+        if (samplingTimeMs) {
+            FLAGS_samples = kTimedSampling;
+        }
+    }
+
     if (kAutoTuneLoops != FLAGS_loops) {
         FLAGS_samples     = 1;
         FLAGS_gpuFrameLag = 0;
     }
 
     if (!FLAGS_writePath.isEmpty()) {
         SkDebugf("Writing files to %s.\n", FLAGS_writePath[0]);
         if (!sk_mkdir(FLAGS_writePath[0])) {
             SkDebugf("Could not create %s. Files won't be written.\n", FLAGS_writePath[0]);
             FLAGS_writePath.set(0, NULL);
@@ skipping 17 matching lines @@
         SkDebugf("ERROR: --key must be passed with an even number of arguments.\n");
         return 1;
     }
     for (int i = 1; i < FLAGS_key.count(); i += 2) {
         log->key(FLAGS_key[i-1], FLAGS_key[i]);
     }
 
     const double overhead = estimate_timer_overhead();
     SkDebugf("Timer overhead: %s\n", HUMANIZE(overhead));
 
-    SkAutoTMalloc<double> samples(FLAGS_samples);
+    SkTArray<double> samples;
 
     if (kAutoTuneLoops != FLAGS_loops) {
         SkDebugf("Fixed number of loops; times would only be misleading so we won't print them.\n");
     } else if (FLAGS_verbose) {
         // No header.
     } else if (FLAGS_quiet) {
         SkDebugf("median\tbench\tconfig\n");
+    } else if (kTimedSampling == FLAGS_samples) {
+        SkDebugf("curr/maxrss\tloops\tmin\tmedian\tmean\tmax\tstddev\tsamples\tconfig\tbench\n");
     } else {
         SkDebugf("curr/maxrss\tloops\tmin\tmedian\tmean\tmax\tstddev\t%-*s\tconfig\tbench\n",
                  FLAGS_samples, "samples");
     }
 
     SkTDArray<Config> configs;
     create_configs(&configs);
 
     int runs = 0;
     BenchmarkStream benchStream;
@@ skipping 11 matching lines @@
             bench->preDraw();
         }
         for (int j = 0; j < targets.count(); j++) {
             // During HWUI output this canvas may be NULL.
             SkCanvas* canvas = targets[j]->getCanvas();
             const char* config = targets[j]->config.name;
 
             targets[j]->setup();
             bench->perCanvasPreDraw(canvas);
 
-            int frameLag;
-            const int loops =
-                targets[j]->needsFrameTiming(&frameLag)
-                ? gpu_bench(targets[j], bench.get(), samples.get(), frameLag)
-                : cpu_bench(overhead, targets[j], bench.get(), samples.get());
+            int maxFrameLag;
+            const int loops = targets[j]->needsFrameTiming(&maxFrameLag)
+                ? setup_gpu_bench(targets[j], bench.get(), maxFrameLag)
+                : setup_cpu_bench(overhead, targets[j], bench.get());
+
+            if (kTimedSampling != FLAGS_samples) {
+                samples.reset(FLAGS_samples);
+                for (int s = 0; s < FLAGS_samples; s++) {
+                    samples[s] = time(loops, bench, targets[j]) / loops;
+                }
+            } else if (samplingTimeMs) {
+                samples.reset();
+                if (FLAGS_verbose) {
+                    SkDebugf("Begin sampling %s for %ims\n",
+                             bench->getUniqueName(), static_cast<int>(samplingTimeMs));
+                }
+                WallTimer timer;
+                timer.start();
+                do {
+                    samples.push_back(time(loops, bench, targets[j]) / loops);
+                    timer.end();
+                } while (timer.fWall < samplingTimeMs);
+            }
 
             bench->perCanvasPostDraw(canvas);
 
             if (Benchmark::kNonRendering_Backend != targets[j]->config.backend &&
                 !FLAGS_writePath.isEmpty() && FLAGS_writePath[0]) {
                 SkString pngFilename = SkOSPath::Join(FLAGS_writePath[0], config);
                 pngFilename = SkOSPath::Join(pngFilename.c_str(), bench->getUniqueName());
                 pngFilename.append(".png");
                 write_canvas_png(targets[j], pngFilename);
             }
 
             if (kFailedLoops == loops) {
                 // Can't be timed.  A warning note has already been printed.
                 continue;
             }
 
-            Stats stats(samples.get(), FLAGS_samples);
+            Stats stats(samples);
             log->config(config);
             log->configOption("name", bench->getName());
             benchStream.fillCurrentOptions(log.get());
             targets[j]->fillOptions(log.get());
             log->metric("min_ms",    stats.min);
             if (runs++ % FLAGS_flushEvery == 0) {
                 log->flush();
             }
 
             if (kAutoTuneLoops != FLAGS_loops) {
                 if (targets.count() == 1) {
                     config = ""; // Only print the config if we run the same bench on more than one.
                 }
                 SkDebugf("%4d/%-4dMB\t%s\t%s\n"
                          , sk_tools::getCurrResidentSetSizeMB()
                          , sk_tools::getMaxResidentSetSizeMB()
                          , bench->getUniqueName()
                          , config);
             } else if (FLAGS_verbose) {
-                for (int i = 0; i < FLAGS_samples; i++) {
+                for (int i = 0; i < samples.count(); i++) {
                     SkDebugf("%s  ", HUMANIZE(samples[i]));
                 }
                 SkDebugf("%s\n", bench->getUniqueName());
             } else if (FLAGS_quiet) {
                 if (targets.count() == 1) {
                     config = ""; // Only print the config if we run the same bench on more than one.
                 }
                 SkDebugf("%s\t%s\t%s\n", HUMANIZE(stats.median), bench->getUniqueName(), config);
             } else {
                 const double stddev_percent = 100 * sqrt(stats.var) / stats.mean;
                 SkDebugf("%4d/%-4dMB\t%d\t%s\t%s\t%s\t%s\t%.0f%%\t%s\t%s\t%s\n"
                         , sk_tools::getCurrResidentSetSizeMB()
                         , sk_tools::getMaxResidentSetSizeMB()
                         , loops
                         , HUMANIZE(stats.min)
                         , HUMANIZE(stats.median)
                         , HUMANIZE(stats.mean)
                         , HUMANIZE(stats.max)
                         , stddev_percent
-                        , stats.plot.c_str()
+                        , kTimedSampling != FLAGS_samples ? stats.plot.c_str()
+                                                          : to_string(samples.count()).c_str()
                         , config
                         , bench->getUniqueName()
                         );
             }
 #if SK_SUPPORT_GPU
             if (FLAGS_gpuStats &&
                 Benchmark::kGPU_Backend == targets[j]->config.backend) {
                 gGrFactory->get(targets[j]->config.ctxType)->printCacheStats();
                 gGrFactory->get(targets[j]->config.ctxType)->printGpuStats();
             }
@@ skipping 23 matching lines @@
 
     return 0;
 }
 
 #if !defined SK_BUILD_FOR_IOS
 int main(int argc, char** argv) {
     SkCommandLineFlags::Parse(argc, argv);
     return nanobench_main();
 }
 #endif