skpbench: run for a fixed duration

tools/skpbench/skpbench.cpp

 /*
  * Copyright 2016 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 #include "GrContextFactory.h"
 #include "SkCanvas.h"
 #include "SkOSFile.h"
@@ skipping 12 matching lines @@
 #include <vector>
 
 /**
  * This is a minimalist program whose sole purpose is to open an skp file, benchmark it on a single
  * config, and exit. It is intended to be used through skpbench.py rather than invoked directly.
  * Limiting the entire process to a single config/skp pair helps to keep the results repeatable.
  *
  * No tiling, looping, or other fanciness is used; it just draws the skp whole into a size-matched
  * render target and syncs the GPU after each draw.
  *
- * The results consist of a fixed amount of samples (--samples). A sample is defined as the number
- * of frames rendered within a set amount of time (--sampleMs).
- *
  * Currently, only GPU configs are supported.
  */
 
-DEFINE_int32(samples, 101, "number of samples to collect");
-DEFINE_int32(sampleMs, 50, "duration of each sample");
+DEFINE_int32(duration, 5000, "number of milliseconds to run the benchmark");
+DEFINE_int32(sampleMs, 50, "minimum duration of a sample");
 DEFINE_bool(fps, false, "use fps instead of ms");
 DEFINE_string(skp, "", "path to a single .skp file to benchmark");
 DEFINE_string(png, "", "if set, save a .png proof to disk at this file location");
 DEFINE_int32(verbosity, 4, "level of verbosity (0=none to 5=debug)");
 DEFINE_bool(suppressHeader, false, "don't print a header row before the results");
 
 static const char* header =
     "  median     accum       max       min   stddev  metric  samples  sample_ms  config    bench";
 
 static const char* resultFormat =
@@ skipping 24 matching lines @@
 static void draw_skp_and_flush(SkCanvas*, const SkPicture*);
 static SkPlatformGpuFence insert_verified_fence(const SkGpuFenceSync*);
 static void wait_fence_and_delete(const SkGpuFenceSync*, SkPlatformGpuFence);
 static bool mkdir_p(const SkString& name);
 static SkString join(const SkCommandLineFlags::StringArray&);
 static void exitf(ExitErr, const char* format, ...);
 
 static void run_benchmark(const SkGpuFenceSync* sync, SkCanvas* canvas, const SkPicture* skp,
                           std::vector<Sample>* samples) {
     using clock = Sample::clock;
-    std::chrono::milliseconds sampleMs(FLAGS_sampleMs);
-
-    samples->clear();
-    samples->resize(FLAGS_samples);
+    const clock::duration sampleDuration = std::chrono::milliseconds(FLAGS_sampleMs);
+    const clock::duration benchDuration = std::chrono::milliseconds(FLAGS_duration);
 
     // Prime the graphics pipe.
     SkPlatformGpuFence frameN_minus_2, frameN_minus_1;
     {
         draw_skp_and_flush(canvas, skp);
         SkPlatformGpuFence frame0 = insert_verified_fence(sync);
 
         draw_skp_and_flush(canvas, skp);
         frameN_minus_2 = insert_verified_fence(sync);
 
         draw_skp_and_flush(canvas, skp);
         frameN_minus_1 = insert_verified_fence(sync);
 
         wait_fence_and_delete(sync, frame0);
     }
 
-    clock::time_point start = clock::now();
+    clock::time_point now = clock::now();
+    const clock::time_point endTime = now + benchDuration;
 
-    for (Sample& sample : *samples) {
-        clock::time_point end;
+    do {
+        clock::time_point sampleStart = now;
+        samples->emplace_back();
+        Sample& sample = samples->back();
+
         do {
             draw_skp_and_flush(canvas, skp);
 
             // Sync the GPU.
             wait_fence_and_delete(sync, frameN_minus_2);
             frameN_minus_2 = frameN_minus_1;
             frameN_minus_1 = insert_verified_fence(sync);
 
-            end = clock::now();
-            sample.fDuration = end - start;
+            now = clock::now();
+            sample.fDuration = now - sampleStart;
             ++sample.fFrames;
-        } while (sample.fDuration < sampleMs);
-
-        if (FLAGS_verbosity >= 5) {
-            fprintf(stderr, "%.4g%s [ms=%.4g frames=%i]\n",
-                            sample.value(), Sample::metric(), sample.ms(), sample.fFrames);
-        }
-
-        start = end;
-    }
+        } while (sample.fDuration < sampleDuration);
+    } while (now < endTime || 0 == samples->size() % 2);
 
     sync->deleteFence(frameN_minus_2);
     sync->deleteFence(frameN_minus_1);
 }
 
 void print_result(const std::vector<Sample>& samples, const char* config, const char* bench)  {
     if (0 == (samples.size() % 2)) {
         exitf(ExitErr::kSoftware, "attempted to gather stats on even number of samples");
     }
 
     Sample accum = Sample();
     std::vector<double> values;
     values.reserve(samples.size());
     for (const Sample& sample : samples) {
         accum.fFrames += sample.fFrames;
         accum.fDuration += sample.fDuration;
         values.push_back(sample.value());
     }
     std::sort(values.begin(), values.end());
-    const double median = values[values.size() / 2];
+    const double medianValue = values[values.size() / 2];
+    const double accumValue = accum.value();
 
-    const double meanValue = accum.value();
     double variance = 0;
-    for (const Sample& sample : samples) {
-        const double delta = sample.value() - meanValue;
+    for (double value : values) {
+        const double delta = value - accumValue;
         variance += delta * delta;
     }
-    variance /= samples.size();
+    variance /= values.size();
     // Technically, this is the relative standard deviation.
-    const double stddev = 100/*%*/ * sqrt(variance) / meanValue;
+    const double stddev = 100/*%*/ * sqrt(variance) / accumValue;
 
-    printf(resultFormat, median, accum.value(), values.back(), values.front(), stddev,
+    printf(resultFormat, medianValue, accumValue, values.back(), values.front(), stddev,
            Sample::metric(), values.size(), FLAGS_sampleMs, config, bench);
     printf("\n");
     fflush(stdout);
 }
 
 int main(int argc, char** argv) {
     SkCommandLineFlags::SetUsage("Use skpbench.py instead. "
                                  "You usually don't want to use this program directly.");
     SkCommandLineFlags::Parse(argc, argv);
 
     if (!FLAGS_suppressHeader) {
         printf("%s\n", header);
     }
-    if (FLAGS_samples <= 0) {
+    if (FLAGS_duration <= 0) {
         exit(0); // This can be used to print the header and quit.
     }
-    if (0 == FLAGS_samples % 2) {
-        fprintf(stderr, "WARNING: even number of samples requested (%i); "
-                        "using %i so there can be a true median.\n",
-                        FLAGS_samples, FLAGS_samples + 1);
-        ++FLAGS_samples;
-    }
 
     // Parse the config.
     const SkCommandLineConfigGpu* config = nullptr; // Initialize for spurious warning.
     SkCommandLineConfigArray configs;
     ParseConfigs(FLAGS_config, &configs);
     if (configs.count() != 1 || !(config = configs[0]->asConfigGpu())) {
         exitf(ExitErr::kUsage, "invalid config %s, must specify one (and only one) GPU config",
                                join(FLAGS_config).c_str());
     }
 
@@ skipping 51 matching lines @@
                                          kPremul_SkAlphaType, sk_ref_sp(config->getColorSpace()));
     uint32_t flags = config->getUseDIText() ? SkSurfaceProps::kUseDeviceIndependentFonts_Flag : 0;
     SkSurfaceProps props(flags, SkSurfaceProps::kLegacyFontHost_InitType);
     sk_sp<SkSurface> surface =
         SkSurface::MakeRenderTarget(ctx, SkBudgeted::kNo, info, config->getSamples(), &props);
     if (!surface) {
         exitf(ExitErr::kUnavailable, "failed to create %ix%i render target for config %s",
                                      width, height, config->getTag().c_str());
     }
 
+    std::vector<Sample> samples;
+    if (FLAGS_sampleMs > 0) {
+        // +1 because we might take one more sample in order to have an odd number.
+        samples.reserve(1 + (FLAGS_duration + FLAGS_sampleMs - 1) / FLAGS_sampleMs);
+    } else {
+        samples.reserve(2 * FLAGS_duration);
+    }
+
     // Run the benchmark.
-    std::vector<Sample> samples;
     SkCanvas* canvas = surface->getCanvas();
     canvas->translate(-skp->cullRect().x(), -skp->cullRect().y());
     run_benchmark(testCtx->fenceSync(), canvas, skp.get(), &samples);
     print_result(samples, config->getTag().c_str(), SkOSPath::Basename(skpfile).c_str());
 
     // Save a proof (if one was requested).
     if (!FLAGS_png.isEmpty()) {
         SkBitmap bmp;
         bmp.setInfo(info);
         if (!surface->getCanvas()->readPixels(&bmp, 0, 0)) {
@@ skipping 52 matching lines @@
 
 static void exitf(ExitErr err, const char* format, ...) {
     fprintf(stderr, ExitErr::kSoftware == err ? "INTERNAL ERROR: " : "ERROR: ");
     va_list args;
     va_start(args, format);
     vfprintf(stderr, format, args);
     va_end(args);
     fprintf(stderr, ExitErr::kSoftware == err ? "; this should never happen.\n": ".\n");
     exit((int)err);
 }