skpbench: run for a fixed duration

tools/skpbench/skpbench.cpp

 /*
  * Copyright 2016 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
+#include "GrAllocator.h"
 #include "GrContextFactory.h"
 #include "SkCanvas.h"
 #include "SkOSFile.h"
 #include "SkPicture.h"
 #include "SkStream.h"
 #include "SkSurface.h"
 #include "SkSurfaceProps.h"
 #include "picture_utils.h"
 #include "flags/SkCommandLineFlags.h"
 #include "flags/SkCommonFlagsConfig.h"
 #include <stdlib.h>
 #include <algorithm>
 #include <array>
 #include <chrono>
 #include <cmath>
 #include <vector>
 
 /**
  * This is a minimalist program whose sole purpose is to open an skp file, benchmark it on a single
  * config, and exit. It is intended to be used through skpbench.py rather than invoked directly.
  * Limiting the entire process to a single config/skp pair helps to keep the results repeatable.
  *
  * No tiling, looping, or other fanciness is used; it just draws the skp whole into a size-matched
  * render target and syncs the GPU after each draw.
  *
- * The results consist of a fixed amount of samples (--samples). A sample is defined as the number
- * of frames rendered within a set amount of time (--sampleMs).
- *
  * Currently, only GPU configs are supported.
  */
 
-DEFINE_int32(samples, 101, "number of samples to collect");
-DEFINE_int32(sampleMs, 50, "duration of each sample");
+DEFINE_int32(duration, 5000, "number of milliseconds to run the benchmark");
+DEFINE_int32(sampleMs, 50, "minimum duration of a sample");
 DEFINE_bool(fps, false, "use fps instead of ms");
 DEFINE_string(skp, "", "path to a single .skp file to benchmark");
 DEFINE_string(png, "", "if set, save a .png proof to disk at this file location");
 DEFINE_int32(verbosity, 4, "level of verbosity (0=none to 5=debug)");
 DEFINE_bool(suppressHeader, false, "don't print a header row before the results");
 
 static const char* header =
     "  median     accum       max       min   stddev  metric  samples  sample_ms  config    bench";
 
 static const char* resultFormat =
@@ skipping 22 matching lines @@
 };
 
 static void draw_skp_and_flush(SkCanvas*, const SkPicture*);
 static SkPlatformGpuFence insert_verified_fence(const SkGpuFenceSync*);
 static void wait_fence_and_delete(const SkGpuFenceSync*, SkPlatformGpuFence);
 static bool mkdir_p(const SkString& name);
 static SkString join(const SkCommandLineFlags::StringArray&);
 static void exitf(ExitErr, const char* format, ...);
 
 static void run_benchmark(const SkGpuFenceSync* sync, SkCanvas* canvas, const SkPicture* skp,
-                          std::vector<Sample>* samples) {
+                          GrTAllocator<Sample>* samples) {

[bsalomon, 2016/09/28 18:51:16]

Why the change to GrTAllocator?

[csmartdalton, 2016/09/28 19:17:54]

Because with "--sampleMs 0" we don't know an upper limit on how many samples
there will be anymore, and the data doesn't need to remain contiguous in the
event that we allocate more space.

It's not a big deal though, is there reason not to use it?

     using clock = Sample::clock;
-    std::chrono::milliseconds sampleMs(FLAGS_sampleMs);
-
-    samples->clear();
-    samples->resize(FLAGS_samples);
+    const clock::duration sampleDuration = std::chrono::milliseconds(FLAGS_sampleMs);
+    const clock::duration benchDuration = std::chrono::milliseconds(FLAGS_duration);
 
     // Prime the graphics pipe.
     SkPlatformGpuFence frameN_minus_2, frameN_minus_1;
     {
         draw_skp_and_flush(canvas, skp);
         SkPlatformGpuFence frame0 = insert_verified_fence(sync);
 
         draw_skp_and_flush(canvas, skp);
         frameN_minus_2 = insert_verified_fence(sync);
 
         draw_skp_and_flush(canvas, skp);
         frameN_minus_1 = insert_verified_fence(sync);
 
         wait_fence_and_delete(sync, frame0);
     }
 
-    clock::time_point start = clock::now();
+    clock::time_point now = clock::now();
+    const clock::time_point endTime = now + benchDuration;
 
-    for (Sample& sample : *samples) {
-        clock::time_point end;
+    do {
+        clock::time_point sampleStart = now;
+        Sample& sample = samples->push_back();
+
         do {
             draw_skp_and_flush(canvas, skp);
 
             // Sync the GPU.
             wait_fence_and_delete(sync, frameN_minus_2);
             frameN_minus_2 = frameN_minus_1;
             frameN_minus_1 = insert_verified_fence(sync);
 
-            end = clock::now();
-            sample.fDuration = end - start;
+            now = clock::now();
+            sample.fDuration = now - sampleStart;
             ++sample.fFrames;
-        } while (sample.fDuration < sampleMs);
-
-        if (FLAGS_verbosity >= 5) {
-            fprintf(stderr, "%.4g%s [ms=%.4g frames=%i]\n",
-                            sample.value(), Sample::metric(), sample.ms(), sample.fFrames);
-        }
-
-        start = end;
-    }
+        } while (sample.fDuration < sampleDuration);
+    } while (now < endTime || 0 == samples->count() % 2);
 
     sync->deleteFence(frameN_minus_2);
     sync->deleteFence(frameN_minus_1);
 }
 
-void print_result(const std::vector<Sample>& samples, const char* config, const char* bench)  {
-    if (0 == (samples.size() % 2)) {
+void print_result(const GrTAllocator<Sample>& samples, const char* config, const char* bench)  {
+    if (0 == (samples.count() % 2)) {
         exitf(ExitErr::kSoftware, "attempted to gather stats on even number of samples");
     }
 
     Sample accum = Sample();
     std::vector<double> values;
-    values.reserve(samples.size());
-    for (const Sample& sample : samples) {
-        accum.fFrames += sample.fFrames;
-        accum.fDuration += sample.fDuration;
-        values.push_back(sample.value());
+    values.reserve(samples.count());
+    for (GrTAllocator<Sample>::Iter it(&samples); it.next();) {
+        accum.fFrames += it->fFrames;
+        accum.fDuration += it->fDuration;
+        values.push_back(it->value());
     }
     std::sort(values.begin(), values.end());
-    const double median = values[values.size() / 2];
+    const double medianValue = values[values.size() / 2];
+    const double accumValue = accum.value();
 
-    const double meanValue = accum.value();
     double variance = 0;
-    for (const Sample& sample : samples) {
-        const double delta = sample.value() - meanValue;
+    for (double value : values) {
+        const double delta = value - accumValue;
         variance += delta * delta;
     }
-    variance /= samples.size();
+    variance /= values.size();
     // Technically, this is the relative standard deviation.
-    const double stddev = 100/*%*/ * sqrt(variance) / meanValue;
+    const double stddev = 100/*%*/ * sqrt(variance) / accumValue;
 
-    printf(resultFormat, median, accum.value(), values.back(), values.front(), stddev,
+    printf(resultFormat, medianValue, accumValue, values.back(), values.front(), stddev,
            Sample::metric(), values.size(), FLAGS_sampleMs, config, bench);
     printf("\n");
     fflush(stdout);
 }
 
 int main(int argc, char** argv) {
     SkCommandLineFlags::SetUsage("Use skpbench.py instead. "
                                  "You usually don't want to use this program directly.");
     SkCommandLineFlags::Parse(argc, argv);
 
     if (!FLAGS_suppressHeader) {
         printf("%s\n", header);
     }
-    if (FLAGS_samples <= 0) {
+    if (FLAGS_duration <= 0) {
         exit(0); // This can be used to print the header and quit.
     }
-    if (0 == FLAGS_samples % 2) {
-        fprintf(stderr, "WARNING: even number of samples requested (%i); "
-                        "using %i so there can be a true median.\n",
-                        FLAGS_samples, FLAGS_samples + 1);
-        ++FLAGS_samples;
-    }
 
     // Parse the config.
     const SkCommandLineConfigGpu* config = nullptr; // Initialize for spurious warning.
     SkCommandLineConfigArray configs;
     ParseConfigs(FLAGS_config, &configs);
     if (configs.count() != 1 || !(config = configs[0]->asConfigGpu())) {
         exitf(ExitErr::kUsage, "invalid config %s, must specify one (and only one) GPU config",
                                join(FLAGS_config).c_str());
     }
 
@@ skipping 51 matching lines @@
                                          kPremul_SkAlphaType, sk_ref_sp(config->getColorSpace()));
     uint32_t flags = config->getUseDIText() ? SkSurfaceProps::kUseDeviceIndependentFonts_Flag : 0;
     SkSurfaceProps props(flags, SkSurfaceProps::kLegacyFontHost_InitType);
     sk_sp<SkSurface> surface =
         SkSurface::MakeRenderTarget(ctx, SkBudgeted::kNo, info, config->getSamples(), &props);
     if (!surface) {
         exitf(ExitErr::kUnavailable, "failed to create %ix%i render target for config %s",
                                      width, height, config->getTag().c_str());
     }
 
+    int reserve;
+    if (FLAGS_sampleMs > 0) {
+        // +1 because we might take one more sample in order to have an odd number.
+        reserve = 1 + (FLAGS_duration + FLAGS_sampleMs - 1) / FLAGS_sampleMs;
+    } else {
+        reserve = 2 * FLAGS_duration;
+    }
+    GrTAllocator<Sample> samples(reserve);
+
     // Run the benchmark.
-    std::vector<Sample> samples;
     SkCanvas* canvas = surface->getCanvas();
     canvas->translate(-skp->cullRect().x(), -skp->cullRect().y());
     run_benchmark(testCtx->fenceSync(), canvas, skp.get(), &samples);
     print_result(samples, config->getTag().c_str(), SkOSPath::Basename(skpfile).c_str());
 
     // Save a proof (if one was requested).
     if (!FLAGS_png.isEmpty()) {
         SkBitmap bmp;
         bmp.setInfo(info);
         if (!surface->getCanvas()->readPixels(&bmp, 0, 0)) {
@@ skipping 52 matching lines @@
 
 static void exitf(ExitErr err, const char* format, ...) {
     fprintf(stderr, ExitErr::kSoftware == err ? "INTERNAL ERROR: " : "ERROR: ");
     va_list args;
     va_start(args, format);
     vfprintf(stderr, format, args);
     va_end(args);
     fprintf(stderr, ExitErr::kSoftware == err ? "; this should never happen.\n": ".\n");
     exit((int)err);
 }