Try for better bench stability.

bench/benchmain.cpp

 
 /*
  * Copyright 2011 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 #if SK_SUPPORT_GPU
 #include "GrContext.h"
@@ skipping 19 matching lines @@
 #include "SkPicture.h"
 #include "SkString.h"
 
 enum BenchMode {
     kNormal_BenchMode,
     kDeferred_BenchMode,
     kDeferredSilent_BenchMode,
     kRecord_BenchMode,
     kPictureRecord_BenchMode
 };
-const char* BenchMode_Name[] = { "normal", "deferred", "deferredSilent", "record", "picturerecord" };
+const char* BenchMode_Name[] = {
+    "normal", "deferred", "deferredSilent", "record", "picturerecord"
+};
 
 ///////////////////////////////////////////////////////////////////////////////
 
 static void erase(SkBitmap& bm) {
     if (bm.config() == SkBitmap::kA8_Config) {
         bm.eraseColor(SK_ColorTRANSPARENT);
     } else {
         bm.eraseColor(SK_ColorWHITE);
     }
 }
@@ skipping 213 matching lines @@
                           "If a test does not match any list entry,\n"
                           "it is skipped unless some list entry starts with ~\n");
 DEFINE_string(mode, "normal",
              "normal:         draw to a normal canvas;\n"
              "deferred:       draw to a deferred canvas;\n"
              "deferredSilent: deferred with silent playback;\n"
              "record:         draw to an SkPicture;\n"
              "picturerecord:  draw from an SkPicture to an SkPicture.\n");
 DEFINE_string(config, "", "Run configs given.  If empty, runs the defaults set in gConfigs.");
 DEFINE_string(logFile, "", "Also write stdout here.");
-DEFINE_int32(benchMs, 20, "Target time in ms to run each benchmark config.");
+DEFINE_int32(minMs, 20,  "Shortest time we'll allow a benchmark to run.");
+DEFINE_int32(maxMs, 4000, "Longest time we'll allow a benchmark to run.");
+DEFINE_double(error, 0.01,
+              "Ratio of subsequent bench measurements must drop within 1±error to converge.");
 DEFINE_string(timeFormat, "%9.2f", "Format to print results, in milliseconds per 1000 loops.");
+DEFINE_bool2(verbose, v, false, "Print more.");
+
+// Has this bench converged?  First arguments are milliseconds / loop iteration,
+// last is overall runtime in milliseconds.
+static bool HasConverged(double prevPerLoop, double currPerLoop, double currRaw) {
+    if (currRaw < FLAGS_minMs) {
+        return false;
+    }
+    const double low = 1 - FLAGS_error, high = 1 + FLAGS_error;
+    const double ratio = currPerLoop / prevPerLoop;
+    return low < ratio && ratio < high;
+}
 
 int tool_main(int argc, char** argv);
 int tool_main(int argc, char** argv) {
 #if SK_ENABLE_INST_COUNT
     gPrintInstCount = true;
 #endif
     SkAutoGraphics ag;
     SkCommandLineFlags::Parse(argc, argv);
 
     // First, parse some flags.
@@ skipping 46 matching lines @@
             }
         }
     }
 #if SK_SUPPORT_GPU
     for (int i = 0; i < configs.count(); ++i) {
         const Config& config = gConfigs[configs[i]];
 
         if (kGPU_Backend == config.backend) {
             GrContext* context = gContextFactory.get(config.contextType);
             if (NULL == context) {
-                SkString error;
-                error.printf("Error creating GrContext for config %s. Config will be skipped.\n",
-                             config.name);
-                logger.logError(error);
+                logger.logError(SkStringPrintf(
+                    "Error creating GrContext for config %s. Config will be skipped.\n",
+                    config.name));
                 configs.remove(i);
                 --i;
                 continue;
             }
             if (config.sampleCount > context->getMaxSampleCount()){
-                SkString error;
-                error.printf("Sample count (%d) for config %s is unsupported. "
-                             "Config will be skipped.\n",
-                             config.sampleCount, config.name);
-                logger.logError(error);
+                logger.logError(SkStringPrintf(
+                    "Sample count (%d) for config %s is unsupported. Config will be skipped.\n",
+                    config.sampleCount, config.name));
                 configs.remove(i);
                 --i;
                 continue;
             }
         }
     }
 #endif
 
     // All flags should be parsed now.  Report our settings.
     if (kIsDebug) {
@@ skipping 111 matching lines @@
             const SkPicture::RecordingFlags kRecordFlags =
                 SkPicture::kUsePathBoundsForClip_RecordingFlag;
 
             if (kNonRendering_Backend != config.backend) {
                 device.reset(make_device(config.config,
                                          dim,
                                          config.backend,
                                          config.sampleCount,
                                          context));
                 if (!device.get()) {
-                    SkString error;
-                    error.printf("Device creation failure for config %s. Will skip.\n", config.name);
-                    logger.logError(error);
+                    logger.logError(SkStringPrintf(
+                        "Device creation failure for config %s. Will skip.\n", config.name));
                     continue;
                 }
 
                 switch(benchMode) {
                     case kDeferredSilent_BenchMode:
                     case kDeferred_BenchMode:
                         canvas.reset(SkDeferredCanvas::Create(device.get()));
                         break;
                     case kRecord_BenchMode:
                         canvas.reset(SkRef(recordTo.beginRecording(dim.fX, dim.fY, kRecordFlags)));
@@ skipping 28 matching lines @@
 
 #if SK_SUPPORT_GPU
             SkGLContextHelper* contextHelper = NULL;
             if (kGPU_Backend == config.backend) {
                 contextHelper = gContextFactory.getGLContext(config.contextType);
             }
             BenchTimer timer(contextHelper);
 #else
             BenchTimer timer;
 #endif
 

[robertphillips, 2013/09/13 20:00:59]

You're sure about this?

[mtklein, 2013/09/13 20:07:46]

Yep.  Shortest way I know to write a double +inf.

+            double previous = 1.0/0.0;
+            bool converged = false;
             bench->setLoops(0);

[robertphillips, 2013/09/13 20:00:59]

if () {

}

[mtklein, 2013/09/13 20:07:46]

Done.

+            if (FLAGS_verbose) SkDebugf("%s %s: ", bench->getName(), config.name);
             do {
                 // Ramp up 1 -> 4 -> 16 -> ... -> ~1 billion.
                 const int loops = bench->getLoops();
-                if (loops >= (1<<30)) {
+                if (loops >= (1<<30) || timer.fWall > FLAGS_maxMs) {
                     // If you find it takes more than a billion loops to get up to 20ms of runtime,
                     // you've got a computer clocked at several THz or have a broken benchmark.  ;)
                     //     "1B ought to be enough for anybody."
-                    SkString str;
-                    str.printf("Can't ramp %s to %dms.\n", bench->getName(), FLAGS_benchMs);
-                    logger.logError(str);
+                    logger.logError(SkStringPrintf(
+                        "Can't get %s %s to converge in %dms.\n",
+                         bench->getName(), config.name, FLAGS_maxMs));
                     break;
                 }
-                bench->setLoops(loops == 0 ? 1 : loops * 4);
+                bench->setLoops(loops == 0 ? 1 : loops * 2);
 
                 if ((benchMode == kRecord_BenchMode || benchMode == kPictureRecord_BenchMode)) {
                     // Clear the recorded commands so that they do not accumulate.
                     canvas.reset(recordTo.beginRecording(dim.fX, dim.fY, kRecordFlags));
                 }
 
                 timer.start();
                 if (NULL != canvas) {
                     canvas->save();
                 }
@@ skipping 16 matching lines @@
 
                 // Stop truncated timers before GL calls complete, and stop the full timers after.
                 timer.truncatedEnd();
 #if SK_SUPPORT_GPU
                 if (NULL != glContext) {
                     context->flush();
                     SK_GL(*glContext, Finish());
                 }
 #endif
                 timer.end();
-            } while (!kIsDebug && timer.fWall < FLAGS_benchMs);  // One loop only in debug mode.
+                const double current = timer.fWall / bench->getLoops();

[robertphillips, 2013/09/13 20:00:59]

if () {

}

[mtklein, 2013/09/13 20:07:46]

Done.

+                if (FLAGS_verbose && current > previous) SkDebugf("↑");

[robertphillips, 2013/09/13 20:00:59]

same here

[mtklein, 2013/09/13 20:07:46]

Done.

+                if (FLAGS_verbose) SkDebugf("%.3g ", current);
+                converged = HasConverged(previous, current, timer.fWall);
+                previous = current;
+            } while (!kIsDebug && !converged);

[robertphillips, 2013/09/13 20:00:59]

and here

[mtklein, 2013/09/13 20:07:46]

Done.

+            if (FLAGS_verbose) SkDebugf("\n");
 
             if (FLAGS_outDir.count() && kNonRendering_Backend != config.backend) {
                 saveFile(bench->getName(),
                          config.name,
                          FLAGS_outDir[0],
                          device->accessBitmap(false));
             }
 
             if (kIsDebug) {
                 // Let's not mislead ourselves by looking at Debug build bench times!
@@ skipping 28 matching lines @@
     gContextFactory.destroyContexts();
 #endif
     return 0;
 }
 
 #if !defined(SK_BUILD_FOR_IOS) && !defined(SK_BUILD_FOR_NACL)
 int main(int argc, char * const argv[]) {
     return tool_main(argc, (char**) argv);
 }
 #endif