New version of the bbh shootout

tools/bbh_shootout.cpp

 /*
  * Copyright 2013 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 #include "BenchTimer.h"
 #include "LazyDecodeBitmap.h"
 #include "PictureBenchmark.h"
 #include "PictureRenderer.h"
 #include "SkBenchmark.h"
 #include "SkForceLinking.h"
 #include "SkGraphics.h"
 #include "SkStream.h"
 #include "SkString.h"
 #include "SkTArray.h"
-#include "TimerData.h"
+#include "SkCommandLineFlags.h"
 
-static const int kNumNormalRecordings = 10;
-static const int kNumRTreeRecordings = 10;
-static const int kNumPlaybacks = 1;
-static const size_t kNumBaseBenchmarks = 3;
-static const size_t kNumTileSizes = 3;
-static const size_t kNumBbhPlaybackBenchmarks = 3;
-static const size_t kNumBenchmarks = kNumBaseBenchmarks + kNumBbhPlaybackBenchmarks;
+typedef sk_tools::PictureRenderer::BBoxHierarchyType BBoxType;
+static const int kBBoxTypeCount = sk_tools::PictureRenderer::kLast_BBoxHierarchyType + 1;
 
-enum BenchmarkType {
-    kNormal_BenchmarkType = 0,
-    kRTree_BenchmarkType,
+
+DEFINE_string2(skps, r, "", "The list of SKPs to benchmark.");
+DEFINE_string(bbh, "", "The set of bbox types to test. If empty, all are tested. "
+                       "Should be one or more of none, quadtree, rtree, tilegrid.");
+DEFINE_int32(record, 100, "Number of times to record each SKP.");
+DEFINE_int32(playback, 1, "Number of times to playback each SKP.");

[tomhudson, 2014/03/10 14:59:15]

The record/playback ratio feels like the opposite of what we see in practice. Do
we really want to bias this test that strongly?

[iancottrell, 2014/03/10 16:22:49]

We don't build a combined metric, so the ratio does not really matter, and all
reported numbers are average for a single iteration.
The record is over 100 times faster than the playback, seemed fair to spend
about the same amount of time on each.

+DEFINE_int32(tilesize, 512, "The size of a tile.");

[tomhudson, 2014/03/10 14:59:15]

Not quite a nit: in Chromium, cc/trees/layer_tree_settings.h still defaults to
256 despite our arguments they should increase it.

Also, apparently Chromium hybrid rendering with Ganesh uses 384 or 416.

But none of the embedders we know about use 512, so it might not be the best
default setting. :(

[iancottrell, 2014/03/10 16:22:49]

Done.

+
+struct Measurement {
+    SkString fName;
+    SkScalar fRecordAverage[kBBoxTypeCount];
+    SkScalar fPlaybackAverage[kBBoxTypeCount];
 };
 
-struct Histogram {
-    Histogram() {
-        // Make fCpuTime negative so that we don't mess with stats:
-        fCpuTime = SkIntToScalar(-1);
-    }
-    SkScalar fCpuTime;
-    SkString fPath;
-};
-
-
-////////////////////////////////////////////////////////////////////////////////
-// Defined below.
-struct BenchmarkControl;
-
-typedef void (*BenchmarkFunction)
-    (const BenchmarkControl&, const SkString&, SkPicture*, BenchTimer*);
-
-static void benchmark_playback(
-    const BenchmarkControl&, const SkString&, SkPicture*, BenchTimer*);
-static void benchmark_recording(
-    const BenchmarkControl&, const SkString&, SkPicture*, BenchTimer*);
-////////////////////////////////////////////////////////////////////////////////
-
-/**
- * Acts as a POD containing information needed to run a benchmark.
- * Provides static methods to poll benchmark info from an index.
- */
-struct BenchmarkControl {
-    SkISize fTileSize;
-    BenchmarkType fType;
-    BenchmarkFunction fFunction;
-    SkString fName;
-
-    /**
-     * Will construct a BenchmarkControl instance from an index between 0 an kNumBenchmarks.
-     */
-    static BenchmarkControl Make(size_t i) {
-        SkASSERT(kNumBenchmarks > i);
-        BenchmarkControl benchControl;
-        benchControl.fTileSize = GetTileSize(i);
-        benchControl.fType = GetBenchmarkType(i);
-        benchControl.fFunction = GetBenchmarkFunc(i);
-        benchControl.fName = GetBenchmarkName(i);
-        return benchControl;
-    }
-
-    enum BaseBenchmarks {
-        kNormalRecord = 0,
-        kRTreeRecord,
-        kNormalPlayback,
-    };
-
-    static SkISize fTileSizes[kNumTileSizes];
-
-    static SkISize GetTileSize(size_t i) {
-        // Two of the base benchmarks don't need a tile size. But to maintain simplicity
-        // down the pipeline we have to let a couple of values unused.
-        if (i < kNumBaseBenchmarks) {
-            return SkISize::Make(256, 256);
-        }
-        if (i >= kNumBaseBenchmarks && i < kNumBenchmarks) {
-            return fTileSizes[i - kNumBaseBenchmarks];
-        }
-        SkASSERT(0);
-        return SkISize::Make(0, 0);
-    }
-
-    static BenchmarkType GetBenchmarkType(size_t i) {
-        if (i < kNumBaseBenchmarks) {
-            switch (i) {
-            case kNormalRecord:
-                return kNormal_BenchmarkType;
-            case kNormalPlayback:
-                return kNormal_BenchmarkType;
-            case kRTreeRecord:
-                return kRTree_BenchmarkType;
-            }
-        }
-        if (i < kNumBenchmarks) {
-            return kRTree_BenchmarkType;
-        }
-        SkASSERT(0);
-        return kRTree_BenchmarkType;
-    }
-
-    static BenchmarkFunction GetBenchmarkFunc(size_t i) {
-        // Base functions.
-        switch (i) {
-            case kNormalRecord:
-                return benchmark_recording;
-            case kNormalPlayback:
-                return benchmark_playback;
-            case kRTreeRecord:
-                return benchmark_recording;
-        }
-        // RTree playbacks
-        if (i < kNumBenchmarks) {
-            return benchmark_playback;
-        }
-        SkASSERT(0);
-        return NULL;
-    }
-
-    static SkString GetBenchmarkName(size_t i) {
-        // Base benchmark names
-        switch (i) {
-            case kNormalRecord:
-                return SkString("normal_recording");
-            case kNormalPlayback:
-                return SkString("normal_playback");
-            case kRTreeRecord:
-                return SkString("rtree_recording");
-        }
-        // RTree benchmark names.
-        if (i < kNumBenchmarks) {
-            SkASSERT(i >= kNumBaseBenchmarks);
-            SkString name;
-            name.printf("rtree_playback_%dx%d",
-                    fTileSizes[i - kNumBaseBenchmarks].fWidth,
-                    fTileSizes[i - kNumBaseBenchmarks].fHeight);
-            return name;
-
-        } else {
-            SkASSERT(0);
-        }
-        return SkString("");
-    }
-
-};
-
-SkISize BenchmarkControl::fTileSizes[kNumTileSizes] = {
-    SkISize::Make(256, 256),
-    SkISize::Make(512, 512),
-    SkISize::Make(1024, 1024),
+const char* kBBoxHierarchyTypeNames[kBBoxTypeCount] = {
+    "none", // kNone_BBoxHierarchyType
+    "quadtree", // kQuadTree_BBoxHierarchyType
+    "rtree", // kRTree_BBoxHierarchyType
+    "tilegrid", // kTileGrid_BBoxHierarchyType
 };
 
 static SkPicture* pic_from_path(const char path[]) {
     SkFILEStream stream(path);
     if (!stream.isValid()) {
         SkDebugf("-- Can't open '%s'\n", path);
         return NULL;
     }
     return SkPicture::CreateFromStream(&stream, &sk_tools::LazyDecodeBitmap);
 }
@@ skipping 12 matching lines @@
     renderer.end();
 
     if (bitmap->getSize() != inBitmap.getSize()) {
         return false;
     }
     return !memcmp(bitmap->getPixels(), inBitmap.getPixels(), bitmap->getSize());
 }
 
 /**
  * This function is the sink to which all work ends up going.
- * Renders the picture into the renderer. It may or may not use an RTree.
+ * Renders the picture into the renderer.

[tomhudson, 2014/03/10 14:59:15]

Nit: I hate comments that use the same root twice in two different word forms
because their meaning can be really unclear. Or tautological. Not to mention
"picture" is misleading here...

[iancottrell, 2014/03/10 16:22:49]

Me too, but it was not my comment :)
I have changed the signature and all the comment to be much clearer, I hope.

  * The renderer is chosen upstream. If we want to measure recording, we will
  * use a RecordPictureRenderer. If we want to measure rendering, we will use a
  * TiledPictureRenderer.
  */
 static void do_benchmark_work(sk_tools::PictureRenderer* renderer,
-        int benchmarkType, const SkString& path, SkPicture* pic,
-        const int numRepeats, const char *msg, BenchTimer* timer) {
-    SkString msgPrefix;
-
-    switch (benchmarkType){
-        case kNormal_BenchmarkType:
-            msgPrefix.set("Normal");
-            renderer->setBBoxHierarchyType(sk_tools::PictureRenderer::kNone_BBoxHierarchyType);
-            break;
-        case kRTree_BenchmarkType:
-            msgPrefix.set("RTree");
-            renderer->setBBoxHierarchyType(sk_tools::PictureRenderer::kRTree_BBoxHierarchyType);
-            break;
-        default:
-            SkASSERT(0);
-            break;
-    }
-
+        BBoxType bBoxType,
+        const SkString& path,
+        SkPicture* pic,
+        const int numRepeats,
+        BenchTimer* timer,
+        bool verify) {

[tomhudson, 2014/03/10 14:59:15]

Nit: is there a more descriptive name? verifyWhat?

[iancottrell, 2014/03/10 16:22:49]

Removed!
The job of this tool is to benchmark, not to verify correctness.

+    const char* msgPrefix = kBBoxHierarchyTypeNames[bBoxType];
+    renderer->setBBoxHierarchyType(bBoxType);
+    renderer->setGridSize(FLAGS_tilesize, FLAGS_tilesize);
     renderer->init(pic);
 
     /**
      * If the renderer is not tiled, assume we are measuring recording.
      */
     bool isPlayback = (NULL != renderer->getTiledRenderer());
+    const char* msg = isPlayback ? "playback" : "record";
     // Will be non-null during RTree picture playback. For correctness test.
     SkBitmap* bitmap = NULL;
 
-    SkDebugf("%s %s %s %d times...\n", msgPrefix.c_str(), msg, path.c_str(), numRepeats);
+    SkDebugf("%s %s %s %d times...\n", msgPrefix, msg, path.c_str(), numRepeats);
     for (int i = 0; i < numRepeats; ++i) {
         // Set up the bitmap.
         SkBitmap** out = NULL;
-        if (i == 0 && kRTree_BenchmarkType == benchmarkType && isPlayback) {
+        if (i == 0 && verify) {
             out = &bitmap;
         }
 
         renderer->setup();
         // Render once to fill caches. Fill bitmap during the first iteration.
         renderer->render(NULL, out);
         // Render again to measure
         timer->start();
         bool result = renderer->render(NULL);
         timer->end();
 
         // We only care about a false result on playback. RecordPictureRenderer::render will always
         // return false because we are passing a NULL file name on purpose; which is fine.
         if (isPlayback && !result) {
             SkDebugf("Error rendering during playback.\n");
         }
     }
     if (bitmap) {
         SkAutoTDelete<SkBitmap> bmDeleter(bitmap);
         if (!compare_picture(path, *bitmap, pic)) {
-            SkDebugf("Error: RTree produced different bitmap\n");
+            SkDebugf("Error: %s produced different bitmap\n", msgPrefix);
         }
     }
 }
 
-/**
- * Call do_benchmark_work with a tiled renderer using the default tile dimensions.
- */
-static void benchmark_playback(
-        const BenchmarkControl& benchControl,
-        const SkString& path, SkPicture* pic, BenchTimer* timer) {
-    sk_tools::TiledPictureRenderer renderer;
-
-    SkString message("tiled_playback");
-    message.appendf("_%dx%d", benchControl.fTileSize.fWidth, benchControl.fTileSize.fHeight);
-    do_benchmark_work(&renderer, benchControl.fType,
-            path, pic, kNumPlaybacks, message.c_str(), timer);
-}
-
-/**
- * Call do_benchmark_work with a RecordPictureRenderer.
- */
-static void benchmark_recording(
-        const BenchmarkControl& benchControl,
-        const SkString& path, SkPicture* pic, BenchTimer* timer) {
-    sk_tools::RecordPictureRenderer renderer;
-    int numRecordings = 0;
-    switch(benchControl.fType) {
-        case kRTree_BenchmarkType:
-            numRecordings = kNumRTreeRecordings;
-            break;
-        case kNormal_BenchmarkType:
-            numRecordings = kNumNormalRecordings;
-            break;
-    }
-    do_benchmark_work(&renderer, benchControl.fType,
-            path, pic, numRecordings, "recording", timer);
-}
-
-/**
- * Takes argc,argv along with one of the benchmark functions defined above.
- * Will loop along all skp files and perform measurments.
- */
-static void benchmark_loop(
-        int argc,
-        char **argv,
-        const BenchmarkControl& benchControl,
-        SkTArray<Histogram>& histogram) {
-    for (int index = 1; index < argc; ++index) {
-        BenchTimer timer;
-        SkString path(argv[index]);
-        SkAutoTUnref<SkPicture> pic(pic_from_path(path.c_str()));
-        if (NULL == pic) {
-            SkDebugf("Couldn't create picture. Ignoring path: %s\n", path.c_str());
-            continue;
-        }
-        benchControl.fFunction(benchControl, path, pic, &timer);
-        histogram[index - 1].fPath = path;
-        histogram[index - 1].fCpuTime = SkDoubleToScalar(timer.fCpu);
-    }
-}
-
 int tool_main(int argc, char** argv);
 int tool_main(int argc, char** argv) {
+    SkCommandLineFlags::Parse(argc, argv);
     SkAutoGraphics ag;
-    SkString usage;
-    usage.printf("Usage: filename [filename]*\n");
-
-    if (argc < 2) {
-        SkDebugf("%s\n", usage.c_str());
-        return -1;
+    bool includeBBoxType[kBBoxTypeCount];
+    for (int bBoxType = 0; bBoxType < kBBoxTypeCount; ++bBoxType) {
+        includeBBoxType[bBoxType] = (FLAGS_bbh.count() == 0) ||
+            FLAGS_bbh.contains(kBBoxHierarchyTypeNames[bBoxType]);
+    }
+    // go through all the pictures
+    SkTArray<Measurement> measurements;
+    for (int index = 0; index < FLAGS_skps.count(); ++index) {
+        const char* path = FLAGS_skps[index];
+        SkPicture* picture = pic_from_path(path);
+        if (NULL == picture) {
+            SkDebugf("Couldn't create picture. Ignoring path: %s\n", path);
+            continue;
+        }
+        Measurement& measurement = measurements.push_back();
+        measurement.fName = path;
+        for (int bBoxType = 0; bBoxType < kBBoxTypeCount; ++bBoxType) {
+            if (!includeBBoxType[bBoxType]) { continue; }
+            if (FLAGS_playback > 0) {
+                sk_tools::TiledPictureRenderer playbackRenderer;
+                BenchTimer playbackTimer;
+                do_benchmark_work(&playbackRenderer, (BBoxType)bBoxType, measurement.fName,
+                                  picture, FLAGS_playback, &playbackTimer, 0 != bBoxType);
+                measurement.fPlaybackAverage[bBoxType] = playbackTimer.fCpu;
+            }
+            if (FLAGS_record > 0) {
+                sk_tools::RecordPictureRenderer recordRenderer;
+                BenchTimer recordTimer;
+                do_benchmark_work(&recordRenderer, (BBoxType)bBoxType, measurement.fName,
+                                  picture, FLAGS_record, &recordTimer, false);
+                measurement.fRecordAverage[bBoxType] = recordTimer.fCpu;
+            }
+        }
     }
 
-    SkTArray<Histogram> histograms[kNumBenchmarks];
-
-    for (size_t i = 0; i < kNumBenchmarks; ++i) {
-        histograms[i].reset(argc - 1);
-        benchmark_loop(argc, argv, BenchmarkControl::Make(i), histograms[i]);
+    Measurement globalMeasurement;
+    for (int bBoxType = 0; bBoxType < kBBoxTypeCount; ++bBoxType) {
+        if (!includeBBoxType[bBoxType]) { continue; }
+        globalMeasurement.fPlaybackAverage[bBoxType] = 0;
+        globalMeasurement.fRecordAverage[bBoxType] = 0;
+        for (int index = 0; index < measurements.count(); ++index) {
+            const Measurement& measurement = measurements[index];
+            globalMeasurement.fPlaybackAverage[bBoxType] +=
+                measurement.fPlaybackAverage[bBoxType];
+            globalMeasurement.fRecordAverage[bBoxType] +=
+                measurement.fRecordAverage[bBoxType];
+        }
+        globalMeasurement.fPlaybackAverage[bBoxType] /= measurements.count();
+        globalMeasurement.fRecordAverage[bBoxType] /= measurements.count();
     }
 
     // Output gnuplot readable histogram data..
     const char* pbTitle = "bbh_shootout_playback.dat";
     const char* recTitle = "bbh_shootout_record.dat";
     SkFILEWStream playbackOut(pbTitle);
     SkFILEWStream recordOut(recTitle);
     recordOut.writeText("# ");
     playbackOut.writeText("# ");
-    for (size_t i = 0; i < kNumBenchmarks; ++i) {
+    for (int bBoxType = 0; bBoxType < kBBoxTypeCount; ++bBoxType) {
+        if (!includeBBoxType[bBoxType]) { continue; }
         SkString out;
-        out.printf("%s ", BenchmarkControl::GetBenchmarkName(i).c_str());
-        if (BenchmarkControl::GetBenchmarkFunc(i) == &benchmark_recording) {
-            recordOut.writeText(out.c_str());
+        out.printf("%s ", kBBoxHierarchyTypeNames[bBoxType]);
+        recordOut.writeText(out.c_str());
+        playbackOut.writeText(out.c_str());
+
+        if (FLAGS_record > 0) {
+            SkDebugf("Average %s recording time: %.3fms\n",
+                kBBoxHierarchyTypeNames[bBoxType],
+                globalMeasurement.fRecordAverage[bBoxType]);
         }
-        if (BenchmarkControl::GetBenchmarkFunc(i) == &benchmark_playback) {
-            playbackOut.writeText(out.c_str());
+        if (FLAGS_playback > 0) {
+            SkDebugf("Average %s playback time: %.3fms\n",
+                kBBoxHierarchyTypeNames[bBoxType],
+                globalMeasurement.fPlaybackAverage[bBoxType]);
         }
     }
     recordOut.writeText("\n");
     playbackOut.writeText("\n");
     // Write to file, and save recording averages.
-    SkScalar avgRecord = SkIntToScalar(0);
-    SkScalar avgPlayback = SkIntToScalar(0);
-    SkScalar avgRecordRTree = SkIntToScalar(0);
-    SkScalar avgPlaybackRTree = SkIntToScalar(0);
-    for (int i = 0; i < argc - 1; ++i) {
+    for (int index = 0; index < measurements.count(); ++index) {
+        const Measurement& measurement = measurements[index];
         SkString pbLine;
         SkString recLine;
-        // ==== Write record info
-        recLine.printf("%d ", i);
-        SkScalar cpuTime = histograms[BenchmarkControl::kNormalRecord][i].fCpuTime;
-        recLine.appendf("%f ", cpuTime);
-        avgRecord += cpuTime;
-        cpuTime = histograms[BenchmarkControl::kRTreeRecord][i].fCpuTime;
-        recLine.appendf("%f", cpuTime);
-        avgRecordRTree += cpuTime;
-        avgPlaybackRTree += cpuTime;
 
-        // ==== Write playback info
-        pbLine.printf("%d ", i);
-        pbLine.appendf("%f ", histograms[2][i].fCpuTime);  // Start with normal playback time.
-        avgPlayback += histograms[kNumBbhPlaybackBenchmarks - 1][i].fCpuTime;
-        avgPlaybackRTree += histograms[kNumBbhPlaybackBenchmarks][i].fCpuTime;
-        // Append all playback benchmark times.
-        for (size_t j = kNumBbhPlaybackBenchmarks; j < kNumBenchmarks; ++j) {
-            pbLine.appendf("%f ", histograms[j][i].fCpuTime);
+        pbLine.printf("%d", index);
+        recLine.printf("%d", index);
+        for (int bBoxType = 0; bBoxType < kBBoxTypeCount; ++bBoxType) {
+            if (!includeBBoxType[bBoxType]) { continue; }
+            pbLine.appendf(" %f", measurement.fPlaybackAverage[bBoxType]);
+            recLine.appendf(" %f", measurement.fRecordAverage[bBoxType]);
         }
-        pbLine.remove(pbLine.size() - 1, 1);  // Remove trailing space from line.
         pbLine.appendf("\n");
         recLine.appendf("\n");
         playbackOut.writeText(pbLine.c_str());
         recordOut.writeText(recLine.c_str());
     }
-    avgRecord /= argc - 1;
-    avgRecordRTree /= argc - 1;
-    avgPlayback /= argc - 1;
-    avgPlaybackRTree /= argc - 1;
-    SkDebugf("Average base recording time: %.3fms\n", avgRecord);
-    SkDebugf("Average recording time with rtree: %.3fms\n", avgRecordRTree);
-    SkDebugf("Average base playback time: %.3fms\n", avgPlayback);
-    SkDebugf("Average playback time with rtree: %.3fms\n", avgPlaybackRTree);
-
     SkDebugf("\nWrote data to gnuplot-readable files: %s %s\n", pbTitle, recTitle);
-
     return 0;
 }
 
 #if !defined(SK_BUILD_FOR_IOS) && !defined(SK_BUILD_FOR_NACL)
 int main(int argc, char** argv) {
     return tool_main(argc, argv);
 }
 #endif