Measure tiled rendering.

tools/bbh_shootout.cpp

Index: tools/bbh_shootout.cpp
diff --git a/tools/bbh_shootout.cpp b/tools/bbh_shootout.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..03ea4f74830d3bf7054595d181745b03e5089634
--- /dev/null
+++ b/tools/bbh_shootout.cpp
@@ -0,0 +1,429 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#include "BenchTimer.h"
+#include "PictureBenchmark.h"
+#include "PictureRenderer.h"
+#include "PictureRenderingFlags.h"
+#include "SkBenchmark.h"
+#include "SkCommandLineFlags.h"
+#include "SkForceLinking.h"
+#include "SkGraphics.h"
+#include "SkStream.h"
+#include "SkString.h"
+#include "TimerData.h"
+
+__SK_FORCE_IMAGE_DECODER_LINKING;
+
+static const int kNumRecordings = SkBENCHLOOP(10);
+static const int kNumPlaybacks = SkBENCHLOOP(5);
+static const size_t kNumBaseBenchmarks = 3;
+static const size_t kNumTileSizes = 3;
+// This is separate from kNumTileSizes because we may add random tile size benchmarks later.
+static const size_t kNumBbhPlaybackBenchmarks = 3;
+static const size_t kNumBenchmarks = kNumBaseBenchmarks + kNumBbhPlaybackBenchmarks;
+
+enum BenchmarkType {
+    kNormal_BenchmarkType = 0,
+    kRTree_BenchmarkType,
+};
+
+struct Histogram {
+    Histogram() {
+        // Make fCpuTime negative so that we don't mess with stats:
+        fCpuTime = SkIntToScalar(-1);
+    }
+    SkScalar fCpuTime;
+    SkString fPath;
+};
+
+typedef void (*BenchmarkFunction)
+    (BenchmarkType, const SkISize&, const SkString&, SkPicture*, BenchTimer*);
+
+// Defined below.
+static void benchmark_playback(
+        BenchmarkType, const SkISize&, const SkString&, SkPicture*, BenchTimer*);
+static void benchmark_recording(
+        BenchmarkType, const SkISize&, const SkString&, SkPicture*, BenchTimer*);
+
+/**
+ * Acts as a POD containing information needed to run a benchmark.
+ * Provides static methods to poll benchmark info from an index.
+ */
+struct BenchmarkControl {
+    SkISize fTileSize;
+    BenchmarkType fType;
+    BenchmarkFunction fFunction;
+    SkString fName;
+
+    static BenchmarkControl Make(size_t i) {
+        BenchmarkControl benchControl;
+        benchControl.fTileSize = getTileSize(i);
+        benchControl.fType = getBenchmarkType(i);
+        benchControl.fFunction = getBenchmarkFunc(i);
+        benchControl.fName = getBenchmarkName(i);
+        return benchControl;
+    }
+
+    enum BaseBenchmarks {
+        kNormalRecord = 0,
+        kNormalPlayback,
+        kRTreeRecord,
+    };
+
+    static SkISize fTileSizes[kNumTileSizes];
+    static SkTArray<Histogram> fHistograms[kNumBenchmarks];
+
+    static SkISize getTileSize(size_t i) {
+        // Two of the base benchmarks don't need a tile size. But to maintain simplicity
+        // down the pipeline we have to let a couple of values unused.
+        if (i < kNumBaseBenchmarks) {
+            return SkISize::Make(256, 256);
+        }
+        if (i >= kNumBaseBenchmarks && i < kNumBenchmarks) {
+            return fTileSizes[i - kNumBaseBenchmarks];
+        }
+        SkASSERT(0);
+        return SkISize::Make(0, 0);
+    }
+
+    static BenchmarkType getBenchmarkType(size_t i) {
+        if (i < kNumBaseBenchmarks) {
+            switch (i) {
+                case kNormalRecord:
+                    return kNormal_BenchmarkType;
+                case kNormalPlayback:
+                    return kNormal_BenchmarkType;
+                case kRTreeRecord:
+                    return kRTree_BenchmarkType;
+            }
+        }
+        if (i < kNumBenchmarks) {
+            return kRTree_BenchmarkType;
+        } else {
+            SkASSERT(0);
+        }
+        return kRTree_BenchmarkType;
+    }
+
+    static BenchmarkFunction getBenchmarkFunc(size_t i) {
+        // Base functions.
+        switch (i) {
+            case kNormalRecord:
+                return benchmark_recording;
+            case kNormalPlayback:
+                return benchmark_playback;
+            case kRTreeRecord:
+                return benchmark_recording;
+        }
+        // RTree playbacks
+        if (i < kNumBenchmarks) {
+            return benchmark_playback;
+        } else {
+            SkASSERT(0);
+        }
+        return NULL;
+    }
+
+    static SkString getBenchmarkName(size_t i) {
+        // Base benchmark names
+        switch (i) {
+            case kNormalRecord:
+                return SkString("normal_recording");
+            case kNormalPlayback:
+                return SkString("normal_playback");
+            case kRTreeRecord:
+                return SkString("rtree_recording");
+        }
+        // RTree benchmark names.
+        if (i < kNumBenchmarks) {
+            if (i < kNumBaseBenchmarks) {
+                SkDebugf("WTF %d", i);
+            }

[sglez, 2013/07/13 03:57:15]

Woops!!!
This if block was me doing printf debugging and forgetting to erase.
Please ignore. Will remove

+            SkASSERT(i >= kNumBaseBenchmarks);
+            SkString name;
+            name.printf("rtree_playback_%dx%d",
+                    fTileSizes[i - kNumBaseBenchmarks].fWidth,
+                    fTileSizes[i - kNumBaseBenchmarks].fHeight);
+            return name;
+
+        } else {
+            SkASSERT(0);
+        }
+        return SkString("");
+    }
+
+};
+
+SkISize BenchmarkControl::fTileSizes[kNumTileSizes] = {
+    SkISize::Make(256, 256),
+    SkISize::Make(512, 512),
+    SkISize::Make(1024, 1024),
+};
+
+// Defined in PictureRenderingFlags.cpp
+extern bool lazy_decode_bitmap(const void* buffer, size_t size, SkBitmap* bitmap);
+
+static SkPicture* pic_from_path(const char path[]) {
+    SkFILEStream stream(path);
+    if (!stream.isValid()) {
+        SkDebugf("-- Can't open '%s'\n", path);
+        return NULL;
+    }
+    return SkPicture::CreateFromStream(&stream, &lazy_decode_bitmap);
+}
+
+/**
+ * This function is the sink to which all work ends up going.
+ * Renders the picture into the renderer. It may or may not use an RTree.
+ * The renderer is chosen upstream. If we want to measure recording, we will
+ * use a RecordPictureRenderer. If we want to measure rendering, we eill use a
+ * TiledPictureRenderer.
+ */
+static void do_benchmark_work(sk_tools::PictureRenderer* renderer,
+        int benchmarkType, const SkString& path, SkPicture* pic,
+        const int numRepeats, const char *msg, BenchTimer* timer) {
+    SkString msgPrefix;
+
+    switch (benchmarkType){
+        case kNormal_BenchmarkType:
+            msgPrefix.set("Normal");
+            renderer->setBBoxHierarchyType(sk_tools::PictureRenderer::kNone_BBoxHierarchyType);
+            break;
+        case kRTree_BenchmarkType:
+            msgPrefix.set("RTree");
+            renderer->setBBoxHierarchyType(sk_tools::PictureRenderer::kRTree_BBoxHierarchyType);
+            break;
+        default:
+            SkASSERT(0);
+            break;
+    }
+
+    renderer->init(pic);
+
+    /**
+     * If the renderer is not tiled, assume we are measuring recording.
+     */
+    bool isPlayback = (NULL != renderer->getTiledRenderer());
+
+    SkDebugf("%s %s %s %d times...\n", msgPrefix.c_str(), msg, path.c_str(), numRepeats);
+    for (int i = 0; i < numRepeats; ++i) {
+        renderer->setup();
+        // Render once to fill caches.
+        renderer->render(NULL);
+        // Render again to measure
+        timer->start();
+        bool result = renderer->render(NULL);
+        timer->end();
+        // We only care about a false result on playback. RecordPictureRenderer::render will always
+        // return false because we are passing a NULL file name on purpose; which is fine.
+        if(isPlayback && !result) {
+            SkDebugf("Error rendering during playback.\n");
+        }
+    }
+    renderer->end();
+}
+
+/**
+ * Call do_benchmark_work with a tiled renderer using the default tile dimensions.
+ */
+static void benchmark_playback(
+        BenchmarkType benchmarkType, const SkISize& tileSize,
+        const SkString& path, SkPicture* pic, BenchTimer* timer) {
+    sk_tools::TiledPictureRenderer renderer;
+
+    SkString message("tiled_playback");
+    message.appendf("_%dx%d", tileSize.fWidth, tileSize.fHeight);
+    do_benchmark_work(&renderer, benchmarkType,
+            path, pic, kNumPlaybacks, message.c_str(), timer);
+}
+
+/**
+ * Call do_benchmark_work with a RecordPictureRenderer.
+ */
+static void benchmark_recording(
+        BenchmarkType benchmarkType, const SkISize& tileSize,
+        const SkString& path, SkPicture* pic, BenchTimer* timer) {
+    sk_tools::RecordPictureRenderer renderer;
+    do_benchmark_work(&renderer, benchmarkType, path, pic, kNumRecordings, "recording", timer);
+}
+
+/**
+ * Takes argc,argv along with one of the benchmark functions defined above.
+ * Will loop along all skp files and perform measurments.
+ *
+ * Returns a SkScalar representing CPU time taken during benchmark.
+ * As a side effect, it spits the timer result to stdout.
+ * Will return -1.0 on error.
+ */
+static SkScalar benchmark_loop(
+        int argc,
+        char **argv,
+        const BenchmarkControl& benchControl,
+        SkTArray<Histogram>& histogram) {
+
+    static const SkString timeFormat("%f");
+    TimerData timerData(timeFormat, timeFormat);
+    for (int index = 1; index < argc; ++index) {
+        BenchTimer timer;
+        SkString path(argv[index]);
+        SkAutoTUnref<SkPicture> pic(pic_from_path(path.c_str()));
+        if (NULL == pic) {
+            SkDebugf("Couldn't create picture. Ignoring path: %s\n", path.c_str());
+            continue;
+        }
+        benchControl.fFunction(benchControl.fType, benchControl.fTileSize, path, pic, &timer);
+        timerData.appendTimes(&timer, argc - 1 == index);
+
+        histogram[index - 1].fPath = path;
+        histogram[index - 1].fCpuTime = timer.fCpu;
+    }
+
+    const SkString timerResult = timerData.getResult(
+            /*logPerIter = */ false,
+            /*printMin = */ false,
+            /*repeatDraw = */ 1,
+            /*configName = */ benchControl.fName.c_str(),
+            /*showWallTime = */ false,
+            /*showTruncatedWallTime = */ false,
+            /*showCpuTime = */ true,
+            /*showTruncatedCpuTime = */ false,
+            /*showGpuTime = */ false);
+
+    const char findStr[] = "= ";
+    int pos = timerResult.find(findStr);
+    if (-1 == pos) {
+        SkDebugf("Unexpected output from TimerData::getResult(...). Unable to parse.");
+        return SkIntToScalar(-1);
+    }
+    SkDebugf("%s\n", timerResult.c_str());
+
+    SkScalar cpuTime = atof(timerResult.c_str() + pos + sizeof(findStr) - 1);
+    if (cpuTime == 0) {  // atof returns 0.0 on error.
+        SkDebugf("Unable to read value from timer result.\n");
+        return SkIntToScalar(-1);
+    }
+    return cpuTime;
+}
+
+static int tool_main(int argc, char** argv) {
+    SkAutoGraphics ag;
+    SkString usage;
+    usage.printf("Usage: filename [filename]*\n");
+
+    if (argc < 2) {
+        SkDebugf("%s\n", usage.c_str());
+        return -1;
+    }
+
+    static SkScalar results[kNumBenchmarks];
+    static SkTArray<Histogram> histograms[kNumBenchmarks];
+
+    for (size_t i = 0; i < kNumBenchmarks; ++i) {
+        histograms[i] = SkTArray<Histogram>(argc - 1);
+        histograms[i].reset(argc - 1);
+        results[i] = benchmark_loop(
+                argc, argv,
+                BenchmarkControl::Make(i),
+                histograms[i]);
+    }
+
+    // Print results
+    SkDebugf("\n");
+    for (size_t i = 0; i < kNumBenchmarks; ++i) {
+        SkDebugf("%s total: %f\n", BenchmarkControl::getBenchmarkName(i).c_str(), results[i]);
+    }
+
+    // Print a rough analysis to stdout:
+    {
+        SkScalar normalRecordResult = results[0];
+        SkScalar normalPlaybackResult = results[1];
+        SkScalar rtreeRecordResult = results[2];
+        SkScalar rtreePlaybackResult = results[3];
+        SkASSERT(normalRecordResult != 0 && normalPlaybackResult != 0);
+        SkDebugf("\n");
+        SkDebugf("Recording: Relative difference: %.4f\n",
+                rtreeRecordResult / normalRecordResult);
+        SkDebugf("Playback (256x256):  Relative difference: %.4f\n",
+                rtreePlaybackResult / normalPlaybackResult);
+        SkScalar times =
+                (kNumPlaybacks * (normalRecordResult - rtreeRecordResult)) /
+                (kNumRecordings * (rtreePlaybackResult - normalPlaybackResult));
+        SkDebugf("Number of playback repetitions for RTree to be worth it: %d (ratio: %.4f)\n",
+                SkScalarCeilToInt(times), times);
+    }
+
+    for (size_t i = 0; i < kNumBenchmarks; ++i) {
+        SkDebugf("%s\n", BenchmarkControl::getBenchmarkName(i).c_str());
+    }
+    // Print min/max times for each benchmark.
+    SkDebugf("\n");
+    SkScalar minMax[kNumBenchmarks][2];
+    enum {
+        kMinBenchmarkTime,
+        kMaxBenchmarkTime
+    };
+    for (size_t i = 0; i < kNumBenchmarks; ++i) {
+        SkString minPath;
+        SkString maxPath;
+        minMax[i][kMinBenchmarkTime] = SK_ScalarMax;
+        minMax[i][kMaxBenchmarkTime] = 0;
+        for (int j = 0; j < argc - 1; ++j) {
+            SkScalar value = histograms[i][j].fCpuTime;
+            if (value < 0) continue;  // skp wasn't found or couldn't be read.
+            if (value < minMax[i][kMinBenchmarkTime]) {
+                minMax[i][kMinBenchmarkTime] = value;
+                minPath = histograms[i][j].fPath;
+            }
+            if (value > minMax[i][kMaxBenchmarkTime]) {
+                minMax[i][kMaxBenchmarkTime] = value;
+                maxPath = histograms[i][j].fPath;
+            }
+        }
+        SkDebugf("%s min is %.3f:\t\t%s\n"
+                "%s max is %.3f:\t\t%s\n",
+                BenchmarkControl::getBenchmarkName(i).c_str(), minMax[i][kMinBenchmarkTime], minPath.c_str(),
+                BenchmarkControl::getBenchmarkName(i).c_str(), minMax[i][kMaxBenchmarkTime], maxPath.c_str());
+    }
+
+    // Output gnuplot readable histogram data..
+    const char* pbTitle = "bbh_shootout_playback.dat";
+    const char* recTitle = "bbh_shootout_record.dat";
+    SkFILEWStream playbackOut(pbTitle);
+    SkFILEWStream recordOut(recTitle);
+    recordOut.writeText("# Index Normal RTree\n");
+    playbackOut.writeText("# Index Normal RTree\n");
+    for (int i = 0; i < argc - 1; ++i) {
+        SkString pbLine;
+        SkString recLine;
+        // ==== Write record info
+        recLine.printf("%d ", i);
+        recLine.appendf("%f ", histograms[0][i].fCpuTime);  // Append normal_record time
+        recLine.appendf("%f", histograms[2][i].fCpuTime);  // Append rtree_record time
+
+        // ==== Write playback info
+        pbLine.printf("%d ", i);
+        pbLine.appendf("%f ", histograms[1][i].fCpuTime);  // Start with normal playback time.
+        // Append all playback benchmark times.
+        for (size_t j = kNumBbhPlaybackBenchmarks; j < kNumBenchmarks; ++j) {
+            pbLine.appendf("%f ", histograms[j][i].fCpuTime);
+        }
+        pbLine.remove(pbLine.size() - 1, 1);  // Remove trailing space from line.
+        pbLine.appendf("\n");
+        recLine.appendf("\n");
+        playbackOut.writeText(pbLine.c_str());
+        recordOut.writeText(recLine.c_str());
+    }
+    SkDebugf("Wrote data to gnuplot-readable files: %s %s\n", pbTitle, recTitle);
+
+    return 0;
+}
+
+int main(int argc, char** argv) {
+    return tool_main(argc, argv);
+}
+