Measure tiled rendering.

tools/bbh_shootout.cpp

Index: tools/bbh_shootout.cpp
diff --git a/tools/bbh_shootout.cpp b/tools/bbh_shootout.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b81137a03d58282f2a5ac29479a80b4772a89f0e
--- /dev/null
+++ b/tools/bbh_shootout.cpp
@@ -0,0 +1,336 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#include "BenchTimer.h"
+#include "PictureBenchmark.h"
+#include "PictureRenderer.h"
+#include "PictureRenderingFlags.h"
+#include "SkCommandLineFlags.h"
+#include "SkBenchmark.h"
+#include "SkForceLinking.h"
+#include "SkStream.h"
+#include "SkString.h"
+#include "SkGraphics.h"

[caryclark, 2013/07/10 15:28:24]

as before, alphabetize

+#include "TimerData.h"
+
+__SK_FORCE_IMAGE_DECODER_LINKING;
+
+static const int kNumRecordings = SkBENCHLOOP(10);
+static const int kNumPlaybacks = SkBENCHLOOP(5);
+static const int kNumTileSizes = 3;
+
+enum BenchmarkType {
+    kNormal_BenchmarkType = 0,
+    kRTree_BenchmarkType,
+};
+
+struct Histogram {
+    int pathIndex;
+    SkScalar cpuTime;
+};
+
+// Defined in PictureRenderingFlags.cpp
+extern bool lazy_decode_bitmap(const void* buffer, size_t size, SkBitmap* bitmap);
+
+static SkPicture* pic_from_path(const char path[]) {
+    SkFILEStream stream(path);
+    if (!stream.isValid()) {
+        SkDebugf("-- Can't open '%s'\n", path);
+        return NULL;
+    }
+    return SkPicture::CreateFromStream(&stream, &lazy_decode_bitmap);
+}
+
+/**
+ * This function is the sink to which all work ends up going.
+ * Renders the picture into the renderer. It may or may not use an RTree.
+ * The renderer is chosen upstream. If we want to measure recording, we will
+ * use a RecordPictureRenderer. If we want to measure rendering, we eill use a
+ * TiledPictureRenderer.
+ */
+static void do_benchmark_work(sk_tools::PictureRenderer* renderer,
+        int benchmarkType, const SkString *path, SkPicture* pic,
+        const int numRepeats, const char *msg, BenchTimer* timer) {
+    SkString msgPrefix;
+
+    switch (benchmarkType){
+        case kNormal_BenchmarkType:
+            msgPrefix.printf("Normal");

[caryclark, 2013/07/10 15:28:24]

this may generate warnings on some platforms, that expect
  printf("%s", "string")
instead.
Until you need the flexibility of printf, use set() instead.

[sglez, 2013/07/11 20:58:30]

Thanks, I didn't know about that.

+            renderer->setBBoxHierarchyType(sk_tools::PictureRenderer::kNone_BBoxHierarchyType);
+            break;
+        case kRTree_BenchmarkType:
+            msgPrefix.printf("RTree");
+            renderer->setBBoxHierarchyType(sk_tools::PictureRenderer::kRTree_BBoxHierarchyType);
+            break;

[caryclark, 2013/07/10 15:28:24]

default:
  SkASSERT(0);

+    }
+
+    renderer->init(pic);
+
+    /**
+     * If the renderer is not tiled, assume we are measuring recording.
+     */
+    bool recording = (NULL == renderer->getTiledRenderer());
+
+    SkDebugf("%s %s %s %d times...\n", msgPrefix.c_str(), msg, path->c_str(), numRepeats);
+    for (int i = 0; i < numRepeats; ++i) {
+        renderer->setup();
+        // Render once to fill caches.
+        renderer->render(NULL);
+        // Render again to measure
+        timer->start();
+        bool result = renderer->render(NULL);
+        timer->end();
+        // We only care about a false result on playback. RecordPictureRenderer::render will always
+        // return false because we are passing a NULL ptr.
+        if(!recording && !result) {
+            SkDebugf("Error rendering (playback).\n");
+        }
+    }
+    renderer->end();
+}
+
+/**
+ * Call do_benchmark_work with a tiled renderer using the default tile dimensions.
+ */
+static void benchmark_playback(
+        BenchmarkType benchmarkType, const int tileSize[2],
+        const SkString* path, SkPicture* pic, BenchTimer* timer) {
+    sk_tools::TiledPictureRenderer renderer;
+
+    SkString message("tiled_playback");
+    message.appendf("_%dx%d", tileSize[0], tileSize[1]);
+    do_benchmark_work(&renderer, benchmarkType,
+            path, pic, kNumPlaybacks, message.c_str(), timer);
+}
+
+/**
+ * Call do_benchmark_work with a RecordPictureRenderer.
+ */
+static void benchmark_recording(
+        BenchmarkType benchmarkType, const int tileSize[2],
+        const SkString* path, SkPicture* pic, BenchTimer* timer) {
+    sk_tools::RecordPictureRenderer renderer;
+    do_benchmark_work(&renderer, benchmarkType, path, pic, kNumRecordings, "recording", timer);
+}
+
+static const SkString perIterTimeFormat("%f");
+static const SkString normalTimeFormat("%f");
+
+/**
+ * Takes argc,argv along with one of the benchmark functions defined above.
+ * Will loop along all skp files and perform measurments.
+ *
+ * Returns a SkScalar representing CPU time taken during benchmark.
+ * As a side effect, it spits the timer result to stdout.
+ * Will return -1.0 on error.
+ */
+static SkScalar benchmark_loop(
+        int argc,
+        char **argv,
+        void (*func)(BenchmarkType, const int[], const SkString*, SkPicture*, BenchTimer*),

[caryclark, 2013/07/10 15:28:24]

skia's convention is to pass const SkString&

[sglez, 2013/07/11 20:58:30]

Sorry about that, I changed some more cases where input params should be const
TYPE&

+        const int tileSize[2],
+        Histogram histogram[],
+        BenchmarkType benchmarkType,
+        const char* configName) {
+    TimerData timerData(perIterTimeFormat, normalTimeFormat);
+    for (int index = 1; index < argc; ++index) {
+        BenchTimer timer;
+        SkString path(argv[index]);
+        SkAutoTUnref<SkPicture> pic(pic_from_path(argv[index]));
+        if (NULL != pic) {
+            func(benchmarkType, tileSize, &path, pic, &timer);
+        }
+        timerData.appendTimes(&timer, index == argc - 1);
+
+        histogram[index - 1].pathIndex = index;
+        histogram[index - 1].cpuTime = timer.fCpu;

[caryclark, 2013/07/10 15:28:24]

if the pic == NULL, what is timer.fCpu? Does it make sense to record data for
pictures that fail to decode? If it does, put some comments here to note your
logic.

[sglez, 2013/07/11 20:58:30]

This was more a lack of logic on my part than anything else.
There is a need to set the cpuTime to an invalid value in the case that the
picture is null.

+    }
+
+    const SkString timerResult = timerData.getResult(
+            /*logPerIter = */ false,
+            /*printMin = */ false,
+            /*repeatDraw = */ 1,
+            /*configName = */ configName,
+            /*showWallTime = */ false,
+            /*showTruncatedWallTime = */ false,
+            /*showCpuTime = */ true,
+            /*showTruncatedCpuTime = */ false,
+            /*showGpuTime = */ false);
+
+    const char findStr[] = "= ";
+    int pos = timerResult.find(findStr);
+    if (-1 == pos) {
+        SkDebugf("Unexpected output from TimerData::getResult(...). Unable to parse.");
+        return -1.0;
+    }
+    SkDebugf("%s\n", timerResult.c_str());
+
+    SkScalar cpuTime = atof(timerResult.c_str() + pos + sizeof(findStr) - 1);
+    if (cpuTime == SkIntToScalar(0)) {  // atof returns 0.0 on error.
+        SkDebugf("Unable to read value from timer result.\n");
+        return -1.0;

[caryclark, 2013/07/10 15:28:24]

return SkIntToScalar(-1);

+    }
+    return cpuTime;
+}
+
+static int tool_main(int argc, char** argv) {
+    SkAutoGraphics ag;
+    SkString usage;
+    usage.printf("Usage: filename [filename]*\n");
+
+    if (argc < 2) {
+        SkDebugf("%s\n", usage.c_str());
+        return 0;
+    }
+
+    static const int tileSizes[kNumTileSizes][2] = {
+        {256, 256},
+        {512, 512},
+        {1024, 1024},
+    };

[caryclark, 2013/07/10 15:28:24]

this seems unnecessarily fragile and will make adding a new tile size tricky.

Start by defining kNumTileSizes as

static const size_t kNumTileSizes = SK_ARRAY_COUNT(tileSizes);

+    static const int kNumBenchmarks = 6;

[caryclark, 2013/07/10 15:28:24]

static const size_t kNumBenchmarks = 3 + kNumTileSizes;

+    static const char* benchNames[kNumBenchmarks] = {
+        "normal_recording",
+        "normal_playback",
+        "rtree_recording",
+        "rtree_playback_256x256",
+        "rtree_playback_512x512",
+        "rtree_playback_1024x1024",

[caryclark, 2013/07/10 15:28:24]

build these rtree_playback strings from the tile sizes

+    };
+    static SkScalar results[kNumBenchmarks];
+    static Histogram *histograms[kNumBenchmarks] = {
+        SkNEW_ARRAY(Histogram, argc - 1),  // normal_recording
+        SkNEW_ARRAY(Histogram, argc - 1),  // normal_playback
+        SkNEW_ARRAY(Histogram, argc - 1),  // rtree_recording
+        SkNEW_ARRAY(Histogram, argc - 1),  // rtree_playback_256x256
+        SkNEW_ARRAY(Histogram, argc - 1),  // rtree_playback_512x512
+        SkNEW_ARRAY(Histogram, argc - 1),  // rtree_playback_1024x1024
+    };

[caryclark, 2013/07/10 15:28:24]

defer array initialization until you use it below

+    static void (*benchmarkFunctions[kNumBenchmarks])
+        (BenchmarkType, const int [kNumBenchmarks], const SkString*, SkPicture*, BenchTimer*) = {
+            benchmark_recording,  // normal_recording
+            benchmark_playback,   // normal_playback
+            benchmark_recording,  // rtree_recording

[caryclark, 2013/07/10 15:28:24]

change this to only contain the first 3 entries so that tile size can be
extendable.

If this array is moved above kNumBenchmarks then the '3' in the comment above
and below could be replaced by SK_ARRAY_COUNT(benchmarkFunctions);

(maybe rename to baseBenchmarkFunctions to distinguish from playback functions
with different tile sizes?)

+            benchmark_playback,   // rtree_playback_256x256
+            benchmark_playback,   // rtree_playback_512x512
+            benchmark_playback,   // rtree_playback_1024x1024
+        };
+
+    for (int i = 0; i < kNumBenchmarks; ++i) {
+        BenchmarkType type;
+        if (i < 2) {
+            type = kNormal_BenchmarkType;
+        } else {
+            type = kRTree_BenchmarkType;
+        }
+        int tileSize[2] = {256, 256};
+        // If our bencmark is of the type rtree_playback_[SIZE]. Set tileSize.
+        if (i >= 3) {
+            tileSize[0] = tileSizes[i - 3][0];
+            tileSize[1] = tileSizes[i - 3][1];

[caryclark, 2013/07/10 15:28:24]

add

benchmarkFunction = benchmark_playback;

+        }

[caryclark, 2013/07/10 15:28:24]

add

histograms[i] = SkNEW_ARRAY(...

+        results[i] = benchmark_loop(
+                argc, argv, benchmarkFunctions[i], tileSize, histograms[i],
+                type, benchNames[i]);
+    }
+
+    // Print results
+    SkDebugf("\n");
+    for (int i = 0; i < kNumBenchmarks; ++i) {
+        SkDebugf("%s total: %f\n", benchNames[i], results[i]);
+    }
+
+    // Print a rough analysis to stdout:
+    {
+        SkScalar normalRecordResult = results[0];
+        SkScalar normalPlaybackResult = results[1];
+        SkScalar rtreeRecordResult = results[2];
+        SkScalar rtreePlaybackResult = results[3];
+        SkASSERT(normalRecordResult != 0 && normalPlaybackResult != 0);
+        SkDebugf("\n");
+        SkDebugf("Recording: Relative difference: %.4f\n",
+                rtreeRecordResult / normalRecordResult);
+        SkDebugf("Playback (256x256):  Relative difference: %.4f\n",
+                rtreePlaybackResult / normalPlaybackResult);
+        SkScalar times =
+                (kNumPlaybacks * (normalRecordResult - rtreeRecordResult)) /
+                (kNumRecordings * (rtreePlaybackResult - normalPlaybackResult));
+        SkDebugf("Number of playback repetitions for RTree to be worth it: %d (ratio: %.4f)\n",
+                SkScalarCeilToInt(times), times);
+    }
+
+    // Print min/max times for each benchmark.
+    SkDebugf("\n");
+    SkScalar minMax[][2] = {

[caryclark, 2013/07/10 15:28:24]

minMax[kNumBenchmarks][...

then

remove initialization

+        // MIN         MAX
+        {SK_ScalarMax, 0},  // normal_recording
+        {SK_ScalarMax, 0},  // normal_playback
+        {SK_ScalarMax, 0},  // rtree_recording
+        {SK_ScalarMax, 0},  // rtree_playback_256x256
+        {SK_ScalarMax, 0},  // rtree_playback_512x512
+        {SK_ScalarMax, 0},  // rtree_playback_1024x1024
+    };
+    for (int i = 0; i < argc - 1; ++i) {
+        for (int j = 0; j < kNumBenchmarks; ++j) {

[caryclark, 2013/07/10 15:28:24]

reverse loop nesting so you can initialize minMax dynamically

+            SkScalar value = histograms[j][i].cpuTime;
+            if (value < minMax[j][0]) {
+                minMax[j][0] = value;
+            }
+            if (value > minMax[j][1]) {
+                minMax[j][1] = value;
+            }
+        }
+    }
+    for (int i = 0; i < kNumBenchmarks; ++i) {

[caryclark, 2013/07/10 15:28:24]

reversing the loop nesting above allows this to be part of the outer loop

+        SkString out;
+        out.printf("%s min is ", benchNames[i]);
+        out.appendf("%f\n", minMax[i][0]);
+        out.appendf("%s max is ", benchNames[i]);
+        out.appendf("%f\n", minMax[i][1]);
+        SkDebugf("%s", out.c_str());

[caryclark, 2013/07/10 15:28:24]

is this more clear to you than if you replaced the printf and appendf with
SkDebugf  ?

[sglez, 2013/07/11 20:58:30]

SkDebugf is much clearer

+    }
+
+    // Output gnuplot readable histogram data..
+    const char* pbTitle = "bbh_shootout_playback.dat";
+    const char* recTitle = "bbh_shootout_record.dat";
+    SkFILEWStream playbackOut(pbTitle);
+    SkFILEWStream recordOut(recTitle);
+    recordOut.writeText("# Index Normal RTree\n");
+    playbackOut.writeText("# Index Normal RTree\n");
+    for (int i = 0; i < argc - 1; ++i) {
+        SkString pbLine;
+        SkString recLine;
+        // ==== Write record info
+        recLine.printf("%d ", i);
+        recLine.appendf("%f ", histograms[0][i].cpuTime);  // Append normal_record time
+        recLine.appendf("%f ", histograms[2][i].cpuTime);  // Append rtree_record time
+
+        // ==== Write playback info
+        pbLine.printf("%d ", i);
+        pbLine.appendf("%f ", histograms[1][i].cpuTime);  // Start with normal playback time.
+        // Append all playback benchmark times.
+        for (int j = 3; j < kNumBenchmarks; ++j) {

[caryclark, 2013/07/10 15:28:24]

replace 3 with some const defined above.

+            pbLine.appendf("%f ", histograms[j][i].cpuTime);
+        }
+        pbLine.appendf("\n");
+        recLine.appendf("\n");
+        playbackOut.writeText(pbLine.c_str());
+        recordOut.writeText(recLine.c_str());
+    }
+    SkDebugf("Wrote data to gnuplot-readable files: %s %s\n", pbTitle, recTitle);
+
+    for (int i = 0; i < kNumBenchmarks; ++i) {
+        SkDELETE(histograms[i]);

[caryclark, 2013/07/10 15:28:24]

this suggests that some auto use is missing here

[sglez, 2013/07/11 20:58:30]

I am now using SkTArrays

+    }
+
+    return 0;
+}
+
+int main(int argc, char** argv) {
+    return tool_main(argc, argv);
+}
+