Measure frame decodes more accurately from ImageDecodeBench

third_party/WebKit/Source/platform/testing/ImageDecodeBench.cpp

Index: third_party/WebKit/Source/platform/testing/ImageDecodeBench.cpp
diff --git a/third_party/WebKit/Source/platform/testing/ImageDecodeBench.cpp b/third_party/WebKit/Source/platform/testing/ImageDecodeBench.cpp
index 11742a16be9d362831a00d0c6d9890075da9d734..94ea1fb1a43e88ed023f1d53b5e3212532e22b25 100644
--- a/third_party/WebKit/Source/platform/testing/ImageDecodeBench.cpp
+++ b/third_party/WebKit/Source/platform/testing/ImageDecodeBench.cpp
@@ -10,6 +10,28 @@
 //   % ninja -C out/Release image_decode_bench &&
 //      ./out/Release/image_decode_bench file [iterations]
 //
+// If --raw-output is specified, the output file provided is formatted for use
+// in a .csv file (comma-separated variable).
+// Each row represents successive frames in an animated image.
+// (frame 0, frame 1, frame 2...)
+// Each column represents a single frame decoded from successive iterations.
+// Iteration 0: (frame 0, frame 1, frame 2...)
+// Iteration 1: (frame 0, frame 1, frame 2...)
+//
+// This means non-animated images will show up as one column.
+//
+// This .csv-formatted output file is a common format for spreadsheet and data
+// visualization programs. A user can take timings before and after a change,
+// import the two .csv files, and overlay their line graphs on top of each other
+// to see how the two measurements compare.
+//
+// Visualizing the measurements can provide insight into things like an
+// intermittent noise source or an unexpected measurement.
+//
+// Each frame is different and so comparing separate frames may not be fruitful.
+// For example, frame 1 may cover a large area and require more time to decode.
+// Frame 2 may have a small rect and be faster to decode.
+//
 // TODO(noel): Consider adding md5 checksum support to WTF. Use it to compute
 // the decoded image frame md5 and output that value.
 //
@@ -17,7 +39,9 @@
 // using the image corpii used to assess Blink image decode performance. Refer
 // to http://crbug.com/398235#c103 and http://crbug.com/258324#c5
 
+#include <limits>
 #include <memory>
+#include <vector>

[Noel Gordon, 2017/07/17 07:38:43]

Use WTF::Vector

 #include "base/command_line.h"
 #include "platform/SharedBuffer.h"
 #include "platform/image-decoders/ImageDecoder.h"
@@ -204,52 +228,175 @@ PassRefPtr<SharedBuffer> ReadFile(const char* file_name) {
   return SharedBuffer::Create(buffer.get(), file_size);
 }
 
-bool DecodeImageData(SharedBuffer* data,
-                     bool color_correction,
-                     size_t packet_size) {
-  std::unique_ptr<ImageDecoder> decoder = ImageDecoder::Create(
-      data, true, ImageDecoder::kAlphaPremultiplied,
-      color_correction ? ColorBehavior::TransformToTargetForTesting()
-                       : ColorBehavior::Ignore());
-  if (!packet_size) {
-    bool all_data_received = true;
-    decoder->SetData(data, all_data_received);
+// This vector represents a single iteration of one (possibly animated) image.

[Noel Gordon, 2017/07/17 07:38:43]

"This"? Which vector are u referring to?

+// Each entry is a single timing of a single frame.
+using FrameTimings = std::vector<double>;
+using IterationsOfFrameTimings = std::vector<FrameTimings>;
 
-    int frame_count = decoder->FrameCount();
-    for (int i = 0; i < frame_count; ++i) {
-      if (!decoder->FrameBufferAtIndex(i))
-        return false;
+void Write2DResultsToFile(const IterationsOfFrameTimings& timings,
+                          const char* file_name) {
+  FILE* fp = fopen(file_name, "wb");
+  if (!fp) {
+    fprintf(stderr, "Can't open file %s\n", file_name);
+    exit(2);
+  }
+
+  FrameTimings first_iteration = timings[0];
+  for (size_t i = 0; i < first_iteration.size(); ++i) {
+    fprintf(fp, "\"Frame %zu\",", i);
+  }
+  fprintf(fp, "\n");
+
+  for (const FrameTimings& iteration : timings) {
+    for (double frame_time : iteration) {
+      fprintf(fp, "%f,", frame_time);
     }
+    fprintf(fp, "\n");
+  }
+  fprintf(fp, "\n");
 
-    return !decoder->Failed();
+  fclose(fp);
+}
+
+class MeanAndMin {

[Noel Gordon, 2017/07/17 07:38:43]

Could we make this a struct please.

+ public:
+  MeanAndMin(double mean, double min) : mean_(mean), min_(min) {}
+  double mean_;
+  double min_;
+};
+
+MeanAndMin GetMeanAndMin(const IterationsOfFrameTimings& timings) {
+  size_t count = 0;
+  double total = 0.0;
+
+  double min = std::numeric_limits<double>::max();
+
+  for (const FrameTimings& iteration : timings) {
+    for (double frame_time : iteration) {
+      ++count;
+      total += frame_time;
+      min = std::min(min, frame_time);
+    }
   }
 
-  RefPtr<SharedBuffer> packet_data = SharedBuffer::Create();
-  size_t position = 0;
-  size_t next_frame_to_decode = 0;
-  while (true) {
-    const char* packet;
-    size_t length = data->GetSomeData(packet, position);
+  return MeanAndMin(total / count, min);
+}
 
-    length = std::min(length, packet_size);
-    packet_data->Append(packet, length);
-    position += length;
+double GetStandardDeviation(const IterationsOfFrameTimings& timings,
+                            double mean) {
+  size_t count = 0;
+  double squared_difference_sum = 0;
+
+  for (const FrameTimings& iteration : timings) {
+    for (double frame_time : iteration) {
+      double difference = frame_time - mean;

[Noel Gordon, 2017/07/17 07:38:43]

1)

{
  double difference = frame_time - mean;
  squared_difference_sum += difference * difference;
  ++count;
}

here perhaps, but ...

2) can you explain why having a single _mean_ for some number of decode bench
iterations of a multi-framed image makes sense?

+      double absolute_difference = fabs(difference);
+      double squared_difference = pow(absolute_difference, 2);
+      ++count;
+      squared_difference_sum += squared_difference;
+    }
+  }
 
-    bool all_data_received = position == data->size();
-    decoder->SetData(packet_data.Get(), all_data_received);
+  return std::sqrt(squared_difference_sum / count);
+}
 
-    size_t frame_count = decoder->FrameCount();
-    for (; next_frame_to_decode < frame_count; ++next_frame_to_decode) {
-      ImageFrame* frame = decoder->FrameBufferAtIndex(next_frame_to_decode);
-      if (frame->GetStatus() != ImageFrame::kFrameComplete)
-        break;
+size_t GetFrameCount(const PassRefPtr<SharedBuffer>& data) {

[Noel Gordon, 2017/07/17 07:38:43]

PassRefPtr ?  Maybe pass by raw ptr.

+  auto decoder =
+      ImageDecoder::Create(data.Get(), true, ImageDecoder::kAlphaPremultiplied,
+                           ColorBehavior::Ignore());
+  if (!decoder) {
+    fprintf(stderr, "Image decode failed\n");
+    exit(3);
+  }
+  size_t frame_count = decoder->FrameCount();
+  if (decoder->Failed()) {
+    fprintf(stderr, "Image decode failed\n");
+    exit(3);
+  }
+  return frame_count;
+}
+
+IterationsOfFrameTimings TimeDecode(const RefPtr<SharedBuffer>& data,
+                                    bool apply_color_correction,
+                                    size_t iterations) {
+  size_t frame_count = GetFrameCount(data);
+
+  IterationsOfFrameTimings timings(iterations, FrameTimings(frame_count, 0.0));
+
+  for (size_t i = 0; i < iterations; ++i) {
+    std::unique_ptr<ImageDecoder> decoder = ImageDecoder::Create(
+        data.Get(), true, ImageDecoder::kAlphaPremultiplied,
+        apply_color_correction ? ColorBehavior::TransformToTargetForTesting()
+                               : ColorBehavior::Ignore());
+    bool all_data_received = true;
+    decoder->SetData(data.Get(), all_data_received);
+    for (size_t frame_index = 0; frame_index < frame_count; ++frame_index) {
+      double start_time = GetCurrentTime();
+      ImageFrame* frame = decoder->FrameBufferAtIndex(frame_index);
+      double elapsed_time = GetCurrentTime() - start_time;
+      if (frame->GetStatus() != ImageFrame::kFrameComplete) {
+        fprintf(stderr, "Image decode failed\n");
+        exit(3);
+      }
+      timings[i][frame_index] = elapsed_time;
     }
+  }

[Noel Gordon, 2017/07/17 07:38:43]

Should we be testing for decoder->Failed() around here?

 
-    if (all_data_received || decoder->Failed())
-      break;
+  return timings;
+}
+
+// This function mimics deferred decoding in Chromium when not all data has been
+// received yet.
+IterationsOfFrameTimings TimePacketedDecode(const RefPtr<SharedBuffer>& data,
+                                            bool apply_color_correction,
+                                            size_t packet_size,
+                                            size_t iterations) {
+  size_t total_frame_count = GetFrameCount(data);
+
+  IterationsOfFrameTimings timings(iterations,
+                                   FrameTimings(total_frame_count, 0.0));
+
+  for (size_t i = 0; i < iterations; ++i) {
+    size_t position = 0;
+    size_t next_frame_to_decode = 0;
+
+    RefPtr<SharedBuffer> packet_data = SharedBuffer::Create();
+    std::unique_ptr<ImageDecoder> decoder = ImageDecoder::Create(
+        packet_data.Get(), false, ImageDecoder::kAlphaPremultiplied,
+        apply_color_correction ? ColorBehavior::TransformToTargetForTesting()
+                               : ColorBehavior::Ignore());
+    while (true) {
+      const char* packet;
+      size_t length = data->GetSomeData(packet, position);
+
+      length = std::min(length, packet_size);
+      packet_data->Append(packet, length);
+      position += length;
+
+      bool all_data_received = position == data->size();
+      decoder->SetData(packet_data.Get(), all_data_received);
+
+      size_t frame_count = decoder->FrameCount();
+      for (; next_frame_to_decode < frame_count; ++next_frame_to_decode) {
+        double start_time = GetCurrentTime();
+        ImageFrame* frame = decoder->FrameBufferAtIndex(next_frame_to_decode);
+        double elapsed_time = GetCurrentTime() - start_time;
+        if (frame->GetStatus() != ImageFrame::kFrameComplete)
+          break;
+        timings[i][next_frame_to_decode] = elapsed_time;
+      }
+
+      if (decoder->Failed()) {
+        fprintf(stderr, "Image decode failed\n");
+        exit(3);
+      }
+
+      if (all_data_received)
+        break;
+    }
   }
 
-  return !decoder->Failed();
+  return timings;
 }
 
 }  // namespace
@@ -260,16 +407,31 @@ int Main(int argc, char* argv[]) {
   // If the platform supports color correction, allow it to be controlled.
 
   bool apply_color_correction = false;
-
   if (argc >= 2 && strcmp(argv[1], "--color-correct") == 0) {
-    apply_color_correction = (--argc, ++argv, true);
+    --argc;
+    ++argv;
+    apply_color_correction = true;
     gfx::ICCProfile profile = gfx::ICCProfileForTestingColorSpin();
     ColorBehavior::SetGlobalTargetColorProfile(profile);
   }
 
+  const char* raw_output_file = nullptr;
+  if (argc >= 2 && strcmp(argv[1], "--raw-output") == 0) {
+    if (argc < 3) {
+      fprintf(stderr,
+              "--raw-output specified without also specifying a file after it");
+      exit(1);
+    }
+
+    raw_output_file = argv[2];
+    argc -= 2;
+    argv += 2;
+  }
+
   if (argc < 2) {
     fprintf(stderr,
-            "Usage: %s [--color-correct] file [iterations] [packetSize]\n",
+            "Usage: %s [--color-correct] [--raw-output output_file] image_file "
+            "[iterations] [packetSize]\n",
             argv[0]);
     exit(1);
   }
@@ -321,33 +483,23 @@ int Main(int argc, char* argv[]) {
 
   data->Data();
 
-  // Warm-up: throw out the first iteration for more consistent results.
-
-  if (!DecodeImageData(data.Get(), apply_color_correction, packet_size)) {
-    fprintf(stderr, "Image decode failed [%s]\n", argv[1]);
-    exit(3);
+  IterationsOfFrameTimings timings;
+  if (packet_size) {
+    timings = TimePacketedDecode(data, apply_color_correction, packet_size,
+                                 iterations);
+  } else {
+    timings = TimeDecode(data, apply_color_correction, iterations);
   }
 
-  // Image decode bench for iterations.
-
-  double total_time = 0.0;
+  if (raw_output_file)
+    Write2DResultsToFile(timings, raw_output_file);
 
-  for (size_t i = 0; i < iterations; ++i) {
-    double start_time = GetCurrentTime();
-    bool decoded =
-        DecodeImageData(data.Get(), apply_color_correction, packet_size);
-    double elapsed_time = GetCurrentTime() - start_time;
-    total_time += elapsed_time;
-    if (!decoded) {
-      fprintf(stderr, "Image decode failed [%s]\n", argv[1]);
-      exit(3);
-    }
-  }
+  MeanAndMin mean_and_min = GetMeanAndMin(timings);
+  double standard_deviation = GetStandardDeviation(timings, mean_and_min.mean_);
 
-  // Results to stdout.
+  printf("min: %f mean: %f standard deviation: %f\n", mean_and_min.min_,
+         mean_and_min.mean_, standard_deviation);
 
-  double average_time = total_time / static_cast<double>(iterations);
-  printf("%f %f\n", total_time, average_time);
   return 0;
 }