Added image pattern comparison logic for test interface and fixture.

remoting/test/test_video_renderer.cc

Index: remoting/test/test_video_renderer.cc
diff --git a/remoting/test/test_video_renderer.cc b/remoting/test/test_video_renderer.cc
index 75dd2c9511ba614425c8230dcebcd5e82bbcb23f..27fa347bd918ef09e5a03a0a65280916c4b429d7 100644
--- a/remoting/test/test_video_renderer.cc
+++ b/remoting/test/test_video_renderer.cc
@@ -4,7 +4,11 @@
 
 #include "remoting/test/test_video_renderer.h"
 
+#include <algorithm>
+#include <cmath>
+
 #include "base/bind.h"
+#include "base/callback_helpers.h"
 #include "base/logging.h"
 #include "base/synchronization/lock.h"
 #include "base/thread_task_runner_handle.h"
@@ -15,6 +19,21 @@
 #include "remoting/proto/video.pb.h"
 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
 
+// RGB Color triplets, and it can be converted to and from RGBA32.
+struct RGBTriplets {
+  RGBTriplets(int red, int green, int blue)
+      : red_(red), green_(green), blue_(blue) {}
+  int red_;

[Sergey Ulanov, 2015/07/10 01:21:56]

don't need _ suffix for struct fields. It's only for class fields

[liaoyuke, 2015/07/10 16:22:30]

Done.

+  int green_;
+  int blue_;
+};
+
+namespace {
+// Used to account for video frame resizing and lossy encoding.
+const int kRectFuzzyThreshold = 2;

[Sergey Ulanov, 2015/07/10 01:21:56]

Maybe call it something like kMaxPositionError? Document this is in pixels

[liaoyuke, 2015/07/10 16:22:30]

Done.

+const double kColorFuzzyThreshold = 0.05;

[Sergey Ulanov, 2015/07/10 01:21:55]

kMaxColorError?
Also document units.

[liaoyuke, 2015/07/10 16:22:30]

Done.

+}  // namespace
+
 namespace remoting {
 namespace test {
 

[liaoyuke, 2015/07/10 16:22:30]

Per discussion with Sergey, the dirty rects are not that stable and the method
should not rely on dirty rects. To get rid of the dependency on dirty rects,
I'll change the comparison logic to every time when a frame is received,
calculate the average color of pixels fall within the expected rect and see if
it matches. This is more computationally intensive, but it's more stable.

@@ -41,10 +60,41 @@ class TestVideoRenderer::Core {
   // when the pattern is matched.
   void SetImagePatternAndMatchedCallback(
       const webrtc::DesktopRect& expected_rect,
-      const RgbaColor& expected_color,
+      const RGBA32& expected_avg_color,
       const base::Closure& image_pattern_matched_callback);
 
+  // Convert a RGB triplets to 32 bit RGBA.
+  static RGBA32 ConvertTripletsToRGBA32(const RGBTriplets& rgb_triplets);
+
+  // Convert a 32 bit RGBA to a RGB triplets.
+  static RGBTriplets ConvertRGBA32ToTriplets(RGBA32 color);
+
+  // Returns true if |candidate_rect| falls within |template_rect|, allowing for
+  // certain amount of error specified by a fuzzy threshold.
+  static bool ContainsRectWithErrorMargin(
+      const webrtc::DesktopRect& template_rect,
+      const webrtc::DesktopRect& candidate_rect,
+      int error_margin);
+
  private:
+  // Expand the |accumulating_rect_| to a minimum rectangle that contains both
+  // |rect| and itself.
+  void MergeRectToAccumulatingRect(const webrtc::DesktopRect& rect);
+
+  // Returns the average color value of pixels fall within |rect|.
+  RGBTriplets CalculateAverageColorTriplets(
+      const webrtc::DesktopRect& rect) const;
+
+  // Compares |candidate_avg_triplets| to |expected_avg_color_|.
+  // Returns true if the root mean square of the errors in the R, G and B
+  // components does not exceed a fuzzy threshold.
+  bool ExpectedColorIsMatched(const RGBTriplets& candidate_avg_triplets) const;
+
+  // Returns true if |expected_rect_| and |accumulating_rect_| contains each
+  // other and expected color is matched.
+  bool ExpectedImagePatternIsMatched(
+      const RGBTriplets& candidate_avg_triplets) const;
+
   // Used to ensure Core methods are called on the same thread.
   base::ThreadChecker thread_checker_;
 
@@ -68,11 +118,10 @@ class TestVideoRenderer::Core {
 
   // Used to store the expected image pattern.
   webrtc::DesktopRect expected_rect_;
-  RgbaColor expected_color_;
+  RGBA32 expected_avg_color_;
 
   // Maintains accumulating image pattern.
   webrtc::DesktopRect accumulating_rect_;
-  RgbaColor accumulating_color_;
 
   // Used to store the callback when expected pattern is matched.
   base::Closure image_pattern_matched_callback_;
@@ -91,6 +140,8 @@ TestVideoRenderer::Core::~Core() {
 
 void TestVideoRenderer::Core::Initialize() {
   DCHECK(thread_checker_.CalledOnValidThread());
+
+  accumulating_rect_ = webrtc::DesktopRect();
 }
 
 void TestVideoRenderer::Core::SetCodecForDecoding(
@@ -172,22 +223,139 @@ void TestVideoRenderer::Core::ProcessVideoPacket(
 
   main_task_runner_->PostTask(FROM_HERE, done);
 
-  // TODO(liaoyuke): Update |accumulating_rect_| and |accumulating_color_|, then
-  // compare to the expected image pattern to check whether the pattern is
-  // matched or not and update |image_pattern_matched| accordingly.
+  // Check to see if a image pattern matched reply is passed in.
+  if (image_pattern_matched_callback_.is_null()) {
+    return;
+  }
+
+  for (webrtc::DesktopRegion::Iterator dirty_region(updated_region_);
+       !dirty_region.IsAtEnd(); dirty_region.Advance()) {
+    webrtc::DesktopRect dirty_rect = dirty_region.rect();
+
+    // Dirty rects not falling within |expected_rect_| doesn't affect the
+    // result.
+    if (ContainsRectWithErrorMargin(expected_rect_, dirty_rect,
+                                    kRectFuzzyThreshold)) {
+      MergeRectToAccumulatingRect(dirty_rect);
+    }
+  }
+
+  if (accumulating_rect_.is_empty()) {
+    return;
+  }
+
+  RGBTriplets accumulating_avg_triplets =
+      CalculateAverageColorTriplets(accumulating_rect_);
+  if (ExpectedImagePatternIsMatched(accumulating_avg_triplets)) {
+    base::ResetAndReturn(&image_pattern_matched_callback_).Run();
+    accumulating_rect_ = webrtc::DesktopRect();
+  }
 }
 
 void TestVideoRenderer::Core::SetImagePatternAndMatchedCallback(
     const webrtc::DesktopRect& expected_rect,
-    const RgbaColor& expected_color,
+    const RGBA32& expected_avg_color,
     const base::Closure& image_pattern_matched_callback) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   expected_rect_ = expected_rect;
-  expected_color_ = expected_color;
+  expected_avg_color_ = expected_avg_color;
   image_pattern_matched_callback_ = image_pattern_matched_callback;
 }
 
+RGBA32 TestVideoRenderer::Core::ConvertTripletsToRGBA32(
+    const RGBTriplets& rgb_triplets) {
+  return 0xFF000000 | (rgb_triplets.red_ << 16) | (rgb_triplets.green_ << 8) |
+         rgb_triplets.blue_;
+}
+
+RGBTriplets TestVideoRenderer::Core::ConvertRGBA32ToTriplets(RGBA32 color) {
+  RGBTriplets rgb_triplets((color >> 16) & 0xFF, (color >> 8) & 0xFF,
+                           color & 0xFF);
+  return rgb_triplets;
+}
+
+bool TestVideoRenderer::Core::ContainsRectWithErrorMargin(
+    const webrtc::DesktopRect& template_rect,
+    const webrtc::DesktopRect& candidate_rect,
+    int error_margin) {
+  return template_rect.ContainsRect(webrtc::DesktopRect::MakeLTRB(
+      candidate_rect.left() + error_margin, candidate_rect.top() + error_margin,
+      candidate_rect.right() - error_margin,
+      candidate_rect.bottom() - error_margin));
+}
+
+void TestVideoRenderer::Core::MergeRectToAccumulatingRect(
+    const webrtc::DesktopRect& rect) {
+  // If |rect| is the first dirty rect that falls within |expected_rect_|,
+  // simply assign |rect| to |accumulating_rect_|.
+  if (accumulating_rect_.is_empty()) {
+    accumulating_rect_ = rect;
+  } else {
+    // Merge |rect| to obtain a larger |accumulating_rect_|.
+    accumulating_rect_ = webrtc::DesktopRect::MakeLTRB(
+        std::min(accumulating_rect_.left(), rect.left()),
+        std::min(accumulating_rect_.top(), rect.top()),
+        std::max(accumulating_rect_.right(), rect.right()),
+        std::max(accumulating_rect_.bottom(), rect.bottom()));
+  }
+}
+
+RGBTriplets TestVideoRenderer::Core::CalculateAverageColorTriplets(
+    const webrtc::DesktopRect& rect) const {
+  int red_sum = 0;
+  int green_sum = 0;
+  int blue_sum = 0;
+
+  // Loop through pixels that fall within |accumulating_rect_| to obtain the
+  // average color triplets.
+  for (int y = rect.top(); y < rect.bottom(); ++y) {
+    uint8_t* ptr =

[Sergey Ulanov, 2015/07/10 01:21:56]

|ptr| is not the best name.
https://google-styleguide.googlecode.com/svn/trunk/cppguide.html#General_Nami...
. Something like |position| or |buffer_pos| would be better

[liaoyuke, 2015/07/10 16:22:30]

Done.

+        buffer_->data() + (y * buffer_->stride() +
+                           rect.left() * webrtc::DesktopFrame::kBytesPerPixel);
+
+    // Pixels of decoded video frame are presented in ARGB format.
+    for (int x = 0; x < rect.width(); ++x) {
+      red_sum += *(ptr + 2);

[Sergey Ulanov, 2015/07/10 01:21:55]

These can be ptr[2], ptr[1] and ptr[0]

[liaoyuke, 2015/07/10 16:22:30]

Done.

+      green_sum += *(ptr + 1);
+      blue_sum += *(ptr + 0);
+      ptr += 4;
+    }
+  }
+
+  int area = screen_size_.width() * screen_size_.height();
+  RGBTriplets rgb_triplets(red_sum / area, green_sum / area, blue_sum / area);
+  return rgb_triplets;
+}
+
+bool TestVideoRenderer::Core::ExpectedColorIsMatched(
+    const RGBTriplets& candidate_avg_triplets) const {
+  RGBTriplets expected_avg_triplets =
+      ConvertRGBA32ToTriplets(expected_avg_color_);
+  double error_sum_squares = 0;
+  double red_error = expected_avg_triplets.red_ - candidate_avg_triplets.red_;
+  double green_error =
+      expected_avg_triplets.green_ - candidate_avg_triplets.green_;
+  double blue_error =
+      expected_avg_triplets.blue_ - candidate_avg_triplets.blue_;
+  error_sum_squares = red_error * red_error + green_error * green_error +
+                      blue_error * blue_error;
+  error_sum_squares /= (255.0 * 255.0);
+
+  return sqrt(error_sum_squares / 3) < kColorFuzzyThreshold;
+}
+
+bool TestVideoRenderer::Core::ExpectedImagePatternIsMatched(
+    const RGBTriplets& candidate_avg_triplets) const {
+  // |accumulating_rect_| must be valid in order to be matched.
+  return accumulating_rect_.width() && accumulating_rect_.height() &&
+         ContainsRectWithErrorMargin(accumulating_rect_, expected_rect_,
+                                     kRectFuzzyThreshold) &&
+         ContainsRectWithErrorMargin(expected_rect_, accumulating_rect_,
+                                     kRectFuzzyThreshold) &&
+         ExpectedColorIsMatched(candidate_avg_triplets);
+}
+
 TestVideoRenderer::TestVideoRenderer()
     : video_decode_thread_(
         new base::Thread("TestVideoRendererVideoDecodingThread")),
@@ -275,15 +443,16 @@ scoped_ptr<webrtc::DesktopFrame> TestVideoRenderer::GetBufferForTest() const {
 
 void TestVideoRenderer::SetImagePatternAndMatchedCallback(
     const webrtc::DesktopRect& expected_rect,
-    const RgbaColor& expected_color,
+    const RGBA32& expected_avg_color,
     const base::Closure& image_pattern_matched_callback) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   DVLOG(2) << "TestVideoRenderer::SetImagePatternAndMatchedCallback() Called";
   video_decode_task_runner_->PostTask(
-      FROM_HERE, base::Bind(&Core::SetImagePatternAndMatchedCallback,
-                            base::Unretained(core_.get()), expected_rect,
-                            expected_color, image_pattern_matched_callback));
+      FROM_HERE,
+      base::Bind(&Core::SetImagePatternAndMatchedCallback,
+                 base::Unretained(core_.get()), expected_rect,
+                 expected_avg_color, image_pattern_matched_callback));
 }
 
 }  // namespace test