Added image pattern comparison logic for test interface and fixture.

remoting/test/test_video_renderer.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "remoting/test/test_video_renderer.h"
 
+#include <algorithm>
+#include <cmath>
+
 #include "base/bind.h"
+#include "base/callback_helpers.h"
 #include "base/logging.h"
 #include "base/synchronization/lock.h"
 #include "base/thread_task_runner_handle.h"
 #include "base/threading/thread.h"
 #include "remoting/codec/video_decoder.h"
 #include "remoting/codec/video_decoder_verbatim.h"
 #include "remoting/codec/video_decoder_vpx.h"
 #include "remoting/proto/video.pb.h"
 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
 
+// RGB Color triplets, and it can be converted to and from RGBA32.
+struct RGBTriplets {
+  RGBTriplets(int red, int green, int blue)
+      : red_(red), green_(green), blue_(blue) {}
+  int red_;

[Sergey Ulanov, 2015/07/10 01:21:56]

don't need _ suffix for struct fields. It's only for class fields

[liaoyuke, 2015/07/10 16:22:30]

Done.

+  int green_;
+  int blue_;
+};
+
+namespace {
+// Used to account for video frame resizing and lossy encoding.
+const int kRectFuzzyThreshold = 2;

[Sergey Ulanov, 2015/07/10 01:21:56]

Maybe call it something like kMaxPositionError? Document this is in pixels

[liaoyuke, 2015/07/10 16:22:30]

Done.

+const double kColorFuzzyThreshold = 0.05;

[Sergey Ulanov, 2015/07/10 01:21:55]

kMaxColorError?
Also document units.

[liaoyuke, 2015/07/10 16:22:30]

Done.

+}  // namespace
+
 namespace remoting {
 namespace test {
 

[liaoyuke, 2015/07/10 16:22:30]

Per discussion with Sergey, the dirty rects are not that stable and the method
should not rely on dirty rects. To get rid of the dependency on dirty rects,
I'll change the comparison logic to every time when a frame is received,
calculate the average color of pixels fall within the expected rect and see if
it matches. This is more computationally intensive, but it's more stable.

 // Implements video decoding functionality.
 class TestVideoRenderer::Core {
  public:
   Core();
   ~Core();
 
   // Initializes the internal structures of the class.
   void Initialize();
 
   // Used to decode video packets.
   void ProcessVideoPacket(scoped_ptr<VideoPacket> packet,
                           const base::Closure& done);
 
   // Initialize a decoder to decode video packets.
   void SetCodecForDecoding(const protocol::ChannelConfig::Codec codec);
 
   // Returns a copy of the current buffer.
   scoped_ptr<webrtc::DesktopFrame> GetBufferForTest() const;
 
   // Set expected image pattern for comparison and the callback will be called
   // when the pattern is matched.
   void SetImagePatternAndMatchedCallback(
       const webrtc::DesktopRect& expected_rect,
-      const RgbaColor& expected_color,
+      const RGBA32& expected_avg_color,
       const base::Closure& image_pattern_matched_callback);
 
+  // Convert a RGB triplets to 32 bit RGBA.
+  static RGBA32 ConvertTripletsToRGBA32(const RGBTriplets& rgb_triplets);
+
+  // Convert a 32 bit RGBA to a RGB triplets.
+  static RGBTriplets ConvertRGBA32ToTriplets(RGBA32 color);
+
+  // Returns true if |candidate_rect| falls within |template_rect|, allowing for
+  // certain amount of error specified by a fuzzy threshold.
+  static bool ContainsRectWithErrorMargin(
+      const webrtc::DesktopRect& template_rect,
+      const webrtc::DesktopRect& candidate_rect,
+      int error_margin);
+
  private:
+  // Expand the |accumulating_rect_| to a minimum rectangle that contains both
+  // |rect| and itself.
+  void MergeRectToAccumulatingRect(const webrtc::DesktopRect& rect);
+
+  // Returns the average color value of pixels fall within |rect|.
+  RGBTriplets CalculateAverageColorTriplets(
+      const webrtc::DesktopRect& rect) const;
+
+  // Compares |candidate_avg_triplets| to |expected_avg_color_|.
+  // Returns true if the root mean square of the errors in the R, G and B
+  // components does not exceed a fuzzy threshold.
+  bool ExpectedColorIsMatched(const RGBTriplets& candidate_avg_triplets) const;
+
+  // Returns true if |expected_rect_| and |accumulating_rect_| contains each
+  // other and expected color is matched.
+  bool ExpectedImagePatternIsMatched(
+      const RGBTriplets& candidate_avg_triplets) const;
+
   // Used to ensure Core methods are called on the same thread.
   base::ThreadChecker thread_checker_;
 
   // Used to decode video packets.
   scoped_ptr<VideoDecoder> decoder_;
 
   // Updated region of the current desktop frame compared to previous one.
   webrtc::DesktopRegion updated_region_;
 
   // Screen size of the remote host.
   webrtc::DesktopSize screen_size_;
 
   // Used to post tasks back to main thread.
   scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
 
   // Used to store decoded video frame.
   scoped_ptr<webrtc::DesktopFrame> buffer_;
 
   // Protects access to |buffer_|.
   mutable base::Lock lock_;
 
   // Used to store the expected image pattern.
   webrtc::DesktopRect expected_rect_;
-  RgbaColor expected_color_;
+  RGBA32 expected_avg_color_;
 
   // Maintains accumulating image pattern.
   webrtc::DesktopRect accumulating_rect_;
-  RgbaColor accumulating_color_;
 
   // Used to store the callback when expected pattern is matched.
   base::Closure image_pattern_matched_callback_;
 
   DISALLOW_COPY_AND_ASSIGN(Core);
 };
 
 TestVideoRenderer::Core::Core()
     : main_task_runner_(base::ThreadTaskRunnerHandle::Get()) {
   thread_checker_.DetachFromThread();
 }
 
 TestVideoRenderer::Core::~Core() {
   DCHECK(thread_checker_.CalledOnValidThread());
 }
 
 void TestVideoRenderer::Core::Initialize() {
   DCHECK(thread_checker_.CalledOnValidThread());
+
+  accumulating_rect_ = webrtc::DesktopRect();
 }
 
 void TestVideoRenderer::Core::SetCodecForDecoding(
     const protocol::ChannelConfig::Codec codec) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (decoder_) {
     LOG(WARNING) << "Decoder is set more than once";
   }
 
@@ skipping 61 matching lines @@
     // Note that the |updated_region_| maintains the changed regions compared to
     // previous video frame.
     decoder_->RenderFrame(screen_size_,
                           webrtc::DesktopRect::MakeWH(screen_size_.width(),
                           screen_size_.height()), buffer_->data(),
                           buffer_->stride(), &updated_region_);
   }
 
   main_task_runner_->PostTask(FROM_HERE, done);
 
-  // TODO(liaoyuke): Update |accumulating_rect_| and |accumulating_color_|, then
-  // compare to the expected image pattern to check whether the pattern is
-  // matched or not and update |image_pattern_matched| accordingly.
+  // Check to see if a image pattern matched reply is passed in.
+  if (image_pattern_matched_callback_.is_null()) {
+    return;
+  }
+
+  for (webrtc::DesktopRegion::Iterator dirty_region(updated_region_);
+       !dirty_region.IsAtEnd(); dirty_region.Advance()) {
+    webrtc::DesktopRect dirty_rect = dirty_region.rect();
+
+    // Dirty rects not falling within |expected_rect_| doesn't affect the
+    // result.
+    if (ContainsRectWithErrorMargin(expected_rect_, dirty_rect,
+                                    kRectFuzzyThreshold)) {
+      MergeRectToAccumulatingRect(dirty_rect);
+    }
+  }
+
+  if (accumulating_rect_.is_empty()) {
+    return;
+  }
+
+  RGBTriplets accumulating_avg_triplets =
+      CalculateAverageColorTriplets(accumulating_rect_);
+  if (ExpectedImagePatternIsMatched(accumulating_avg_triplets)) {
+    base::ResetAndReturn(&image_pattern_matched_callback_).Run();
+    accumulating_rect_ = webrtc::DesktopRect();
+  }
 }
 
 void TestVideoRenderer::Core::SetImagePatternAndMatchedCallback(
     const webrtc::DesktopRect& expected_rect,
-    const RgbaColor& expected_color,
+    const RGBA32& expected_avg_color,
     const base::Closure& image_pattern_matched_callback) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   expected_rect_ = expected_rect;
-  expected_color_ = expected_color;
+  expected_avg_color_ = expected_avg_color;
   image_pattern_matched_callback_ = image_pattern_matched_callback;
 }
 
+RGBA32 TestVideoRenderer::Core::ConvertTripletsToRGBA32(
+    const RGBTriplets& rgb_triplets) {
+  return 0xFF000000 | (rgb_triplets.red_ << 16) | (rgb_triplets.green_ << 8) |
+         rgb_triplets.blue_;
+}
+
+RGBTriplets TestVideoRenderer::Core::ConvertRGBA32ToTriplets(RGBA32 color) {
+  RGBTriplets rgb_triplets((color >> 16) & 0xFF, (color >> 8) & 0xFF,
+                           color & 0xFF);
+  return rgb_triplets;
+}
+
+bool TestVideoRenderer::Core::ContainsRectWithErrorMargin(
+    const webrtc::DesktopRect& template_rect,
+    const webrtc::DesktopRect& candidate_rect,
+    int error_margin) {
+  return template_rect.ContainsRect(webrtc::DesktopRect::MakeLTRB(
+      candidate_rect.left() + error_margin, candidate_rect.top() + error_margin,
+      candidate_rect.right() - error_margin,
+      candidate_rect.bottom() - error_margin));
+}
+
+void TestVideoRenderer::Core::MergeRectToAccumulatingRect(
+    const webrtc::DesktopRect& rect) {
+  // If |rect| is the first dirty rect that falls within |expected_rect_|,
+  // simply assign |rect| to |accumulating_rect_|.
+  if (accumulating_rect_.is_empty()) {
+    accumulating_rect_ = rect;
+  } else {
+    // Merge |rect| to obtain a larger |accumulating_rect_|.
+    accumulating_rect_ = webrtc::DesktopRect::MakeLTRB(
+        std::min(accumulating_rect_.left(), rect.left()),
+        std::min(accumulating_rect_.top(), rect.top()),
+        std::max(accumulating_rect_.right(), rect.right()),
+        std::max(accumulating_rect_.bottom(), rect.bottom()));
+  }
+}
+
+RGBTriplets TestVideoRenderer::Core::CalculateAverageColorTriplets(
+    const webrtc::DesktopRect& rect) const {
+  int red_sum = 0;
+  int green_sum = 0;
+  int blue_sum = 0;
+
+  // Loop through pixels that fall within |accumulating_rect_| to obtain the
+  // average color triplets.
+  for (int y = rect.top(); y < rect.bottom(); ++y) {
+    uint8_t* ptr =

[Sergey Ulanov, 2015/07/10 01:21:56]

|ptr| is not the best name.
https://google-styleguide.googlecode.com/svn/trunk/cppguide.html#General_Nami...
. Something like |position| or |buffer_pos| would be better

[liaoyuke, 2015/07/10 16:22:30]

Done.

+        buffer_->data() + (y * buffer_->stride() +
+                           rect.left() * webrtc::DesktopFrame::kBytesPerPixel);
+
+    // Pixels of decoded video frame are presented in ARGB format.
+    for (int x = 0; x < rect.width(); ++x) {
+      red_sum += *(ptr + 2);

[Sergey Ulanov, 2015/07/10 01:21:55]

These can be ptr[2], ptr[1] and ptr[0]

[liaoyuke, 2015/07/10 16:22:30]

Done.

+      green_sum += *(ptr + 1);
+      blue_sum += *(ptr + 0);
+      ptr += 4;
+    }
+  }
+
+  int area = screen_size_.width() * screen_size_.height();
+  RGBTriplets rgb_triplets(red_sum / area, green_sum / area, blue_sum / area);
+  return rgb_triplets;
+}
+
+bool TestVideoRenderer::Core::ExpectedColorIsMatched(
+    const RGBTriplets& candidate_avg_triplets) const {
+  RGBTriplets expected_avg_triplets =
+      ConvertRGBA32ToTriplets(expected_avg_color_);
+  double error_sum_squares = 0;
+  double red_error = expected_avg_triplets.red_ - candidate_avg_triplets.red_;
+  double green_error =
+      expected_avg_triplets.green_ - candidate_avg_triplets.green_;
+  double blue_error =
+      expected_avg_triplets.blue_ - candidate_avg_triplets.blue_;
+  error_sum_squares = red_error * red_error + green_error * green_error +
+                      blue_error * blue_error;
+  error_sum_squares /= (255.0 * 255.0);
+
+  return sqrt(error_sum_squares / 3) < kColorFuzzyThreshold;
+}
+
+bool TestVideoRenderer::Core::ExpectedImagePatternIsMatched(
+    const RGBTriplets& candidate_avg_triplets) const {
+  // |accumulating_rect_| must be valid in order to be matched.
+  return accumulating_rect_.width() && accumulating_rect_.height() &&
+         ContainsRectWithErrorMargin(accumulating_rect_, expected_rect_,
+                                     kRectFuzzyThreshold) &&
+         ContainsRectWithErrorMargin(expected_rect_, accumulating_rect_,
+                                     kRectFuzzyThreshold) &&
+         ExpectedColorIsMatched(candidate_avg_triplets);
+}
+
 TestVideoRenderer::TestVideoRenderer()
     : video_decode_thread_(
         new base::Thread("TestVideoRendererVideoDecodingThread")),
       weak_factory_(this) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   core_.reset(new Core());
   if (!video_decode_thread_->Start()) {
     LOG(ERROR) << "Cannot start TestVideoRenderer";
   } else {
@@ skipping 67 matching lines @@
 }
 
 scoped_ptr<webrtc::DesktopFrame> TestVideoRenderer::GetBufferForTest() const {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   return core_->GetBufferForTest();
 }
 
 void TestVideoRenderer::SetImagePatternAndMatchedCallback(
     const webrtc::DesktopRect& expected_rect,
-    const RgbaColor& expected_color,
+    const RGBA32& expected_avg_color,
     const base::Closure& image_pattern_matched_callback) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   DVLOG(2) << "TestVideoRenderer::SetImagePatternAndMatchedCallback() Called";
   video_decode_task_runner_->PostTask(
-      FROM_HERE, base::Bind(&Core::SetImagePatternAndMatchedCallback,
-                            base::Unretained(core_.get()), expected_rect,
-                            expected_color, image_pattern_matched_callback));
+      FROM_HERE,
+      base::Bind(&Core::SetImagePatternAndMatchedCallback,
+                 base::Unretained(core_.get()), expected_rect,
+                 expected_avg_color, image_pattern_matched_callback));
 }
 
 }  // namespace test
 }  // namespace remoting