[WebRTC] Add ScreenCapturerDifferWrapper to share Differ across ScreenCapturers

webrtc/modules/desktop_capture/screen_capturer_differ_wrapper.cc

+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/desktop_capture/screen_capturer_differ_wrapper.h"
+
+#include <algorithm>
+#include <utility>
+
+#include "webrtc/base/checks.h"
+#include "webrtc/base/timeutils.h"
+#include "webrtc/modules/desktop_capture/desktop_geometry.h"
+#include "webrtc/modules/desktop_capture/differ_block.h"
+
+namespace webrtc {
+
+namespace {
+
+static const int kBlockXOffset = kBlockSize * DesktopFrame::kBytesPerPixel;
+
+// Returns true if (0, 0) - (|width|, |height|) vector in |old_buffer| and
+// |new_buffer| are equal. |width| should be less than 32
+// (defined by kBlockSize), otherwise BlockDifference() should be used.
+bool PartialBlockDifference(const uint8_t* old_buffer,
+                            const uint8_t* new_buffer,
+                            int width,
+                            int height,
+                            int stride) {
+  RTC_DCHECK(width < kBlockSize);
+  const int width_bytes = width * DesktopFrame::kBytesPerPixel;
+  for (int i = 0; i < height; i++) {
+    if (memcmp(old_buffer, new_buffer, width_bytes) != 0) {
+      return true;
+    }
+    old_buffer += stride;
+    new_buffer += stride;
+  }
+  return false;
+}
+
+// Compares block-columns in the range of [0, |x_end| - |x_start|), in
+// a row in the range of [row_top, row_top + row_height), starts from
+// |old_buffer| and |new_buffer|, and outputs updated regions into |output|.
+// |width| is the DesktopFrame::size()::width(), |stride| is the
+// DesktopFrame::stride().
+void CompareRow(const uint8_t* old_buffer,
+                const uint8_t* new_buffer,
+                const int x_start,
+                const int x_end,
+                const int row_top,
+                const int row_height,
+                const int width,
+                const int stride,
+                DesktopRegion* const output) {
+  const int row_bottom = row_top + row_height;
+  const int column_right = std::min(x_end * kBlockSize, width);
+
+  // The first block-column in a continuous dirty area in current block-row.
+  int first_dirty_x_block = -1;
+
+  // We always need to add dirty area into |output| in the last block, so handle
+  // it separatedly.
+  for (int x = x_start; x < x_end - 1; x++) {
+    if (BlockDifference(old_buffer, new_buffer, row_height, stride)) {
+      if (first_dirty_x_block == -1) {
+        // This is the first dirty block in a continuous dirty area.
+        first_dirty_x_block = x;
+      }
+    } else if (first_dirty_x_block != -1) {
+      // The block on the left is the last dirty block in a continuous
+      // dirty area.
+      output->AddRect(DesktopRect::MakeLTRB(first_dirty_x_block * kBlockSize,
+                                            row_top, x * kBlockSize,
+                                            row_bottom));
+      first_dirty_x_block = -1;
+    }
+    old_buffer += kBlockXOffset;
+    new_buffer += kBlockXOffset;
+  }
+
+  bool last_block_diff;
+  if (x_end * kBlockSize > width) {
+    // The last one is a partial vector.
+    last_block_diff = PartialBlockDifference(
+        old_buffer, new_buffer, column_right - (x_end - 1) * kBlockSize,
+        row_height, stride);
+  } else {
+    last_block_diff = BlockDifference(
+        old_buffer, new_buffer, row_height, stride);
+  }
+  if (last_block_diff) {
+    if (first_dirty_x_block == -1) {
+      output->AddRect(DesktopRect::MakeLTRB((x_end - 1) * kBlockSize, row_top,
+                                            column_right, row_bottom));
+    } else {
+      output->AddRect(DesktopRect::MakeLTRB(first_dirty_x_block * kBlockSize,
+                                            row_top, column_right, row_bottom));
+    }
+  } else if (first_dirty_x_block != -1) {
+    output->AddRect(DesktopRect::MakeLTRB(first_dirty_x_block * kBlockSize,
+                                          row_top, (x_end - 1) * kBlockSize,
+                                          row_bottom));
+  }
+}
+
+// Compares |rect| area in |old_frame| and |new_frame|, and outputs dirty
+// regions into |output|.
+void CompareFrames(const DesktopFrame& old_frame,
+                   const DesktopFrame& new_frame,
+                   DesktopRect rect,
+                   DesktopRegion* const output) {
+  RTC_DCHECK(old_frame.size().equals(new_frame.size()));
+  RTC_DCHECK_EQ(old_frame.stride(), new_frame.stride());
+  rect.IntersectWith(DesktopRect::MakeSize(old_frame.size()));
+
+  const int x_start = rect.left() / kBlockSize;
+  const int y_start = rect.top() / kBlockSize;
+  const int x_end = (rect.right() + kBlockSize - 1) / kBlockSize;
+  const int y_end = (rect.bottom() + kBlockSize - 1) / kBlockSize;
+  const DesktopVector block_top_left(x_start * kBlockSize,
+                                     y_start * kBlockSize);
+  // Offset from the start of one block-row to the next.
+  const int block_y_stride = old_frame.stride() * kBlockSize;
+  const uint8_t* prev_block_row_start =
+      old_frame.GetFrameDataAtPos(block_top_left);
+  const uint8_t* curr_block_row_start =
+      new_frame.GetFrameDataAtPos(block_top_left);
+
+  int row_top = y_start * kBlockSize;
+  // The last row may have a different height, so we handle it separately.
+  for (int y = y_start; y < y_end - 1; y++) {
+    CompareRow(prev_block_row_start, curr_block_row_start, x_start, x_end,

[Sergey Ulanov, 2016/08/18 05:07:58]

This function compares whole 32x32 blocks and as far as I can tell each block
may be compared multiple times - once for each rectangle that intersects with
that block. I think that's suboptimal and should be avoided.
Another issue is that the updated_region() returned by
ScreenCapturerDifferWrapper in many cases will be bigger that then region
returned by the underlying capturer. I'd like to avoid this too.
What I would prefer is to only compare intersection of each block with the
covering rect in the source updated_region(). But there may be other solutions.
Let's discuss this offline.

[Hzj_jie, 2016/08/18 08:50:31]

Yes, I aware this issue, as this is a disadvantage of removing bool differ-info
array. I have tried to compare exact area, by separating 32 x 32 blocks starting
from top-left of a rectangle, and do not expand pixels at the right and bottom.
But the performance is much worse (about twice time is used to execute the test
cases, since we need to exclude comparison and random data generation time, the
real perf could be 1/3 or even worse). The performance drop is coming from 1.
partial block difference, 2. DesktopRegion, i.e. adding very random desktop
rectangles to a DesktopRegion will cause the count of its Rows to increase
significantly. But current solution ensures there are at most 24 rows for a
monitor with 768 vertical resolution. I believe the impact from the latter,
DesktopRegion, is even important.

I have not tried several other combinations, but considering the implementation
of DesktopRegion, it may have similar performance drop.

I have also compared the performance between current implementation and Differ,
they two have comparable performance. (Differ is slightly faster, < 5%.)

I will be OOO on Thursday and Friday. Let's talk about it next Monday.

[Hzj_jie, 2016/08/23 00:54:59]

After reducing the range of several random values (say, from 500 to 50 updated
rectangles), the performance of the new implementation (not 32 pixels alignment)
is much better, but no consistent improvement has been found by using Differ. I
believe it's because less DesktopRegion rows will be added. And by using Differ,
it should have a consistent behavior no matter - how many updated regions
returned, - how much overlap between the updated regions returned.

The performance is too OS implementation dependent, as we have no idea how OS
returns updated region. I am now using an exact area without 32 pixels alignment
as you suggested. We can revise this decision on each platform later.

+               row_top, kBlockSize, old_frame.size().width(),
+               old_frame.stride(), output);
+    row_top += kBlockSize;
+    prev_block_row_start += block_y_stride;
+    curr_block_row_start += block_y_stride;
+  }
+  CompareRow(prev_block_row_start, curr_block_row_start, x_start, x_end,
+             row_top,
+             std::min(kBlockSize,
+                      old_frame.size().height() - (y_end - 1) * kBlockSize),
+             old_frame.size().width(), old_frame.stride(), output);
+}
+
+}  // namespace
+
+ScreenCapturerDifferWrapper::ScreenCapturerDifferWrapper(
+    std::unique_ptr<ScreenCapturer> base_capturer,
+    bool diff_entire_frame)
+    : base_capturer_(std::move(base_capturer)),
+      diff_entire_frame_(diff_entire_frame) {
+  RTC_DCHECK(base_capturer_);
+}
+
+ScreenCapturerDifferWrapper::~ScreenCapturerDifferWrapper() {}
+
+void ScreenCapturerDifferWrapper::Start(DesktopCapturer::Callback* callback) {
+  callback_ = callback;
+  base_capturer_->Start(this);
+}
+
+void ScreenCapturerDifferWrapper::SetSharedMemoryFactory(
+    std::unique_ptr<SharedMemoryFactory> shared_memory_factory) {
+  base_capturer_->SetSharedMemoryFactory(std::move(shared_memory_factory));
+}
+
+void ScreenCapturerDifferWrapper::Capture(const DesktopRegion& region) {
+  base_capturer_->Capture(region);
+}
+
+bool ScreenCapturerDifferWrapper::GetScreenList(ScreenList* screens) {
+  return base_capturer_->GetScreenList(screens);
+}
+
+bool ScreenCapturerDifferWrapper::SelectScreen(ScreenId id) {
+  return base_capturer_->SelectScreen(id);
+}
+
+void ScreenCapturerDifferWrapper::OnCaptureResult(
+    Result result,
+    std::unique_ptr<DesktopFrame> input_frame) {
+  int64_t start_time_nanos = rtc::TimeNanos();
+  if (!input_frame) {
+    callback_->OnCaptureResult(result, nullptr);
+    return;
+  }
+
+  std::unique_ptr<SharedDesktopFrame> frame =
+      SharedDesktopFrame::Wrap(std::move(input_frame));
+  if (result == Result::SUCCESS) {
+    if (last_frame_ &&
+        (last_frame_->size().width() != frame->size().width() ||
+         last_frame_->size().height() != frame->size().height() ||
+         last_frame_->stride() != frame->stride())) {
+      last_frame_.reset();
+    }
+
+    if (last_frame_) {
+      if (diff_entire_frame_) {
+        CompareFrames(*last_frame_, *frame,
+                      DesktopRect::MakeSize(frame->size()),
+                      frame->mutable_updated_region());
+      } else {
+        DesktopRegion hints;
+        hints.Swap(frame->GetUnderlyingFrame()->mutable_updated_region());
+        for (DesktopRegion::Iterator it(hints); !it.IsAtEnd(); it.Advance()) {
+          // Note, here we always expect the underlying capturer won't return a
+          // dirty region which excceeds the DesktopFrame::size().
+          CompareFrames(*last_frame_, *frame, it.rect(),
+                        frame->mutable_updated_region());
+        }
+      }
+    } else {
+      frame->mutable_updated_region()->SetRect(
+          DesktopRect::MakeSize(frame->size()));
+    }
+    last_frame_ = frame->Share();
+  } else {
+    last_frame_.reset();
+  }
+
+  frame->set_capture_time_ms(frame->GetUnderlyingFrame()->capture_time_ms() +

[Sergey Ulanov, 2016/08/18 05:07:58]

don't really need ->GetUnderlyingFrame(). The value is copied when sharing a
frame.

[Hzj_jie, 2016/08/23 00:54:59]

This SharedDesktopFrame is generated above @ line 195, so the underlying
ScreenCapturer won't set the capture_time_ms to the SharedDesktopFrame, but
|input_frame|. But |input_frame| has been moved also in line 195, so we can only
use this GetUnderlyingFrame() to access the original capture_time_ms().

+                             (rtc::TimeNanos() - start_time_nanos) /
+                                 rtc::kNumNanosecsPerMillisec);
+  callback_->OnCaptureResult(result, std::move(frame));
+}
+
+}  // namespace webrtc