Fix cursor missing in tabCapture on OSX Sierra

content/browser/media/capture/cursor_renderer_mac.mm

Index: content/browser/media/capture/cursor_renderer_mac.mm
diff --git a/content/browser/media/capture/cursor_renderer_mac.mm b/content/browser/media/capture/cursor_renderer_mac.mm
index c7c85f6ea087b229f33fcb3aea86dde3febeafb7..22238310b7c6024e55e8a71e0d12867aa4917cc8 100644
--- a/content/browser/media/capture/cursor_renderer_mac.mm
+++ b/content/browser/media/capture/cursor_renderer_mac.mm
@@ -4,189 +4,82 @@
 
 #include "content/browser/media/capture/cursor_renderer_mac.h"
 
-#include <ApplicationServices/ApplicationServices.h>
 #include <Cocoa/Cocoa.h>
 #include <CoreFoundation/CoreFoundation.h>
-#include <stdint.h>
-
-#include <cmath>
 
 #include "base/logging.h"
+#include "base/memory/ptr_util.h"
+#include "ui/gfx/image/image.h"
 
 namespace content {
 
-namespace {
-
-// RGBA format on cursor bitmap
-const int kBytesPerPixel = 4;
-
-inline int clip_byte(int x) {
-  return std::max(0, std::min(x, 255));
-}
-
-inline int alpha_blend(int alpha, int src, int dst) {
-  return (src * alpha + dst * (255 - alpha)) / 255;
-}
-
-}  // namespace
-
 // static
 std::unique_ptr<CursorRenderer> CursorRenderer::Create(gfx::NativeView view) {
-  return std::unique_ptr<CursorRenderer>(new CursorRendererMac(view));
+  return base::MakeUnique<CursorRendererMac>(view);
 }
 
-CursorRendererMac::CursorRendererMac(NSView* view)
-    : view_(view), weak_factory_(this) {
-  Clear();
-}
+CursorRendererMac::CursorRendererMac(gfx::NativeView view)
+    : CursorRenderer(view, kCursorEnabledOnMouseMovement),
+      view_(view),
+      last_known_cursor_location_(gfx::Point()) {}
 
 CursorRendererMac::~CursorRendererMac() {}
 
-base::WeakPtr<CursorRenderer> CursorRendererMac::GetWeakPtr() {
-  return weak_factory_.GetWeakPtr();
+bool CursorRendererMac::IsCapturedViewActive() {
+  if (![[view_ window] isKeyWindow]) {
+    DVLOG(2) << "Window currently inactive";
+    return false;
+  }
+  return true;
 }
 
-void CursorRendererMac::Clear() {
-  last_cursor_data_.reset();
-  last_cursor_width_ = 0;
-  last_cursor_height_ = 0;
-  last_mouse_location_x_ = 0;
-  last_mouse_location_y_ = 0;
-  last_mouse_movement_timestamp_ = base::TimeTicks();
+gfx::Size CursorRendererMac::GetCapturedViewSize() {
+  NSRect frame_rect = [view_ bounds];
+  return gfx::Size(frame_rect.size.width, frame_rect.size.height);
 }
 
-// Polls mouse cursor location and image and returns whether the mouse
-// cursor should be rendered on the frame.
-bool CursorRendererMac::SnapshotCursorState(const gfx::Rect& region_in_frame) {
+gfx::Point CursorRendererMac::GetCursorPositionInView() {
   // Mouse location in window co-ordinates.
   NSPoint mouse_window_location =
       [view_ window].mouseLocationOutsideOfEventStream;
   // Mouse location with respect to the web contents.

[miu, 2016/12/27 23:21:03]

nit: s/the web contents/the view within the window/

[braveyao, 2017/01/04 01:57:49]

Done.

-  NSPoint mouse_tab_location =
+  NSPoint mouse_view_location =
       [view_ convertPoint:mouse_window_location fromView:nil];
 
-  // Mouse co-ordinates directly comparable against frame co-ordinates
-  // after translation.
-  if (mouse_tab_location.x < 0 || mouse_tab_location.y < 0 ||
-      mouse_tab_location.x > region_in_frame.width() ||
-      mouse_tab_location.y > region_in_frame.height()) {
-    VLOG(2) << "Mouse outside content region";
-    return false;
-  }
+  // Revert y coordinate to unify with Aura.

[miu, 2016/12/27 23:21:03]

s/Revert/Invert/

[braveyao, 2017/01/04 01:57:49]

Done.

+  gfx::Point cursor_position_in_view(
+      mouse_view_location.x,
+      GetCapturedViewSize().height() - mouse_view_location.y);
 
-  if (![[view_ window] isKeyWindow]) {
-    VLOG(2) << "Window currently inactive";
-    return false;
+  // Update cursor movement info to CursorRenderer.

[miu, 2016/12/27 23:21:03]

This is an unexpected side-effect of calling GetCursorPositionInView(). Could
you instead use a mouse event listener, like in CursorRendererAura? To do this
in the Mac/Cocoa world, you'll need to add a "tracking area" to the
|captured_view_| in CursorRendererMac's ctor.

References:

https://developer.apple.com/library/content/documentation/Cocoa/Conceptual/Ev...

https://developer.apple.com/reference/appkit/nstrackingarea

I think, for our purposes, the NSTrackingArea would be created/added this way.
In the ctor:

  CursorRendererMac(...) : ...,
    // Note: All of |captured_view_|, |tracker_| and |tracking_area_| should be
base::scoped_nsobject<T> to ensure these objects remain alive until the dtor is
called.
    tracker_([[CursorRendererMouseTracker alloc] initWithOwner:this]),
    tracking_area_(
        [[NSTrackingArea alloc] initWithRect:NSZeroRect
             options: (NSTrackingMouseMoved | NSTrackingActiveAlways |
                       NSTrackingInVisibleRect)
             owner:tracker_.get()
             userInfo:nil]) {
    [captured_view_ addTrackingArea:tracking_area_.get()];
  }

And, in the dtor:

  [captured_view_ removeTrackingArea:tracking_area_.get()];

Also, you'll need to create an Objective-C class that will receive the
|-mouseMoved| method calls for the tracking area and pass them back to
CursorRendererMac:

  @interface CursorRendererMouseTracker
  - (id)initWithOwner:(CursorRendererMac* owner);
  - (void)mouseMoved:(NSEvent*)event;
  @private
    CursorRendererMac* owner_;
  @end

  @implementation CursorRendererMouseTracker

  - (id)initWithOwner:(CursorRendererMac* owner) {
    self = [super init];
    if (self)
      owner_ = owner;
    return self;
  }

  - (void)mouseMoved:(NSEvent*)theEvent {
    NSPoint locationInView = [captured_view_ convertPoint:[theEvent
locationInWindow] fromView:nil];
    // Invert Y coord too???
    owner_->OnMouseMoved(gfx::Point(locationInView.x, locationInView.y),
                         ui::EventTimeFromNative(theEvent));
  }

  @end  // @implementation CursorRendererMouseTracker

[braveyao, 2017/01/04 01:57:49]

Done.

+  if (cursor_position_in_view != last_known_cursor_location_) {
+    last_known_cursor_location_ = cursor_position_in_view;
+    OnMouseMoved(cursor_position_in_view, base::TimeTicks::Now());
   }
 
-  if ((base::TimeTicks::Now() - last_mouse_movement_timestamp_).InSeconds() >
-          MAX_IDLE_TIME_SECONDS &&
-      std::abs(mouse_tab_location.x - last_mouse_location_x_) <
-          MIN_MOVEMENT_PIXELS &&
-      std::abs(mouse_tab_location.y - last_mouse_location_y_) <
-          MIN_MOVEMENT_PIXELS) {
-    VLOG(2) << "No mouse movement in a while";
-    return false;
-  }
+  return cursor_position_in_view;
+}
 
-  // Mouse cursor position within the frame.
-  cursor_position_in_frame_ =
-      gfx::Point(region_in_frame.x() + mouse_tab_location.x,
-                 region_in_frame.y() + mouse_tab_location.y);
+gfx::NativeCursor CursorRendererMac::GetLastKnownCursor() {
+  // Grab system cursor.
+  return [NSCursor currentSystemCursor];
+}
 
+SkBitmap CursorRendererMac::GetLastKnownCursorImage() {
   // Grab system cursor.
   NSCursor* nscursor = [NSCursor currentSystemCursor];
-  NSPoint nshotspot = [nscursor hotSpot];
   NSImage* nsimage = [nscursor image];
-  NSSize nssize = [nsimage size];
-
-  // The cursor co-ordinates in the window and the video frame co-ordinates are
-  // inverted along y-axis. We render the cursor inverse vertically on the
-  // frame. Hence the inversion on hotspot offset here.
-  cursor_position_in_frame_.Offset(-nshotspot.x,
-                                   -(nssize.height - nshotspot.y));
-  last_cursor_width_ = nssize.width;
-  last_cursor_height_ = nssize.height;
-
-  CGImageRef cg_image =
-      [nsimage CGImageForProposedRect:NULL context:nil hints:nil];
-  if (!cg_image)
-    return false;
-
-  if (CGImageGetBitsPerPixel(cg_image) != kBytesPerPixel * 8 ||
-      CGImageGetBytesPerRow(cg_image) !=
-          static_cast<size_t>(kBytesPerPixel * nssize.width) ||
-      CGImageGetBitsPerComponent(cg_image) != 8) {
-    return false;
-  }
 
-  CGDataProviderRef provider = CGImageGetDataProvider(cg_image);
-  CFDataRef image_data_ref = CGDataProviderCopyData(provider);
-  if (!image_data_ref)
-    return false;
-  last_cursor_data_.reset(image_data_ref, base::scoped_policy::ASSUME);
-
-  if (std::abs(mouse_tab_location.x - last_mouse_location_x_) >
-          MIN_MOVEMENT_PIXELS ||
-      std::abs(mouse_tab_location.y - last_mouse_location_y_) >
-          MIN_MOVEMENT_PIXELS) {
-    last_mouse_movement_timestamp_ = base::TimeTicks::Now();
-    last_mouse_location_x_ = mouse_tab_location.x;
-    last_mouse_location_y_ = mouse_tab_location.y;
-  }
-  return true;
+  gfx::Image cursor_image = gfx::Image([nsimage retain]);
+  return *(cursor_image.ToSkBitmap());
 }
 
-// Helper function to composite a RGBA cursor bitmap on a YUV420 video frame.
-void CursorRendererMac::RenderOnVideoFrame(
-    const scoped_refptr<media::VideoFrame>& target) const {
-  DCHECK(target);
-  DCHECK(last_cursor_data_);
-  const uint8_t* cursor_data_ =
-      reinterpret_cast<const uint8_t*>(CFDataGetBytePtr(last_cursor_data_));
-
-  gfx::Rect visible_rect = target->visible_rect();
-  gfx::Rect rect =
-      gfx::IntersectRects(gfx::Rect(last_cursor_width_, last_cursor_height_) +
-                              gfx::Vector2d(cursor_position_in_frame_.x(),
-                                            cursor_position_in_frame_.y()),
-                          visible_rect);
-
-  for (int y = rect.y() + 1; y <= rect.bottom(); ++y) {
-    int cursor_y = rect.bottom() - y;
-    int inverted_y = visible_rect.bottom() - y;
-    uint8_t* yplane =
-        target->data(media::VideoFrame::kYPlane) +
-        inverted_y * target->row_bytes(media::VideoFrame::kYPlane);
-    uint8_t* uplane =
-        target->data(media::VideoFrame::kUPlane) +
-        (inverted_y / 2) * target->row_bytes(media::VideoFrame::kUPlane);
-    uint8_t* vplane =
-        target->data(media::VideoFrame::kVPlane) +
-        (inverted_y / 2) * target->row_bytes(media::VideoFrame::kVPlane);
-    for (int x = rect.x(); x < rect.right(); ++x) {
-      int cursor_x = x - rect.x();
-      int byte_pos = cursor_y * last_cursor_width_ * kBytesPerPixel +
-                     cursor_x * kBytesPerPixel;
-      int color_r = cursor_data_[byte_pos];
-      int color_g = cursor_data_[byte_pos + 1];
-      int color_b = cursor_data_[byte_pos + 2];
-      int alpha = cursor_data_[byte_pos + 3];
-      int color_y = clip_byte(
-          ((color_r * 66 + color_g * 129 + color_b * 25 + 128) >> 8) + 16);
-      yplane[x] = alpha_blend(alpha, color_y, yplane[x]);
-
-      // Only sample U and V at even coordinates.
-      if ((x % 2 == 0) && (y % 2 == 0)) {
-        int color_u = clip_byte(
-            ((color_r * -38 + color_g * -74 + color_b * 112 + 128) >> 8) + 128);
-        int color_v = clip_byte(
-            ((color_r * 112 + color_g * -94 + color_b * -18 + 128) >> 8) + 128);
-        uplane[x / 2] = alpha_blend(alpha, color_u, uplane[x / 2]);
-        vplane[x / 2] = alpha_blend(alpha, color_v, vplane[x / 2]);
-      }
-    }
-  }
+gfx::Point CursorRendererMac::GetLastKnownCursorHotPoint() {
+  // Grab system cursor.
+  NSCursor* nscursor = [NSCursor currentSystemCursor];
+  NSPoint nshotspot = [nscursor hotSpot];
+
+  return gfx::Point(nshotspot.x, nshotspot.y);
 }
 
 }  // namespace content