| OLD | NEW |
| 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 // | |
| 5 // Implementation notes: This needs to work on a variety of hardware | |
| 6 // configurations where the speed of the CPU and GPU greatly affect overall | |
| 7 // performance. Spanning several threads, the process of capturing has been | |
| 8 // split up into four conceptual stages: | |
| 9 // | |
| 10 // 1. Reserve Buffer: Before a frame can be captured, a slot in the client's | |
| 11 // shared-memory IPC buffer is reserved. There are only a few of these; | |
| 12 // when they run out, it indicates that the downstream client -- likely a | |
| 13 // video encoder -- is the performance bottleneck, and that the rate of | |
| 14 // frame capture should be throttled back. | |
| 15 // | |
| 16 // 2. Capture: A bitmap is snapshotted/copied from the RenderWidget's backing | |
| 17 // store. This is initiated on the UI BrowserThread, and often occurs | |
| 18 // asynchronously. Where supported, the GPU scales and color converts | |
| 19 // frames to our desired size, and the readback happens directly into the | |
| 20 // shared-memory buffer. But this is not always possible, particularly when | |
| 21 // accelerated compositing is disabled. | |
| 22 // | |
| 23 // 3. Render (if needed): If the web contents cannot be captured directly into | |
| 24 // our target size and color format, scaling and colorspace conversion must | |
| 25 // be done on the CPU. A dedicated thread is used for this operation, to | |
| 26 // avoid blocking the UI thread. The Render stage always reads from a | |
| 27 // bitmap returned by Capture, and writes into the reserved slot in the | |
| 28 // shared-memory buffer. | |
| 29 // | |
| 30 // 4. Deliver: The rendered video frame is returned to the client (which | |
| 31 // implements the VideoCaptureDevice::Client interface). Because all | |
| 32 // paths have written the frame into the IPC buffer, this step should | |
| 33 // never need to do an additional copy of the pixel data. | |
| 34 // | |
| 35 // In the best-performing case, the Render step is bypassed: Capture produces | |
| 36 // ready-to-Deliver frames. But when accelerated readback is not possible, the | |
| 37 // system is designed so that Capture and Render may run concurrently. A timing | |
| 38 // diagram helps illustrate this point (@30 FPS): | |
| 39 // | |
| 40 // Time: 0ms 33ms 66ms 99ms | |
| 41 // thread1: |-Capture-f1------v |-Capture-f2------v |-Capture-f3----v |-Capt | |
| 42 // thread2: |-Render-f1-----v |-Render-f2-----v |-Render-f3 | |
| 43 // | |
| 44 // In the above example, both capturing and rendering *each* take almost the | |
| 45 // full 33 ms available between frames, yet we see that the required throughput | |
| 46 // is obtained. | |
| 47 | 4 |
| 48 #include "content/browser/media/capture/web_contents_video_capture_device.h" | 5 #include "content/browser/media/capture/web_contents_video_capture_device.h" |
| 49 | 6 |
| 50 #include <stdint.h> | 7 #include <stdint.h> |
| 51 | 8 |
| 52 #include <algorithm> | 9 #include <algorithm> |
| 53 #include <memory> | 10 #include <memory> |
| 54 #include <utility> | 11 #include <string> |
| 55 | 12 |
| 56 #include "base/bind.h" | 13 #include "base/bind.h" |
| 57 #include "base/callback_helpers.h" | 14 #include "base/callback_helpers.h" |
| 58 #include "base/location.h" | 15 #include "base/location.h" |
| 59 #include "base/logging.h" | 16 #include "base/logging.h" |
| 60 #include "base/macros.h" | 17 #include "base/macros.h" |
| 61 #include "base/memory/ptr_util.h" | |
| 62 #include "base/memory/weak_ptr.h" | 18 #include "base/memory/weak_ptr.h" |
| 63 #include "base/sequenced_task_runner.h" | |
| 64 #include "base/single_thread_task_runner.h" | |
| 65 #include "base/threading/thread.h" | |
| 66 #include "base/threading/thread_checker.h" | |
| 67 #include "base/time/time.h" | 19 #include "base/time/time.h" |
| 68 #include "build/build_config.h" | 20 #include "build/build_config.h" |
| 69 #include "content/browser/media/capture/cursor_renderer.h" | 21 #include "content/browser/media/capture/cursor_renderer.h" |
| 70 #include "content/browser/media/capture/web_contents_tracker.h" | 22 #include "content/browser/media/capture/web_contents_tracker.h" |
| 71 #include "content/browser/media/capture/window_activity_tracker.h" | 23 #include "content/browser/media/capture/window_activity_tracker.h" |
| 72 #include "content/browser/renderer_host/render_widget_host_impl.h" | 24 #include "content/browser/renderer_host/render_widget_host_impl.h" |
| 73 #include "content/browser/renderer_host/render_widget_host_view_base.h" | 25 #include "content/browser/renderer_host/render_widget_host_view_base.h" |
| 74 #include "content/public/browser/browser_thread.h" | 26 #include "content/public/browser/browser_thread.h" |
| 75 #include "content/public/browser/render_process_host.h" | 27 #include "content/public/browser/render_process_host.h" |
| 76 #include "content/public/browser/render_widget_host_view.h" | 28 #include "content/public/browser/render_widget_host_view.h" |
| 77 #include "content/public/browser/render_widget_host_view_frame_subscriber.h" | 29 #include "content/public/browser/render_widget_host_view_frame_subscriber.h" |
| 78 #include "content/public/browser/web_contents.h" | 30 #include "content/public/browser/web_contents.h" |
| 79 #include "content/public/browser/web_contents_media_capture_id.h" | 31 #include "content/public/browser/web_contents_media_capture_id.h" |
| 80 #include "media/base/bind_to_current_loop.h" | |
| 81 #include "media/base/video_frame_metadata.h" | 32 #include "media/base/video_frame_metadata.h" |
| 82 #include "media/base/video_util.h" | |
| 83 #include "media/capture/content/screen_capture_device_core.h" | 33 #include "media/capture/content/screen_capture_device_core.h" |
| 84 #include "media/capture/content/thread_safe_capture_oracle.h" | 34 #include "media/capture/content/thread_safe_capture_oracle.h" |
| 85 #include "media/capture/content/video_capture_oracle.h" | 35 #include "media/capture/content/video_capture_oracle.h" |
| 86 #include "media/capture/video_capture_types.h" | 36 #include "media/capture/video_capture_types.h" |
| 87 #include "skia/ext/image_operations.h" | |
| 88 #include "third_party/skia/include/core/SkBitmap.h" | |
| 89 #include "third_party/skia/include/core/SkColor.h" | |
| 90 #include "ui/base/layout.h" | 37 #include "ui/base/layout.h" |
| 91 #include "ui/gfx/geometry/dip_util.h" | 38 #include "ui/gfx/geometry/dip_util.h" |
| 92 #include "ui/gfx/geometry/size_conversions.h" | 39 #include "ui/gfx/geometry/size_conversions.h" |
| 93 | 40 |
| 94 namespace content { | 41 namespace content { |
| 95 | 42 |
| 96 namespace { | 43 namespace { |
| 97 | 44 |
| 98 enum InteractiveModeSettings { | 45 // Minimum amount of time for which there should be no animation detected |
| 99 // Minimum amount of time for which there should be no animation detected | 46 // to consider interactive mode being active. This is to prevent very brief |
| 100 // to consider interactive mode being active. This is to prevent very brief | 47 // periods of animated content not being detected (due to CPU fluctations for |
| 101 // periods of animated content not being detected (due to CPU fluctations for | 48 // example) from causing a flip-flop on interactive mode. |
| 102 // example) from causing a flip-flop on interactive mode. | 49 constexpr int64_t kMinPeriodNoAnimationMillis = 3000; |
| 103 kMinPeriodNoAnimationMillis = 3000 | |
| 104 }; | |
| 105 | |
| 106 void DeleteOnWorkerThread(std::unique_ptr<base::Thread> render_thread, | |
| 107 const base::Closure& callback) { | |
| 108 render_thread.reset(); | |
| 109 | |
| 110 // After thread join call the callback on UI thread. | |
| 111 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, callback); | |
| 112 } | |
| 113 | 50 |
| 114 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible | 51 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible |
| 115 // with RenderWidgetHostViewFrameSubscriber. We create one per event type. | 52 // with RenderWidgetHostViewFrameSubscriber. One is created per event type. |
| 116 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber { | 53 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber { |
| 117 public: | 54 public: |
| 118 FrameSubscriber(media::VideoCaptureOracle::Event event_type, | 55 FrameSubscriber(media::VideoCaptureOracle::Event event_type, |
| 119 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle, | 56 scoped_refptr<media::ThreadSafeCaptureOracle> oracle, |
| 120 base::WeakPtr<content::CursorRenderer> cursor_renderer, | 57 base::WeakPtr<content::CursorRenderer> cursor_renderer, |
| 121 base::WeakPtr<content::WindowActivityTracker> tracker) | 58 base::WeakPtr<content::WindowActivityTracker> tracker) |
| 122 : event_type_(event_type), | 59 : event_type_(event_type), |
| 123 oracle_proxy_(oracle), | 60 oracle_proxy_(std::move(oracle)), |
| 124 cursor_renderer_(cursor_renderer), | 61 cursor_renderer_(cursor_renderer), |
| 125 window_activity_tracker_(tracker), | 62 window_activity_tracker_(tracker), |
| 126 weak_ptr_factory_(this) {} | 63 weak_ptr_factory_(this) {} |
| 127 | 64 |
| 128 bool ShouldCaptureFrame( | 65 bool ShouldCaptureFrame( |
| 129 const gfx::Rect& damage_rect, | 66 const gfx::Rect& damage_rect, |
| 130 base::TimeTicks present_time, | 67 base::TimeTicks present_time, |
| 131 scoped_refptr<media::VideoFrame>* storage, | 68 scoped_refptr<media::VideoFrame>* storage, |
| 132 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback* | 69 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback* |
| 133 deliver_frame_cb) override; | 70 deliver_frame_cb) override; |
| 134 | 71 |
| 135 static void DidCaptureFrame( | 72 static void DidCaptureFrame( |
| 136 base::WeakPtr<FrameSubscriber> frame_subscriber_, | 73 base::WeakPtr<FrameSubscriber> frame_subscriber_, |
| 137 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& | 74 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& |
| 138 capture_frame_cb, | 75 capture_frame_cb, |
| 139 scoped_refptr<media::VideoFrame> frame, | 76 scoped_refptr<media::VideoFrame> frame, |
| 140 base::TimeTicks timestamp, | 77 base::TimeTicks timestamp, |
| 141 const gfx::Rect& region_in_frame, | 78 const gfx::Rect& region_in_frame, |
| 142 bool success); | 79 bool success); |
| 143 | 80 |
| 144 bool IsUserInteractingWithContent(); | 81 bool IsUserInteractingWithContent(); |
| 145 | 82 |
| 146 private: | 83 private: |
| 147 const media::VideoCaptureOracle::Event event_type_; | 84 const media::VideoCaptureOracle::Event event_type_; |
| 148 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; | 85 const scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; |
| 149 // We need a weak pointer since FrameSubscriber is owned externally and | 86 // We need a weak pointer since FrameSubscriber is owned externally and |
| 150 // may outlive the cursor renderer. | 87 // may outlive the cursor renderer. |
| 151 base::WeakPtr<CursorRenderer> cursor_renderer_; | 88 const base::WeakPtr<CursorRenderer> cursor_renderer_; |
| 152 // We need a weak pointer since FrameSubscriber is owned externally and | 89 // We need a weak pointer since FrameSubscriber is owned externally and |
| 153 // may outlive the ui activity tracker. | 90 // may outlive the ui activity tracker. |
| 154 base::WeakPtr<WindowActivityTracker> window_activity_tracker_; | 91 const base::WeakPtr<WindowActivityTracker> window_activity_tracker_; |
| 155 base::WeakPtrFactory<FrameSubscriber> weak_ptr_factory_; | 92 base::WeakPtrFactory<FrameSubscriber> weak_ptr_factory_; |
| 156 }; | 93 }; |
| 157 | 94 |
| 158 // ContentCaptureSubscription is the relationship between a RenderWidgetHost | 95 // ContentCaptureSubscription is the relationship between a RenderWidgetHost |
| 159 // whose content is updating, a subscriber that is deciding which of these | 96 // whose content is updating, a subscriber that is deciding which of these |
| 160 // updates to capture (and where to deliver them to), and a callback that | 97 // updates to capture (and where to deliver them to), and a callback that |
| 161 // knows how to do the capture and prepare the result for delivery. | 98 // knows how to do the capture and prepare the result for delivery. |
| 162 // | 99 // |
| 163 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in | 100 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in |
| 164 // the RenderWidgetHostView, to process compositor updates, and (b) occasionally | 101 // the RenderWidgetHostView, to process compositor updates, and (b) occasionally |
| 165 // initiating forced, non-event-driven captures needed by downstream consumers | 102 // using the capture callback to force additional captures that are needed for |
| 166 // that request "refresh frames" of unchanged content. | 103 // things like mouse cursor rendering updates and/or refresh frames. |
| 167 // | 104 // |
| 168 // All of this happens on the UI thread, although the | 105 // All of this happens on the UI thread, although the |
| 169 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates | 106 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates |
| 170 // autonomously on some other thread. | 107 // autonomously on some other thread. |
| 171 class ContentCaptureSubscription { | 108 class ContentCaptureSubscription { |
| 172 public: | 109 public: |
| 173 typedef base::Callback<void( | 110 using CaptureCallback = base::Callback<void( |
| 174 const base::TimeTicks&, | 111 base::TimeTicks, |
| 175 const scoped_refptr<media::VideoFrame>&, | 112 scoped_refptr<media::VideoFrame>, |
| 176 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&)> | 113 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&)>; |
| 177 CaptureCallback; | |
| 178 | 114 |
| 179 // Create a subscription. Whenever a manual capture is required, the | 115 // Create a subscription. Whenever a non-compositor capture is required, the |
| 180 // subscription will invoke |capture_callback| on the UI thread to do the | 116 // subscription will invoke |capture_callback| on the UI thread to do the |
| 181 // work. | 117 // work. |
| 182 ContentCaptureSubscription( | 118 ContentCaptureSubscription( |
| 183 const RenderWidgetHost& source, | 119 const RenderWidgetHost& source, |
| 184 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 120 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
| 185 const CaptureCallback& capture_callback); | 121 const CaptureCallback& capture_callback); |
| 186 ~ContentCaptureSubscription(); | 122 ~ContentCaptureSubscription(); |
| 187 | 123 |
| 124 // Called when a condition occurs that requires a refresh frame be |
| 125 // captured. The subscribers will determine whether a frame was recently |
| 126 // captured, or if a non-compositor initiated capture must be made. |
| 188 void MaybeCaptureForRefresh(); | 127 void MaybeCaptureForRefresh(); |
| 189 | 128 |
| 190 private: | 129 private: |
| 191 // Called for active frame refresh requests, or mouse activity events. | 130 // Called for active frame refresh requests, or mouse activity events. |
| 192 void OnEvent(FrameSubscriber* subscriber); | 131 void OnEvent(FrameSubscriber* subscriber); |
| 193 | 132 |
| 194 // Maintain a weak reference to the RenderWidgetHost (via its routing ID), | 133 // Maintain a weak reference to the RenderWidgetHost (via its routing ID), |
| 195 // since the instance could be destroyed externally during the lifetime of | 134 // since the instance could be destroyed externally during the lifetime of |
| 196 // |this|. | 135 // |this|. |
| 197 const int render_process_id_; | 136 const int render_process_id_; |
| 198 const int render_widget_id_; | 137 const int render_widget_id_; |
| 199 | 138 |
| 200 std::unique_ptr<FrameSubscriber> refresh_subscriber_; | 139 std::unique_ptr<FrameSubscriber> refresh_subscriber_; |
| 201 std::unique_ptr<FrameSubscriber> mouse_activity_subscriber_; | 140 std::unique_ptr<FrameSubscriber> mouse_activity_subscriber_; |
| 202 CaptureCallback capture_callback_; | 141 CaptureCallback capture_callback_; |
| 203 | 142 |
| 204 // Responsible for tracking the cursor state and input events to make | 143 // Responsible for tracking the cursor state and input events to make |
| 205 // decisions and then render the mouse cursor on the video frame after | 144 // decisions and then render the mouse cursor on the video frame after |
| 206 // capture is completed. | 145 // capture is completed. |
| 207 std::unique_ptr<content::CursorRenderer> cursor_renderer_; | 146 std::unique_ptr<content::CursorRenderer> cursor_renderer_; |
| 208 | 147 |
| 209 // Responsible for tracking the UI events and making a decision on whether | 148 // Responsible for tracking the UI events and making a decision on whether |
| 210 // user is actively interacting with content. | 149 // user is actively interacting with content. |
| 211 std::unique_ptr<content::WindowActivityTracker> window_activity_tracker_; | 150 std::unique_ptr<content::WindowActivityTracker> window_activity_tracker_; |
| 212 | 151 |
| 213 DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription); | 152 DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription); |
| 214 }; | 153 }; |
| 215 | 154 |
| 216 // Render the SkBitmap |input| into the given VideoFrame buffer |output|, then | |
| 217 // invoke |done_cb| to indicate success or failure. |input| is expected to be | |
| 218 // ARGB. |output| must be YV12 or I420. Colorspace conversion is always done. | |
| 219 // Scaling and letterboxing will be done as needed. | |
| 220 // | |
| 221 // This software implementation should be used only when GPU acceleration of | |
| 222 // these activities is not possible. This operation may be expensive (tens to | |
| 223 // hundreds of milliseconds), so the caller should ensure that it runs on a | |
| 224 // thread where such a pause would cause UI jank. | |
| 225 void RenderVideoFrame( | |
| 226 const SkBitmap& input, | |
| 227 const scoped_refptr<media::VideoFrame>& output, | |
| 228 const base::Callback<void(const gfx::Rect&, bool)>& done_cb); | |
| 229 | |
| 230 // Renews capture subscriptions based on feedback from WebContentsTracker, and | 155 // Renews capture subscriptions based on feedback from WebContentsTracker, and |
| 231 // also executes copying of the backing store on the UI BrowserThread. | 156 // also executes non-compositor-initiated frame captures. |
| 232 class WebContentsCaptureMachine : public media::VideoCaptureMachine { | 157 class WebContentsCaptureMachine : public media::VideoCaptureMachine { |
| 233 public: | 158 public: |
| 234 WebContentsCaptureMachine(int render_process_id, | 159 WebContentsCaptureMachine(int render_process_id, |
| 235 int main_render_frame_id, | 160 int main_render_frame_id, |
| 236 bool enable_auto_throttling); | 161 bool enable_auto_throttling); |
| 237 ~WebContentsCaptureMachine() override; | 162 ~WebContentsCaptureMachine() override; |
| 238 | 163 |
| 239 // VideoCaptureMachine overrides. | 164 // VideoCaptureMachine overrides. |
| 240 void Start(const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 165 void Start(const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, |
| 241 const media::VideoCaptureParams& params, | 166 const media::VideoCaptureParams& params, |
| 242 const base::Callback<void(bool)> callback) override; | 167 const base::Callback<void(bool)> callback) override; |
| 243 void Suspend() override; | 168 void Suspend() override; |
| 244 void Resume() override; | 169 void Resume() override; |
| 245 void Stop(const base::Closure& callback) override; | 170 void Stop(const base::Closure& callback) override; |
| 246 bool IsAutoThrottlingEnabled() const override { | 171 bool IsAutoThrottlingEnabled() const override { |
| 247 return auto_throttling_enabled_; | 172 return auto_throttling_enabled_; |
| 248 } | 173 } |
| 249 void MaybeCaptureForRefresh() override; | 174 void MaybeCaptureForRefresh() override; |
| 250 | 175 |
| 251 // Starts a copy from the backing store or the composited surface. Must be run | 176 private: |
| 252 // on the UI BrowserThread. |deliver_frame_cb| will be run when the operation | 177 // Called for cases where events other than compositor updates require a frame |
| 253 // completes. The copy will occur to |target|. | 178 // capture from the composited surface. Must be run on the UI BrowserThread. |
| 179 // |deliver_frame_cb| will be run when the operation completes. The copy will |
| 180 // populate |target|. |
| 254 // | 181 // |
| 255 // This may be used as a ContentCaptureSubscription::CaptureCallback. | 182 // This method is only used by a ContentCaptureSubscription::CaptureCallback. |
| 256 void Capture(const base::TimeTicks& start_time, | 183 void Capture(base::TimeTicks start_time, |
| 257 const scoped_refptr<media::VideoFrame>& target, | 184 scoped_refptr<media::VideoFrame> target, |
| 258 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 185 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
| 259 deliver_frame_cb); | 186 deliver_frame_cb); |
| 260 | 187 |
| 261 private: | 188 bool InternalStart(scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
| 262 bool InternalStart( | 189 const media::VideoCaptureParams& params); |
| 263 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | |
| 264 const media::VideoCaptureParams& params); | |
| 265 void InternalSuspend(); | 190 void InternalSuspend(); |
| 266 void InternalResume(); | 191 void InternalResume(); |
| 267 void InternalStop(const base::Closure& callback); | 192 void InternalStop(const base::Closure& callback); |
| 268 void InternalMaybeCaptureForRefresh(); | 193 void InternalMaybeCaptureForRefresh(); |
| 269 bool IsStarted() const; | 194 bool IsStarted() const; |
| 270 | 195 |
| 271 // Computes the preferred size of the target RenderWidget for optimal capture. | 196 // Computes the preferred size of the target RenderWidget for optimal capture. |
| 272 gfx::Size ComputeOptimalViewSize() const; | 197 gfx::Size ComputeOptimalViewSize() const; |
| 273 | 198 |
| 274 // Response callback for RenderWidgetHost::CopyFromBackingStore(). | |
| 275 void DidCopyFromBackingStore( | |
| 276 const base::TimeTicks& start_time, | |
| 277 const scoped_refptr<media::VideoFrame>& target, | |
| 278 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | |
| 279 deliver_frame_cb, | |
| 280 const SkBitmap& bitmap, | |
| 281 ReadbackResponse response); | |
| 282 | |
| 283 // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame(). | 199 // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame(). |
| 284 void DidCopyFromCompositingSurfaceToVideoFrame( | 200 void DidCopyToVideoFrame( |
| 285 const base::TimeTicks& start_time, | 201 const base::TimeTicks& start_time, |
| 286 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 202 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
| 287 deliver_frame_cb, | 203 deliver_frame_cb, |
| 288 const gfx::Rect& region_in_frame, | 204 const gfx::Rect& region_in_frame, |
| 289 bool success); | 205 bool success); |
| 290 | 206 |
| 291 // Remove the old subscription, and attempt to start a new one if |had_target| | 207 // Remove the old subscription, and attempt to start a new one if |had_target| |
| 292 // is true. | 208 // is true. |
| 293 void RenewFrameSubscription(bool had_target); | 209 void RenewFrameSubscription(bool had_target); |
| 294 | 210 |
| 295 // Called whenever the render widget is resized. | 211 // Called whenever the render widget is resized. |
| 296 void UpdateCaptureSize(); | 212 void UpdateCaptureSize(); |
| 297 | 213 |
| 298 // Parameters saved in constructor. | 214 // Parameters saved in constructor. |
| 299 const int initial_render_process_id_; | 215 const int initial_render_process_id_; |
| 300 const int initial_main_render_frame_id_; | 216 const int initial_main_render_frame_id_; |
| 301 | 217 |
| 302 // Tracks events and calls back to RenewFrameSubscription() to maintain | 218 // Tracks events and calls back to RenewFrameSubscription() to maintain |
| 303 // capture on the correct RenderWidgetHost. | 219 // capture on the correct RenderWidgetHost. |
| 304 const scoped_refptr<WebContentsTracker> tracker_; | 220 const scoped_refptr<WebContentsTracker> tracker_; |
| 305 | 221 |
| 306 // Set to false to prevent the capture size from automatically adjusting in | 222 // Set to false to prevent the capture size from automatically adjusting in |
| 307 // response to end-to-end utilization. This is enabled via the throttling | 223 // response to end-to-end utilization. This is enabled via the throttling |
| 308 // option in the WebContentsVideoCaptureDevice device ID. | 224 // option in the WebContentsVideoCaptureDevice device ID. |
| 309 const bool auto_throttling_enabled_; | 225 const bool auto_throttling_enabled_; |
| 310 | 226 |
| 311 // A dedicated worker thread on which SkBitmap->VideoFrame conversion will | |
| 312 // occur. Only used when this activity cannot be done on the GPU. | |
| 313 std::unique_ptr<base::Thread> render_thread_; | |
| 314 | |
| 315 // Makes all the decisions about which frames to copy, and how. | 227 // Makes all the decisions about which frames to copy, and how. |
| 316 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; | 228 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; |
| 317 | 229 |
| 318 // Video capture parameters that this machine is started with. | 230 // Video capture parameters that this machine is started with. |
| 319 media::VideoCaptureParams capture_params_; | 231 media::VideoCaptureParams capture_params_; |
| 320 | 232 |
| 321 // Responsible for forwarding events from the active RenderWidgetHost to the | 233 // Responsible for forwarding events from the active RenderWidgetHost to the |
| 322 // oracle, and initiating captures accordingly. | 234 // oracle, and initiating captures accordingly. |
| 323 std::unique_ptr<ContentCaptureSubscription> subscription_; | 235 std::unique_ptr<ContentCaptureSubscription> subscription_; |
| 324 | 236 |
| (...skipping 11 matching lines...) Expand all Loading... |
| 336 | 248 |
| 337 bool FrameSubscriber::ShouldCaptureFrame( | 249 bool FrameSubscriber::ShouldCaptureFrame( |
| 338 const gfx::Rect& damage_rect, | 250 const gfx::Rect& damage_rect, |
| 339 base::TimeTicks present_time, | 251 base::TimeTicks present_time, |
| 340 scoped_refptr<media::VideoFrame>* storage, | 252 scoped_refptr<media::VideoFrame>* storage, |
| 341 DeliverFrameCallback* deliver_frame_cb) { | 253 DeliverFrameCallback* deliver_frame_cb) { |
| 342 TRACE_EVENT1("gpu.capture", "FrameSubscriber::ShouldCaptureFrame", "instance", | 254 TRACE_EVENT1("gpu.capture", "FrameSubscriber::ShouldCaptureFrame", "instance", |
| 343 this); | 255 this); |
| 344 | 256 |
| 345 media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb; | 257 media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb; |
| 258 if (!oracle_proxy_->ObserveEventAndDecideCapture( |
| 259 event_type_, damage_rect, present_time, storage, &capture_frame_cb)) { |
| 260 return false; |
| 261 } |
| 346 | 262 |
| 347 bool oracle_decision = oracle_proxy_->ObserveEventAndDecideCapture( | 263 DCHECK(*storage); |
| 348 event_type_, damage_rect, present_time, storage, &capture_frame_cb); | 264 DCHECK(!capture_frame_cb.is_null()); |
| 349 | 265 *deliver_frame_cb = |
| 350 if (!capture_frame_cb.is_null()) | 266 base::Bind(&FrameSubscriber::DidCaptureFrame, |
| 351 *deliver_frame_cb = | 267 weak_ptr_factory_.GetWeakPtr(), capture_frame_cb, *storage); |
| 352 base::Bind(&FrameSubscriber::DidCaptureFrame, | 268 return true; |
| 353 weak_ptr_factory_.GetWeakPtr(), capture_frame_cb, *storage); | |
| 354 return oracle_decision; | |
| 355 } | 269 } |
| 356 | 270 |
| 357 void FrameSubscriber::DidCaptureFrame( | 271 void FrameSubscriber::DidCaptureFrame( |
| 358 base::WeakPtr<FrameSubscriber> frame_subscriber_, | 272 base::WeakPtr<FrameSubscriber> frame_subscriber_, |
| 359 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& | 273 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& |
| 360 capture_frame_cb, | 274 capture_frame_cb, |
| 361 scoped_refptr<media::VideoFrame> frame, | 275 scoped_refptr<media::VideoFrame> frame, |
| 362 base::TimeTicks timestamp, | 276 base::TimeTicks timestamp, |
| 363 const gfx::Rect& region_in_frame, | 277 const gfx::Rect& region_in_frame, |
| 364 bool success) { | 278 bool success) { |
| 365 // We can get a callback in the shutdown sequence for the browser main loop | |
| 366 // and this can result in a DCHECK failure. Avoid this by doing DCHECK only | |
| 367 // on success. | |
| 368 if (success) { | 279 if (success) { |
| 280 // We can get a callback in the shutdown sequence for the browser main loop |
| 281 // and this can result in a DCHECK failure. Avoid this by doing DCHECK only |
| 282 // on success. |
| 369 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 283 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 370 // TODO(isheriff): Unclear if taking a snapshot of cursor here affects user | 284 |
| 371 // experience in any particular scenarios. Doing it prior to capture may | |
| 372 // require evaluating region_in_frame in this file. | |
| 373 if (frame_subscriber_ && frame_subscriber_->cursor_renderer_) { | 285 if (frame_subscriber_ && frame_subscriber_->cursor_renderer_) { |
| 374 CursorRenderer* cursor_renderer = | 286 CursorRenderer* cursor_renderer = |
| 375 frame_subscriber_->cursor_renderer_.get(); | 287 frame_subscriber_->cursor_renderer_.get(); |
| 376 if (cursor_renderer->SnapshotCursorState(region_in_frame)) | 288 if (cursor_renderer->SnapshotCursorState(region_in_frame)) |
| 377 cursor_renderer->RenderOnVideoFrame(frame); | 289 cursor_renderer->RenderOnVideoFrame(frame); |
| 378 frame->metadata()->SetBoolean( | 290 frame->metadata()->SetBoolean( |
| 379 media::VideoFrameMetadata::INTERACTIVE_CONTENT, | 291 media::VideoFrameMetadata::INTERACTIVE_CONTENT, |
| 380 frame_subscriber_->IsUserInteractingWithContent()); | 292 frame_subscriber_->IsUserInteractingWithContent()); |
| 381 } | 293 } |
| 382 } | 294 } |
| 383 capture_frame_cb.Run(std::move(frame), timestamp, success); | 295 capture_frame_cb.Run(std::move(frame), timestamp, success); |
| 384 } | 296 } |
| 385 | 297 |
| 386 bool FrameSubscriber::IsUserInteractingWithContent() { | 298 bool FrameSubscriber::IsUserInteractingWithContent() { |
| 299 bool ui_activity = window_activity_tracker_ && |
| 300 window_activity_tracker_->IsUiInteractionActive(); |
| 387 bool interactive_mode = false; | 301 bool interactive_mode = false; |
| 388 bool ui_activity = false; | |
| 389 if (window_activity_tracker_) { | |
| 390 ui_activity = window_activity_tracker_->IsUiInteractionActive(); | |
| 391 } | |
| 392 if (cursor_renderer_) { | 302 if (cursor_renderer_) { |
| 393 bool animation_active = | 303 bool animation_active = |
| 394 (base::TimeTicks::Now() - | 304 (base::TimeTicks::Now() - |
| 395 oracle_proxy_->last_time_animation_was_detected()) < | 305 oracle_proxy_->last_time_animation_was_detected()) < |
| 396 base::TimeDelta::FromMilliseconds(kMinPeriodNoAnimationMillis); | 306 base::TimeDelta::FromMilliseconds(kMinPeriodNoAnimationMillis); |
| 397 if (ui_activity && !animation_active) { | 307 if (ui_activity && !animation_active) { |
| 398 interactive_mode = true; | 308 interactive_mode = true; |
| 399 } else if (animation_active && window_activity_tracker_.get()) { | 309 } else if (animation_active && window_activity_tracker_) { |
| 400 window_activity_tracker_->Reset(); | 310 window_activity_tracker_->Reset(); |
| 401 } | 311 } |
| 402 } | 312 } |
| 403 return interactive_mode; | 313 return interactive_mode; |
| 404 } | 314 } |
| 405 | 315 |
| 406 ContentCaptureSubscription::ContentCaptureSubscription( | 316 ContentCaptureSubscription::ContentCaptureSubscription( |
| 407 const RenderWidgetHost& source, | 317 const RenderWidgetHost& source, |
| 408 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 318 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
| 409 const CaptureCallback& capture_callback) | 319 const CaptureCallback& capture_callback) |
| 410 : render_process_id_(source.GetProcess()->GetID()), | 320 : render_process_id_(source.GetProcess()->GetID()), |
| 411 render_widget_id_(source.GetRoutingID()), | 321 render_widget_id_(source.GetRoutingID()), |
| 412 capture_callback_(capture_callback) { | 322 capture_callback_(capture_callback) { |
| 413 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 323 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 414 | 324 |
| 415 RenderWidgetHostView* const view = source.GetView(); | 325 RenderWidgetHostView* const view = source.GetView(); |
| 416 #if defined(USE_AURA) || defined(OS_MACOSX) | 326 #if defined(USE_AURA) || defined(OS_MACOSX) |
| 417 if (view) { | 327 if (view) { |
| 418 cursor_renderer_ = CursorRenderer::Create(view->GetNativeView()); | 328 cursor_renderer_ = CursorRenderer::Create(view->GetNativeView()); |
| 419 window_activity_tracker_ = | 329 window_activity_tracker_ = |
| 420 WindowActivityTracker::Create(view->GetNativeView()); | 330 WindowActivityTracker::Create(view->GetNativeView()); |
| 421 } | 331 } |
| 422 #endif | 332 #endif |
| 333 // Subscriptions for refresh frames and mouse cursor updates. When events |
| 334 // outside of the normal content change/compositing workflow occur, these |
| 335 // decide whether extra frames need to be captured. Such extra frame captures |
| 336 // are initiated by running the |capture_callback_|. |
| 423 refresh_subscriber_.reset(new FrameSubscriber( | 337 refresh_subscriber_.reset(new FrameSubscriber( |
| 424 media::VideoCaptureOracle::kActiveRefreshRequest, oracle_proxy, | 338 media::VideoCaptureOracle::kActiveRefreshRequest, oracle_proxy, |
| 425 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() | 339 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() |
| 426 : base::WeakPtr<CursorRenderer>(), | 340 : base::WeakPtr<CursorRenderer>(), |
| 427 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() | 341 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() |
| 428 : base::WeakPtr<WindowActivityTracker>())); | 342 : base::WeakPtr<WindowActivityTracker>())); |
| 429 mouse_activity_subscriber_.reset(new FrameSubscriber( | 343 mouse_activity_subscriber_.reset(new FrameSubscriber( |
| 430 media::VideoCaptureOracle::kMouseCursorUpdate, oracle_proxy, | 344 media::VideoCaptureOracle::kMouseCursorUpdate, oracle_proxy, |
| 431 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() | 345 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() |
| 432 : base::WeakPtr<CursorRenderer>(), | 346 : base::WeakPtr<CursorRenderer>(), |
| 433 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() | 347 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() |
| 434 : base::WeakPtr<WindowActivityTracker>())); | 348 : base::WeakPtr<WindowActivityTracker>())); |
| 435 | 349 |
| 436 // Subscribe to compositor updates. These will be serviced directly by the | 350 // Main capture path: Subscribing to compositor updates. This means that any |
| 437 // oracle. | 351 // time the tab content has changed and compositing has taken place, the |
| 352 // RenderWidgetHostView will consult the FrameSubscriber (which consults the |
| 353 // oracle) to determine whether a frame should be captured. If so, the |
| 354 // RenderWidgetHostView will initiate the frame capture and NOT the |
| 355 // |capture_callback_|. |
| 438 if (view) { | 356 if (view) { |
| 439 std::unique_ptr<RenderWidgetHostViewFrameSubscriber> subscriber( | 357 std::unique_ptr<RenderWidgetHostViewFrameSubscriber> subscriber( |
| 440 new FrameSubscriber( | 358 new FrameSubscriber( |
| 441 media::VideoCaptureOracle::kCompositorUpdate, oracle_proxy, | 359 media::VideoCaptureOracle::kCompositorUpdate, oracle_proxy, |
| 442 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() | 360 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() |
| 443 : base::WeakPtr<CursorRenderer>(), | 361 : base::WeakPtr<CursorRenderer>(), |
| 444 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() | 362 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() |
| 445 : base::WeakPtr<WindowActivityTracker>())); | 363 : base::WeakPtr<WindowActivityTracker>())); |
| 446 view->BeginFrameSubscription(std::move(subscriber)); | 364 view->BeginFrameSubscription(std::move(subscriber)); |
| 447 } | 365 } |
| 448 | 366 |
| 449 // Subscribe to mouse movement and mouse cursor update events. | 367 // Begin monitoring for user activity that is used to signal "interactive |
| 368 // content." |
| 450 if (window_activity_tracker_) { | 369 if (window_activity_tracker_) { |
| 451 window_activity_tracker_->RegisterMouseInteractionObserver( | 370 window_activity_tracker_->RegisterMouseInteractionObserver( |
| 452 base::Bind(&ContentCaptureSubscription::OnEvent, base::Unretained(this), | 371 base::Bind(&ContentCaptureSubscription::OnEvent, base::Unretained(this), |
| 453 mouse_activity_subscriber_.get())); | 372 mouse_activity_subscriber_.get())); |
| 454 } | 373 } |
| 455 } | 374 } |
| 456 | 375 |
| 457 ContentCaptureSubscription::~ContentCaptureSubscription() { | 376 ContentCaptureSubscription::~ContentCaptureSubscription() { |
| 458 // If the BrowserThreads have been torn down, then the browser is in the final | 377 // If the BrowserThreads have been torn down, then the browser is in the final |
| 459 // stages of exiting and it is dangerous to take any further action. We must | 378 // stages of exiting and it is dangerous to take any further action. We must |
| 460 // return early. http://crbug.com/396413 | 379 // return early. http://crbug.com/396413 |
| 461 if (!BrowserThread::IsMessageLoopValid(BrowserThread::UI)) | 380 if (!BrowserThread::IsMessageLoopValid(BrowserThread::UI)) |
| 462 return; | 381 return; |
| 463 | 382 |
| 464 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 383 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 465 RenderWidgetHost* const source = | 384 RenderWidgetHost* const source = |
| 466 RenderWidgetHost::FromID(render_process_id_, render_widget_id_); | 385 RenderWidgetHost::FromID(render_process_id_, render_widget_id_); |
| 467 RenderWidgetHostView* const view = source ? source->GetView() : NULL; | 386 RenderWidgetHostView* const view = source ? source->GetView() : nullptr; |
| 468 if (view) | 387 if (view) |
| 469 view->EndFrameSubscription(); | 388 view->EndFrameSubscription(); |
| 470 } | 389 } |
| 471 | 390 |
| 472 void ContentCaptureSubscription::MaybeCaptureForRefresh() { | 391 void ContentCaptureSubscription::MaybeCaptureForRefresh() { |
| 473 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 392 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 474 OnEvent(refresh_subscriber_.get()); | 393 OnEvent(refresh_subscriber_.get()); |
| 475 } | 394 } |
| 476 | 395 |
| 477 void ContentCaptureSubscription::OnEvent(FrameSubscriber* subscriber) { | 396 void ContentCaptureSubscription::OnEvent(FrameSubscriber* subscriber) { |
| 478 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 397 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 479 TRACE_EVENT0("gpu.capture", "ContentCaptureSubscription::OnEvent"); | 398 TRACE_EVENT0("gpu.capture", "ContentCaptureSubscription::OnEvent"); |
| 480 | 399 |
| 481 scoped_refptr<media::VideoFrame> frame; | 400 scoped_refptr<media::VideoFrame> frame; |
| 482 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb; | 401 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb; |
| 483 | 402 |
| 484 const base::TimeTicks start_time = base::TimeTicks::Now(); | 403 const base::TimeTicks start_time = base::TimeTicks::Now(); |
| 485 DCHECK(subscriber == refresh_subscriber_.get() || | 404 DCHECK(subscriber == refresh_subscriber_.get() || |
| 486 subscriber == mouse_activity_subscriber_.get()); | 405 subscriber == mouse_activity_subscriber_.get()); |
| 487 if (subscriber->ShouldCaptureFrame(gfx::Rect(), start_time, &frame, | 406 if (subscriber->ShouldCaptureFrame(gfx::Rect(), start_time, &frame, |
| 488 &deliver_frame_cb)) { | 407 &deliver_frame_cb)) { |
| 489 capture_callback_.Run(start_time, frame, deliver_frame_cb); | 408 capture_callback_.Run(start_time, frame, deliver_frame_cb); |
| 490 } | 409 } |
| 491 } | 410 } |
| 492 | 411 |
| 493 void RenderVideoFrame( | |
| 494 const SkBitmap& input, | |
| 495 const scoped_refptr<media::VideoFrame>& output, | |
| 496 const base::Callback<void(const gfx::Rect&, bool)>& done_cb) { | |
| 497 base::ScopedClosureRunner failure_handler( | |
| 498 base::Bind(done_cb, gfx::Rect(), false)); | |
| 499 | |
| 500 SkAutoLockPixels locker(input); | |
| 501 | |
| 502 // Sanity-check the captured bitmap. | |
| 503 if (input.empty() || !input.readyToDraw() || | |
| 504 input.colorType() != kN32_SkColorType || input.width() < 2 || | |
| 505 input.height() < 2) { | |
| 506 DVLOG(1) << "input unacceptable (size=" << input.getSize() | |
| 507 << ", ready=" << input.readyToDraw() | |
| 508 << ", colorType=" << input.colorType() << ')'; | |
| 509 return; | |
| 510 } | |
| 511 | |
| 512 // Sanity-check the output buffer. | |
| 513 if (output->format() != media::PIXEL_FORMAT_I420) { | |
| 514 NOTREACHED(); | |
| 515 return; | |
| 516 } | |
| 517 | |
| 518 // Calculate the width and height of the content region in the |output|, based | |
| 519 // on the aspect ratio of |input|. | |
| 520 const gfx::Rect region_in_frame = media::ComputeLetterboxRegion( | |
| 521 output->visible_rect(), gfx::Size(input.width(), input.height())); | |
| 522 | |
| 523 // Scale the bitmap to the required size, if necessary. | |
| 524 SkBitmap scaled_bitmap; | |
| 525 if (input.width() != region_in_frame.width() || | |
| 526 input.height() != region_in_frame.height()) { | |
| 527 skia::ImageOperations::ResizeMethod method; | |
| 528 if (input.width() < region_in_frame.width() || | |
| 529 input.height() < region_in_frame.height()) { | |
| 530 // Avoid box filtering when magnifying, because it's actually | |
| 531 // nearest-neighbor. | |
| 532 method = skia::ImageOperations::RESIZE_HAMMING1; | |
| 533 } else { | |
| 534 method = skia::ImageOperations::RESIZE_BOX; | |
| 535 } | |
| 536 | |
| 537 TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", output.get(), | |
| 538 "Scale"); | |
| 539 scaled_bitmap = skia::ImageOperations::Resize( | |
| 540 input, method, region_in_frame.width(), region_in_frame.height()); | |
| 541 } else { | |
| 542 scaled_bitmap = input; | |
| 543 } | |
| 544 | |
| 545 TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", output.get(), "YUV"); | |
| 546 { | |
| 547 // Align to 2x2 pixel boundaries, as required by | |
| 548 // media::CopyRGBToVideoFrame(). | |
| 549 const gfx::Rect region_in_yv12_frame( | |
| 550 region_in_frame.x() & ~1, region_in_frame.y() & ~1, | |
| 551 region_in_frame.width() & ~1, region_in_frame.height() & ~1); | |
| 552 if (region_in_yv12_frame.IsEmpty()) | |
| 553 return; | |
| 554 | |
| 555 SkAutoLockPixels scaled_bitmap_locker(scaled_bitmap); | |
| 556 media::CopyRGBToVideoFrame( | |
| 557 reinterpret_cast<uint8_t*>(scaled_bitmap.getPixels()), | |
| 558 scaled_bitmap.rowBytes(), region_in_yv12_frame, output.get()); | |
| 559 } | |
| 560 | |
| 561 // The result is now ready. | |
| 562 ignore_result(failure_handler.Release()); | |
| 563 done_cb.Run(region_in_frame, true); | |
| 564 } | |
| 565 | |
| 566 WebContentsCaptureMachine::WebContentsCaptureMachine( | 412 WebContentsCaptureMachine::WebContentsCaptureMachine( |
| 567 int render_process_id, | 413 int render_process_id, |
| 568 int main_render_frame_id, | 414 int main_render_frame_id, |
| 569 bool enable_auto_throttling) | 415 bool enable_auto_throttling) |
| 570 : initial_render_process_id_(render_process_id), | 416 : initial_render_process_id_(render_process_id), |
| 571 initial_main_render_frame_id_(main_render_frame_id), | 417 initial_main_render_frame_id_(main_render_frame_id), |
| 572 tracker_(new WebContentsTracker(true)), | 418 tracker_(new WebContentsTracker(true)), |
| 573 auto_throttling_enabled_(enable_auto_throttling), | 419 auto_throttling_enabled_(enable_auto_throttling), |
| 574 frame_capture_active_(true), | 420 frame_capture_active_(true), |
| 575 weak_ptr_factory_(this) { | 421 weak_ptr_factory_(this) { |
| (...skipping 15 matching lines...) Expand all Loading... |
| 591 const base::Callback<void(bool)> callback) { | 437 const base::Callback<void(bool)> callback) { |
| 592 // Starts the capture machine asynchronously. | 438 // Starts the capture machine asynchronously. |
| 593 BrowserThread::PostTaskAndReplyWithResult( | 439 BrowserThread::PostTaskAndReplyWithResult( |
| 594 BrowserThread::UI, FROM_HERE, | 440 BrowserThread::UI, FROM_HERE, |
| 595 base::Bind(&WebContentsCaptureMachine::InternalStart, | 441 base::Bind(&WebContentsCaptureMachine::InternalStart, |
| 596 base::Unretained(this), oracle_proxy, params), | 442 base::Unretained(this), oracle_proxy, params), |
| 597 callback); | 443 callback); |
| 598 } | 444 } |
| 599 | 445 |
| 600 bool WebContentsCaptureMachine::InternalStart( | 446 bool WebContentsCaptureMachine::InternalStart( |
| 601 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 447 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
| 602 const media::VideoCaptureParams& params) { | 448 const media::VideoCaptureParams& params) { |
| 603 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 449 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 604 DCHECK(!IsStarted()); | 450 DCHECK(!IsStarted()); |
| 605 | 451 |
| 606 DCHECK(oracle_proxy); | 452 DCHECK(oracle_proxy); |
| 607 oracle_proxy_ = oracle_proxy; | 453 oracle_proxy_ = std::move(oracle_proxy); |
| 608 capture_params_ = params; | 454 capture_params_ = params; |
| 609 | 455 |
| 610 render_thread_.reset(new base::Thread("WebContentsVideo_RenderThread")); | |
| 611 if (!render_thread_->Start()) { | |
| 612 DVLOG(1) << "Failed to spawn render thread."; | |
| 613 render_thread_.reset(); | |
| 614 return false; | |
| 615 } | |
| 616 | |
| 617 // Note: Creation of the first WeakPtr in the following statement will cause | 456 // Note: Creation of the first WeakPtr in the following statement will cause |
| 618 // IsStarted() to return true from now on. | 457 // IsStarted() to return true from now on. |
| 619 tracker_->SetResizeChangeCallback( | 458 tracker_->SetResizeChangeCallback( |
| 620 base::Bind(&WebContentsCaptureMachine::UpdateCaptureSize, | 459 base::Bind(&WebContentsCaptureMachine::UpdateCaptureSize, |
| 621 weak_ptr_factory_.GetWeakPtr())); | 460 weak_ptr_factory_.GetWeakPtr())); |
| 622 tracker_->Start(initial_render_process_id_, initial_main_render_frame_id_, | 461 tracker_->Start(initial_render_process_id_, initial_main_render_frame_id_, |
| 623 base::Bind(&WebContentsCaptureMachine::RenewFrameSubscription, | 462 base::Bind(&WebContentsCaptureMachine::RenewFrameSubscription, |
| 624 weak_ptr_factory_.GetWeakPtr())); | 463 weak_ptr_factory_.GetWeakPtr())); |
| 625 if (WebContents* contents = tracker_->web_contents()) | 464 if (WebContents* contents = tracker_->web_contents()) |
| 626 contents->IncrementCapturerCount(ComputeOptimalViewSize()); | 465 contents->IncrementCapturerCount(ComputeOptimalViewSize()); |
| (...skipping 26 matching lines...) Expand all Loading... |
| 653 void WebContentsCaptureMachine::InternalResume() { | 492 void WebContentsCaptureMachine::InternalResume() { |
| 654 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 493 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 655 if (frame_capture_active_) | 494 if (frame_capture_active_) |
| 656 return; | 495 return; |
| 657 frame_capture_active_ = true; | 496 frame_capture_active_ = true; |
| 658 if (IsStarted()) | 497 if (IsStarted()) |
| 659 RenewFrameSubscription(true); | 498 RenewFrameSubscription(true); |
| 660 } | 499 } |
| 661 | 500 |
| 662 void WebContentsCaptureMachine::Stop(const base::Closure& callback) { | 501 void WebContentsCaptureMachine::Stop(const base::Closure& callback) { |
| 663 // Stops the capture machine asynchronously. | |
| 664 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, | 502 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, |
| 665 base::Bind(&WebContentsCaptureMachine::InternalStop, | 503 base::Bind(&WebContentsCaptureMachine::InternalStop, |
| 666 base::Unretained(this), callback)); | 504 base::Unretained(this), callback)); |
| 667 } | 505 } |
| 668 | 506 |
| 669 void WebContentsCaptureMachine::InternalStop(const base::Closure& callback) { | 507 void WebContentsCaptureMachine::InternalStop(const base::Closure& callback) { |
| 670 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 508 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 671 | 509 |
| 672 if (!IsStarted()) { | 510 base::ScopedClosureRunner done_runner(callback); |
| 673 callback.Run(); | 511 |
| 512 if (!IsStarted()) |
| 674 return; | 513 return; |
| 675 } | |
| 676 | 514 |
| 677 // The following cancels any outstanding callbacks and causes IsStarted() to | 515 // The following cancels any outstanding callbacks and causes IsStarted() to |
| 678 // return false from here onward. | 516 // return false from here onward. |
| 679 weak_ptr_factory_.InvalidateWeakPtrs(); | 517 weak_ptr_factory_.InvalidateWeakPtrs(); |
| 680 | 518 |
| 681 RenewFrameSubscription(false); | 519 RenewFrameSubscription(false); |
| 682 if (WebContents* contents = tracker_->web_contents()) | 520 if (WebContents* contents = tracker_->web_contents()) |
| 683 contents->DecrementCapturerCount(); | 521 contents->DecrementCapturerCount(); |
| 684 tracker_->Stop(); | 522 tracker_->Stop(); |
| 685 | |
| 686 // The render thread cannot be stopped on the UI thread, so post a message | |
| 687 // to the thread pool used for blocking operations. | |
| 688 if (render_thread_) { | |
| 689 BrowserThread::PostBlockingPoolTask( | |
| 690 FROM_HERE, base::Bind(&DeleteOnWorkerThread, | |
| 691 base::Passed(&render_thread_), callback)); | |
| 692 } | |
| 693 } | 523 } |
| 694 | 524 |
| 695 void WebContentsCaptureMachine::MaybeCaptureForRefresh() { | 525 void WebContentsCaptureMachine::MaybeCaptureForRefresh() { |
| 696 BrowserThread::PostTask( | 526 BrowserThread::PostTask( |
| 697 BrowserThread::UI, FROM_HERE, | 527 BrowserThread::UI, FROM_HERE, |
| 698 base::Bind(&WebContentsCaptureMachine::InternalMaybeCaptureForRefresh, | 528 base::Bind(&WebContentsCaptureMachine::InternalMaybeCaptureForRefresh, |
| 699 // Use of Unretained() is safe here since this task must run | 529 // Use of Unretained() is safe here since this task must run |
| 700 // before InternalStop(). | 530 // before InternalStop(). |
| 701 base::Unretained(this))); | 531 base::Unretained(this))); |
| 702 } | 532 } |
| 703 | 533 |
| 704 void WebContentsCaptureMachine::InternalMaybeCaptureForRefresh() { | 534 void WebContentsCaptureMachine::InternalMaybeCaptureForRefresh() { |
| 705 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 535 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 706 if (IsStarted() && subscription_) | 536 if (IsStarted() && subscription_) |
| 707 subscription_->MaybeCaptureForRefresh(); | 537 subscription_->MaybeCaptureForRefresh(); |
| 708 } | 538 } |
| 709 | 539 |
| 710 void WebContentsCaptureMachine::Capture( | 540 void WebContentsCaptureMachine::Capture( |
| 711 const base::TimeTicks& start_time, | 541 base::TimeTicks start_time, |
| 712 const scoped_refptr<media::VideoFrame>& target, | 542 scoped_refptr<media::VideoFrame> target, |
| 713 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 543 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
| 714 deliver_frame_cb) { | 544 deliver_frame_cb) { |
| 715 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 545 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 716 | 546 |
| 717 RenderWidgetHost* rwh = tracker_->GetTargetRenderWidgetHost(); | 547 RenderWidgetHost* rwh = tracker_->GetTargetRenderWidgetHost(); |
| 718 RenderWidgetHostViewBase* view = | 548 RenderWidgetHostViewBase* view = |
| 719 rwh ? static_cast<RenderWidgetHostViewBase*>(rwh->GetView()) : NULL; | 549 rwh ? static_cast<RenderWidgetHostViewBase*>(rwh->GetView()) : nullptr; |
| 720 if (!view) { | 550 if (!view || !view->CanCopyToVideoFrame()) { |
| 721 deliver_frame_cb.Run(base::TimeTicks(), gfx::Rect(), false); | 551 deliver_frame_cb.Run(base::TimeTicks(), gfx::Rect(), false); |
| 722 return; | 552 return; |
| 723 } | 553 } |
| 724 | 554 |
| 725 const gfx::Size view_size = view->GetViewBounds().size(); | 555 const gfx::Size view_size = view->GetViewBounds().size(); |
| 726 if (view->CanCopyToVideoFrame()) { | 556 view->CopyFromCompositingSurfaceToVideoFrame( |
| 727 view->CopyFromCompositingSurfaceToVideoFrame( | 557 gfx::Rect(view_size), std::move(target), |
| 728 gfx::Rect(view_size), target, | 558 base::Bind(&WebContentsCaptureMachine::DidCopyToVideoFrame, |
| 729 base::Bind(&WebContentsCaptureMachine:: | 559 weak_ptr_factory_.GetWeakPtr(), start_time, deliver_frame_cb)); |
| 730 DidCopyFromCompositingSurfaceToVideoFrame, | |
| 731 weak_ptr_factory_.GetWeakPtr(), start_time, | |
| 732 deliver_frame_cb)); | |
| 733 } else { | |
| 734 const gfx::Size fitted_size = | |
| 735 view_size.IsEmpty() | |
| 736 ? gfx::Size() | |
| 737 : media::ComputeLetterboxRegion(target->visible_rect(), view_size) | |
| 738 .size(); | |
| 739 rwh->CopyFromBackingStore( | |
| 740 gfx::Rect(), | |
| 741 fitted_size, // Size here is a request not always honored. | |
| 742 base::Bind(&WebContentsCaptureMachine::DidCopyFromBackingStore, | |
| 743 weak_ptr_factory_.GetWeakPtr(), start_time, target, | |
| 744 deliver_frame_cb), | |
| 745 kN32_SkColorType); | |
| 746 } | |
| 747 } | 560 } |
| 748 | 561 |
| 749 gfx::Size WebContentsCaptureMachine::ComputeOptimalViewSize() const { | 562 gfx::Size WebContentsCaptureMachine::ComputeOptimalViewSize() const { |
| 750 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 563 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 751 | 564 |
| 752 // TODO(miu): Propagate capture frame size changes as new "preferred size" | 565 // TODO(miu): Propagate capture frame size changes as new "preferred size" |
| 753 // updates, rather than just using the max frame size. | 566 // updates, rather than just using the max frame size. |
| 754 // http://crbug.com/350491 | 567 // http://crbug.com/350491 |
| 755 gfx::Size optimal_size = oracle_proxy_->max_frame_size(); | 568 gfx::Size optimal_size = oracle_proxy_->max_frame_size(); |
| 756 | 569 |
| (...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 789 break; | 602 break; |
| 790 } | 603 } |
| 791 } | 604 } |
| 792 | 605 |
| 793 // If the ratio between physical and logical pixels is greater than 1:1, | 606 // If the ratio between physical and logical pixels is greater than 1:1, |
| 794 // shrink |optimal_size| by that amount. Then, when external code resizes the | 607 // shrink |optimal_size| by that amount. Then, when external code resizes the |
| 795 // render widget to the "preferred size," the widget will be physically | 608 // render widget to the "preferred size," the widget will be physically |
| 796 // rendered at the exact capture size, thereby eliminating unnecessary scaling | 609 // rendered at the exact capture size, thereby eliminating unnecessary scaling |
| 797 // operations in the graphics pipeline. | 610 // operations in the graphics pipeline. |
| 798 RenderWidgetHost* const rwh = tracker_->GetTargetRenderWidgetHost(); | 611 RenderWidgetHost* const rwh = tracker_->GetTargetRenderWidgetHost(); |
| 799 RenderWidgetHostView* const rwhv = rwh ? rwh->GetView() : NULL; | 612 RenderWidgetHostView* const rwhv = rwh ? rwh->GetView() : nullptr; |
| 800 if (rwhv) { | 613 if (rwhv) { |
| 801 const gfx::NativeView view = rwhv->GetNativeView(); | 614 const gfx::NativeView view = rwhv->GetNativeView(); |
| 802 const float scale = ui::GetScaleFactorForNativeView(view); | 615 const float scale = ui::GetScaleFactorForNativeView(view); |
| 803 if (scale > 1.0f) { | 616 if (scale > 1.0f) { |
| 804 const gfx::Size shrunk_size = | 617 const gfx::Size shrunk_size = |
| 805 gfx::ScaleToFlooredSize(optimal_size, 1.0f / scale); | 618 gfx::ScaleToFlooredSize(optimal_size, 1.0f / scale); |
| 806 if (shrunk_size.width() > 0 && shrunk_size.height() > 0) | 619 if (shrunk_size.width() > 0 && shrunk_size.height() > 0) |
| 807 optimal_size = shrunk_size; | 620 optimal_size = shrunk_size; |
| 808 } | 621 } |
| 809 } | 622 } |
| 810 | 623 |
| 811 VLOG(1) << "Computed optimal target size: " << optimal_size.ToString(); | 624 VLOG(1) << "Computed optimal target size: " << optimal_size.ToString(); |
| 812 return optimal_size; | 625 return optimal_size; |
| 813 } | 626 } |
| 814 | 627 |
| 815 void WebContentsCaptureMachine::DidCopyFromBackingStore( | 628 void WebContentsCaptureMachine::DidCopyToVideoFrame( |
| 816 const base::TimeTicks& start_time, | |
| 817 const scoped_refptr<media::VideoFrame>& target, | |
| 818 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | |
| 819 deliver_frame_cb, | |
| 820 const SkBitmap& bitmap, | |
| 821 ReadbackResponse response) { | |
| 822 DCHECK_CURRENTLY_ON(BrowserThread::UI); | |
| 823 | |
| 824 DCHECK(render_thread_); | |
| 825 if (response == READBACK_SUCCESS) { | |
| 826 TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", target.get(), | |
| 827 "Render"); | |
| 828 render_thread_->task_runner()->PostTask( | |
| 829 FROM_HERE, media::BindToCurrentLoop( | |
| 830 base::Bind(&RenderVideoFrame, bitmap, target, | |
| 831 base::Bind(deliver_frame_cb, start_time)))); | |
| 832 } else { | |
| 833 // Capture can fail due to transient issues, so just skip this frame. | |
| 834 DVLOG(1) << "CopyFromBackingStore failed; skipping frame."; | |
| 835 deliver_frame_cb.Run(start_time, gfx::Rect(), false); | |
| 836 } | |
| 837 } | |
| 838 | |
| 839 void WebContentsCaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame( | |
| 840 const base::TimeTicks& start_time, | 629 const base::TimeTicks& start_time, |
| 841 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 630 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
| 842 deliver_frame_cb, | 631 deliver_frame_cb, |
| 843 const gfx::Rect& region_in_frame, | 632 const gfx::Rect& region_in_frame, |
| 844 bool success) { | 633 bool success) { |
| 845 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 634 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 846 | 635 |
| 847 // Capture can fail due to transient issues, so just skip this frame. | 636 // Capture can fail due to transient issues, so just skip this frame. |
| 848 DVLOG_IF(1, !success) << "CopyFromCompositingSurface failed; skipping frame."; | 637 DVLOG_IF(1, !success) << "CopyFromCompositingSurface failed; skipping frame."; |
| 849 deliver_frame_cb.Run(start_time, region_in_frame, success); | 638 deliver_frame_cb.Run(start_time, region_in_frame, success); |
| (...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 918 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() { | 707 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() { |
| 919 DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying."; | 708 DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying."; |
| 920 } | 709 } |
| 921 | 710 |
| 922 // static | 711 // static |
| 923 std::unique_ptr<media::VideoCaptureDevice> | 712 std::unique_ptr<media::VideoCaptureDevice> |
| 924 WebContentsVideoCaptureDevice::Create(const std::string& device_id) { | 713 WebContentsVideoCaptureDevice::Create(const std::string& device_id) { |
| 925 // Parse device_id into render_process_id and main_render_frame_id. | 714 // Parse device_id into render_process_id and main_render_frame_id. |
| 926 WebContentsMediaCaptureId media_id; | 715 WebContentsMediaCaptureId media_id; |
| 927 if (!WebContentsMediaCaptureId::Parse(device_id, &media_id)) { | 716 if (!WebContentsMediaCaptureId::Parse(device_id, &media_id)) { |
| 928 return NULL; | 717 return nullptr; |
| 929 } | 718 } |
| 930 | 719 |
| 931 return std::unique_ptr<media::VideoCaptureDevice>( | 720 return std::unique_ptr<media::VideoCaptureDevice>( |
| 932 new WebContentsVideoCaptureDevice(media_id.render_process_id, | 721 new WebContentsVideoCaptureDevice(media_id.render_process_id, |
| 933 media_id.main_render_frame_id, | 722 media_id.main_render_frame_id, |
| 934 media_id.enable_auto_throttling)); | 723 media_id.enable_auto_throttling)); |
| 935 } | 724 } |
| 936 | 725 |
| 937 void WebContentsVideoCaptureDevice::AllocateAndStart( | 726 void WebContentsVideoCaptureDevice::AllocateAndStart( |
| 938 const media::VideoCaptureParams& params, | 727 const media::VideoCaptureParams& params, |
| (...skipping 17 matching lines...) Expand all Loading... |
| 956 void WebContentsVideoCaptureDevice::StopAndDeAllocate() { | 745 void WebContentsVideoCaptureDevice::StopAndDeAllocate() { |
| 957 core_->StopAndDeAllocate(); | 746 core_->StopAndDeAllocate(); |
| 958 } | 747 } |
| 959 | 748 |
| 960 void WebContentsVideoCaptureDevice::OnUtilizationReport(int frame_feedback_id, | 749 void WebContentsVideoCaptureDevice::OnUtilizationReport(int frame_feedback_id, |
| 961 double utilization) { | 750 double utilization) { |
| 962 core_->OnConsumerReportingUtilization(frame_feedback_id, utilization); | 751 core_->OnConsumerReportingUtilization(frame_feedback_id, utilization); |
| 963 } | 752 } |
| 964 | 753 |
| 965 } // namespace content | 754 } // namespace content |
| OLD | NEW |