| OLD | NEW |
| 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 // | |
| 5 // Implementation notes: This needs to work on a variety of hardware | |
| 6 // configurations where the speed of the CPU and GPU greatly affect overall | |
| 7 // performance. Spanning several threads, the process of capturing has been | |
| 8 // split up into four conceptual stages: | |
| 9 // | |
| 10 // 1. Reserve Buffer: Before a frame can be captured, a slot in the client's | |
| 11 // shared-memory IPC buffer is reserved. There are only a few of these; | |
| 12 // when they run out, it indicates that the downstream client -- likely a | |
| 13 // video encoder -- is the performance bottleneck, and that the rate of | |
| 14 // frame capture should be throttled back. | |
| 15 // | |
| 16 // 2. Capture: A bitmap is snapshotted/copied from the RenderWidget's backing | |
| 17 // store. This is initiated on the UI BrowserThread, and often occurs | |
| 18 // asynchronously. Where supported, the GPU scales and color converts | |
| 19 // frames to our desired size, and the readback happens directly into the | |
| 20 // shared-memory buffer. But this is not always possible, particularly when | |
| 21 // accelerated compositing is disabled. | |
| 22 // | |
| 23 // 3. Render (if needed): If the web contents cannot be captured directly into | |
| 24 // our target size and color format, scaling and colorspace conversion must | |
| 25 // be done on the CPU. A dedicated thread is used for this operation, to | |
| 26 // avoid blocking the UI thread. The Render stage always reads from a | |
| 27 // bitmap returned by Capture, and writes into the reserved slot in the | |
| 28 // shared-memory buffer. | |
| 29 // | |
| 30 // 4. Deliver: The rendered video frame is returned to the client (which | |
| 31 // implements the VideoCaptureDevice::Client interface). Because all | |
| 32 // paths have written the frame into the IPC buffer, this step should | |
| 33 // never need to do an additional copy of the pixel data. | |
| 34 // | |
| 35 // In the best-performing case, the Render step is bypassed: Capture produces | |
| 36 // ready-to-Deliver frames. But when accelerated readback is not possible, the | |
| 37 // system is designed so that Capture and Render may run concurrently. A timing | |
| 38 // diagram helps illustrate this point (@30 FPS): | |
| 39 // | |
| 40 // Time: 0ms 33ms 66ms 99ms | |
| 41 // thread1: |-Capture-f1------v |-Capture-f2------v |-Capture-f3----v |-Capt | |
| 42 // thread2: |-Render-f1-----v |-Render-f2-----v |-Render-f3 | |
| 43 // | |
| 44 // In the above example, both capturing and rendering *each* take almost the | |
| 45 // full 33 ms available between frames, yet we see that the required throughput | |
| 46 // is obtained. | |
| 47 | 4 |
| 48 #include "content/browser/media/capture/web_contents_video_capture_device.h" | 5 #include "content/browser/media/capture/web_contents_video_capture_device.h" |
| 49 | 6 |
| 50 #include <stdint.h> | 7 #include <stdint.h> |
| 51 | 8 |
| 52 #include <algorithm> | 9 #include <algorithm> |
| 53 #include <memory> | 10 #include <memory> |
| 54 #include <utility> | 11 #include <string> |
| 55 | 12 |
| 56 #include "base/bind.h" | 13 #include "base/bind.h" |
| 57 #include "base/callback_helpers.h" | 14 #include "base/callback_helpers.h" |
| 58 #include "base/location.h" | 15 #include "base/location.h" |
| 59 #include "base/logging.h" | 16 #include "base/logging.h" |
| 60 #include "base/macros.h" | 17 #include "base/macros.h" |
| 61 #include "base/memory/ptr_util.h" | |
| 62 #include "base/memory/weak_ptr.h" | 18 #include "base/memory/weak_ptr.h" |
| 63 #include "base/sequenced_task_runner.h" | |
| 64 #include "base/single_thread_task_runner.h" | |
| 65 #include "base/threading/thread.h" | |
| 66 #include "base/threading/thread_checker.h" | |
| 67 #include "base/time/time.h" | 19 #include "base/time/time.h" |
| 68 #include "build/build_config.h" | 20 #include "build/build_config.h" |
| 69 #include "content/browser/media/capture/cursor_renderer.h" | 21 #include "content/browser/media/capture/cursor_renderer.h" |
| 70 #include "content/browser/media/capture/web_contents_tracker.h" | 22 #include "content/browser/media/capture/web_contents_tracker.h" |
| 71 #include "content/browser/media/capture/window_activity_tracker.h" | 23 #include "content/browser/media/capture/window_activity_tracker.h" |
| 72 #include "content/browser/renderer_host/render_widget_host_impl.h" | 24 #include "content/browser/renderer_host/render_widget_host_impl.h" |
| 73 #include "content/browser/renderer_host/render_widget_host_view_base.h" | 25 #include "content/browser/renderer_host/render_widget_host_view_base.h" |
| 74 #include "content/public/browser/browser_thread.h" | 26 #include "content/public/browser/browser_thread.h" |
| 75 #include "content/public/browser/render_process_host.h" | 27 #include "content/public/browser/render_process_host.h" |
| 76 #include "content/public/browser/render_widget_host_view.h" | 28 #include "content/public/browser/render_widget_host_view.h" |
| 77 #include "content/public/browser/render_widget_host_view_frame_subscriber.h" | 29 #include "content/public/browser/render_widget_host_view_frame_subscriber.h" |
| 78 #include "content/public/browser/web_contents.h" | 30 #include "content/public/browser/web_contents.h" |
| 79 #include "content/public/browser/web_contents_media_capture_id.h" | 31 #include "content/public/browser/web_contents_media_capture_id.h" |
| 80 #include "media/base/bind_to_current_loop.h" | |
| 81 #include "media/base/video_frame_metadata.h" | 32 #include "media/base/video_frame_metadata.h" |
| 82 #include "media/base/video_util.h" | |
| 83 #include "media/capture/content/screen_capture_device_core.h" | 33 #include "media/capture/content/screen_capture_device_core.h" |
| 84 #include "media/capture/content/thread_safe_capture_oracle.h" | 34 #include "media/capture/content/thread_safe_capture_oracle.h" |
| 85 #include "media/capture/content/video_capture_oracle.h" | 35 #include "media/capture/content/video_capture_oracle.h" |
| 86 #include "media/capture/video_capture_types.h" | 36 #include "media/capture/video_capture_types.h" |
| 87 #include "skia/ext/image_operations.h" | |
| 88 #include "third_party/skia/include/core/SkBitmap.h" | |
| 89 #include "third_party/skia/include/core/SkColor.h" | |
| 90 #include "ui/base/layout.h" | 37 #include "ui/base/layout.h" |
| 91 #include "ui/gfx/geometry/dip_util.h" | 38 #include "ui/gfx/geometry/dip_util.h" |
| 92 #include "ui/gfx/geometry/size_conversions.h" | 39 #include "ui/gfx/geometry/size_conversions.h" |
| 93 | 40 |
| 94 namespace content { | 41 namespace content { |
| 95 | 42 |
| 96 namespace { | 43 namespace { |
| 97 | 44 |
| 98 enum InteractiveModeSettings { | 45 // Minimum amount of time for which there should be no animation detected |
| 99 // Minimum amount of time for which there should be no animation detected | 46 // to consider interactive mode being active. This is to prevent very brief |
| 100 // to consider interactive mode being active. This is to prevent very brief | 47 // periods of animated content not being detected (due to CPU fluctations for |
| 101 // periods of animated content not being detected (due to CPU fluctations for | 48 // example) from causing a flip-flop on interactive mode. |
| 102 // example) from causing a flip-flop on interactive mode. | 49 constexpr int64_t kMinPeriodNoAnimationMillis = 3000; |
| 103 kMinPeriodNoAnimationMillis = 3000 | |
| 104 }; | |
| 105 | |
| 106 void DeleteOnWorkerThread(std::unique_ptr<base::Thread> render_thread, | |
| 107 const base::Closure& callback) { | |
| 108 render_thread.reset(); | |
| 109 | |
| 110 // After thread join call the callback on UI thread. | |
| 111 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, callback); | |
| 112 } | |
| 113 | 50 |
| 114 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible | 51 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible |
| 115 // with RenderWidgetHostViewFrameSubscriber. We create one per event type. | 52 // with RenderWidgetHostViewFrameSubscriber. One is created per event type. |
| 116 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber { | 53 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber { |
| 117 public: | 54 public: |
| 118 FrameSubscriber(media::VideoCaptureOracle::Event event_type, | 55 FrameSubscriber(media::VideoCaptureOracle::Event event_type, |
| 119 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle, | 56 scoped_refptr<media::ThreadSafeCaptureOracle> oracle, |
| 120 base::WeakPtr<content::CursorRenderer> cursor_renderer, | 57 base::WeakPtr<content::CursorRenderer> cursor_renderer, |
| 121 base::WeakPtr<content::WindowActivityTracker> tracker) | 58 base::WeakPtr<content::WindowActivityTracker> tracker) |
| 122 : event_type_(event_type), | 59 : event_type_(event_type), |
| 123 oracle_proxy_(oracle), | 60 oracle_proxy_(std::move(oracle)), |
| 124 cursor_renderer_(cursor_renderer), | 61 cursor_renderer_(cursor_renderer), |
| 125 window_activity_tracker_(tracker), | 62 window_activity_tracker_(tracker), |
| 126 weak_ptr_factory_(this) {} | 63 weak_ptr_factory_(this) {} |
| 127 | 64 |
| 128 bool ShouldCaptureFrame( | 65 bool ShouldCaptureFrame( |
| 129 const gfx::Rect& damage_rect, | 66 const gfx::Rect& damage_rect, |
| 130 base::TimeTicks present_time, | 67 base::TimeTicks present_time, |
| 131 scoped_refptr<media::VideoFrame>* storage, | 68 scoped_refptr<media::VideoFrame>* storage, |
| 132 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback* | 69 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback* |
| 133 deliver_frame_cb) override; | 70 deliver_frame_cb) override; |
| 134 | 71 |
| 135 static void DidCaptureFrame( | 72 static void DidCaptureFrame( |
| 136 base::WeakPtr<FrameSubscriber> frame_subscriber_, | 73 base::WeakPtr<FrameSubscriber> frame_subscriber_, |
| 137 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& | 74 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& |
| 138 capture_frame_cb, | 75 capture_frame_cb, |
| 139 scoped_refptr<media::VideoFrame> frame, | 76 scoped_refptr<media::VideoFrame> frame, |
| 140 base::TimeTicks timestamp, | 77 base::TimeTicks timestamp, |
| 141 const gfx::Rect& region_in_frame, | 78 const gfx::Rect& region_in_frame, |
| 142 bool success); | 79 bool success); |
| 143 | 80 |
| 144 bool IsUserInteractingWithContent(); | 81 bool IsUserInteractingWithContent(); |
| 145 | 82 |
| 146 private: | 83 private: |
| 147 const media::VideoCaptureOracle::Event event_type_; | 84 const media::VideoCaptureOracle::Event event_type_; |
| 148 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; | 85 const scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; |
| 149 // We need a weak pointer since FrameSubscriber is owned externally and | 86 // We need a weak pointer since FrameSubscriber is owned externally and |
| 150 // may outlive the cursor renderer. | 87 // may outlive the cursor renderer. |
| 151 base::WeakPtr<CursorRenderer> cursor_renderer_; | 88 const base::WeakPtr<CursorRenderer> cursor_renderer_; |
| 152 // We need a weak pointer since FrameSubscriber is owned externally and | 89 // We need a weak pointer since FrameSubscriber is owned externally and |
| 153 // may outlive the ui activity tracker. | 90 // may outlive the ui activity tracker. |
| 154 base::WeakPtr<WindowActivityTracker> window_activity_tracker_; | 91 const base::WeakPtr<WindowActivityTracker> window_activity_tracker_; |
| 155 base::WeakPtrFactory<FrameSubscriber> weak_ptr_factory_; | 92 base::WeakPtrFactory<FrameSubscriber> weak_ptr_factory_; |
| 156 }; | 93 }; |
| 157 | 94 |
| 158 // ContentCaptureSubscription is the relationship between a RenderWidgetHostView | 95 // ContentCaptureSubscription is the relationship between a RenderWidgetHostView |
| 159 // whose content is updating, a subscriber that is deciding which of these | 96 // whose content is updating, a subscriber that is deciding which of these |
| 160 // updates to capture (and where to deliver them to), and a callback that | 97 // updates to capture (and where to deliver them to), and a callback that |
| 161 // knows how to do the capture and prepare the result for delivery. | 98 // knows how to do the capture and prepare the result for delivery. |
| 162 // | 99 // |
| 163 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in | 100 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in |
| 164 // the RenderWidgetHostView, to process compositor updates, and (b) occasionally | 101 // the RenderWidgetHostView, to process compositor updates, and (b) occasionally |
| 165 // initiating forced, non-event-driven captures needed by downstream consumers | 102 // using the capture callback to force additional captures that are needed for |
| 166 // that request "refresh frames" of unchanged content. | 103 // things like mouse cursor rendering updates and/or refresh frames. |
| 167 // | 104 // |
| 168 // All of this happens on the UI thread, although the | 105 // All of this happens on the UI thread, although the |
| 169 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates | 106 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates |
| 170 // autonomously on some other thread. | 107 // autonomously on some other thread. |
| 171 class ContentCaptureSubscription { | 108 class ContentCaptureSubscription { |
| 172 public: | 109 public: |
| 173 typedef base::Callback<void( | 110 using CaptureCallback = base::Callback<void( |
| 174 const base::TimeTicks&, | 111 base::TimeTicks, |
| 175 const scoped_refptr<media::VideoFrame>&, | 112 scoped_refptr<media::VideoFrame>, |
| 176 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&)> | 113 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&)>; |
| 177 CaptureCallback; | |
| 178 | 114 |
| 179 // Create a subscription. Whenever a manual capture is required, the | 115 // Create a subscription. Whenever a non-compositor capture is required, the |
| 180 // subscription will invoke |capture_callback| on the UI thread to do the | 116 // subscription will invoke |capture_callback| on the UI thread to do the |
| 181 // work. | 117 // work. |
| 182 ContentCaptureSubscription( | 118 ContentCaptureSubscription( |
| 183 base::WeakPtr<RenderWidgetHostViewBase> source_view, | 119 base::WeakPtr<RenderWidgetHostViewBase> source_view, |
| 184 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 120 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
| 185 const CaptureCallback& capture_callback); | 121 const CaptureCallback& capture_callback); |
| 186 ~ContentCaptureSubscription(); | 122 ~ContentCaptureSubscription(); |
| 187 | 123 |
| 124 // Called when a condition occurs that requires a refresh frame be |
| 125 // captured. The subscribers will determine whether a frame was recently |
| 126 // captured, or if a non-compositor initiated capture must be made. |
| 188 void MaybeCaptureForRefresh(); | 127 void MaybeCaptureForRefresh(); |
| 189 | 128 |
| 190 private: | 129 private: |
| 191 // Called for active frame refresh requests, or mouse activity events. | 130 // Called for active frame refresh requests, or mouse activity events. |
| 192 void OnEvent(FrameSubscriber* subscriber); | 131 void OnEvent(FrameSubscriber* subscriber); |
| 193 | 132 |
| 194 const base::WeakPtr<RenderWidgetHostViewBase> source_view_; | 133 const base::WeakPtr<RenderWidgetHostViewBase> source_view_; |
| 195 | 134 |
| 196 std::unique_ptr<FrameSubscriber> refresh_subscriber_; | 135 std::unique_ptr<FrameSubscriber> refresh_subscriber_; |
| 197 std::unique_ptr<FrameSubscriber> mouse_activity_subscriber_; | 136 std::unique_ptr<FrameSubscriber> mouse_activity_subscriber_; |
| 198 CaptureCallback capture_callback_; | 137 CaptureCallback capture_callback_; |
| 199 | 138 |
| 200 // Responsible for tracking the cursor state and input events to make | 139 // Responsible for tracking the cursor state and input events to make |
| 201 // decisions and then render the mouse cursor on the video frame after | 140 // decisions and then render the mouse cursor on the video frame after |
| 202 // capture is completed. | 141 // capture is completed. |
| 203 std::unique_ptr<content::CursorRenderer> cursor_renderer_; | 142 std::unique_ptr<content::CursorRenderer> cursor_renderer_; |
| 204 | 143 |
| 205 // Responsible for tracking the UI events and making a decision on whether | 144 // Responsible for tracking the UI events and making a decision on whether |
| 206 // user is actively interacting with content. | 145 // user is actively interacting with content. |
| 207 std::unique_ptr<content::WindowActivityTracker> window_activity_tracker_; | 146 std::unique_ptr<content::WindowActivityTracker> window_activity_tracker_; |
| 208 | 147 |
| 209 DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription); | 148 DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription); |
| 210 }; | 149 }; |
| 211 | 150 |
| 212 // Render the SkBitmap |input| into the given VideoFrame buffer |output|, then | |
| 213 // invoke |done_cb| to indicate success or failure. |input| is expected to be | |
| 214 // ARGB. |output| must be YV12 or I420. Colorspace conversion is always done. | |
| 215 // Scaling and letterboxing will be done as needed. | |
| 216 // | |
| 217 // This software implementation should be used only when GPU acceleration of | |
| 218 // these activities is not possible. This operation may be expensive (tens to | |
| 219 // hundreds of milliseconds), so the caller should ensure that it runs on a | |
| 220 // thread where such a pause would cause UI jank. | |
| 221 void RenderVideoFrame( | |
| 222 const SkBitmap& input, | |
| 223 const scoped_refptr<media::VideoFrame>& output, | |
| 224 const base::Callback<void(const gfx::Rect&, bool)>& done_cb); | |
| 225 | |
| 226 // Renews capture subscriptions based on feedback from WebContentsTracker, and | 151 // Renews capture subscriptions based on feedback from WebContentsTracker, and |
| 227 // also executes copying of the backing store on the UI BrowserThread. | 152 // also executes non-compositor-initiated frame captures. |
| 228 class WebContentsCaptureMachine : public media::VideoCaptureMachine { | 153 class WebContentsCaptureMachine : public media::VideoCaptureMachine { |
| 229 public: | 154 public: |
| 230 WebContentsCaptureMachine(int render_process_id, | 155 WebContentsCaptureMachine(int render_process_id, |
| 231 int main_render_frame_id, | 156 int main_render_frame_id, |
| 232 bool enable_auto_throttling); | 157 bool enable_auto_throttling); |
| 233 ~WebContentsCaptureMachine() override; | 158 ~WebContentsCaptureMachine() override; |
| 234 | 159 |
| 235 // VideoCaptureMachine overrides. | 160 // VideoCaptureMachine overrides. |
| 236 void Start(const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 161 void Start(const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, |
| 237 const media::VideoCaptureParams& params, | 162 const media::VideoCaptureParams& params, |
| 238 const base::Callback<void(bool)> callback) override; | 163 const base::Callback<void(bool)> callback) override; |
| 239 void Suspend() override; | 164 void Suspend() override; |
| 240 void Resume() override; | 165 void Resume() override; |
| 241 void Stop(const base::Closure& callback) override; | 166 void Stop(const base::Closure& callback) override; |
| 242 bool IsAutoThrottlingEnabled() const override { | 167 bool IsAutoThrottlingEnabled() const override { |
| 243 return auto_throttling_enabled_; | 168 return auto_throttling_enabled_; |
| 244 } | 169 } |
| 245 void MaybeCaptureForRefresh() override; | 170 void MaybeCaptureForRefresh() override; |
| 246 | 171 |
| 247 // Starts a copy from the backing store or the composited surface. Must be run | 172 private: |
| 248 // on the UI BrowserThread. |deliver_frame_cb| will be run when the operation | 173 // Called for cases where events other than compositor updates require a frame |
| 249 // completes. The copy will occur to |target|. | 174 // capture from the composited surface. Must be run on the UI BrowserThread. |
| 175 // |deliver_frame_cb| will be run when the operation completes. The copy will |
| 176 // populate |target|. |
| 250 // | 177 // |
| 251 // This may be used as a ContentCaptureSubscription::CaptureCallback. | 178 // This method is only used by a ContentCaptureSubscription::CaptureCallback. |
| 252 void Capture(const base::TimeTicks& start_time, | 179 void Capture(base::TimeTicks start_time, |
| 253 const scoped_refptr<media::VideoFrame>& target, | 180 scoped_refptr<media::VideoFrame> target, |
| 254 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 181 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
| 255 deliver_frame_cb); | 182 deliver_frame_cb); |
| 256 | 183 |
| 257 private: | 184 bool InternalStart(scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
| 258 bool InternalStart( | 185 const media::VideoCaptureParams& params); |
| 259 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | |
| 260 const media::VideoCaptureParams& params); | |
| 261 void InternalSuspend(); | 186 void InternalSuspend(); |
| 262 void InternalResume(); | 187 void InternalResume(); |
| 263 void InternalStop(const base::Closure& callback); | 188 void InternalStop(const base::Closure& callback); |
| 264 void InternalMaybeCaptureForRefresh(); | 189 void InternalMaybeCaptureForRefresh(); |
| 265 bool IsStarted() const; | 190 bool IsStarted() const; |
| 266 | 191 |
| 267 // Computes the preferred size of the target RenderWidget for optimal capture. | 192 // Computes the preferred size of the target RenderWidget for optimal capture. |
| 268 gfx::Size ComputeOptimalViewSize() const; | 193 gfx::Size ComputeOptimalViewSize() const; |
| 269 | 194 |
| 270 // Response callback for RenderWidgetHost::CopyFromBackingStore(). | |
| 271 void DidCopyFromBackingStore( | |
| 272 const base::TimeTicks& start_time, | |
| 273 const scoped_refptr<media::VideoFrame>& target, | |
| 274 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | |
| 275 deliver_frame_cb, | |
| 276 const SkBitmap& bitmap, | |
| 277 ReadbackResponse response); | |
| 278 | |
| 279 // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame(). | 195 // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame(). |
| 280 void DidCopyFromCompositingSurfaceToVideoFrame( | 196 void DidCopyToVideoFrame( |
| 281 const base::TimeTicks& start_time, | 197 const base::TimeTicks& start_time, |
| 282 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 198 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
| 283 deliver_frame_cb, | 199 deliver_frame_cb, |
| 284 const gfx::Rect& region_in_frame, | 200 const gfx::Rect& region_in_frame, |
| 285 bool success); | 201 bool success); |
| 286 | 202 |
| 287 // Remove the old subscription, and attempt to start a new one. If | 203 // Remove the old subscription, and attempt to start a new one. If |
| 288 // |is_still_tracking| is false, emit an error rather than attempt to start a | 204 // |is_still_tracking| is false, emit an error rather than attempt to start a |
| 289 // new subscription. | 205 // new subscription. |
| 290 void RenewFrameSubscription(bool is_still_tracking); | 206 void RenewFrameSubscription(bool is_still_tracking); |
| 291 | 207 |
| 292 // Called whenever the render widget is resized. | 208 // Called whenever the render widget is resized. |
| 293 void UpdateCaptureSize(); | 209 void UpdateCaptureSize(); |
| 294 | 210 |
| 295 // Parameters saved in constructor. | 211 // Parameters saved in constructor. |
| 296 const int initial_render_process_id_; | 212 const int initial_render_process_id_; |
| 297 const int initial_main_render_frame_id_; | 213 const int initial_main_render_frame_id_; |
| 298 | 214 |
| 299 // Tracks events and calls back to RenewFrameSubscription() to maintain | 215 // Tracks events and calls back to RenewFrameSubscription() to maintain |
| 300 // capture on the correct RenderWidgetHostView. | 216 // capture on the correct RenderWidgetHostView. |
| 301 const scoped_refptr<WebContentsTracker> tracker_; | 217 const scoped_refptr<WebContentsTracker> tracker_; |
| 302 | 218 |
| 303 // Set to false to prevent the capture size from automatically adjusting in | 219 // Set to false to prevent the capture size from automatically adjusting in |
| 304 // response to end-to-end utilization. This is enabled via the throttling | 220 // response to end-to-end utilization. This is enabled via the throttling |
| 305 // option in the WebContentsVideoCaptureDevice device ID. | 221 // option in the WebContentsVideoCaptureDevice device ID. |
| 306 const bool auto_throttling_enabled_; | 222 const bool auto_throttling_enabled_; |
| 307 | 223 |
| 308 // A dedicated worker thread on which SkBitmap->VideoFrame conversion will | |
| 309 // occur. Only used when this activity cannot be done on the GPU. | |
| 310 std::unique_ptr<base::Thread> render_thread_; | |
| 311 | |
| 312 // Makes all the decisions about which frames to copy, and how. | 224 // Makes all the decisions about which frames to copy, and how. |
| 313 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; | 225 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; |
| 314 | 226 |
| 315 // Video capture parameters that this machine is started with. | 227 // Video capture parameters that this machine is started with. |
| 316 media::VideoCaptureParams capture_params_; | 228 media::VideoCaptureParams capture_params_; |
| 317 | 229 |
| 318 // Responsible for forwarding events from the active RenderWidgetHost to the | 230 // Responsible for forwarding events from the active RenderWidgetHost to the |
| 319 // oracle, and initiating captures accordingly. | 231 // oracle, and initiating captures accordingly. |
| 320 std::unique_ptr<ContentCaptureSubscription> subscription_; | 232 std::unique_ptr<ContentCaptureSubscription> subscription_; |
| 321 | 233 |
| (...skipping 11 matching lines...) Expand all Loading... |
| 333 | 245 |
| 334 bool FrameSubscriber::ShouldCaptureFrame( | 246 bool FrameSubscriber::ShouldCaptureFrame( |
| 335 const gfx::Rect& damage_rect, | 247 const gfx::Rect& damage_rect, |
| 336 base::TimeTicks present_time, | 248 base::TimeTicks present_time, |
| 337 scoped_refptr<media::VideoFrame>* storage, | 249 scoped_refptr<media::VideoFrame>* storage, |
| 338 DeliverFrameCallback* deliver_frame_cb) { | 250 DeliverFrameCallback* deliver_frame_cb) { |
| 339 TRACE_EVENT1("gpu.capture", "FrameSubscriber::ShouldCaptureFrame", "instance", | 251 TRACE_EVENT1("gpu.capture", "FrameSubscriber::ShouldCaptureFrame", "instance", |
| 340 this); | 252 this); |
| 341 | 253 |
| 342 media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb; | 254 media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb; |
| 255 if (!oracle_proxy_->ObserveEventAndDecideCapture( |
| 256 event_type_, damage_rect, present_time, storage, &capture_frame_cb)) { |
| 257 return false; |
| 258 } |
| 343 | 259 |
| 344 bool oracle_decision = oracle_proxy_->ObserveEventAndDecideCapture( | 260 DCHECK(*storage); |
| 345 event_type_, damage_rect, present_time, storage, &capture_frame_cb); | 261 DCHECK(!capture_frame_cb.is_null()); |
| 346 | 262 *deliver_frame_cb = |
| 347 if (!capture_frame_cb.is_null()) | 263 base::Bind(&FrameSubscriber::DidCaptureFrame, |
| 348 *deliver_frame_cb = | 264 weak_ptr_factory_.GetWeakPtr(), capture_frame_cb, *storage); |
| 349 base::Bind(&FrameSubscriber::DidCaptureFrame, | 265 return true; |
| 350 weak_ptr_factory_.GetWeakPtr(), capture_frame_cb, *storage); | |
| 351 return oracle_decision; | |
| 352 } | 266 } |
| 353 | 267 |
| 354 void FrameSubscriber::DidCaptureFrame( | 268 void FrameSubscriber::DidCaptureFrame( |
| 355 base::WeakPtr<FrameSubscriber> frame_subscriber_, | 269 base::WeakPtr<FrameSubscriber> frame_subscriber_, |
| 356 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& | 270 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& |
| 357 capture_frame_cb, | 271 capture_frame_cb, |
| 358 scoped_refptr<media::VideoFrame> frame, | 272 scoped_refptr<media::VideoFrame> frame, |
| 359 base::TimeTicks timestamp, | 273 base::TimeTicks timestamp, |
| 360 const gfx::Rect& region_in_frame, | 274 const gfx::Rect& region_in_frame, |
| 361 bool success) { | 275 bool success) { |
| 362 // We can get a callback in the shutdown sequence for the browser main loop | |
| 363 // and this can result in a DCHECK failure. Avoid this by doing DCHECK only | |
| 364 // on success. | |
| 365 if (success) { | 276 if (success) { |
| 277 // We can get a callback in the shutdown sequence for the browser main loop |
| 278 // and this can result in a DCHECK failure. Avoid this by doing DCHECK only |
| 279 // on success. |
| 366 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 280 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 367 // TODO(isheriff): Unclear if taking a snapshot of cursor here affects user | 281 |
| 368 // experience in any particular scenarios. Doing it prior to capture may | |
| 369 // require evaluating region_in_frame in this file. | |
| 370 if (frame_subscriber_ && frame_subscriber_->cursor_renderer_) { | 282 if (frame_subscriber_ && frame_subscriber_->cursor_renderer_) { |
| 371 CursorRenderer* cursor_renderer = | 283 CursorRenderer* cursor_renderer = |
| 372 frame_subscriber_->cursor_renderer_.get(); | 284 frame_subscriber_->cursor_renderer_.get(); |
| 373 if (cursor_renderer->SnapshotCursorState(region_in_frame)) | 285 if (cursor_renderer->SnapshotCursorState(region_in_frame)) |
| 374 cursor_renderer->RenderOnVideoFrame(frame); | 286 cursor_renderer->RenderOnVideoFrame(frame); |
| 375 frame->metadata()->SetBoolean( | 287 frame->metadata()->SetBoolean( |
| 376 media::VideoFrameMetadata::INTERACTIVE_CONTENT, | 288 media::VideoFrameMetadata::INTERACTIVE_CONTENT, |
| 377 frame_subscriber_->IsUserInteractingWithContent()); | 289 frame_subscriber_->IsUserInteractingWithContent()); |
| 378 } | 290 } |
| 379 } | 291 } |
| 380 capture_frame_cb.Run(std::move(frame), timestamp, success); | 292 capture_frame_cb.Run(std::move(frame), timestamp, success); |
| 381 } | 293 } |
| 382 | 294 |
| 383 bool FrameSubscriber::IsUserInteractingWithContent() { | 295 bool FrameSubscriber::IsUserInteractingWithContent() { |
| 296 bool ui_activity = window_activity_tracker_ && |
| 297 window_activity_tracker_->IsUiInteractionActive(); |
| 384 bool interactive_mode = false; | 298 bool interactive_mode = false; |
| 385 bool ui_activity = false; | |
| 386 if (window_activity_tracker_) { | |
| 387 ui_activity = window_activity_tracker_->IsUiInteractionActive(); | |
| 388 } | |
| 389 if (cursor_renderer_) { | 299 if (cursor_renderer_) { |
| 390 bool animation_active = | 300 bool animation_active = |
| 391 (base::TimeTicks::Now() - | 301 (base::TimeTicks::Now() - |
| 392 oracle_proxy_->last_time_animation_was_detected()) < | 302 oracle_proxy_->last_time_animation_was_detected()) < |
| 393 base::TimeDelta::FromMilliseconds(kMinPeriodNoAnimationMillis); | 303 base::TimeDelta::FromMilliseconds(kMinPeriodNoAnimationMillis); |
| 394 if (ui_activity && !animation_active) { | 304 if (ui_activity && !animation_active) { |
| 395 interactive_mode = true; | 305 interactive_mode = true; |
| 396 } else if (animation_active && window_activity_tracker_.get()) { | 306 } else if (animation_active && window_activity_tracker_) { |
| 397 window_activity_tracker_->Reset(); | 307 window_activity_tracker_->Reset(); |
| 398 } | 308 } |
| 399 } | 309 } |
| 400 return interactive_mode; | 310 return interactive_mode; |
| 401 } | 311 } |
| 402 | 312 |
| 403 ContentCaptureSubscription::ContentCaptureSubscription( | 313 ContentCaptureSubscription::ContentCaptureSubscription( |
| 404 base::WeakPtr<RenderWidgetHostViewBase> source_view, | 314 base::WeakPtr<RenderWidgetHostViewBase> source_view, |
| 405 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 315 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
| 406 const CaptureCallback& capture_callback) | 316 const CaptureCallback& capture_callback) |
| 407 : source_view_(source_view), capture_callback_(capture_callback) { | 317 : source_view_(source_view), capture_callback_(capture_callback) { |
| 408 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 318 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 409 DCHECK(source_view_); | 319 DCHECK(source_view_); |
| 410 | 320 |
| 411 #if defined(USE_AURA) || defined(OS_MACOSX) | 321 #if defined(USE_AURA) || defined(OS_MACOSX) |
| 412 cursor_renderer_ = CursorRenderer::Create(source_view_->GetNativeView()); | 322 cursor_renderer_ = CursorRenderer::Create(source_view_->GetNativeView()); |
| 413 window_activity_tracker_ = | 323 window_activity_tracker_ = |
| 414 WindowActivityTracker::Create(source_view_->GetNativeView()); | 324 WindowActivityTracker::Create(source_view_->GetNativeView()); |
| 415 #endif | 325 #endif |
| 326 // Subscriptions for refresh frames and mouse cursor updates. When events |
| 327 // outside of the normal content change/compositing workflow occur, these |
| 328 // decide whether extra frames need to be captured. Such extra frame captures |
| 329 // are initiated by running the |capture_callback_|. |
| 416 refresh_subscriber_.reset(new FrameSubscriber( | 330 refresh_subscriber_.reset(new FrameSubscriber( |
| 417 media::VideoCaptureOracle::kActiveRefreshRequest, oracle_proxy, | 331 media::VideoCaptureOracle::kActiveRefreshRequest, oracle_proxy, |
| 418 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() | 332 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() |
| 419 : base::WeakPtr<CursorRenderer>(), | 333 : base::WeakPtr<CursorRenderer>(), |
| 420 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() | 334 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() |
| 421 : base::WeakPtr<WindowActivityTracker>())); | 335 : base::WeakPtr<WindowActivityTracker>())); |
| 422 mouse_activity_subscriber_.reset(new FrameSubscriber( | 336 mouse_activity_subscriber_.reset(new FrameSubscriber( |
| 423 media::VideoCaptureOracle::kMouseCursorUpdate, oracle_proxy, | 337 media::VideoCaptureOracle::kMouseCursorUpdate, oracle_proxy, |
| 424 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() | 338 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() |
| 425 : base::WeakPtr<CursorRenderer>(), | 339 : base::WeakPtr<CursorRenderer>(), |
| 426 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() | 340 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() |
| 427 : base::WeakPtr<WindowActivityTracker>())); | 341 : base::WeakPtr<WindowActivityTracker>())); |
| 428 | 342 |
| 429 // Subscribe to compositor updates. These will be serviced directly by the | 343 // Main capture path: Subscribing to compositor updates. This means that any |
| 430 // oracle. | 344 // time the tab content has changed and compositing has taken place, the |
| 345 // RenderWidgetHostView will consult the FrameSubscriber (which consults the |
| 346 // oracle) to determine whether a frame should be captured. If so, the |
| 347 // RenderWidgetHostView will initiate the frame capture and NOT the |
| 348 // |capture_callback_|. |
| 431 std::unique_ptr<RenderWidgetHostViewFrameSubscriber> subscriber( | 349 std::unique_ptr<RenderWidgetHostViewFrameSubscriber> subscriber( |
| 432 new FrameSubscriber( | 350 new FrameSubscriber( |
| 433 media::VideoCaptureOracle::kCompositorUpdate, oracle_proxy, | 351 media::VideoCaptureOracle::kCompositorUpdate, oracle_proxy, |
| 434 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() | 352 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() |
| 435 : base::WeakPtr<CursorRenderer>(), | 353 : base::WeakPtr<CursorRenderer>(), |
| 436 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() | 354 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() |
| 437 : base::WeakPtr<WindowActivityTracker>())); | 355 : base::WeakPtr<WindowActivityTracker>())); |
| 438 source_view_->BeginFrameSubscription(std::move(subscriber)); | 356 source_view_->BeginFrameSubscription(std::move(subscriber)); |
| 439 | 357 |
| 440 // Subscribe to mouse movement and mouse cursor update events. | 358 // Begin monitoring for user activity that is used to signal "interactive |
| 359 // content." |
| 441 if (window_activity_tracker_) { | 360 if (window_activity_tracker_) { |
| 442 window_activity_tracker_->RegisterMouseInteractionObserver( | 361 window_activity_tracker_->RegisterMouseInteractionObserver( |
| 443 base::Bind(&ContentCaptureSubscription::OnEvent, base::Unretained(this), | 362 base::Bind(&ContentCaptureSubscription::OnEvent, base::Unretained(this), |
| 444 mouse_activity_subscriber_.get())); | 363 mouse_activity_subscriber_.get())); |
| 445 } | 364 } |
| 446 } | 365 } |
| 447 | 366 |
| 448 ContentCaptureSubscription::~ContentCaptureSubscription() { | 367 ContentCaptureSubscription::~ContentCaptureSubscription() { |
| 449 // If the BrowserThreads have been torn down, then the browser is in the final | 368 // If the BrowserThreads have been torn down, then the browser is in the final |
| 450 // stages of exiting and it is dangerous to take any further action. We must | 369 // stages of exiting and it is dangerous to take any further action. We must |
| (...skipping 25 matching lines...) Expand all Loading... |
| 476 | 395 |
| 477 const base::TimeTicks start_time = base::TimeTicks::Now(); | 396 const base::TimeTicks start_time = base::TimeTicks::Now(); |
| 478 DCHECK(subscriber == refresh_subscriber_.get() || | 397 DCHECK(subscriber == refresh_subscriber_.get() || |
| 479 subscriber == mouse_activity_subscriber_.get()); | 398 subscriber == mouse_activity_subscriber_.get()); |
| 480 if (subscriber->ShouldCaptureFrame(gfx::Rect(), start_time, &frame, | 399 if (subscriber->ShouldCaptureFrame(gfx::Rect(), start_time, &frame, |
| 481 &deliver_frame_cb)) { | 400 &deliver_frame_cb)) { |
| 482 capture_callback_.Run(start_time, frame, deliver_frame_cb); | 401 capture_callback_.Run(start_time, frame, deliver_frame_cb); |
| 483 } | 402 } |
| 484 } | 403 } |
| 485 | 404 |
| 486 void RenderVideoFrame( | |
| 487 const SkBitmap& input, | |
| 488 const scoped_refptr<media::VideoFrame>& output, | |
| 489 const base::Callback<void(const gfx::Rect&, bool)>& done_cb) { | |
| 490 base::ScopedClosureRunner failure_handler( | |
| 491 base::Bind(done_cb, gfx::Rect(), false)); | |
| 492 | |
| 493 SkAutoLockPixels locker(input); | |
| 494 | |
| 495 // Sanity-check the captured bitmap. | |
| 496 if (input.empty() || !input.readyToDraw() || | |
| 497 input.colorType() != kN32_SkColorType || input.width() < 2 || | |
| 498 input.height() < 2) { | |
| 499 DVLOG(1) << "input unacceptable (size=" << input.getSize() | |
| 500 << ", ready=" << input.readyToDraw() | |
| 501 << ", colorType=" << input.colorType() << ')'; | |
| 502 return; | |
| 503 } | |
| 504 | |
| 505 // Sanity-check the output buffer. | |
| 506 if (output->format() != media::PIXEL_FORMAT_I420) { | |
| 507 NOTREACHED(); | |
| 508 return; | |
| 509 } | |
| 510 | |
| 511 // Calculate the width and height of the content region in the |output|, based | |
| 512 // on the aspect ratio of |input|. | |
| 513 const gfx::Rect region_in_frame = media::ComputeLetterboxRegion( | |
| 514 output->visible_rect(), gfx::Size(input.width(), input.height())); | |
| 515 | |
| 516 // Scale the bitmap to the required size, if necessary. | |
| 517 SkBitmap scaled_bitmap; | |
| 518 if (input.width() != region_in_frame.width() || | |
| 519 input.height() != region_in_frame.height()) { | |
| 520 skia::ImageOperations::ResizeMethod method; | |
| 521 if (input.width() < region_in_frame.width() || | |
| 522 input.height() < region_in_frame.height()) { | |
| 523 // Avoid box filtering when magnifying, because it's actually | |
| 524 // nearest-neighbor. | |
| 525 method = skia::ImageOperations::RESIZE_HAMMING1; | |
| 526 } else { | |
| 527 method = skia::ImageOperations::RESIZE_BOX; | |
| 528 } | |
| 529 | |
| 530 TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", output.get(), | |
| 531 "Scale"); | |
| 532 scaled_bitmap = skia::ImageOperations::Resize( | |
| 533 input, method, region_in_frame.width(), region_in_frame.height()); | |
| 534 } else { | |
| 535 scaled_bitmap = input; | |
| 536 } | |
| 537 | |
| 538 TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", output.get(), "YUV"); | |
| 539 { | |
| 540 // Align to 2x2 pixel boundaries, as required by | |
| 541 // media::CopyRGBToVideoFrame(). | |
| 542 const gfx::Rect region_in_yv12_frame( | |
| 543 region_in_frame.x() & ~1, region_in_frame.y() & ~1, | |
| 544 region_in_frame.width() & ~1, region_in_frame.height() & ~1); | |
| 545 if (region_in_yv12_frame.IsEmpty()) | |
| 546 return; | |
| 547 | |
| 548 SkAutoLockPixels scaled_bitmap_locker(scaled_bitmap); | |
| 549 media::CopyRGBToVideoFrame( | |
| 550 reinterpret_cast<uint8_t*>(scaled_bitmap.getPixels()), | |
| 551 scaled_bitmap.rowBytes(), region_in_yv12_frame, output.get()); | |
| 552 } | |
| 553 | |
| 554 // The result is now ready. | |
| 555 ignore_result(failure_handler.Release()); | |
| 556 done_cb.Run(region_in_frame, true); | |
| 557 } | |
| 558 | |
| 559 WebContentsCaptureMachine::WebContentsCaptureMachine( | 405 WebContentsCaptureMachine::WebContentsCaptureMachine( |
| 560 int render_process_id, | 406 int render_process_id, |
| 561 int main_render_frame_id, | 407 int main_render_frame_id, |
| 562 bool enable_auto_throttling) | 408 bool enable_auto_throttling) |
| 563 : initial_render_process_id_(render_process_id), | 409 : initial_render_process_id_(render_process_id), |
| 564 initial_main_render_frame_id_(main_render_frame_id), | 410 initial_main_render_frame_id_(main_render_frame_id), |
| 565 tracker_(new WebContentsTracker(true)), | 411 tracker_(new WebContentsTracker(true)), |
| 566 auto_throttling_enabled_(enable_auto_throttling), | 412 auto_throttling_enabled_(enable_auto_throttling), |
| 567 frame_capture_active_(true), | 413 frame_capture_active_(true), |
| 568 weak_ptr_factory_(this) { | 414 weak_ptr_factory_(this) { |
| (...skipping 15 matching lines...) Expand all Loading... |
| 584 const base::Callback<void(bool)> callback) { | 430 const base::Callback<void(bool)> callback) { |
| 585 // Starts the capture machine asynchronously. | 431 // Starts the capture machine asynchronously. |
| 586 BrowserThread::PostTaskAndReplyWithResult( | 432 BrowserThread::PostTaskAndReplyWithResult( |
| 587 BrowserThread::UI, FROM_HERE, | 433 BrowserThread::UI, FROM_HERE, |
| 588 base::Bind(&WebContentsCaptureMachine::InternalStart, | 434 base::Bind(&WebContentsCaptureMachine::InternalStart, |
| 589 base::Unretained(this), oracle_proxy, params), | 435 base::Unretained(this), oracle_proxy, params), |
| 590 callback); | 436 callback); |
| 591 } | 437 } |
| 592 | 438 |
| 593 bool WebContentsCaptureMachine::InternalStart( | 439 bool WebContentsCaptureMachine::InternalStart( |
| 594 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 440 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
| 595 const media::VideoCaptureParams& params) { | 441 const media::VideoCaptureParams& params) { |
| 596 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 442 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 597 DCHECK(!IsStarted()); | 443 DCHECK(!IsStarted()); |
| 598 | 444 |
| 599 DCHECK(oracle_proxy); | 445 DCHECK(oracle_proxy); |
| 600 oracle_proxy_ = oracle_proxy; | 446 oracle_proxy_ = std::move(oracle_proxy); |
| 601 capture_params_ = params; | 447 capture_params_ = params; |
| 602 | 448 |
| 603 render_thread_.reset(new base::Thread("WebContentsVideo_RenderThread")); | |
| 604 if (!render_thread_->Start()) { | |
| 605 DVLOG(1) << "Failed to spawn render thread."; | |
| 606 render_thread_.reset(); | |
| 607 return false; | |
| 608 } | |
| 609 | |
| 610 // Note: Creation of the first WeakPtr in the following statement will cause | 449 // Note: Creation of the first WeakPtr in the following statement will cause |
| 611 // IsStarted() to return true from now on. | 450 // IsStarted() to return true from now on. |
| 612 tracker_->SetResizeChangeCallback( | 451 tracker_->SetResizeChangeCallback( |
| 613 base::Bind(&WebContentsCaptureMachine::UpdateCaptureSize, | 452 base::Bind(&WebContentsCaptureMachine::UpdateCaptureSize, |
| 614 weak_ptr_factory_.GetWeakPtr())); | 453 weak_ptr_factory_.GetWeakPtr())); |
| 615 tracker_->Start(initial_render_process_id_, initial_main_render_frame_id_, | 454 tracker_->Start(initial_render_process_id_, initial_main_render_frame_id_, |
| 616 base::Bind(&WebContentsCaptureMachine::RenewFrameSubscription, | 455 base::Bind(&WebContentsCaptureMachine::RenewFrameSubscription, |
| 617 weak_ptr_factory_.GetWeakPtr())); | 456 weak_ptr_factory_.GetWeakPtr())); |
| 618 if (WebContents* contents = tracker_->web_contents()) | 457 if (WebContents* contents = tracker_->web_contents()) |
| 619 contents->IncrementCapturerCount(ComputeOptimalViewSize()); | 458 contents->IncrementCapturerCount(ComputeOptimalViewSize()); |
| (...skipping 26 matching lines...) Expand all Loading... |
| 646 void WebContentsCaptureMachine::InternalResume() { | 485 void WebContentsCaptureMachine::InternalResume() { |
| 647 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 486 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 648 if (frame_capture_active_) | 487 if (frame_capture_active_) |
| 649 return; | 488 return; |
| 650 frame_capture_active_ = true; | 489 frame_capture_active_ = true; |
| 651 if (IsStarted()) | 490 if (IsStarted()) |
| 652 RenewFrameSubscription(tracker_->is_still_tracking()); | 491 RenewFrameSubscription(tracker_->is_still_tracking()); |
| 653 } | 492 } |
| 654 | 493 |
| 655 void WebContentsCaptureMachine::Stop(const base::Closure& callback) { | 494 void WebContentsCaptureMachine::Stop(const base::Closure& callback) { |
| 656 // Stops the capture machine asynchronously. | |
| 657 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, | 495 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, |
| 658 base::Bind(&WebContentsCaptureMachine::InternalStop, | 496 base::Bind(&WebContentsCaptureMachine::InternalStop, |
| 659 base::Unretained(this), callback)); | 497 base::Unretained(this), callback)); |
| 660 } | 498 } |
| 661 | 499 |
| 662 void WebContentsCaptureMachine::InternalStop(const base::Closure& callback) { | 500 void WebContentsCaptureMachine::InternalStop(const base::Closure& callback) { |
| 663 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 501 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 664 | 502 |
| 665 if (!IsStarted()) { | 503 base::ScopedClosureRunner done_runner(callback); |
| 666 callback.Run(); | 504 |
| 505 if (!IsStarted()) |
| 667 return; | 506 return; |
| 668 } | |
| 669 | 507 |
| 670 // The following cancels any outstanding callbacks and causes IsStarted() to | 508 // The following cancels any outstanding callbacks and causes IsStarted() to |
| 671 // return false from here onward. | 509 // return false from here onward. |
| 672 weak_ptr_factory_.InvalidateWeakPtrs(); | 510 weak_ptr_factory_.InvalidateWeakPtrs(); |
| 673 | 511 |
| 674 RenewFrameSubscription(false); | 512 RenewFrameSubscription(false); |
| 675 if (WebContents* contents = tracker_->web_contents()) | 513 if (WebContents* contents = tracker_->web_contents()) |
| 676 contents->DecrementCapturerCount(); | 514 contents->DecrementCapturerCount(); |
| 677 tracker_->Stop(); | 515 tracker_->Stop(); |
| 678 | |
| 679 // The render thread cannot be stopped on the UI thread, so post a message | |
| 680 // to the thread pool used for blocking operations. | |
| 681 if (render_thread_) { | |
| 682 BrowserThread::PostBlockingPoolTask( | |
| 683 FROM_HERE, base::Bind(&DeleteOnWorkerThread, | |
| 684 base::Passed(&render_thread_), callback)); | |
| 685 } | |
| 686 } | 516 } |
| 687 | 517 |
| 688 void WebContentsCaptureMachine::MaybeCaptureForRefresh() { | 518 void WebContentsCaptureMachine::MaybeCaptureForRefresh() { |
| 689 BrowserThread::PostTask( | 519 BrowserThread::PostTask( |
| 690 BrowserThread::UI, FROM_HERE, | 520 BrowserThread::UI, FROM_HERE, |
| 691 base::Bind(&WebContentsCaptureMachine::InternalMaybeCaptureForRefresh, | 521 base::Bind(&WebContentsCaptureMachine::InternalMaybeCaptureForRefresh, |
| 692 // Use of Unretained() is safe here since this task must run | 522 // Use of Unretained() is safe here since this task must run |
| 693 // before InternalStop(). | 523 // before InternalStop(). |
| 694 base::Unretained(this))); | 524 base::Unretained(this))); |
| 695 } | 525 } |
| 696 | 526 |
| 697 void WebContentsCaptureMachine::InternalMaybeCaptureForRefresh() { | 527 void WebContentsCaptureMachine::InternalMaybeCaptureForRefresh() { |
| 698 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 528 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 699 if (IsStarted() && subscription_) | 529 if (IsStarted() && subscription_) |
| 700 subscription_->MaybeCaptureForRefresh(); | 530 subscription_->MaybeCaptureForRefresh(); |
| 701 } | 531 } |
| 702 | 532 |
| 703 void WebContentsCaptureMachine::Capture( | 533 void WebContentsCaptureMachine::Capture( |
| 704 const base::TimeTicks& start_time, | 534 base::TimeTicks start_time, |
| 705 const scoped_refptr<media::VideoFrame>& target, | 535 scoped_refptr<media::VideoFrame> target, |
| 706 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 536 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
| 707 deliver_frame_cb) { | 537 deliver_frame_cb) { |
| 708 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 538 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 709 | 539 |
| 710 auto* const view = | 540 auto* const view = |
| 711 static_cast<RenderWidgetHostViewBase*>(tracker_->GetTargetView()); | 541 static_cast<RenderWidgetHostViewBase*>(tracker_->GetTargetView()); |
| 712 if (!view) { | 542 if (!view || !view->CanCopyToVideoFrame()) { |
| 713 deliver_frame_cb.Run(base::TimeTicks(), gfx::Rect(), false); | 543 deliver_frame_cb.Run(base::TimeTicks(), gfx::Rect(), false); |
| 714 return; | 544 return; |
| 715 } | 545 } |
| 716 | 546 |
| 717 const gfx::Size view_size = view->GetViewBounds().size(); | 547 const gfx::Size view_size = view->GetViewBounds().size(); |
| 718 if (view->CanCopyToVideoFrame()) { | 548 view->CopyFromCompositingSurfaceToVideoFrame( |
| 719 view->CopyFromCompositingSurfaceToVideoFrame( | 549 gfx::Rect(view_size), std::move(target), |
| 720 gfx::Rect(view_size), target, | 550 base::Bind(&WebContentsCaptureMachine::DidCopyToVideoFrame, |
| 721 base::Bind(&WebContentsCaptureMachine:: | 551 weak_ptr_factory_.GetWeakPtr(), start_time, deliver_frame_cb)); |
| 722 DidCopyFromCompositingSurfaceToVideoFrame, | |
| 723 weak_ptr_factory_.GetWeakPtr(), start_time, | |
| 724 deliver_frame_cb)); | |
| 725 } else { | |
| 726 const gfx::Size fitted_size = | |
| 727 view_size.IsEmpty() | |
| 728 ? gfx::Size() | |
| 729 : media::ComputeLetterboxRegion(target->visible_rect(), view_size) | |
| 730 .size(); | |
| 731 if (auto* rwh = view->GetRenderWidgetHost()) { | |
| 732 rwh->CopyFromBackingStore( | |
| 733 gfx::Rect(), | |
| 734 fitted_size, // Size here is a request not always honored. | |
| 735 base::Bind(&WebContentsCaptureMachine::DidCopyFromBackingStore, | |
| 736 weak_ptr_factory_.GetWeakPtr(), start_time, target, | |
| 737 deliver_frame_cb), | |
| 738 kN32_SkColorType); | |
| 739 } | |
| 740 } | |
| 741 } | 552 } |
| 742 | 553 |
| 743 gfx::Size WebContentsCaptureMachine::ComputeOptimalViewSize() const { | 554 gfx::Size WebContentsCaptureMachine::ComputeOptimalViewSize() const { |
| 744 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 555 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 745 | 556 |
| 746 // TODO(miu): Propagate capture frame size changes as new "preferred size" | 557 // TODO(miu): Propagate capture frame size changes as new "preferred size" |
| 747 // updates, rather than just using the max frame size. | 558 // updates, rather than just using the max frame size. |
| 748 // http://crbug.com/350491 | 559 // http://crbug.com/350491 |
| 749 gfx::Size optimal_size = oracle_proxy_->max_frame_size(); | 560 gfx::Size optimal_size = oracle_proxy_->max_frame_size(); |
| 750 | 561 |
| (...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 797 gfx::ScaleToFlooredSize(optimal_size, 1.0f / scale); | 608 gfx::ScaleToFlooredSize(optimal_size, 1.0f / scale); |
| 798 if (shrunk_size.width() > 0 && shrunk_size.height() > 0) | 609 if (shrunk_size.width() > 0 && shrunk_size.height() > 0) |
| 799 optimal_size = shrunk_size; | 610 optimal_size = shrunk_size; |
| 800 } | 611 } |
| 801 } | 612 } |
| 802 | 613 |
| 803 VLOG(1) << "Computed optimal target size: " << optimal_size.ToString(); | 614 VLOG(1) << "Computed optimal target size: " << optimal_size.ToString(); |
| 804 return optimal_size; | 615 return optimal_size; |
| 805 } | 616 } |
| 806 | 617 |
| 807 void WebContentsCaptureMachine::DidCopyFromBackingStore( | 618 void WebContentsCaptureMachine::DidCopyToVideoFrame( |
| 808 const base::TimeTicks& start_time, | |
| 809 const scoped_refptr<media::VideoFrame>& target, | |
| 810 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | |
| 811 deliver_frame_cb, | |
| 812 const SkBitmap& bitmap, | |
| 813 ReadbackResponse response) { | |
| 814 DCHECK_CURRENTLY_ON(BrowserThread::UI); | |
| 815 | |
| 816 DCHECK(render_thread_); | |
| 817 if (response == READBACK_SUCCESS) { | |
| 818 TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", target.get(), | |
| 819 "Render"); | |
| 820 render_thread_->task_runner()->PostTask( | |
| 821 FROM_HERE, media::BindToCurrentLoop( | |
| 822 base::Bind(&RenderVideoFrame, bitmap, target, | |
| 823 base::Bind(deliver_frame_cb, start_time)))); | |
| 824 } else { | |
| 825 // Capture can fail due to transient issues, so just skip this frame. | |
| 826 DVLOG(1) << "CopyFromBackingStore failed; skipping frame."; | |
| 827 deliver_frame_cb.Run(start_time, gfx::Rect(), false); | |
| 828 } | |
| 829 } | |
| 830 | |
| 831 void WebContentsCaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame( | |
| 832 const base::TimeTicks& start_time, | 619 const base::TimeTicks& start_time, |
| 833 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 620 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
| 834 deliver_frame_cb, | 621 deliver_frame_cb, |
| 835 const gfx::Rect& region_in_frame, | 622 const gfx::Rect& region_in_frame, |
| 836 bool success) { | 623 bool success) { |
| 837 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 624 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
| 838 | 625 |
| 839 // Capture can fail due to transient issues, so just skip this frame. | 626 // Capture can fail due to transient issues, so just skip this frame. |
| 840 DVLOG_IF(1, !success) << "CopyFromCompositingSurface failed; skipping frame."; | 627 DVLOG_IF(1, !success) << "CopyFromCompositingSurface failed; skipping frame."; |
| 841 deliver_frame_cb.Run(start_time, region_in_frame, success); | 628 deliver_frame_cb.Run(start_time, region_in_frame, success); |
| (...skipping 74 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 916 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() { | 703 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() { |
| 917 DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying."; | 704 DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying."; |
| 918 } | 705 } |
| 919 | 706 |
| 920 // static | 707 // static |
| 921 std::unique_ptr<media::VideoCaptureDevice> | 708 std::unique_ptr<media::VideoCaptureDevice> |
| 922 WebContentsVideoCaptureDevice::Create(const std::string& device_id) { | 709 WebContentsVideoCaptureDevice::Create(const std::string& device_id) { |
| 923 // Parse device_id into render_process_id and main_render_frame_id. | 710 // Parse device_id into render_process_id and main_render_frame_id. |
| 924 WebContentsMediaCaptureId media_id; | 711 WebContentsMediaCaptureId media_id; |
| 925 if (!WebContentsMediaCaptureId::Parse(device_id, &media_id)) { | 712 if (!WebContentsMediaCaptureId::Parse(device_id, &media_id)) { |
| 926 return NULL; | 713 return nullptr; |
| 927 } | 714 } |
| 928 | 715 |
| 929 return std::unique_ptr<media::VideoCaptureDevice>( | 716 return std::unique_ptr<media::VideoCaptureDevice>( |
| 930 new WebContentsVideoCaptureDevice(media_id.render_process_id, | 717 new WebContentsVideoCaptureDevice(media_id.render_process_id, |
| 931 media_id.main_render_frame_id, | 718 media_id.main_render_frame_id, |
| 932 media_id.enable_auto_throttling)); | 719 media_id.enable_auto_throttling)); |
| 933 } | 720 } |
| 934 | 721 |
| 935 void WebContentsVideoCaptureDevice::AllocateAndStart( | 722 void WebContentsVideoCaptureDevice::AllocateAndStart( |
| 936 const media::VideoCaptureParams& params, | 723 const media::VideoCaptureParams& params, |
| (...skipping 17 matching lines...) Expand all Loading... |
| 954 void WebContentsVideoCaptureDevice::StopAndDeAllocate() { | 741 void WebContentsVideoCaptureDevice::StopAndDeAllocate() { |
| 955 core_->StopAndDeAllocate(); | 742 core_->StopAndDeAllocate(); |
| 956 } | 743 } |
| 957 | 744 |
| 958 void WebContentsVideoCaptureDevice::OnUtilizationReport(int frame_feedback_id, | 745 void WebContentsVideoCaptureDevice::OnUtilizationReport(int frame_feedback_id, |
| 959 double utilization) { | 746 double utilization) { |
| 960 core_->OnConsumerReportingUtilization(frame_feedback_id, utilization); | 747 core_->OnConsumerReportingUtilization(frame_feedback_id, utilization); |
| 961 } | 748 } |
| 962 | 749 |
| 963 } // namespace content | 750 } // namespace content |
| OLD | NEW |