OLD | NEW |
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 // | |
5 // Implementation notes: This needs to work on a variety of hardware | |
6 // configurations where the speed of the CPU and GPU greatly affect overall | |
7 // performance. Spanning several threads, the process of capturing has been | |
8 // split up into four conceptual stages: | |
9 // | |
10 // 1. Reserve Buffer: Before a frame can be captured, a slot in the client's | |
11 // shared-memory IPC buffer is reserved. There are only a few of these; | |
12 // when they run out, it indicates that the downstream client -- likely a | |
13 // video encoder -- is the performance bottleneck, and that the rate of | |
14 // frame capture should be throttled back. | |
15 // | |
16 // 2. Capture: A bitmap is snapshotted/copied from the RenderWidget's backing | |
17 // store. This is initiated on the UI BrowserThread, and often occurs | |
18 // asynchronously. Where supported, the GPU scales and color converts | |
19 // frames to our desired size, and the readback happens directly into the | |
20 // shared-memory buffer. But this is not always possible, particularly when | |
21 // accelerated compositing is disabled. | |
22 // | |
23 // 3. Render (if needed): If the web contents cannot be captured directly into | |
24 // our target size and color format, scaling and colorspace conversion must | |
25 // be done on the CPU. A dedicated thread is used for this operation, to | |
26 // avoid blocking the UI thread. The Render stage always reads from a | |
27 // bitmap returned by Capture, and writes into the reserved slot in the | |
28 // shared-memory buffer. | |
29 // | |
30 // 4. Deliver: The rendered video frame is returned to the client (which | |
31 // implements the VideoCaptureDevice::Client interface). Because all | |
32 // paths have written the frame into the IPC buffer, this step should | |
33 // never need to do an additional copy of the pixel data. | |
34 // | |
35 // In the best-performing case, the Render step is bypassed: Capture produces | |
36 // ready-to-Deliver frames. But when accelerated readback is not possible, the | |
37 // system is designed so that Capture and Render may run concurrently. A timing | |
38 // diagram helps illustrate this point (@30 FPS): | |
39 // | |
40 // Time: 0ms 33ms 66ms 99ms | |
41 // thread1: |-Capture-f1------v |-Capture-f2------v |-Capture-f3----v |-Capt | |
42 // thread2: |-Render-f1-----v |-Render-f2-----v |-Render-f3 | |
43 // | |
44 // In the above example, both capturing and rendering *each* take almost the | |
45 // full 33 ms available between frames, yet we see that the required throughput | |
46 // is obtained. | |
47 | 4 |
48 #include "content/browser/media/capture/web_contents_video_capture_device.h" | 5 #include "content/browser/media/capture/web_contents_video_capture_device.h" |
49 | 6 |
50 #include <stdint.h> | 7 #include <stdint.h> |
51 | 8 |
52 #include <algorithm> | 9 #include <algorithm> |
53 #include <memory> | 10 #include <memory> |
54 #include <utility> | 11 #include <string> |
55 | 12 |
56 #include "base/bind.h" | 13 #include "base/bind.h" |
57 #include "base/callback_helpers.h" | 14 #include "base/callback_helpers.h" |
58 #include "base/location.h" | 15 #include "base/location.h" |
59 #include "base/logging.h" | 16 #include "base/logging.h" |
60 #include "base/macros.h" | 17 #include "base/macros.h" |
61 #include "base/memory/ptr_util.h" | |
62 #include "base/memory/weak_ptr.h" | 18 #include "base/memory/weak_ptr.h" |
63 #include "base/sequenced_task_runner.h" | |
64 #include "base/single_thread_task_runner.h" | |
65 #include "base/threading/thread.h" | |
66 #include "base/threading/thread_checker.h" | |
67 #include "base/time/time.h" | 19 #include "base/time/time.h" |
68 #include "build/build_config.h" | 20 #include "build/build_config.h" |
69 #include "content/browser/media/capture/cursor_renderer.h" | 21 #include "content/browser/media/capture/cursor_renderer.h" |
70 #include "content/browser/media/capture/web_contents_tracker.h" | 22 #include "content/browser/media/capture/web_contents_tracker.h" |
71 #include "content/browser/media/capture/window_activity_tracker.h" | 23 #include "content/browser/media/capture/window_activity_tracker.h" |
72 #include "content/browser/renderer_host/render_widget_host_impl.h" | 24 #include "content/browser/renderer_host/render_widget_host_impl.h" |
73 #include "content/browser/renderer_host/render_widget_host_view_base.h" | 25 #include "content/browser/renderer_host/render_widget_host_view_base.h" |
74 #include "content/public/browser/browser_thread.h" | 26 #include "content/public/browser/browser_thread.h" |
75 #include "content/public/browser/render_process_host.h" | 27 #include "content/public/browser/render_process_host.h" |
76 #include "content/public/browser/render_widget_host_view.h" | 28 #include "content/public/browser/render_widget_host_view.h" |
77 #include "content/public/browser/render_widget_host_view_frame_subscriber.h" | 29 #include "content/public/browser/render_widget_host_view_frame_subscriber.h" |
78 #include "content/public/browser/web_contents.h" | 30 #include "content/public/browser/web_contents.h" |
79 #include "content/public/browser/web_contents_media_capture_id.h" | 31 #include "content/public/browser/web_contents_media_capture_id.h" |
80 #include "media/base/bind_to_current_loop.h" | |
81 #include "media/base/video_frame_metadata.h" | 32 #include "media/base/video_frame_metadata.h" |
82 #include "media/base/video_util.h" | |
83 #include "media/capture/content/screen_capture_device_core.h" | 33 #include "media/capture/content/screen_capture_device_core.h" |
84 #include "media/capture/content/thread_safe_capture_oracle.h" | 34 #include "media/capture/content/thread_safe_capture_oracle.h" |
85 #include "media/capture/content/video_capture_oracle.h" | 35 #include "media/capture/content/video_capture_oracle.h" |
86 #include "media/capture/video_capture_types.h" | 36 #include "media/capture/video_capture_types.h" |
87 #include "skia/ext/image_operations.h" | |
88 #include "third_party/skia/include/core/SkBitmap.h" | |
89 #include "third_party/skia/include/core/SkColor.h" | |
90 #include "ui/base/layout.h" | 37 #include "ui/base/layout.h" |
91 #include "ui/gfx/geometry/dip_util.h" | 38 #include "ui/gfx/geometry/dip_util.h" |
92 #include "ui/gfx/geometry/size_conversions.h" | 39 #include "ui/gfx/geometry/size_conversions.h" |
93 | 40 |
94 namespace content { | 41 namespace content { |
95 | 42 |
96 namespace { | 43 namespace { |
97 | 44 |
98 enum InteractiveModeSettings { | 45 // Minimum amount of time for which there should be no animation detected |
99 // Minimum amount of time for which there should be no animation detected | 46 // to consider interactive mode being active. This is to prevent very brief |
100 // to consider interactive mode being active. This is to prevent very brief | 47 // periods of animated content not being detected (due to CPU fluctations for |
101 // periods of animated content not being detected (due to CPU fluctations for | 48 // example) from causing a flip-flop on interactive mode. |
102 // example) from causing a flip-flop on interactive mode. | 49 constexpr int64_t kMinPeriodNoAnimationMillis = 3000; |
103 kMinPeriodNoAnimationMillis = 3000 | |
104 }; | |
105 | |
106 void DeleteOnWorkerThread(std::unique_ptr<base::Thread> render_thread, | |
107 const base::Closure& callback) { | |
108 render_thread.reset(); | |
109 | |
110 // After thread join call the callback on UI thread. | |
111 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, callback); | |
112 } | |
113 | 50 |
114 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible | 51 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible |
115 // with RenderWidgetHostViewFrameSubscriber. We create one per event type. | 52 // with RenderWidgetHostViewFrameSubscriber. One is created per event type. |
116 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber { | 53 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber { |
117 public: | 54 public: |
118 FrameSubscriber(media::VideoCaptureOracle::Event event_type, | 55 FrameSubscriber(media::VideoCaptureOracle::Event event_type, |
119 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle, | 56 scoped_refptr<media::ThreadSafeCaptureOracle> oracle, |
120 base::WeakPtr<content::CursorRenderer> cursor_renderer, | 57 base::WeakPtr<content::CursorRenderer> cursor_renderer, |
121 base::WeakPtr<content::WindowActivityTracker> tracker) | 58 base::WeakPtr<content::WindowActivityTracker> tracker) |
122 : event_type_(event_type), | 59 : event_type_(event_type), |
123 oracle_proxy_(oracle), | 60 oracle_proxy_(std::move(oracle)), |
124 cursor_renderer_(cursor_renderer), | 61 cursor_renderer_(cursor_renderer), |
125 window_activity_tracker_(tracker), | 62 window_activity_tracker_(tracker), |
126 weak_ptr_factory_(this) {} | 63 weak_ptr_factory_(this) {} |
127 | 64 |
128 bool ShouldCaptureFrame( | 65 bool ShouldCaptureFrame( |
129 const gfx::Rect& damage_rect, | 66 const gfx::Rect& damage_rect, |
130 base::TimeTicks present_time, | 67 base::TimeTicks present_time, |
131 scoped_refptr<media::VideoFrame>* storage, | 68 scoped_refptr<media::VideoFrame>* storage, |
132 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback* | 69 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback* |
133 deliver_frame_cb) override; | 70 deliver_frame_cb) override; |
134 | 71 |
135 static void DidCaptureFrame( | 72 static void DidCaptureFrame( |
136 base::WeakPtr<FrameSubscriber> frame_subscriber_, | 73 base::WeakPtr<FrameSubscriber> frame_subscriber_, |
137 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& | 74 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& |
138 capture_frame_cb, | 75 capture_frame_cb, |
139 scoped_refptr<media::VideoFrame> frame, | 76 scoped_refptr<media::VideoFrame> frame, |
140 base::TimeTicks timestamp, | 77 base::TimeTicks timestamp, |
141 const gfx::Rect& region_in_frame, | 78 const gfx::Rect& region_in_frame, |
142 bool success); | 79 bool success); |
143 | 80 |
144 bool IsUserInteractingWithContent(); | 81 bool IsUserInteractingWithContent(); |
145 | 82 |
146 private: | 83 private: |
147 const media::VideoCaptureOracle::Event event_type_; | 84 const media::VideoCaptureOracle::Event event_type_; |
148 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; | 85 const scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; |
149 // We need a weak pointer since FrameSubscriber is owned externally and | 86 // We need a weak pointer since FrameSubscriber is owned externally and |
150 // may outlive the cursor renderer. | 87 // may outlive the cursor renderer. |
151 base::WeakPtr<CursorRenderer> cursor_renderer_; | 88 const base::WeakPtr<CursorRenderer> cursor_renderer_; |
152 // We need a weak pointer since FrameSubscriber is owned externally and | 89 // We need a weak pointer since FrameSubscriber is owned externally and |
153 // may outlive the ui activity tracker. | 90 // may outlive the ui activity tracker. |
154 base::WeakPtr<WindowActivityTracker> window_activity_tracker_; | 91 const base::WeakPtr<WindowActivityTracker> window_activity_tracker_; |
155 base::WeakPtrFactory<FrameSubscriber> weak_ptr_factory_; | 92 base::WeakPtrFactory<FrameSubscriber> weak_ptr_factory_; |
156 }; | 93 }; |
157 | 94 |
158 // ContentCaptureSubscription is the relationship between a RenderWidgetHostView | 95 // ContentCaptureSubscription is the relationship between a RenderWidgetHostView |
159 // whose content is updating, a subscriber that is deciding which of these | 96 // whose content is updating, a subscriber that is deciding which of these |
160 // updates to capture (and where to deliver them to), and a callback that | 97 // updates to capture (and where to deliver them to), and a callback that |
161 // knows how to do the capture and prepare the result for delivery. | 98 // knows how to do the capture and prepare the result for delivery. |
162 // | 99 // |
163 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in | 100 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in |
164 // the RenderWidgetHostView, to process compositor updates, and (b) occasionally | 101 // the RenderWidgetHostView, to process compositor updates, and (b) occasionally |
165 // initiating forced, non-event-driven captures needed by downstream consumers | 102 // using the capture callback to force additional captures that are needed for |
166 // that request "refresh frames" of unchanged content. | 103 // things like mouse cursor rendering updates and/or refresh frames. |
167 // | 104 // |
168 // All of this happens on the UI thread, although the | 105 // All of this happens on the UI thread, although the |
169 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates | 106 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates |
170 // autonomously on some other thread. | 107 // autonomously on some other thread. |
171 class ContentCaptureSubscription { | 108 class ContentCaptureSubscription { |
172 public: | 109 public: |
173 typedef base::Callback<void( | 110 using CaptureCallback = base::Callback<void( |
174 const base::TimeTicks&, | 111 base::TimeTicks, |
175 const scoped_refptr<media::VideoFrame>&, | 112 scoped_refptr<media::VideoFrame>, |
176 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&)> | 113 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&)>; |
177 CaptureCallback; | |
178 | 114 |
179 // Create a subscription. Whenever a manual capture is required, the | 115 // Create a subscription. Whenever a non-compositor capture is required, the |
180 // subscription will invoke |capture_callback| on the UI thread to do the | 116 // subscription will invoke |capture_callback| on the UI thread to do the |
181 // work. | 117 // work. |
182 ContentCaptureSubscription( | 118 ContentCaptureSubscription( |
183 base::WeakPtr<RenderWidgetHostViewBase> source_view, | 119 base::WeakPtr<RenderWidgetHostViewBase> source_view, |
184 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 120 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
185 const CaptureCallback& capture_callback); | 121 const CaptureCallback& capture_callback); |
186 ~ContentCaptureSubscription(); | 122 ~ContentCaptureSubscription(); |
187 | 123 |
| 124 // Called when a condition occurs that requires a refresh frame be |
| 125 // captured. The subscribers will determine whether a frame was recently |
| 126 // captured, or if a non-compositor initiated capture must be made. |
188 void MaybeCaptureForRefresh(); | 127 void MaybeCaptureForRefresh(); |
189 | 128 |
190 private: | 129 private: |
191 // Called for active frame refresh requests, or mouse activity events. | 130 // Called for active frame refresh requests, or mouse activity events. |
192 void OnEvent(FrameSubscriber* subscriber); | 131 void OnEvent(FrameSubscriber* subscriber); |
193 | 132 |
194 const base::WeakPtr<RenderWidgetHostViewBase> source_view_; | 133 const base::WeakPtr<RenderWidgetHostViewBase> source_view_; |
195 | 134 |
196 std::unique_ptr<FrameSubscriber> refresh_subscriber_; | 135 std::unique_ptr<FrameSubscriber> refresh_subscriber_; |
197 std::unique_ptr<FrameSubscriber> mouse_activity_subscriber_; | 136 std::unique_ptr<FrameSubscriber> mouse_activity_subscriber_; |
198 CaptureCallback capture_callback_; | 137 CaptureCallback capture_callback_; |
199 | 138 |
200 // Responsible for tracking the cursor state and input events to make | 139 // Responsible for tracking the cursor state and input events to make |
201 // decisions and then render the mouse cursor on the video frame after | 140 // decisions and then render the mouse cursor on the video frame after |
202 // capture is completed. | 141 // capture is completed. |
203 std::unique_ptr<content::CursorRenderer> cursor_renderer_; | 142 std::unique_ptr<content::CursorRenderer> cursor_renderer_; |
204 | 143 |
205 // Responsible for tracking the UI events and making a decision on whether | 144 // Responsible for tracking the UI events and making a decision on whether |
206 // user is actively interacting with content. | 145 // user is actively interacting with content. |
207 std::unique_ptr<content::WindowActivityTracker> window_activity_tracker_; | 146 std::unique_ptr<content::WindowActivityTracker> window_activity_tracker_; |
208 | 147 |
209 DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription); | 148 DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription); |
210 }; | 149 }; |
211 | 150 |
212 // Render the SkBitmap |input| into the given VideoFrame buffer |output|, then | |
213 // invoke |done_cb| to indicate success or failure. |input| is expected to be | |
214 // ARGB. |output| must be YV12 or I420. Colorspace conversion is always done. | |
215 // Scaling and letterboxing will be done as needed. | |
216 // | |
217 // This software implementation should be used only when GPU acceleration of | |
218 // these activities is not possible. This operation may be expensive (tens to | |
219 // hundreds of milliseconds), so the caller should ensure that it runs on a | |
220 // thread where such a pause would cause UI jank. | |
221 void RenderVideoFrame( | |
222 const SkBitmap& input, | |
223 const scoped_refptr<media::VideoFrame>& output, | |
224 const base::Callback<void(const gfx::Rect&, bool)>& done_cb); | |
225 | |
226 // Renews capture subscriptions based on feedback from WebContentsTracker, and | 151 // Renews capture subscriptions based on feedback from WebContentsTracker, and |
227 // also executes copying of the backing store on the UI BrowserThread. | 152 // also executes non-compositor-initiated frame captures. |
228 class WebContentsCaptureMachine : public media::VideoCaptureMachine { | 153 class WebContentsCaptureMachine : public media::VideoCaptureMachine { |
229 public: | 154 public: |
230 WebContentsCaptureMachine(int render_process_id, | 155 WebContentsCaptureMachine(int render_process_id, |
231 int main_render_frame_id, | 156 int main_render_frame_id, |
232 bool enable_auto_throttling); | 157 bool enable_auto_throttling); |
233 ~WebContentsCaptureMachine() override; | 158 ~WebContentsCaptureMachine() override; |
234 | 159 |
235 // VideoCaptureMachine overrides. | 160 // VideoCaptureMachine overrides. |
236 void Start(const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 161 void Start(const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, |
237 const media::VideoCaptureParams& params, | 162 const media::VideoCaptureParams& params, |
238 const base::Callback<void(bool)> callback) override; | 163 const base::Callback<void(bool)> callback) override; |
239 void Suspend() override; | 164 void Suspend() override; |
240 void Resume() override; | 165 void Resume() override; |
241 void Stop(const base::Closure& callback) override; | 166 void Stop(const base::Closure& callback) override; |
242 bool IsAutoThrottlingEnabled() const override { | 167 bool IsAutoThrottlingEnabled() const override { |
243 return auto_throttling_enabled_; | 168 return auto_throttling_enabled_; |
244 } | 169 } |
245 void MaybeCaptureForRefresh() override; | 170 void MaybeCaptureForRefresh() override; |
246 | 171 |
247 // Starts a copy from the backing store or the composited surface. Must be run | 172 private: |
248 // on the UI BrowserThread. |deliver_frame_cb| will be run when the operation | 173 // Called for cases where events other than compositor updates require a frame |
249 // completes. The copy will occur to |target|. | 174 // capture from the composited surface. Must be run on the UI BrowserThread. |
| 175 // |deliver_frame_cb| will be run when the operation completes. The copy will |
| 176 // populate |target|. |
250 // | 177 // |
251 // This may be used as a ContentCaptureSubscription::CaptureCallback. | 178 // This method is only used by a ContentCaptureSubscription::CaptureCallback. |
252 void Capture(const base::TimeTicks& start_time, | 179 void Capture(base::TimeTicks start_time, |
253 const scoped_refptr<media::VideoFrame>& target, | 180 scoped_refptr<media::VideoFrame> target, |
254 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 181 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
255 deliver_frame_cb); | 182 deliver_frame_cb); |
256 | 183 |
257 private: | 184 bool InternalStart(scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
258 bool InternalStart( | 185 const media::VideoCaptureParams& params); |
259 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | |
260 const media::VideoCaptureParams& params); | |
261 void InternalSuspend(); | 186 void InternalSuspend(); |
262 void InternalResume(); | 187 void InternalResume(); |
263 void InternalStop(const base::Closure& callback); | 188 void InternalStop(const base::Closure& callback); |
264 void InternalMaybeCaptureForRefresh(); | 189 void InternalMaybeCaptureForRefresh(); |
265 bool IsStarted() const; | 190 bool IsStarted() const; |
266 | 191 |
267 // Computes the preferred size of the target RenderWidget for optimal capture. | 192 // Computes the preferred size of the target RenderWidget for optimal capture. |
268 gfx::Size ComputeOptimalViewSize() const; | 193 gfx::Size ComputeOptimalViewSize() const; |
269 | 194 |
270 // Response callback for RenderWidgetHost::CopyFromBackingStore(). | |
271 void DidCopyFromBackingStore( | |
272 const base::TimeTicks& start_time, | |
273 const scoped_refptr<media::VideoFrame>& target, | |
274 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | |
275 deliver_frame_cb, | |
276 const SkBitmap& bitmap, | |
277 ReadbackResponse response); | |
278 | |
279 // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame(). | 195 // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame(). |
280 void DidCopyFromCompositingSurfaceToVideoFrame( | 196 void DidCopyToVideoFrame( |
281 const base::TimeTicks& start_time, | 197 const base::TimeTicks& start_time, |
282 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 198 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
283 deliver_frame_cb, | 199 deliver_frame_cb, |
284 const gfx::Rect& region_in_frame, | 200 const gfx::Rect& region_in_frame, |
285 bool success); | 201 bool success); |
286 | 202 |
287 // Remove the old subscription, and attempt to start a new one. If | 203 // Remove the old subscription, and attempt to start a new one. If |
288 // |is_still_tracking| is false, emit an error rather than attempt to start a | 204 // |is_still_tracking| is false, emit an error rather than attempt to start a |
289 // new subscription. | 205 // new subscription. |
290 void RenewFrameSubscription(bool is_still_tracking); | 206 void RenewFrameSubscription(bool is_still_tracking); |
291 | 207 |
292 // Called whenever the render widget is resized. | 208 // Called whenever the render widget is resized. |
293 void UpdateCaptureSize(); | 209 void UpdateCaptureSize(); |
294 | 210 |
295 // Parameters saved in constructor. | 211 // Parameters saved in constructor. |
296 const int initial_render_process_id_; | 212 const int initial_render_process_id_; |
297 const int initial_main_render_frame_id_; | 213 const int initial_main_render_frame_id_; |
298 | 214 |
299 // Tracks events and calls back to RenewFrameSubscription() to maintain | 215 // Tracks events and calls back to RenewFrameSubscription() to maintain |
300 // capture on the correct RenderWidgetHostView. | 216 // capture on the correct RenderWidgetHostView. |
301 const scoped_refptr<WebContentsTracker> tracker_; | 217 const scoped_refptr<WebContentsTracker> tracker_; |
302 | 218 |
303 // Set to false to prevent the capture size from automatically adjusting in | 219 // Set to false to prevent the capture size from automatically adjusting in |
304 // response to end-to-end utilization. This is enabled via the throttling | 220 // response to end-to-end utilization. This is enabled via the throttling |
305 // option in the WebContentsVideoCaptureDevice device ID. | 221 // option in the WebContentsVideoCaptureDevice device ID. |
306 const bool auto_throttling_enabled_; | 222 const bool auto_throttling_enabled_; |
307 | 223 |
308 // A dedicated worker thread on which SkBitmap->VideoFrame conversion will | |
309 // occur. Only used when this activity cannot be done on the GPU. | |
310 std::unique_ptr<base::Thread> render_thread_; | |
311 | |
312 // Makes all the decisions about which frames to copy, and how. | 224 // Makes all the decisions about which frames to copy, and how. |
313 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; | 225 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; |
314 | 226 |
315 // Video capture parameters that this machine is started with. | 227 // Video capture parameters that this machine is started with. |
316 media::VideoCaptureParams capture_params_; | 228 media::VideoCaptureParams capture_params_; |
317 | 229 |
318 // Responsible for forwarding events from the active RenderWidgetHost to the | 230 // Responsible for forwarding events from the active RenderWidgetHost to the |
319 // oracle, and initiating captures accordingly. | 231 // oracle, and initiating captures accordingly. |
320 std::unique_ptr<ContentCaptureSubscription> subscription_; | 232 std::unique_ptr<ContentCaptureSubscription> subscription_; |
321 | 233 |
(...skipping 11 matching lines...) Expand all Loading... |
333 | 245 |
334 bool FrameSubscriber::ShouldCaptureFrame( | 246 bool FrameSubscriber::ShouldCaptureFrame( |
335 const gfx::Rect& damage_rect, | 247 const gfx::Rect& damage_rect, |
336 base::TimeTicks present_time, | 248 base::TimeTicks present_time, |
337 scoped_refptr<media::VideoFrame>* storage, | 249 scoped_refptr<media::VideoFrame>* storage, |
338 DeliverFrameCallback* deliver_frame_cb) { | 250 DeliverFrameCallback* deliver_frame_cb) { |
339 TRACE_EVENT1("gpu.capture", "FrameSubscriber::ShouldCaptureFrame", "instance", | 251 TRACE_EVENT1("gpu.capture", "FrameSubscriber::ShouldCaptureFrame", "instance", |
340 this); | 252 this); |
341 | 253 |
342 media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb; | 254 media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb; |
| 255 if (!oracle_proxy_->ObserveEventAndDecideCapture( |
| 256 event_type_, damage_rect, present_time, storage, &capture_frame_cb)) { |
| 257 return false; |
| 258 } |
343 | 259 |
344 bool oracle_decision = oracle_proxy_->ObserveEventAndDecideCapture( | 260 DCHECK(*storage); |
345 event_type_, damage_rect, present_time, storage, &capture_frame_cb); | 261 DCHECK(!capture_frame_cb.is_null()); |
346 | 262 *deliver_frame_cb = |
347 if (!capture_frame_cb.is_null()) | 263 base::Bind(&FrameSubscriber::DidCaptureFrame, |
348 *deliver_frame_cb = | 264 weak_ptr_factory_.GetWeakPtr(), capture_frame_cb, *storage); |
349 base::Bind(&FrameSubscriber::DidCaptureFrame, | 265 return true; |
350 weak_ptr_factory_.GetWeakPtr(), capture_frame_cb, *storage); | |
351 return oracle_decision; | |
352 } | 266 } |
353 | 267 |
354 void FrameSubscriber::DidCaptureFrame( | 268 void FrameSubscriber::DidCaptureFrame( |
355 base::WeakPtr<FrameSubscriber> frame_subscriber_, | 269 base::WeakPtr<FrameSubscriber> frame_subscriber_, |
356 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& | 270 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& |
357 capture_frame_cb, | 271 capture_frame_cb, |
358 scoped_refptr<media::VideoFrame> frame, | 272 scoped_refptr<media::VideoFrame> frame, |
359 base::TimeTicks timestamp, | 273 base::TimeTicks timestamp, |
360 const gfx::Rect& region_in_frame, | 274 const gfx::Rect& region_in_frame, |
361 bool success) { | 275 bool success) { |
362 // We can get a callback in the shutdown sequence for the browser main loop | |
363 // and this can result in a DCHECK failure. Avoid this by doing DCHECK only | |
364 // on success. | |
365 if (success) { | 276 if (success) { |
| 277 // We can get a callback in the shutdown sequence for the browser main loop |
| 278 // and this can result in a DCHECK failure. Avoid this by doing DCHECK only |
| 279 // on success. |
366 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 280 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
367 // TODO(isheriff): Unclear if taking a snapshot of cursor here affects user | 281 |
368 // experience in any particular scenarios. Doing it prior to capture may | |
369 // require evaluating region_in_frame in this file. | |
370 if (frame_subscriber_ && frame_subscriber_->cursor_renderer_) { | 282 if (frame_subscriber_ && frame_subscriber_->cursor_renderer_) { |
371 CursorRenderer* cursor_renderer = | 283 CursorRenderer* cursor_renderer = |
372 frame_subscriber_->cursor_renderer_.get(); | 284 frame_subscriber_->cursor_renderer_.get(); |
373 if (cursor_renderer->SnapshotCursorState(region_in_frame)) | 285 if (cursor_renderer->SnapshotCursorState(region_in_frame)) |
374 cursor_renderer->RenderOnVideoFrame(frame); | 286 cursor_renderer->RenderOnVideoFrame(frame); |
375 frame->metadata()->SetBoolean( | 287 frame->metadata()->SetBoolean( |
376 media::VideoFrameMetadata::INTERACTIVE_CONTENT, | 288 media::VideoFrameMetadata::INTERACTIVE_CONTENT, |
377 frame_subscriber_->IsUserInteractingWithContent()); | 289 frame_subscriber_->IsUserInteractingWithContent()); |
378 } | 290 } |
379 } | 291 } |
380 capture_frame_cb.Run(std::move(frame), timestamp, success); | 292 capture_frame_cb.Run(std::move(frame), timestamp, success); |
381 } | 293 } |
382 | 294 |
383 bool FrameSubscriber::IsUserInteractingWithContent() { | 295 bool FrameSubscriber::IsUserInteractingWithContent() { |
| 296 bool ui_activity = window_activity_tracker_ && |
| 297 window_activity_tracker_->IsUiInteractionActive(); |
384 bool interactive_mode = false; | 298 bool interactive_mode = false; |
385 bool ui_activity = false; | |
386 if (window_activity_tracker_) { | |
387 ui_activity = window_activity_tracker_->IsUiInteractionActive(); | |
388 } | |
389 if (cursor_renderer_) { | 299 if (cursor_renderer_) { |
390 bool animation_active = | 300 bool animation_active = |
391 (base::TimeTicks::Now() - | 301 (base::TimeTicks::Now() - |
392 oracle_proxy_->last_time_animation_was_detected()) < | 302 oracle_proxy_->last_time_animation_was_detected()) < |
393 base::TimeDelta::FromMilliseconds(kMinPeriodNoAnimationMillis); | 303 base::TimeDelta::FromMilliseconds(kMinPeriodNoAnimationMillis); |
394 if (ui_activity && !animation_active) { | 304 if (ui_activity && !animation_active) { |
395 interactive_mode = true; | 305 interactive_mode = true; |
396 } else if (animation_active && window_activity_tracker_.get()) { | 306 } else if (animation_active && window_activity_tracker_) { |
397 window_activity_tracker_->Reset(); | 307 window_activity_tracker_->Reset(); |
398 } | 308 } |
399 } | 309 } |
400 return interactive_mode; | 310 return interactive_mode; |
401 } | 311 } |
402 | 312 |
403 ContentCaptureSubscription::ContentCaptureSubscription( | 313 ContentCaptureSubscription::ContentCaptureSubscription( |
404 base::WeakPtr<RenderWidgetHostViewBase> source_view, | 314 base::WeakPtr<RenderWidgetHostViewBase> source_view, |
405 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 315 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
406 const CaptureCallback& capture_callback) | 316 const CaptureCallback& capture_callback) |
407 : source_view_(source_view), capture_callback_(capture_callback) { | 317 : source_view_(source_view), capture_callback_(capture_callback) { |
408 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 318 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
409 DCHECK(source_view_); | 319 DCHECK(source_view_); |
410 | 320 |
411 #if defined(USE_AURA) || defined(OS_MACOSX) | 321 #if defined(USE_AURA) || defined(OS_MACOSX) |
412 cursor_renderer_ = CursorRenderer::Create(source_view_->GetNativeView()); | 322 cursor_renderer_ = CursorRenderer::Create(source_view_->GetNativeView()); |
413 window_activity_tracker_ = | 323 window_activity_tracker_ = |
414 WindowActivityTracker::Create(source_view_->GetNativeView()); | 324 WindowActivityTracker::Create(source_view_->GetNativeView()); |
415 #endif | 325 #endif |
| 326 // Subscriptions for refresh frames and mouse cursor updates. When events |
| 327 // outside of the normal content change/compositing workflow occur, these |
| 328 // decide whether extra frames need to be captured. Such extra frame captures |
| 329 // are initiated by running the |capture_callback_|. |
416 refresh_subscriber_.reset(new FrameSubscriber( | 330 refresh_subscriber_.reset(new FrameSubscriber( |
417 media::VideoCaptureOracle::kActiveRefreshRequest, oracle_proxy, | 331 media::VideoCaptureOracle::kActiveRefreshRequest, oracle_proxy, |
418 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() | 332 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() |
419 : base::WeakPtr<CursorRenderer>(), | 333 : base::WeakPtr<CursorRenderer>(), |
420 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() | 334 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() |
421 : base::WeakPtr<WindowActivityTracker>())); | 335 : base::WeakPtr<WindowActivityTracker>())); |
422 mouse_activity_subscriber_.reset(new FrameSubscriber( | 336 mouse_activity_subscriber_.reset(new FrameSubscriber( |
423 media::VideoCaptureOracle::kMouseCursorUpdate, oracle_proxy, | 337 media::VideoCaptureOracle::kMouseCursorUpdate, oracle_proxy, |
424 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() | 338 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() |
425 : base::WeakPtr<CursorRenderer>(), | 339 : base::WeakPtr<CursorRenderer>(), |
426 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() | 340 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() |
427 : base::WeakPtr<WindowActivityTracker>())); | 341 : base::WeakPtr<WindowActivityTracker>())); |
428 | 342 |
429 // Subscribe to compositor updates. These will be serviced directly by the | 343 // Main capture path: Subscribing to compositor updates. This means that any |
430 // oracle. | 344 // time the tab content has changed and compositing has taken place, the |
| 345 // RenderWidgetHostView will consult the FrameSubscriber (which consults the |
| 346 // oracle) to determine whether a frame should be captured. If so, the |
| 347 // RenderWidgetHostView will initiate the frame capture and NOT the |
| 348 // |capture_callback_|. |
431 std::unique_ptr<RenderWidgetHostViewFrameSubscriber> subscriber( | 349 std::unique_ptr<RenderWidgetHostViewFrameSubscriber> subscriber( |
432 new FrameSubscriber( | 350 new FrameSubscriber( |
433 media::VideoCaptureOracle::kCompositorUpdate, oracle_proxy, | 351 media::VideoCaptureOracle::kCompositorUpdate, oracle_proxy, |
434 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() | 352 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() |
435 : base::WeakPtr<CursorRenderer>(), | 353 : base::WeakPtr<CursorRenderer>(), |
436 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() | 354 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() |
437 : base::WeakPtr<WindowActivityTracker>())); | 355 : base::WeakPtr<WindowActivityTracker>())); |
438 source_view_->BeginFrameSubscription(std::move(subscriber)); | 356 source_view_->BeginFrameSubscription(std::move(subscriber)); |
439 | 357 |
440 // Subscribe to mouse movement and mouse cursor update events. | 358 // Begin monitoring for user activity that is used to signal "interactive |
| 359 // content." |
441 if (window_activity_tracker_) { | 360 if (window_activity_tracker_) { |
442 window_activity_tracker_->RegisterMouseInteractionObserver( | 361 window_activity_tracker_->RegisterMouseInteractionObserver( |
443 base::Bind(&ContentCaptureSubscription::OnEvent, base::Unretained(this), | 362 base::Bind(&ContentCaptureSubscription::OnEvent, base::Unretained(this), |
444 mouse_activity_subscriber_.get())); | 363 mouse_activity_subscriber_.get())); |
445 } | 364 } |
446 } | 365 } |
447 | 366 |
448 ContentCaptureSubscription::~ContentCaptureSubscription() { | 367 ContentCaptureSubscription::~ContentCaptureSubscription() { |
449 // If the BrowserThreads have been torn down, then the browser is in the final | 368 // If the BrowserThreads have been torn down, then the browser is in the final |
450 // stages of exiting and it is dangerous to take any further action. We must | 369 // stages of exiting and it is dangerous to take any further action. We must |
(...skipping 25 matching lines...) Expand all Loading... |
476 | 395 |
477 const base::TimeTicks start_time = base::TimeTicks::Now(); | 396 const base::TimeTicks start_time = base::TimeTicks::Now(); |
478 DCHECK(subscriber == refresh_subscriber_.get() || | 397 DCHECK(subscriber == refresh_subscriber_.get() || |
479 subscriber == mouse_activity_subscriber_.get()); | 398 subscriber == mouse_activity_subscriber_.get()); |
480 if (subscriber->ShouldCaptureFrame(gfx::Rect(), start_time, &frame, | 399 if (subscriber->ShouldCaptureFrame(gfx::Rect(), start_time, &frame, |
481 &deliver_frame_cb)) { | 400 &deliver_frame_cb)) { |
482 capture_callback_.Run(start_time, frame, deliver_frame_cb); | 401 capture_callback_.Run(start_time, frame, deliver_frame_cb); |
483 } | 402 } |
484 } | 403 } |
485 | 404 |
486 void RenderVideoFrame( | |
487 const SkBitmap& input, | |
488 const scoped_refptr<media::VideoFrame>& output, | |
489 const base::Callback<void(const gfx::Rect&, bool)>& done_cb) { | |
490 base::ScopedClosureRunner failure_handler( | |
491 base::Bind(done_cb, gfx::Rect(), false)); | |
492 | |
493 SkAutoLockPixels locker(input); | |
494 | |
495 // Sanity-check the captured bitmap. | |
496 if (input.empty() || !input.readyToDraw() || | |
497 input.colorType() != kN32_SkColorType || input.width() < 2 || | |
498 input.height() < 2) { | |
499 DVLOG(1) << "input unacceptable (size=" << input.getSize() | |
500 << ", ready=" << input.readyToDraw() | |
501 << ", colorType=" << input.colorType() << ')'; | |
502 return; | |
503 } | |
504 | |
505 // Sanity-check the output buffer. | |
506 if (output->format() != media::PIXEL_FORMAT_I420) { | |
507 NOTREACHED(); | |
508 return; | |
509 } | |
510 | |
511 // Calculate the width and height of the content region in the |output|, based | |
512 // on the aspect ratio of |input|. | |
513 const gfx::Rect region_in_frame = media::ComputeLetterboxRegion( | |
514 output->visible_rect(), gfx::Size(input.width(), input.height())); | |
515 | |
516 // Scale the bitmap to the required size, if necessary. | |
517 SkBitmap scaled_bitmap; | |
518 if (input.width() != region_in_frame.width() || | |
519 input.height() != region_in_frame.height()) { | |
520 skia::ImageOperations::ResizeMethod method; | |
521 if (input.width() < region_in_frame.width() || | |
522 input.height() < region_in_frame.height()) { | |
523 // Avoid box filtering when magnifying, because it's actually | |
524 // nearest-neighbor. | |
525 method = skia::ImageOperations::RESIZE_HAMMING1; | |
526 } else { | |
527 method = skia::ImageOperations::RESIZE_BOX; | |
528 } | |
529 | |
530 TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", output.get(), | |
531 "Scale"); | |
532 scaled_bitmap = skia::ImageOperations::Resize( | |
533 input, method, region_in_frame.width(), region_in_frame.height()); | |
534 } else { | |
535 scaled_bitmap = input; | |
536 } | |
537 | |
538 TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", output.get(), "YUV"); | |
539 { | |
540 // Align to 2x2 pixel boundaries, as required by | |
541 // media::CopyRGBToVideoFrame(). | |
542 const gfx::Rect region_in_yv12_frame( | |
543 region_in_frame.x() & ~1, region_in_frame.y() & ~1, | |
544 region_in_frame.width() & ~1, region_in_frame.height() & ~1); | |
545 if (region_in_yv12_frame.IsEmpty()) | |
546 return; | |
547 | |
548 SkAutoLockPixels scaled_bitmap_locker(scaled_bitmap); | |
549 media::CopyRGBToVideoFrame( | |
550 reinterpret_cast<uint8_t*>(scaled_bitmap.getPixels()), | |
551 scaled_bitmap.rowBytes(), region_in_yv12_frame, output.get()); | |
552 } | |
553 | |
554 // The result is now ready. | |
555 ignore_result(failure_handler.Release()); | |
556 done_cb.Run(region_in_frame, true); | |
557 } | |
558 | |
559 WebContentsCaptureMachine::WebContentsCaptureMachine( | 405 WebContentsCaptureMachine::WebContentsCaptureMachine( |
560 int render_process_id, | 406 int render_process_id, |
561 int main_render_frame_id, | 407 int main_render_frame_id, |
562 bool enable_auto_throttling) | 408 bool enable_auto_throttling) |
563 : initial_render_process_id_(render_process_id), | 409 : initial_render_process_id_(render_process_id), |
564 initial_main_render_frame_id_(main_render_frame_id), | 410 initial_main_render_frame_id_(main_render_frame_id), |
565 tracker_(new WebContentsTracker(true)), | 411 tracker_(new WebContentsTracker(true)), |
566 auto_throttling_enabled_(enable_auto_throttling), | 412 auto_throttling_enabled_(enable_auto_throttling), |
567 frame_capture_active_(true), | 413 frame_capture_active_(true), |
568 weak_ptr_factory_(this) { | 414 weak_ptr_factory_(this) { |
(...skipping 15 matching lines...) Expand all Loading... |
584 const base::Callback<void(bool)> callback) { | 430 const base::Callback<void(bool)> callback) { |
585 // Starts the capture machine asynchronously. | 431 // Starts the capture machine asynchronously. |
586 BrowserThread::PostTaskAndReplyWithResult( | 432 BrowserThread::PostTaskAndReplyWithResult( |
587 BrowserThread::UI, FROM_HERE, | 433 BrowserThread::UI, FROM_HERE, |
588 base::Bind(&WebContentsCaptureMachine::InternalStart, | 434 base::Bind(&WebContentsCaptureMachine::InternalStart, |
589 base::Unretained(this), oracle_proxy, params), | 435 base::Unretained(this), oracle_proxy, params), |
590 callback); | 436 callback); |
591 } | 437 } |
592 | 438 |
593 bool WebContentsCaptureMachine::InternalStart( | 439 bool WebContentsCaptureMachine::InternalStart( |
594 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 440 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
595 const media::VideoCaptureParams& params) { | 441 const media::VideoCaptureParams& params) { |
596 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 442 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
597 DCHECK(!IsStarted()); | 443 DCHECK(!IsStarted()); |
598 | 444 |
599 DCHECK(oracle_proxy); | 445 DCHECK(oracle_proxy); |
600 oracle_proxy_ = oracle_proxy; | 446 oracle_proxy_ = std::move(oracle_proxy); |
601 capture_params_ = params; | 447 capture_params_ = params; |
602 | 448 |
603 render_thread_.reset(new base::Thread("WebContentsVideo_RenderThread")); | |
604 if (!render_thread_->Start()) { | |
605 DVLOG(1) << "Failed to spawn render thread."; | |
606 render_thread_.reset(); | |
607 return false; | |
608 } | |
609 | |
610 // Note: Creation of the first WeakPtr in the following statement will cause | 449 // Note: Creation of the first WeakPtr in the following statement will cause |
611 // IsStarted() to return true from now on. | 450 // IsStarted() to return true from now on. |
612 tracker_->SetResizeChangeCallback( | 451 tracker_->SetResizeChangeCallback( |
613 base::Bind(&WebContentsCaptureMachine::UpdateCaptureSize, | 452 base::Bind(&WebContentsCaptureMachine::UpdateCaptureSize, |
614 weak_ptr_factory_.GetWeakPtr())); | 453 weak_ptr_factory_.GetWeakPtr())); |
615 tracker_->Start(initial_render_process_id_, initial_main_render_frame_id_, | 454 tracker_->Start(initial_render_process_id_, initial_main_render_frame_id_, |
616 base::Bind(&WebContentsCaptureMachine::RenewFrameSubscription, | 455 base::Bind(&WebContentsCaptureMachine::RenewFrameSubscription, |
617 weak_ptr_factory_.GetWeakPtr())); | 456 weak_ptr_factory_.GetWeakPtr())); |
618 if (WebContents* contents = tracker_->web_contents()) | 457 if (WebContents* contents = tracker_->web_contents()) |
619 contents->IncrementCapturerCount(ComputeOptimalViewSize()); | 458 contents->IncrementCapturerCount(ComputeOptimalViewSize()); |
(...skipping 26 matching lines...) Expand all Loading... |
646 void WebContentsCaptureMachine::InternalResume() { | 485 void WebContentsCaptureMachine::InternalResume() { |
647 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 486 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
648 if (frame_capture_active_) | 487 if (frame_capture_active_) |
649 return; | 488 return; |
650 frame_capture_active_ = true; | 489 frame_capture_active_ = true; |
651 if (IsStarted()) | 490 if (IsStarted()) |
652 RenewFrameSubscription(tracker_->is_still_tracking()); | 491 RenewFrameSubscription(tracker_->is_still_tracking()); |
653 } | 492 } |
654 | 493 |
655 void WebContentsCaptureMachine::Stop(const base::Closure& callback) { | 494 void WebContentsCaptureMachine::Stop(const base::Closure& callback) { |
656 // Stops the capture machine asynchronously. | |
657 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, | 495 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, |
658 base::Bind(&WebContentsCaptureMachine::InternalStop, | 496 base::Bind(&WebContentsCaptureMachine::InternalStop, |
659 base::Unretained(this), callback)); | 497 base::Unretained(this), callback)); |
660 } | 498 } |
661 | 499 |
662 void WebContentsCaptureMachine::InternalStop(const base::Closure& callback) { | 500 void WebContentsCaptureMachine::InternalStop(const base::Closure& callback) { |
663 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 501 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
664 | 502 |
665 if (!IsStarted()) { | 503 base::ScopedClosureRunner done_runner(callback); |
666 callback.Run(); | 504 |
| 505 if (!IsStarted()) |
667 return; | 506 return; |
668 } | |
669 | 507 |
670 // The following cancels any outstanding callbacks and causes IsStarted() to | 508 // The following cancels any outstanding callbacks and causes IsStarted() to |
671 // return false from here onward. | 509 // return false from here onward. |
672 weak_ptr_factory_.InvalidateWeakPtrs(); | 510 weak_ptr_factory_.InvalidateWeakPtrs(); |
673 | 511 |
674 RenewFrameSubscription(false); | 512 RenewFrameSubscription(false); |
675 if (WebContents* contents = tracker_->web_contents()) | 513 if (WebContents* contents = tracker_->web_contents()) |
676 contents->DecrementCapturerCount(); | 514 contents->DecrementCapturerCount(); |
677 tracker_->Stop(); | 515 tracker_->Stop(); |
678 | |
679 // The render thread cannot be stopped on the UI thread, so post a message | |
680 // to the thread pool used for blocking operations. | |
681 if (render_thread_) { | |
682 BrowserThread::PostBlockingPoolTask( | |
683 FROM_HERE, base::Bind(&DeleteOnWorkerThread, | |
684 base::Passed(&render_thread_), callback)); | |
685 } | |
686 } | 516 } |
687 | 517 |
688 void WebContentsCaptureMachine::MaybeCaptureForRefresh() { | 518 void WebContentsCaptureMachine::MaybeCaptureForRefresh() { |
689 BrowserThread::PostTask( | 519 BrowserThread::PostTask( |
690 BrowserThread::UI, FROM_HERE, | 520 BrowserThread::UI, FROM_HERE, |
691 base::Bind(&WebContentsCaptureMachine::InternalMaybeCaptureForRefresh, | 521 base::Bind(&WebContentsCaptureMachine::InternalMaybeCaptureForRefresh, |
692 // Use of Unretained() is safe here since this task must run | 522 // Use of Unretained() is safe here since this task must run |
693 // before InternalStop(). | 523 // before InternalStop(). |
694 base::Unretained(this))); | 524 base::Unretained(this))); |
695 } | 525 } |
696 | 526 |
697 void WebContentsCaptureMachine::InternalMaybeCaptureForRefresh() { | 527 void WebContentsCaptureMachine::InternalMaybeCaptureForRefresh() { |
698 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 528 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
699 if (IsStarted() && subscription_) | 529 if (IsStarted() && subscription_) |
700 subscription_->MaybeCaptureForRefresh(); | 530 subscription_->MaybeCaptureForRefresh(); |
701 } | 531 } |
702 | 532 |
703 void WebContentsCaptureMachine::Capture( | 533 void WebContentsCaptureMachine::Capture( |
704 const base::TimeTicks& start_time, | 534 base::TimeTicks start_time, |
705 const scoped_refptr<media::VideoFrame>& target, | 535 scoped_refptr<media::VideoFrame> target, |
706 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 536 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
707 deliver_frame_cb) { | 537 deliver_frame_cb) { |
708 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 538 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
709 | 539 |
710 auto* const view = | 540 auto* const view = |
711 static_cast<RenderWidgetHostViewBase*>(tracker_->GetTargetView()); | 541 static_cast<RenderWidgetHostViewBase*>(tracker_->GetTargetView()); |
712 if (!view) { | 542 if (!view || !view->CanCopyToVideoFrame()) { |
713 deliver_frame_cb.Run(base::TimeTicks(), gfx::Rect(), false); | 543 deliver_frame_cb.Run(base::TimeTicks(), gfx::Rect(), false); |
714 return; | 544 return; |
715 } | 545 } |
716 | 546 |
717 const gfx::Size view_size = view->GetViewBounds().size(); | 547 const gfx::Size view_size = view->GetViewBounds().size(); |
718 if (view->CanCopyToVideoFrame()) { | 548 view->CopyFromCompositingSurfaceToVideoFrame( |
719 view->CopyFromCompositingSurfaceToVideoFrame( | 549 gfx::Rect(view_size), std::move(target), |
720 gfx::Rect(view_size), target, | 550 base::Bind(&WebContentsCaptureMachine::DidCopyToVideoFrame, |
721 base::Bind(&WebContentsCaptureMachine:: | 551 weak_ptr_factory_.GetWeakPtr(), start_time, deliver_frame_cb)); |
722 DidCopyFromCompositingSurfaceToVideoFrame, | |
723 weak_ptr_factory_.GetWeakPtr(), start_time, | |
724 deliver_frame_cb)); | |
725 } else { | |
726 const gfx::Size fitted_size = | |
727 view_size.IsEmpty() | |
728 ? gfx::Size() | |
729 : media::ComputeLetterboxRegion(target->visible_rect(), view_size) | |
730 .size(); | |
731 if (auto* rwh = view->GetRenderWidgetHost()) { | |
732 rwh->CopyFromBackingStore( | |
733 gfx::Rect(), | |
734 fitted_size, // Size here is a request not always honored. | |
735 base::Bind(&WebContentsCaptureMachine::DidCopyFromBackingStore, | |
736 weak_ptr_factory_.GetWeakPtr(), start_time, target, | |
737 deliver_frame_cb), | |
738 kN32_SkColorType); | |
739 } | |
740 } | |
741 } | 552 } |
742 | 553 |
743 gfx::Size WebContentsCaptureMachine::ComputeOptimalViewSize() const { | 554 gfx::Size WebContentsCaptureMachine::ComputeOptimalViewSize() const { |
744 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 555 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
745 | 556 |
746 // TODO(miu): Propagate capture frame size changes as new "preferred size" | 557 // TODO(miu): Propagate capture frame size changes as new "preferred size" |
747 // updates, rather than just using the max frame size. | 558 // updates, rather than just using the max frame size. |
748 // http://crbug.com/350491 | 559 // http://crbug.com/350491 |
749 gfx::Size optimal_size = oracle_proxy_->max_frame_size(); | 560 gfx::Size optimal_size = oracle_proxy_->max_frame_size(); |
750 | 561 |
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
797 gfx::ScaleToFlooredSize(optimal_size, 1.0f / scale); | 608 gfx::ScaleToFlooredSize(optimal_size, 1.0f / scale); |
798 if (shrunk_size.width() > 0 && shrunk_size.height() > 0) | 609 if (shrunk_size.width() > 0 && shrunk_size.height() > 0) |
799 optimal_size = shrunk_size; | 610 optimal_size = shrunk_size; |
800 } | 611 } |
801 } | 612 } |
802 | 613 |
803 VLOG(1) << "Computed optimal target size: " << optimal_size.ToString(); | 614 VLOG(1) << "Computed optimal target size: " << optimal_size.ToString(); |
804 return optimal_size; | 615 return optimal_size; |
805 } | 616 } |
806 | 617 |
807 void WebContentsCaptureMachine::DidCopyFromBackingStore( | 618 void WebContentsCaptureMachine::DidCopyToVideoFrame( |
808 const base::TimeTicks& start_time, | |
809 const scoped_refptr<media::VideoFrame>& target, | |
810 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | |
811 deliver_frame_cb, | |
812 const SkBitmap& bitmap, | |
813 ReadbackResponse response) { | |
814 DCHECK_CURRENTLY_ON(BrowserThread::UI); | |
815 | |
816 DCHECK(render_thread_); | |
817 if (response == READBACK_SUCCESS) { | |
818 TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", target.get(), | |
819 "Render"); | |
820 render_thread_->task_runner()->PostTask( | |
821 FROM_HERE, media::BindToCurrentLoop( | |
822 base::Bind(&RenderVideoFrame, bitmap, target, | |
823 base::Bind(deliver_frame_cb, start_time)))); | |
824 } else { | |
825 // Capture can fail due to transient issues, so just skip this frame. | |
826 DVLOG(1) << "CopyFromBackingStore failed; skipping frame."; | |
827 deliver_frame_cb.Run(start_time, gfx::Rect(), false); | |
828 } | |
829 } | |
830 | |
831 void WebContentsCaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame( | |
832 const base::TimeTicks& start_time, | 619 const base::TimeTicks& start_time, |
833 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 620 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
834 deliver_frame_cb, | 621 deliver_frame_cb, |
835 const gfx::Rect& region_in_frame, | 622 const gfx::Rect& region_in_frame, |
836 bool success) { | 623 bool success) { |
837 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 624 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
838 | 625 |
839 // Capture can fail due to transient issues, so just skip this frame. | 626 // Capture can fail due to transient issues, so just skip this frame. |
840 DVLOG_IF(1, !success) << "CopyFromCompositingSurface failed; skipping frame."; | 627 DVLOG_IF(1, !success) << "CopyFromCompositingSurface failed; skipping frame."; |
841 deliver_frame_cb.Run(start_time, region_in_frame, success); | 628 deliver_frame_cb.Run(start_time, region_in_frame, success); |
(...skipping 74 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
916 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() { | 703 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() { |
917 DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying."; | 704 DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying."; |
918 } | 705 } |
919 | 706 |
920 // static | 707 // static |
921 std::unique_ptr<media::VideoCaptureDevice> | 708 std::unique_ptr<media::VideoCaptureDevice> |
922 WebContentsVideoCaptureDevice::Create(const std::string& device_id) { | 709 WebContentsVideoCaptureDevice::Create(const std::string& device_id) { |
923 // Parse device_id into render_process_id and main_render_frame_id. | 710 // Parse device_id into render_process_id and main_render_frame_id. |
924 WebContentsMediaCaptureId media_id; | 711 WebContentsMediaCaptureId media_id; |
925 if (!WebContentsMediaCaptureId::Parse(device_id, &media_id)) { | 712 if (!WebContentsMediaCaptureId::Parse(device_id, &media_id)) { |
926 return NULL; | 713 return nullptr; |
927 } | 714 } |
928 | 715 |
929 return std::unique_ptr<media::VideoCaptureDevice>( | 716 return std::unique_ptr<media::VideoCaptureDevice>( |
930 new WebContentsVideoCaptureDevice(media_id.render_process_id, | 717 new WebContentsVideoCaptureDevice(media_id.render_process_id, |
931 media_id.main_render_frame_id, | 718 media_id.main_render_frame_id, |
932 media_id.enable_auto_throttling)); | 719 media_id.enable_auto_throttling)); |
933 } | 720 } |
934 | 721 |
935 void WebContentsVideoCaptureDevice::AllocateAndStart( | 722 void WebContentsVideoCaptureDevice::AllocateAndStart( |
936 const media::VideoCaptureParams& params, | 723 const media::VideoCaptureParams& params, |
(...skipping 17 matching lines...) Expand all Loading... |
954 void WebContentsVideoCaptureDevice::StopAndDeAllocate() { | 741 void WebContentsVideoCaptureDevice::StopAndDeAllocate() { |
955 core_->StopAndDeAllocate(); | 742 core_->StopAndDeAllocate(); |
956 } | 743 } |
957 | 744 |
958 void WebContentsVideoCaptureDevice::OnUtilizationReport(int frame_feedback_id, | 745 void WebContentsVideoCaptureDevice::OnUtilizationReport(int frame_feedback_id, |
959 double utilization) { | 746 double utilization) { |
960 core_->OnConsumerReportingUtilization(frame_feedback_id, utilization); | 747 core_->OnConsumerReportingUtilization(frame_feedback_id, utilization); |
961 } | 748 } |
962 | 749 |
963 } // namespace content | 750 } // namespace content |
OLD | NEW |