OLD | NEW |
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 // | |
5 // Implementation notes: This needs to work on a variety of hardware | |
6 // configurations where the speed of the CPU and GPU greatly affect overall | |
7 // performance. Spanning several threads, the process of capturing has been | |
8 // split up into four conceptual stages: | |
9 // | |
10 // 1. Reserve Buffer: Before a frame can be captured, a slot in the client's | |
11 // shared-memory IPC buffer is reserved. There are only a few of these; | |
12 // when they run out, it indicates that the downstream client -- likely a | |
13 // video encoder -- is the performance bottleneck, and that the rate of | |
14 // frame capture should be throttled back. | |
15 // | |
16 // 2. Capture: A bitmap is snapshotted/copied from the RenderWidget's backing | |
17 // store. This is initiated on the UI BrowserThread, and often occurs | |
18 // asynchronously. Where supported, the GPU scales and color converts | |
19 // frames to our desired size, and the readback happens directly into the | |
20 // shared-memory buffer. But this is not always possible, particularly when | |
21 // accelerated compositing is disabled. | |
22 // | |
23 // 3. Render (if needed): If the web contents cannot be captured directly into | |
24 // our target size and color format, scaling and colorspace conversion must | |
25 // be done on the CPU. A dedicated thread is used for this operation, to | |
26 // avoid blocking the UI thread. The Render stage always reads from a | |
27 // bitmap returned by Capture, and writes into the reserved slot in the | |
28 // shared-memory buffer. | |
29 // | |
30 // 4. Deliver: The rendered video frame is returned to the client (which | |
31 // implements the VideoCaptureDevice::Client interface). Because all | |
32 // paths have written the frame into the IPC buffer, this step should | |
33 // never need to do an additional copy of the pixel data. | |
34 // | |
35 // In the best-performing case, the Render step is bypassed: Capture produces | |
36 // ready-to-Deliver frames. But when accelerated readback is not possible, the | |
37 // system is designed so that Capture and Render may run concurrently. A timing | |
38 // diagram helps illustrate this point (@30 FPS): | |
39 // | |
40 // Time: 0ms 33ms 66ms 99ms | |
41 // thread1: |-Capture-f1------v |-Capture-f2------v |-Capture-f3----v |-Capt | |
42 // thread2: |-Render-f1-----v |-Render-f2-----v |-Render-f3 | |
43 // | |
44 // In the above example, both capturing and rendering *each* take almost the | |
45 // full 33 ms available between frames, yet we see that the required throughput | |
46 // is obtained. | |
47 | 4 |
48 #include "content/browser/media/capture/web_contents_video_capture_device.h" | 5 #include "content/browser/media/capture/web_contents_video_capture_device.h" |
49 | 6 |
50 #include <stdint.h> | 7 #include <stdint.h> |
51 | 8 |
52 #include <algorithm> | 9 #include <algorithm> |
53 #include <memory> | 10 #include <memory> |
54 #include <utility> | 11 #include <string> |
55 | 12 |
56 #include "base/bind.h" | 13 #include "base/bind.h" |
57 #include "base/callback_helpers.h" | 14 #include "base/callback_helpers.h" |
58 #include "base/location.h" | 15 #include "base/location.h" |
59 #include "base/logging.h" | 16 #include "base/logging.h" |
60 #include "base/macros.h" | 17 #include "base/macros.h" |
61 #include "base/memory/ptr_util.h" | |
62 #include "base/memory/weak_ptr.h" | 18 #include "base/memory/weak_ptr.h" |
63 #include "base/sequenced_task_runner.h" | |
64 #include "base/single_thread_task_runner.h" | |
65 #include "base/threading/thread.h" | |
66 #include "base/threading/thread_checker.h" | |
67 #include "base/time/time.h" | 19 #include "base/time/time.h" |
68 #include "build/build_config.h" | 20 #include "build/build_config.h" |
69 #include "content/browser/media/capture/cursor_renderer.h" | 21 #include "content/browser/media/capture/cursor_renderer.h" |
70 #include "content/browser/media/capture/web_contents_tracker.h" | 22 #include "content/browser/media/capture/web_contents_tracker.h" |
71 #include "content/browser/media/capture/window_activity_tracker.h" | 23 #include "content/browser/media/capture/window_activity_tracker.h" |
72 #include "content/browser/renderer_host/render_widget_host_impl.h" | 24 #include "content/browser/renderer_host/render_widget_host_impl.h" |
73 #include "content/browser/renderer_host/render_widget_host_view_base.h" | 25 #include "content/browser/renderer_host/render_widget_host_view_base.h" |
74 #include "content/public/browser/browser_thread.h" | 26 #include "content/public/browser/browser_thread.h" |
75 #include "content/public/browser/render_process_host.h" | 27 #include "content/public/browser/render_process_host.h" |
76 #include "content/public/browser/render_widget_host_view.h" | 28 #include "content/public/browser/render_widget_host_view.h" |
77 #include "content/public/browser/render_widget_host_view_frame_subscriber.h" | 29 #include "content/public/browser/render_widget_host_view_frame_subscriber.h" |
78 #include "content/public/browser/web_contents.h" | 30 #include "content/public/browser/web_contents.h" |
79 #include "content/public/browser/web_contents_media_capture_id.h" | 31 #include "content/public/browser/web_contents_media_capture_id.h" |
80 #include "media/base/bind_to_current_loop.h" | |
81 #include "media/base/video_frame_metadata.h" | 32 #include "media/base/video_frame_metadata.h" |
82 #include "media/base/video_util.h" | |
83 #include "media/capture/content/screen_capture_device_core.h" | 33 #include "media/capture/content/screen_capture_device_core.h" |
84 #include "media/capture/content/thread_safe_capture_oracle.h" | 34 #include "media/capture/content/thread_safe_capture_oracle.h" |
85 #include "media/capture/content/video_capture_oracle.h" | 35 #include "media/capture/content/video_capture_oracle.h" |
86 #include "media/capture/video_capture_types.h" | 36 #include "media/capture/video_capture_types.h" |
87 #include "skia/ext/image_operations.h" | |
88 #include "third_party/skia/include/core/SkBitmap.h" | |
89 #include "third_party/skia/include/core/SkColor.h" | |
90 #include "ui/base/layout.h" | 37 #include "ui/base/layout.h" |
91 #include "ui/gfx/geometry/dip_util.h" | 38 #include "ui/gfx/geometry/dip_util.h" |
92 #include "ui/gfx/geometry/size_conversions.h" | 39 #include "ui/gfx/geometry/size_conversions.h" |
93 | 40 |
94 namespace content { | 41 namespace content { |
95 | 42 |
96 namespace { | 43 namespace { |
97 | 44 |
98 enum InteractiveModeSettings { | 45 // Minimum amount of time for which there should be no animation detected |
99 // Minimum amount of time for which there should be no animation detected | 46 // to consider interactive mode being active. This is to prevent very brief |
100 // to consider interactive mode being active. This is to prevent very brief | 47 // periods of animated content not being detected (due to CPU fluctations for |
101 // periods of animated content not being detected (due to CPU fluctations for | 48 // example) from causing a flip-flop on interactive mode. |
102 // example) from causing a flip-flop on interactive mode. | 49 constexpr int64_t kMinPeriodNoAnimationMillis = 3000; |
103 kMinPeriodNoAnimationMillis = 3000 | |
104 }; | |
105 | |
106 void DeleteOnWorkerThread(std::unique_ptr<base::Thread> render_thread, | |
107 const base::Closure& callback) { | |
108 render_thread.reset(); | |
109 | |
110 // After thread join call the callback on UI thread. | |
111 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, callback); | |
112 } | |
113 | 50 |
114 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible | 51 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible |
115 // with RenderWidgetHostViewFrameSubscriber. We create one per event type. | 52 // with RenderWidgetHostViewFrameSubscriber. One is created per event type. |
116 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber { | 53 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber { |
117 public: | 54 public: |
118 FrameSubscriber(media::VideoCaptureOracle::Event event_type, | 55 FrameSubscriber(media::VideoCaptureOracle::Event event_type, |
119 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle, | 56 scoped_refptr<media::ThreadSafeCaptureOracle> oracle, |
120 base::WeakPtr<content::CursorRenderer> cursor_renderer, | 57 base::WeakPtr<content::CursorRenderer> cursor_renderer, |
121 base::WeakPtr<content::WindowActivityTracker> tracker) | 58 base::WeakPtr<content::WindowActivityTracker> tracker) |
122 : event_type_(event_type), | 59 : event_type_(event_type), |
123 oracle_proxy_(oracle), | 60 oracle_proxy_(std::move(oracle)), |
124 cursor_renderer_(cursor_renderer), | 61 cursor_renderer_(cursor_renderer), |
125 window_activity_tracker_(tracker), | 62 window_activity_tracker_(tracker), |
126 weak_ptr_factory_(this) {} | 63 weak_ptr_factory_(this) {} |
127 | 64 |
128 bool ShouldCaptureFrame( | 65 bool ShouldCaptureFrame( |
129 const gfx::Rect& damage_rect, | 66 const gfx::Rect& damage_rect, |
130 base::TimeTicks present_time, | 67 base::TimeTicks present_time, |
131 scoped_refptr<media::VideoFrame>* storage, | 68 scoped_refptr<media::VideoFrame>* storage, |
132 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback* | 69 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback* |
133 deliver_frame_cb) override; | 70 deliver_frame_cb) override; |
134 | 71 |
135 static void DidCaptureFrame( | 72 static void DidCaptureFrame( |
136 base::WeakPtr<FrameSubscriber> frame_subscriber_, | 73 base::WeakPtr<FrameSubscriber> frame_subscriber_, |
137 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& | 74 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& |
138 capture_frame_cb, | 75 capture_frame_cb, |
139 scoped_refptr<media::VideoFrame> frame, | 76 scoped_refptr<media::VideoFrame> frame, |
140 base::TimeTicks timestamp, | 77 base::TimeTicks timestamp, |
141 const gfx::Rect& region_in_frame, | 78 const gfx::Rect& region_in_frame, |
142 bool success); | 79 bool success); |
143 | 80 |
144 bool IsUserInteractingWithContent(); | 81 bool IsUserInteractingWithContent(); |
145 | 82 |
146 private: | 83 private: |
147 const media::VideoCaptureOracle::Event event_type_; | 84 const media::VideoCaptureOracle::Event event_type_; |
148 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; | 85 const scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; |
149 // We need a weak pointer since FrameSubscriber is owned externally and | 86 // We need a weak pointer since FrameSubscriber is owned externally and |
150 // may outlive the cursor renderer. | 87 // may outlive the cursor renderer. |
151 base::WeakPtr<CursorRenderer> cursor_renderer_; | 88 const base::WeakPtr<CursorRenderer> cursor_renderer_; |
152 // We need a weak pointer since FrameSubscriber is owned externally and | 89 // We need a weak pointer since FrameSubscriber is owned externally and |
153 // may outlive the ui activity tracker. | 90 // may outlive the ui activity tracker. |
154 base::WeakPtr<WindowActivityTracker> window_activity_tracker_; | 91 const base::WeakPtr<WindowActivityTracker> window_activity_tracker_; |
155 base::WeakPtrFactory<FrameSubscriber> weak_ptr_factory_; | 92 base::WeakPtrFactory<FrameSubscriber> weak_ptr_factory_; |
156 }; | 93 }; |
157 | 94 |
158 // ContentCaptureSubscription is the relationship between a RenderWidgetHost | 95 // ContentCaptureSubscription is the relationship between a RenderWidgetHost |
159 // whose content is updating, a subscriber that is deciding which of these | 96 // whose content is updating, a subscriber that is deciding which of these |
160 // updates to capture (and where to deliver them to), and a callback that | 97 // updates to capture (and where to deliver them to), and a callback that |
161 // knows how to do the capture and prepare the result for delivery. | 98 // knows how to do the capture and prepare the result for delivery. |
162 // | 99 // |
163 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in | 100 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in |
164 // the RenderWidgetHostView, to process compositor updates, and (b) occasionally | 101 // the RenderWidgetHostView, to process compositor updates, and (b) occasionally |
165 // initiating forced, non-event-driven captures needed by downstream consumers | 102 // using the capture callback to force additional captures that are needed for |
166 // that request "refresh frames" of unchanged content. | 103 // things like mouse cursor rendering updates and/or refresh frames. |
167 // | 104 // |
168 // All of this happens on the UI thread, although the | 105 // All of this happens on the UI thread, although the |
169 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates | 106 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates |
170 // autonomously on some other thread. | 107 // autonomously on some other thread. |
171 class ContentCaptureSubscription { | 108 class ContentCaptureSubscription { |
172 public: | 109 public: |
173 typedef base::Callback<void( | 110 using CaptureCallback = base::Callback<void( |
174 const base::TimeTicks&, | 111 base::TimeTicks, |
175 const scoped_refptr<media::VideoFrame>&, | 112 scoped_refptr<media::VideoFrame>, |
176 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&)> | 113 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&)>; |
177 CaptureCallback; | |
178 | 114 |
179 // Create a subscription. Whenever a manual capture is required, the | 115 // Create a subscription. Whenever a non-compositor capture is required, the |
180 // subscription will invoke |capture_callback| on the UI thread to do the | 116 // subscription will invoke |capture_callback| on the UI thread to do the |
181 // work. | 117 // work. |
182 ContentCaptureSubscription( | 118 ContentCaptureSubscription( |
183 const RenderWidgetHost& source, | 119 const RenderWidgetHost& source, |
184 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 120 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
185 const CaptureCallback& capture_callback); | 121 const CaptureCallback& capture_callback); |
186 ~ContentCaptureSubscription(); | 122 ~ContentCaptureSubscription(); |
187 | 123 |
| 124 // Called when a condition occurs that requires a refresh frame be |
| 125 // captured. The subscribers will determine whether a frame was recently |
| 126 // captured, or if a non-compositor initiated capture must be made. |
188 void MaybeCaptureForRefresh(); | 127 void MaybeCaptureForRefresh(); |
189 | 128 |
190 private: | 129 private: |
191 // Called for active frame refresh requests, or mouse activity events. | 130 // Called for active frame refresh requests, or mouse activity events. |
192 void OnEvent(FrameSubscriber* subscriber); | 131 void OnEvent(FrameSubscriber* subscriber); |
193 | 132 |
194 // Maintain a weak reference to the RenderWidgetHost (via its routing ID), | 133 // Maintain a weak reference to the RenderWidgetHost (via its routing ID), |
195 // since the instance could be destroyed externally during the lifetime of | 134 // since the instance could be destroyed externally during the lifetime of |
196 // |this|. | 135 // |this|. |
197 const int render_process_id_; | 136 const int render_process_id_; |
198 const int render_widget_id_; | 137 const int render_widget_id_; |
199 | 138 |
200 std::unique_ptr<FrameSubscriber> refresh_subscriber_; | 139 std::unique_ptr<FrameSubscriber> refresh_subscriber_; |
201 std::unique_ptr<FrameSubscriber> mouse_activity_subscriber_; | 140 std::unique_ptr<FrameSubscriber> mouse_activity_subscriber_; |
202 CaptureCallback capture_callback_; | 141 CaptureCallback capture_callback_; |
203 | 142 |
204 // Responsible for tracking the cursor state and input events to make | 143 // Responsible for tracking the cursor state and input events to make |
205 // decisions and then render the mouse cursor on the video frame after | 144 // decisions and then render the mouse cursor on the video frame after |
206 // capture is completed. | 145 // capture is completed. |
207 std::unique_ptr<content::CursorRenderer> cursor_renderer_; | 146 std::unique_ptr<content::CursorRenderer> cursor_renderer_; |
208 | 147 |
209 // Responsible for tracking the UI events and making a decision on whether | 148 // Responsible for tracking the UI events and making a decision on whether |
210 // user is actively interacting with content. | 149 // user is actively interacting with content. |
211 std::unique_ptr<content::WindowActivityTracker> window_activity_tracker_; | 150 std::unique_ptr<content::WindowActivityTracker> window_activity_tracker_; |
212 | 151 |
213 DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription); | 152 DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription); |
214 }; | 153 }; |
215 | 154 |
216 // Render the SkBitmap |input| into the given VideoFrame buffer |output|, then | |
217 // invoke |done_cb| to indicate success or failure. |input| is expected to be | |
218 // ARGB. |output| must be YV12 or I420. Colorspace conversion is always done. | |
219 // Scaling and letterboxing will be done as needed. | |
220 // | |
221 // This software implementation should be used only when GPU acceleration of | |
222 // these activities is not possible. This operation may be expensive (tens to | |
223 // hundreds of milliseconds), so the caller should ensure that it runs on a | |
224 // thread where such a pause would cause UI jank. | |
225 void RenderVideoFrame( | |
226 const SkBitmap& input, | |
227 const scoped_refptr<media::VideoFrame>& output, | |
228 const base::Callback<void(const gfx::Rect&, bool)>& done_cb); | |
229 | |
230 // Renews capture subscriptions based on feedback from WebContentsTracker, and | 155 // Renews capture subscriptions based on feedback from WebContentsTracker, and |
231 // also executes copying of the backing store on the UI BrowserThread. | 156 // also executes non-compositor-initiated frame captures. |
232 class WebContentsCaptureMachine : public media::VideoCaptureMachine { | 157 class WebContentsCaptureMachine : public media::VideoCaptureMachine { |
233 public: | 158 public: |
234 WebContentsCaptureMachine(int render_process_id, | 159 WebContentsCaptureMachine(int render_process_id, |
235 int main_render_frame_id, | 160 int main_render_frame_id, |
236 bool enable_auto_throttling); | 161 bool enable_auto_throttling); |
237 ~WebContentsCaptureMachine() override; | 162 ~WebContentsCaptureMachine() override; |
238 | 163 |
239 // VideoCaptureMachine overrides. | 164 // VideoCaptureMachine overrides. |
240 void Start(const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 165 void Start(const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, |
241 const media::VideoCaptureParams& params, | 166 const media::VideoCaptureParams& params, |
242 const base::Callback<void(bool)> callback) override; | 167 const base::Callback<void(bool)> callback) override; |
243 void Suspend() override; | 168 void Suspend() override; |
244 void Resume() override; | 169 void Resume() override; |
245 void Stop(const base::Closure& callback) override; | 170 void Stop(const base::Closure& callback) override; |
246 bool IsAutoThrottlingEnabled() const override { | 171 bool IsAutoThrottlingEnabled() const override { |
247 return auto_throttling_enabled_; | 172 return auto_throttling_enabled_; |
248 } | 173 } |
249 void MaybeCaptureForRefresh() override; | 174 void MaybeCaptureForRefresh() override; |
250 | 175 |
251 // Starts a copy from the backing store or the composited surface. Must be run | 176 private: |
252 // on the UI BrowserThread. |deliver_frame_cb| will be run when the operation | 177 // Called for cases where events other than compositor updates require a frame |
253 // completes. The copy will occur to |target|. | 178 // capture from the composited surface. Must be run on the UI BrowserThread. |
| 179 // |deliver_frame_cb| will be run when the operation completes. The copy will |
| 180 // populate |target|. |
254 // | 181 // |
255 // This may be used as a ContentCaptureSubscription::CaptureCallback. | 182 // This method is only used by a ContentCaptureSubscription::CaptureCallback. |
256 void Capture(const base::TimeTicks& start_time, | 183 void Capture(base::TimeTicks start_time, |
257 const scoped_refptr<media::VideoFrame>& target, | 184 scoped_refptr<media::VideoFrame> target, |
258 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 185 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
259 deliver_frame_cb); | 186 deliver_frame_cb); |
260 | 187 |
261 private: | 188 bool InternalStart(scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
262 bool InternalStart( | 189 const media::VideoCaptureParams& params); |
263 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | |
264 const media::VideoCaptureParams& params); | |
265 void InternalSuspend(); | 190 void InternalSuspend(); |
266 void InternalResume(); | 191 void InternalResume(); |
267 void InternalStop(const base::Closure& callback); | 192 void InternalStop(const base::Closure& callback); |
268 void InternalMaybeCaptureForRefresh(); | 193 void InternalMaybeCaptureForRefresh(); |
269 bool IsStarted() const; | 194 bool IsStarted() const; |
270 | 195 |
271 // Computes the preferred size of the target RenderWidget for optimal capture. | 196 // Computes the preferred size of the target RenderWidget for optimal capture. |
272 gfx::Size ComputeOptimalViewSize() const; | 197 gfx::Size ComputeOptimalViewSize() const; |
273 | 198 |
274 // Response callback for RenderWidgetHost::CopyFromBackingStore(). | |
275 void DidCopyFromBackingStore( | |
276 const base::TimeTicks& start_time, | |
277 const scoped_refptr<media::VideoFrame>& target, | |
278 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | |
279 deliver_frame_cb, | |
280 const SkBitmap& bitmap, | |
281 ReadbackResponse response); | |
282 | |
283 // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame(). | 199 // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame(). |
284 void DidCopyFromCompositingSurfaceToVideoFrame( | 200 void DidCopyToVideoFrame( |
285 const base::TimeTicks& start_time, | 201 const base::TimeTicks& start_time, |
286 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 202 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
287 deliver_frame_cb, | 203 deliver_frame_cb, |
288 const gfx::Rect& region_in_frame, | 204 const gfx::Rect& region_in_frame, |
289 bool success); | 205 bool success); |
290 | 206 |
291 // Remove the old subscription, and attempt to start a new one if |had_target| | 207 // Remove the old subscription, and attempt to start a new one if |had_target| |
292 // is true. | 208 // is true. |
293 void RenewFrameSubscription(bool had_target); | 209 void RenewFrameSubscription(bool had_target); |
294 | 210 |
295 // Called whenever the render widget is resized. | 211 // Called whenever the render widget is resized. |
296 void UpdateCaptureSize(); | 212 void UpdateCaptureSize(); |
297 | 213 |
298 // Parameters saved in constructor. | 214 // Parameters saved in constructor. |
299 const int initial_render_process_id_; | 215 const int initial_render_process_id_; |
300 const int initial_main_render_frame_id_; | 216 const int initial_main_render_frame_id_; |
301 | 217 |
302 // Tracks events and calls back to RenewFrameSubscription() to maintain | 218 // Tracks events and calls back to RenewFrameSubscription() to maintain |
303 // capture on the correct RenderWidgetHost. | 219 // capture on the correct RenderWidgetHost. |
304 const scoped_refptr<WebContentsTracker> tracker_; | 220 const scoped_refptr<WebContentsTracker> tracker_; |
305 | 221 |
306 // Set to false to prevent the capture size from automatically adjusting in | 222 // Set to false to prevent the capture size from automatically adjusting in |
307 // response to end-to-end utilization. This is enabled via the throttling | 223 // response to end-to-end utilization. This is enabled via the throttling |
308 // option in the WebContentsVideoCaptureDevice device ID. | 224 // option in the WebContentsVideoCaptureDevice device ID. |
309 const bool auto_throttling_enabled_; | 225 const bool auto_throttling_enabled_; |
310 | 226 |
311 // A dedicated worker thread on which SkBitmap->VideoFrame conversion will | |
312 // occur. Only used when this activity cannot be done on the GPU. | |
313 std::unique_ptr<base::Thread> render_thread_; | |
314 | |
315 // Makes all the decisions about which frames to copy, and how. | 227 // Makes all the decisions about which frames to copy, and how. |
316 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; | 228 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_; |
317 | 229 |
318 // Video capture parameters that this machine is started with. | 230 // Video capture parameters that this machine is started with. |
319 media::VideoCaptureParams capture_params_; | 231 media::VideoCaptureParams capture_params_; |
320 | 232 |
321 // Responsible for forwarding events from the active RenderWidgetHost to the | 233 // Responsible for forwarding events from the active RenderWidgetHost to the |
322 // oracle, and initiating captures accordingly. | 234 // oracle, and initiating captures accordingly. |
323 std::unique_ptr<ContentCaptureSubscription> subscription_; | 235 std::unique_ptr<ContentCaptureSubscription> subscription_; |
324 | 236 |
(...skipping 11 matching lines...) Expand all Loading... |
336 | 248 |
337 bool FrameSubscriber::ShouldCaptureFrame( | 249 bool FrameSubscriber::ShouldCaptureFrame( |
338 const gfx::Rect& damage_rect, | 250 const gfx::Rect& damage_rect, |
339 base::TimeTicks present_time, | 251 base::TimeTicks present_time, |
340 scoped_refptr<media::VideoFrame>* storage, | 252 scoped_refptr<media::VideoFrame>* storage, |
341 DeliverFrameCallback* deliver_frame_cb) { | 253 DeliverFrameCallback* deliver_frame_cb) { |
342 TRACE_EVENT1("gpu.capture", "FrameSubscriber::ShouldCaptureFrame", "instance", | 254 TRACE_EVENT1("gpu.capture", "FrameSubscriber::ShouldCaptureFrame", "instance", |
343 this); | 255 this); |
344 | 256 |
345 media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb; | 257 media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb; |
| 258 if (!oracle_proxy_->ObserveEventAndDecideCapture( |
| 259 event_type_, damage_rect, present_time, storage, &capture_frame_cb)) { |
| 260 return false; |
| 261 } |
346 | 262 |
347 bool oracle_decision = oracle_proxy_->ObserveEventAndDecideCapture( | 263 DCHECK(*storage); |
348 event_type_, damage_rect, present_time, storage, &capture_frame_cb); | 264 DCHECK(!capture_frame_cb.is_null()); |
349 | 265 *deliver_frame_cb = |
350 if (!capture_frame_cb.is_null()) | 266 base::Bind(&FrameSubscriber::DidCaptureFrame, |
351 *deliver_frame_cb = | 267 weak_ptr_factory_.GetWeakPtr(), capture_frame_cb, *storage); |
352 base::Bind(&FrameSubscriber::DidCaptureFrame, | 268 return true; |
353 weak_ptr_factory_.GetWeakPtr(), capture_frame_cb, *storage); | |
354 return oracle_decision; | |
355 } | 269 } |
356 | 270 |
357 void FrameSubscriber::DidCaptureFrame( | 271 void FrameSubscriber::DidCaptureFrame( |
358 base::WeakPtr<FrameSubscriber> frame_subscriber_, | 272 base::WeakPtr<FrameSubscriber> frame_subscriber_, |
359 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& | 273 const media::ThreadSafeCaptureOracle::CaptureFrameCallback& |
360 capture_frame_cb, | 274 capture_frame_cb, |
361 scoped_refptr<media::VideoFrame> frame, | 275 scoped_refptr<media::VideoFrame> frame, |
362 base::TimeTicks timestamp, | 276 base::TimeTicks timestamp, |
363 const gfx::Rect& region_in_frame, | 277 const gfx::Rect& region_in_frame, |
364 bool success) { | 278 bool success) { |
365 // We can get a callback in the shutdown sequence for the browser main loop | |
366 // and this can result in a DCHECK failure. Avoid this by doing DCHECK only | |
367 // on success. | |
368 if (success) { | 279 if (success) { |
| 280 // We can get a callback in the shutdown sequence for the browser main loop |
| 281 // and this can result in a DCHECK failure. Avoid this by doing DCHECK only |
| 282 // on success. |
369 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 283 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
370 // TODO(isheriff): Unclear if taking a snapshot of cursor here affects user | 284 |
371 // experience in any particular scenarios. Doing it prior to capture may | |
372 // require evaluating region_in_frame in this file. | |
373 if (frame_subscriber_ && frame_subscriber_->cursor_renderer_) { | 285 if (frame_subscriber_ && frame_subscriber_->cursor_renderer_) { |
374 CursorRenderer* cursor_renderer = | 286 CursorRenderer* cursor_renderer = |
375 frame_subscriber_->cursor_renderer_.get(); | 287 frame_subscriber_->cursor_renderer_.get(); |
376 if (cursor_renderer->SnapshotCursorState(region_in_frame)) | 288 if (cursor_renderer->SnapshotCursorState(region_in_frame)) |
377 cursor_renderer->RenderOnVideoFrame(frame); | 289 cursor_renderer->RenderOnVideoFrame(frame); |
378 frame->metadata()->SetBoolean( | 290 frame->metadata()->SetBoolean( |
379 media::VideoFrameMetadata::INTERACTIVE_CONTENT, | 291 media::VideoFrameMetadata::INTERACTIVE_CONTENT, |
380 frame_subscriber_->IsUserInteractingWithContent()); | 292 frame_subscriber_->IsUserInteractingWithContent()); |
381 } | 293 } |
382 } | 294 } |
383 capture_frame_cb.Run(std::move(frame), timestamp, success); | 295 capture_frame_cb.Run(std::move(frame), timestamp, success); |
384 } | 296 } |
385 | 297 |
386 bool FrameSubscriber::IsUserInteractingWithContent() { | 298 bool FrameSubscriber::IsUserInteractingWithContent() { |
| 299 bool ui_activity = window_activity_tracker_ && |
| 300 window_activity_tracker_->IsUiInteractionActive(); |
387 bool interactive_mode = false; | 301 bool interactive_mode = false; |
388 bool ui_activity = false; | |
389 if (window_activity_tracker_) { | |
390 ui_activity = window_activity_tracker_->IsUiInteractionActive(); | |
391 } | |
392 if (cursor_renderer_) { | 302 if (cursor_renderer_) { |
393 bool animation_active = | 303 bool animation_active = |
394 (base::TimeTicks::Now() - | 304 (base::TimeTicks::Now() - |
395 oracle_proxy_->last_time_animation_was_detected()) < | 305 oracle_proxy_->last_time_animation_was_detected()) < |
396 base::TimeDelta::FromMilliseconds(kMinPeriodNoAnimationMillis); | 306 base::TimeDelta::FromMilliseconds(kMinPeriodNoAnimationMillis); |
397 if (ui_activity && !animation_active) { | 307 if (ui_activity && !animation_active) { |
398 interactive_mode = true; | 308 interactive_mode = true; |
399 } else if (animation_active && window_activity_tracker_.get()) { | 309 } else if (animation_active && window_activity_tracker_) { |
400 window_activity_tracker_->Reset(); | 310 window_activity_tracker_->Reset(); |
401 } | 311 } |
402 } | 312 } |
403 return interactive_mode; | 313 return interactive_mode; |
404 } | 314 } |
405 | 315 |
406 ContentCaptureSubscription::ContentCaptureSubscription( | 316 ContentCaptureSubscription::ContentCaptureSubscription( |
407 const RenderWidgetHost& source, | 317 const RenderWidgetHost& source, |
408 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 318 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
409 const CaptureCallback& capture_callback) | 319 const CaptureCallback& capture_callback) |
410 : render_process_id_(source.GetProcess()->GetID()), | 320 : render_process_id_(source.GetProcess()->GetID()), |
411 render_widget_id_(source.GetRoutingID()), | 321 render_widget_id_(source.GetRoutingID()), |
412 capture_callback_(capture_callback) { | 322 capture_callback_(capture_callback) { |
413 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 323 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
414 | 324 |
415 RenderWidgetHostView* const view = source.GetView(); | 325 RenderWidgetHostView* const view = source.GetView(); |
416 #if defined(USE_AURA) || defined(OS_MACOSX) | 326 #if defined(USE_AURA) || defined(OS_MACOSX) |
417 if (view) { | 327 if (view) { |
418 cursor_renderer_ = CursorRenderer::Create(view->GetNativeView()); | 328 cursor_renderer_ = CursorRenderer::Create(view->GetNativeView()); |
419 window_activity_tracker_ = | 329 window_activity_tracker_ = |
420 WindowActivityTracker::Create(view->GetNativeView()); | 330 WindowActivityTracker::Create(view->GetNativeView()); |
421 } | 331 } |
422 #endif | 332 #endif |
| 333 // Subscriptions for refresh frames and mouse cursor updates. When events |
| 334 // outside of the normal content change/compositing workflow occur, these |
| 335 // decide whether extra frames need to be captured. Such extra frame captures |
| 336 // are initiated by running the |capture_callback_|. |
423 refresh_subscriber_.reset(new FrameSubscriber( | 337 refresh_subscriber_.reset(new FrameSubscriber( |
424 media::VideoCaptureOracle::kActiveRefreshRequest, oracle_proxy, | 338 media::VideoCaptureOracle::kActiveRefreshRequest, oracle_proxy, |
425 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() | 339 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() |
426 : base::WeakPtr<CursorRenderer>(), | 340 : base::WeakPtr<CursorRenderer>(), |
427 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() | 341 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() |
428 : base::WeakPtr<WindowActivityTracker>())); | 342 : base::WeakPtr<WindowActivityTracker>())); |
429 mouse_activity_subscriber_.reset(new FrameSubscriber( | 343 mouse_activity_subscriber_.reset(new FrameSubscriber( |
430 media::VideoCaptureOracle::kMouseCursorUpdate, oracle_proxy, | 344 media::VideoCaptureOracle::kMouseCursorUpdate, oracle_proxy, |
431 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() | 345 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() |
432 : base::WeakPtr<CursorRenderer>(), | 346 : base::WeakPtr<CursorRenderer>(), |
433 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() | 347 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() |
434 : base::WeakPtr<WindowActivityTracker>())); | 348 : base::WeakPtr<WindowActivityTracker>())); |
435 | 349 |
436 // Subscribe to compositor updates. These will be serviced directly by the | 350 // Main capture path: Subscribing to compositor updates. This means that any |
437 // oracle. | 351 // time the tab content has changed and compositing has taken place, the |
| 352 // RenderWidgetHostView will consult the FrameSubscriber (which consults the |
| 353 // oracle) to determine whether a frame should be captured. If so, the |
| 354 // RenderWidgetHostView will initiate the frame capture and NOT the |
| 355 // |capture_callback_|. |
438 if (view) { | 356 if (view) { |
439 std::unique_ptr<RenderWidgetHostViewFrameSubscriber> subscriber( | 357 std::unique_ptr<RenderWidgetHostViewFrameSubscriber> subscriber( |
440 new FrameSubscriber( | 358 new FrameSubscriber( |
441 media::VideoCaptureOracle::kCompositorUpdate, oracle_proxy, | 359 media::VideoCaptureOracle::kCompositorUpdate, oracle_proxy, |
442 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() | 360 cursor_renderer_ ? cursor_renderer_->GetWeakPtr() |
443 : base::WeakPtr<CursorRenderer>(), | 361 : base::WeakPtr<CursorRenderer>(), |
444 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() | 362 window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr() |
445 : base::WeakPtr<WindowActivityTracker>())); | 363 : base::WeakPtr<WindowActivityTracker>())); |
446 view->BeginFrameSubscription(std::move(subscriber)); | 364 view->BeginFrameSubscription(std::move(subscriber)); |
447 } | 365 } |
448 | 366 |
449 // Subscribe to mouse movement and mouse cursor update events. | 367 // Begin monitoring for user activity that is used to signal "interactive |
| 368 // content." |
450 if (window_activity_tracker_) { | 369 if (window_activity_tracker_) { |
451 window_activity_tracker_->RegisterMouseInteractionObserver( | 370 window_activity_tracker_->RegisterMouseInteractionObserver( |
452 base::Bind(&ContentCaptureSubscription::OnEvent, base::Unretained(this), | 371 base::Bind(&ContentCaptureSubscription::OnEvent, base::Unretained(this), |
453 mouse_activity_subscriber_.get())); | 372 mouse_activity_subscriber_.get())); |
454 } | 373 } |
455 } | 374 } |
456 | 375 |
457 ContentCaptureSubscription::~ContentCaptureSubscription() { | 376 ContentCaptureSubscription::~ContentCaptureSubscription() { |
458 // If the BrowserThreads have been torn down, then the browser is in the final | 377 // If the BrowserThreads have been torn down, then the browser is in the final |
459 // stages of exiting and it is dangerous to take any further action. We must | 378 // stages of exiting and it is dangerous to take any further action. We must |
460 // return early. http://crbug.com/396413 | 379 // return early. http://crbug.com/396413 |
461 if (!BrowserThread::IsMessageLoopValid(BrowserThread::UI)) | 380 if (!BrowserThread::IsMessageLoopValid(BrowserThread::UI)) |
462 return; | 381 return; |
463 | 382 |
464 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 383 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
465 RenderWidgetHost* const source = | 384 RenderWidgetHost* const source = |
466 RenderWidgetHost::FromID(render_process_id_, render_widget_id_); | 385 RenderWidgetHost::FromID(render_process_id_, render_widget_id_); |
467 RenderWidgetHostView* const view = source ? source->GetView() : NULL; | 386 RenderWidgetHostView* const view = source ? source->GetView() : nullptr; |
468 if (view) | 387 if (view) |
469 view->EndFrameSubscription(); | 388 view->EndFrameSubscription(); |
470 } | 389 } |
471 | 390 |
472 void ContentCaptureSubscription::MaybeCaptureForRefresh() { | 391 void ContentCaptureSubscription::MaybeCaptureForRefresh() { |
473 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 392 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
474 OnEvent(refresh_subscriber_.get()); | 393 OnEvent(refresh_subscriber_.get()); |
475 } | 394 } |
476 | 395 |
477 void ContentCaptureSubscription::OnEvent(FrameSubscriber* subscriber) { | 396 void ContentCaptureSubscription::OnEvent(FrameSubscriber* subscriber) { |
478 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 397 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
479 TRACE_EVENT0("gpu.capture", "ContentCaptureSubscription::OnEvent"); | 398 TRACE_EVENT0("gpu.capture", "ContentCaptureSubscription::OnEvent"); |
480 | 399 |
481 scoped_refptr<media::VideoFrame> frame; | 400 scoped_refptr<media::VideoFrame> frame; |
482 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb; | 401 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb; |
483 | 402 |
484 const base::TimeTicks start_time = base::TimeTicks::Now(); | 403 const base::TimeTicks start_time = base::TimeTicks::Now(); |
485 DCHECK(subscriber == refresh_subscriber_.get() || | 404 DCHECK(subscriber == refresh_subscriber_.get() || |
486 subscriber == mouse_activity_subscriber_.get()); | 405 subscriber == mouse_activity_subscriber_.get()); |
487 if (subscriber->ShouldCaptureFrame(gfx::Rect(), start_time, &frame, | 406 if (subscriber->ShouldCaptureFrame(gfx::Rect(), start_time, &frame, |
488 &deliver_frame_cb)) { | 407 &deliver_frame_cb)) { |
489 capture_callback_.Run(start_time, frame, deliver_frame_cb); | 408 capture_callback_.Run(start_time, frame, deliver_frame_cb); |
490 } | 409 } |
491 } | 410 } |
492 | 411 |
493 void RenderVideoFrame( | |
494 const SkBitmap& input, | |
495 const scoped_refptr<media::VideoFrame>& output, | |
496 const base::Callback<void(const gfx::Rect&, bool)>& done_cb) { | |
497 base::ScopedClosureRunner failure_handler( | |
498 base::Bind(done_cb, gfx::Rect(), false)); | |
499 | |
500 SkAutoLockPixels locker(input); | |
501 | |
502 // Sanity-check the captured bitmap. | |
503 if (input.empty() || !input.readyToDraw() || | |
504 input.colorType() != kN32_SkColorType || input.width() < 2 || | |
505 input.height() < 2) { | |
506 DVLOG(1) << "input unacceptable (size=" << input.getSize() | |
507 << ", ready=" << input.readyToDraw() | |
508 << ", colorType=" << input.colorType() << ')'; | |
509 return; | |
510 } | |
511 | |
512 // Sanity-check the output buffer. | |
513 if (output->format() != media::PIXEL_FORMAT_I420) { | |
514 NOTREACHED(); | |
515 return; | |
516 } | |
517 | |
518 // Calculate the width and height of the content region in the |output|, based | |
519 // on the aspect ratio of |input|. | |
520 const gfx::Rect region_in_frame = media::ComputeLetterboxRegion( | |
521 output->visible_rect(), gfx::Size(input.width(), input.height())); | |
522 | |
523 // Scale the bitmap to the required size, if necessary. | |
524 SkBitmap scaled_bitmap; | |
525 if (input.width() != region_in_frame.width() || | |
526 input.height() != region_in_frame.height()) { | |
527 skia::ImageOperations::ResizeMethod method; | |
528 if (input.width() < region_in_frame.width() || | |
529 input.height() < region_in_frame.height()) { | |
530 // Avoid box filtering when magnifying, because it's actually | |
531 // nearest-neighbor. | |
532 method = skia::ImageOperations::RESIZE_HAMMING1; | |
533 } else { | |
534 method = skia::ImageOperations::RESIZE_BOX; | |
535 } | |
536 | |
537 TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", output.get(), | |
538 "Scale"); | |
539 scaled_bitmap = skia::ImageOperations::Resize( | |
540 input, method, region_in_frame.width(), region_in_frame.height()); | |
541 } else { | |
542 scaled_bitmap = input; | |
543 } | |
544 | |
545 TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", output.get(), "YUV"); | |
546 { | |
547 // Align to 2x2 pixel boundaries, as required by | |
548 // media::CopyRGBToVideoFrame(). | |
549 const gfx::Rect region_in_yv12_frame( | |
550 region_in_frame.x() & ~1, region_in_frame.y() & ~1, | |
551 region_in_frame.width() & ~1, region_in_frame.height() & ~1); | |
552 if (region_in_yv12_frame.IsEmpty()) | |
553 return; | |
554 | |
555 SkAutoLockPixels scaled_bitmap_locker(scaled_bitmap); | |
556 media::CopyRGBToVideoFrame( | |
557 reinterpret_cast<uint8_t*>(scaled_bitmap.getPixels()), | |
558 scaled_bitmap.rowBytes(), region_in_yv12_frame, output.get()); | |
559 } | |
560 | |
561 // The result is now ready. | |
562 ignore_result(failure_handler.Release()); | |
563 done_cb.Run(region_in_frame, true); | |
564 } | |
565 | |
566 WebContentsCaptureMachine::WebContentsCaptureMachine( | 412 WebContentsCaptureMachine::WebContentsCaptureMachine( |
567 int render_process_id, | 413 int render_process_id, |
568 int main_render_frame_id, | 414 int main_render_frame_id, |
569 bool enable_auto_throttling) | 415 bool enable_auto_throttling) |
570 : initial_render_process_id_(render_process_id), | 416 : initial_render_process_id_(render_process_id), |
571 initial_main_render_frame_id_(main_render_frame_id), | 417 initial_main_render_frame_id_(main_render_frame_id), |
572 tracker_(new WebContentsTracker(true)), | 418 tracker_(new WebContentsTracker(true)), |
573 auto_throttling_enabled_(enable_auto_throttling), | 419 auto_throttling_enabled_(enable_auto_throttling), |
574 frame_capture_active_(true), | 420 frame_capture_active_(true), |
575 weak_ptr_factory_(this) { | 421 weak_ptr_factory_(this) { |
(...skipping 15 matching lines...) Expand all Loading... |
591 const base::Callback<void(bool)> callback) { | 437 const base::Callback<void(bool)> callback) { |
592 // Starts the capture machine asynchronously. | 438 // Starts the capture machine asynchronously. |
593 BrowserThread::PostTaskAndReplyWithResult( | 439 BrowserThread::PostTaskAndReplyWithResult( |
594 BrowserThread::UI, FROM_HERE, | 440 BrowserThread::UI, FROM_HERE, |
595 base::Bind(&WebContentsCaptureMachine::InternalStart, | 441 base::Bind(&WebContentsCaptureMachine::InternalStart, |
596 base::Unretained(this), oracle_proxy, params), | 442 base::Unretained(this), oracle_proxy, params), |
597 callback); | 443 callback); |
598 } | 444 } |
599 | 445 |
600 bool WebContentsCaptureMachine::InternalStart( | 446 bool WebContentsCaptureMachine::InternalStart( |
601 const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy, | 447 scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy, |
602 const media::VideoCaptureParams& params) { | 448 const media::VideoCaptureParams& params) { |
603 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 449 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
604 DCHECK(!IsStarted()); | 450 DCHECK(!IsStarted()); |
605 | 451 |
606 DCHECK(oracle_proxy); | 452 DCHECK(oracle_proxy); |
607 oracle_proxy_ = oracle_proxy; | 453 oracle_proxy_ = std::move(oracle_proxy); |
608 capture_params_ = params; | 454 capture_params_ = params; |
609 | 455 |
610 render_thread_.reset(new base::Thread("WebContentsVideo_RenderThread")); | |
611 if (!render_thread_->Start()) { | |
612 DVLOG(1) << "Failed to spawn render thread."; | |
613 render_thread_.reset(); | |
614 return false; | |
615 } | |
616 | |
617 // Note: Creation of the first WeakPtr in the following statement will cause | 456 // Note: Creation of the first WeakPtr in the following statement will cause |
618 // IsStarted() to return true from now on. | 457 // IsStarted() to return true from now on. |
619 tracker_->SetResizeChangeCallback( | 458 tracker_->SetResizeChangeCallback( |
620 base::Bind(&WebContentsCaptureMachine::UpdateCaptureSize, | 459 base::Bind(&WebContentsCaptureMachine::UpdateCaptureSize, |
621 weak_ptr_factory_.GetWeakPtr())); | 460 weak_ptr_factory_.GetWeakPtr())); |
622 tracker_->Start(initial_render_process_id_, initial_main_render_frame_id_, | 461 tracker_->Start(initial_render_process_id_, initial_main_render_frame_id_, |
623 base::Bind(&WebContentsCaptureMachine::RenewFrameSubscription, | 462 base::Bind(&WebContentsCaptureMachine::RenewFrameSubscription, |
624 weak_ptr_factory_.GetWeakPtr())); | 463 weak_ptr_factory_.GetWeakPtr())); |
625 if (WebContents* contents = tracker_->web_contents()) | 464 if (WebContents* contents = tracker_->web_contents()) |
626 contents->IncrementCapturerCount(ComputeOptimalViewSize()); | 465 contents->IncrementCapturerCount(ComputeOptimalViewSize()); |
(...skipping 26 matching lines...) Expand all Loading... |
653 void WebContentsCaptureMachine::InternalResume() { | 492 void WebContentsCaptureMachine::InternalResume() { |
654 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 493 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
655 if (frame_capture_active_) | 494 if (frame_capture_active_) |
656 return; | 495 return; |
657 frame_capture_active_ = true; | 496 frame_capture_active_ = true; |
658 if (IsStarted()) | 497 if (IsStarted()) |
659 RenewFrameSubscription(true); | 498 RenewFrameSubscription(true); |
660 } | 499 } |
661 | 500 |
662 void WebContentsCaptureMachine::Stop(const base::Closure& callback) { | 501 void WebContentsCaptureMachine::Stop(const base::Closure& callback) { |
663 // Stops the capture machine asynchronously. | |
664 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, | 502 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, |
665 base::Bind(&WebContentsCaptureMachine::InternalStop, | 503 base::Bind(&WebContentsCaptureMachine::InternalStop, |
666 base::Unretained(this), callback)); | 504 base::Unretained(this), callback)); |
667 } | 505 } |
668 | 506 |
669 void WebContentsCaptureMachine::InternalStop(const base::Closure& callback) { | 507 void WebContentsCaptureMachine::InternalStop(const base::Closure& callback) { |
670 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 508 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
671 | 509 |
672 if (!IsStarted()) { | 510 base::ScopedClosureRunner done_runner(callback); |
673 callback.Run(); | 511 |
| 512 if (!IsStarted()) |
674 return; | 513 return; |
675 } | |
676 | 514 |
677 // The following cancels any outstanding callbacks and causes IsStarted() to | 515 // The following cancels any outstanding callbacks and causes IsStarted() to |
678 // return false from here onward. | 516 // return false from here onward. |
679 weak_ptr_factory_.InvalidateWeakPtrs(); | 517 weak_ptr_factory_.InvalidateWeakPtrs(); |
680 | 518 |
681 RenewFrameSubscription(false); | 519 RenewFrameSubscription(false); |
682 if (WebContents* contents = tracker_->web_contents()) | 520 if (WebContents* contents = tracker_->web_contents()) |
683 contents->DecrementCapturerCount(); | 521 contents->DecrementCapturerCount(); |
684 tracker_->Stop(); | 522 tracker_->Stop(); |
685 | |
686 // The render thread cannot be stopped on the UI thread, so post a message | |
687 // to the thread pool used for blocking operations. | |
688 if (render_thread_) { | |
689 BrowserThread::PostBlockingPoolTask( | |
690 FROM_HERE, base::Bind(&DeleteOnWorkerThread, | |
691 base::Passed(&render_thread_), callback)); | |
692 } | |
693 } | 523 } |
694 | 524 |
695 void WebContentsCaptureMachine::MaybeCaptureForRefresh() { | 525 void WebContentsCaptureMachine::MaybeCaptureForRefresh() { |
696 BrowserThread::PostTask( | 526 BrowserThread::PostTask( |
697 BrowserThread::UI, FROM_HERE, | 527 BrowserThread::UI, FROM_HERE, |
698 base::Bind(&WebContentsCaptureMachine::InternalMaybeCaptureForRefresh, | 528 base::Bind(&WebContentsCaptureMachine::InternalMaybeCaptureForRefresh, |
699 // Use of Unretained() is safe here since this task must run | 529 // Use of Unretained() is safe here since this task must run |
700 // before InternalStop(). | 530 // before InternalStop(). |
701 base::Unretained(this))); | 531 base::Unretained(this))); |
702 } | 532 } |
703 | 533 |
704 void WebContentsCaptureMachine::InternalMaybeCaptureForRefresh() { | 534 void WebContentsCaptureMachine::InternalMaybeCaptureForRefresh() { |
705 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 535 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
706 if (IsStarted() && subscription_) | 536 if (IsStarted() && subscription_) |
707 subscription_->MaybeCaptureForRefresh(); | 537 subscription_->MaybeCaptureForRefresh(); |
708 } | 538 } |
709 | 539 |
710 void WebContentsCaptureMachine::Capture( | 540 void WebContentsCaptureMachine::Capture( |
711 const base::TimeTicks& start_time, | 541 base::TimeTicks start_time, |
712 const scoped_refptr<media::VideoFrame>& target, | 542 scoped_refptr<media::VideoFrame> target, |
713 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 543 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
714 deliver_frame_cb) { | 544 deliver_frame_cb) { |
715 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 545 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
716 | 546 |
717 RenderWidgetHost* rwh = tracker_->GetTargetRenderWidgetHost(); | 547 RenderWidgetHost* rwh = tracker_->GetTargetRenderWidgetHost(); |
718 RenderWidgetHostViewBase* view = | 548 RenderWidgetHostViewBase* view = |
719 rwh ? static_cast<RenderWidgetHostViewBase*>(rwh->GetView()) : NULL; | 549 rwh ? static_cast<RenderWidgetHostViewBase*>(rwh->GetView()) : nullptr; |
720 if (!view) { | 550 if (!view || !view->CanCopyToVideoFrame()) { |
721 deliver_frame_cb.Run(base::TimeTicks(), gfx::Rect(), false); | 551 deliver_frame_cb.Run(base::TimeTicks(), gfx::Rect(), false); |
722 return; | 552 return; |
723 } | 553 } |
724 | 554 |
725 const gfx::Size view_size = view->GetViewBounds().size(); | 555 const gfx::Size view_size = view->GetViewBounds().size(); |
726 if (view->CanCopyToVideoFrame()) { | 556 view->CopyFromCompositingSurfaceToVideoFrame( |
727 view->CopyFromCompositingSurfaceToVideoFrame( | 557 gfx::Rect(view_size), std::move(target), |
728 gfx::Rect(view_size), target, | 558 base::Bind(&WebContentsCaptureMachine::DidCopyToVideoFrame, |
729 base::Bind(&WebContentsCaptureMachine:: | 559 weak_ptr_factory_.GetWeakPtr(), start_time, deliver_frame_cb)); |
730 DidCopyFromCompositingSurfaceToVideoFrame, | |
731 weak_ptr_factory_.GetWeakPtr(), start_time, | |
732 deliver_frame_cb)); | |
733 } else { | |
734 const gfx::Size fitted_size = | |
735 view_size.IsEmpty() | |
736 ? gfx::Size() | |
737 : media::ComputeLetterboxRegion(target->visible_rect(), view_size) | |
738 .size(); | |
739 rwh->CopyFromBackingStore( | |
740 gfx::Rect(), | |
741 fitted_size, // Size here is a request not always honored. | |
742 base::Bind(&WebContentsCaptureMachine::DidCopyFromBackingStore, | |
743 weak_ptr_factory_.GetWeakPtr(), start_time, target, | |
744 deliver_frame_cb), | |
745 kN32_SkColorType); | |
746 } | |
747 } | 560 } |
748 | 561 |
749 gfx::Size WebContentsCaptureMachine::ComputeOptimalViewSize() const { | 562 gfx::Size WebContentsCaptureMachine::ComputeOptimalViewSize() const { |
750 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 563 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
751 | 564 |
752 // TODO(miu): Propagate capture frame size changes as new "preferred size" | 565 // TODO(miu): Propagate capture frame size changes as new "preferred size" |
753 // updates, rather than just using the max frame size. | 566 // updates, rather than just using the max frame size. |
754 // http://crbug.com/350491 | 567 // http://crbug.com/350491 |
755 gfx::Size optimal_size = oracle_proxy_->max_frame_size(); | 568 gfx::Size optimal_size = oracle_proxy_->max_frame_size(); |
756 | 569 |
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
789 break; | 602 break; |
790 } | 603 } |
791 } | 604 } |
792 | 605 |
793 // If the ratio between physical and logical pixels is greater than 1:1, | 606 // If the ratio between physical and logical pixels is greater than 1:1, |
794 // shrink |optimal_size| by that amount. Then, when external code resizes the | 607 // shrink |optimal_size| by that amount. Then, when external code resizes the |
795 // render widget to the "preferred size," the widget will be physically | 608 // render widget to the "preferred size," the widget will be physically |
796 // rendered at the exact capture size, thereby eliminating unnecessary scaling | 609 // rendered at the exact capture size, thereby eliminating unnecessary scaling |
797 // operations in the graphics pipeline. | 610 // operations in the graphics pipeline. |
798 RenderWidgetHost* const rwh = tracker_->GetTargetRenderWidgetHost(); | 611 RenderWidgetHost* const rwh = tracker_->GetTargetRenderWidgetHost(); |
799 RenderWidgetHostView* const rwhv = rwh ? rwh->GetView() : NULL; | 612 RenderWidgetHostView* const rwhv = rwh ? rwh->GetView() : nullptr; |
800 if (rwhv) { | 613 if (rwhv) { |
801 const gfx::NativeView view = rwhv->GetNativeView(); | 614 const gfx::NativeView view = rwhv->GetNativeView(); |
802 const float scale = ui::GetScaleFactorForNativeView(view); | 615 const float scale = ui::GetScaleFactorForNativeView(view); |
803 if (scale > 1.0f) { | 616 if (scale > 1.0f) { |
804 const gfx::Size shrunk_size = | 617 const gfx::Size shrunk_size = |
805 gfx::ScaleToFlooredSize(optimal_size, 1.0f / scale); | 618 gfx::ScaleToFlooredSize(optimal_size, 1.0f / scale); |
806 if (shrunk_size.width() > 0 && shrunk_size.height() > 0) | 619 if (shrunk_size.width() > 0 && shrunk_size.height() > 0) |
807 optimal_size = shrunk_size; | 620 optimal_size = shrunk_size; |
808 } | 621 } |
809 } | 622 } |
810 | 623 |
811 VLOG(1) << "Computed optimal target size: " << optimal_size.ToString(); | 624 VLOG(1) << "Computed optimal target size: " << optimal_size.ToString(); |
812 return optimal_size; | 625 return optimal_size; |
813 } | 626 } |
814 | 627 |
815 void WebContentsCaptureMachine::DidCopyFromBackingStore( | 628 void WebContentsCaptureMachine::DidCopyToVideoFrame( |
816 const base::TimeTicks& start_time, | |
817 const scoped_refptr<media::VideoFrame>& target, | |
818 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | |
819 deliver_frame_cb, | |
820 const SkBitmap& bitmap, | |
821 ReadbackResponse response) { | |
822 DCHECK_CURRENTLY_ON(BrowserThread::UI); | |
823 | |
824 DCHECK(render_thread_); | |
825 if (response == READBACK_SUCCESS) { | |
826 TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", target.get(), | |
827 "Render"); | |
828 render_thread_->task_runner()->PostTask( | |
829 FROM_HERE, media::BindToCurrentLoop( | |
830 base::Bind(&RenderVideoFrame, bitmap, target, | |
831 base::Bind(deliver_frame_cb, start_time)))); | |
832 } else { | |
833 // Capture can fail due to transient issues, so just skip this frame. | |
834 DVLOG(1) << "CopyFromBackingStore failed; skipping frame."; | |
835 deliver_frame_cb.Run(start_time, gfx::Rect(), false); | |
836 } | |
837 } | |
838 | |
839 void WebContentsCaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame( | |
840 const base::TimeTicks& start_time, | 629 const base::TimeTicks& start_time, |
841 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | 630 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& |
842 deliver_frame_cb, | 631 deliver_frame_cb, |
843 const gfx::Rect& region_in_frame, | 632 const gfx::Rect& region_in_frame, |
844 bool success) { | 633 bool success) { |
845 DCHECK_CURRENTLY_ON(BrowserThread::UI); | 634 DCHECK_CURRENTLY_ON(BrowserThread::UI); |
846 | 635 |
847 // Capture can fail due to transient issues, so just skip this frame. | 636 // Capture can fail due to transient issues, so just skip this frame. |
848 DVLOG_IF(1, !success) << "CopyFromCompositingSurface failed; skipping frame."; | 637 DVLOG_IF(1, !success) << "CopyFromCompositingSurface failed; skipping frame."; |
849 deliver_frame_cb.Run(start_time, region_in_frame, success); | 638 deliver_frame_cb.Run(start_time, region_in_frame, success); |
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
918 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() { | 707 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() { |
919 DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying."; | 708 DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying."; |
920 } | 709 } |
921 | 710 |
922 // static | 711 // static |
923 std::unique_ptr<media::VideoCaptureDevice> | 712 std::unique_ptr<media::VideoCaptureDevice> |
924 WebContentsVideoCaptureDevice::Create(const std::string& device_id) { | 713 WebContentsVideoCaptureDevice::Create(const std::string& device_id) { |
925 // Parse device_id into render_process_id and main_render_frame_id. | 714 // Parse device_id into render_process_id and main_render_frame_id. |
926 WebContentsMediaCaptureId media_id; | 715 WebContentsMediaCaptureId media_id; |
927 if (!WebContentsMediaCaptureId::Parse(device_id, &media_id)) { | 716 if (!WebContentsMediaCaptureId::Parse(device_id, &media_id)) { |
928 return NULL; | 717 return nullptr; |
929 } | 718 } |
930 | 719 |
931 return std::unique_ptr<media::VideoCaptureDevice>( | 720 return std::unique_ptr<media::VideoCaptureDevice>( |
932 new WebContentsVideoCaptureDevice(media_id.render_process_id, | 721 new WebContentsVideoCaptureDevice(media_id.render_process_id, |
933 media_id.main_render_frame_id, | 722 media_id.main_render_frame_id, |
934 media_id.enable_auto_throttling)); | 723 media_id.enable_auto_throttling)); |
935 } | 724 } |
936 | 725 |
937 void WebContentsVideoCaptureDevice::AllocateAndStart( | 726 void WebContentsVideoCaptureDevice::AllocateAndStart( |
938 const media::VideoCaptureParams& params, | 727 const media::VideoCaptureParams& params, |
(...skipping 17 matching lines...) Expand all Loading... |
956 void WebContentsVideoCaptureDevice::StopAndDeAllocate() { | 745 void WebContentsVideoCaptureDevice::StopAndDeAllocate() { |
957 core_->StopAndDeAllocate(); | 746 core_->StopAndDeAllocate(); |
958 } | 747 } |
959 | 748 |
960 void WebContentsVideoCaptureDevice::OnUtilizationReport(int frame_feedback_id, | 749 void WebContentsVideoCaptureDevice::OnUtilizationReport(int frame_feedback_id, |
961 double utilization) { | 750 double utilization) { |
962 core_->OnConsumerReportingUtilization(frame_feedback_id, utilization); | 751 core_->OnConsumerReportingUtilization(frame_feedback_id, utilization); |
963 } | 752 } |
964 | 753 |
965 } // namespace content | 754 } // namespace content |
OLD | NEW |