| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 // | |
| 5 // Implementation notes: This needs to work on a variety of hardware | |
| 6 // configurations where the speed of the CPU and GPU greatly affect overall | |
| 7 // performance. Spanning several threads, the process of capturing has been | |
| 8 // split up into four conceptual stages: | |
| 9 // | |
| 10 // 1. Reserve Buffer: Before a frame can be captured, a slot in the client's | |
| 11 // shared-memory IPC buffer is reserved. There are only a few of these; | |
| 12 // when they run out, it indicates that the downstream client -- likely a | |
| 13 // video encoder -- is the performance bottleneck, and that the rate of | |
| 14 // frame capture should be throttled back. | |
| 15 // | |
| 16 // 2. Capture: A bitmap is snapshotted/copied from the RenderView's backing | |
| 17 // store. This is initiated on the UI BrowserThread, and often occurs | |
| 18 // asynchronously. Where supported, the GPU scales and color converts | |
| 19 // frames to our desired size, and the readback happens directly into the | |
| 20 // shared-memory buffer. But this is not always possible, particularly when | |
| 21 // accelerated compositing is disabled. | |
| 22 // | |
| 23 // 3. Render (if needed): If the web contents cannot be captured directly into | |
| 24 // our target size and color format, scaling and colorspace conversion must | |
| 25 // be done on the CPU. A dedicated thread is used for this operation, to | |
| 26 // avoid blocking the UI thread. The Render stage always reads from a | |
| 27 // bitmap returned by Capture, and writes into the reserved slot in the | |
| 28 // shared-memory buffer. | |
| 29 // | |
| 30 // 4. Deliver: The rendered video frame is returned to the client (which | |
| 31 // implements the VideoCaptureDevice::Client interface). Because all | |
| 32 // paths have written the frame into the IPC buffer, this step should | |
| 33 // never need to do an additional copy of the pixel data. | |
| 34 // | |
| 35 // In the best-performing case, the Render step is bypassed: Capture produces | |
| 36 // ready-to-Deliver frames. But when accelerated readback is not possible, the | |
| 37 // system is designed so that Capture and Render may run concurrently. A timing | |
| 38 // diagram helps illustrate this point (@30 FPS): | |
| 39 // | |
| 40 // Time: 0ms 33ms 66ms 99ms | |
| 41 // thread1: |-Capture-f1------v |-Capture-f2------v |-Capture-f3----v |-Capt | |
| 42 // thread2: |-Render-f1-----v |-Render-f2-----v |-Render-f3 | |
| 43 // | |
| 44 // In the above example, both capturing and rendering *each* take almost the | |
| 45 // full 33 ms available between frames, yet we see that the required throughput | |
| 46 // is obtained. | |
| 47 // | |
| 48 // Turning on verbose logging will cause the effective frame rate to be logged | |
| 49 // at 5-second intervals. | |
| 50 | |
| 51 #include "content/browser/renderer_host/media/web_contents_video_capture_device.
h" | |
| 52 | |
| 53 #include "base/basictypes.h" | |
| 54 #include "base/bind.h" | |
| 55 #include "base/callback_helpers.h" | |
| 56 #include "base/logging.h" | |
| 57 #include "base/memory/scoped_ptr.h" | |
| 58 #include "base/memory/weak_ptr.h" | |
| 59 #include "base/message_loop/message_loop_proxy.h" | |
| 60 #include "base/metrics/histogram.h" | |
| 61 #include "base/sequenced_task_runner.h" | |
| 62 #include "base/threading/thread.h" | |
| 63 #include "base/threading/thread_checker.h" | |
| 64 #include "base/time/time.h" | |
| 65 #include "content/browser/renderer_host/media/content_video_capture_device_core.
h" | |
| 66 #include "content/browser/renderer_host/media/video_capture_oracle.h" | |
| 67 #include "content/browser/renderer_host/media/web_contents_capture_util.h" | |
| 68 #include "content/browser/renderer_host/render_widget_host_impl.h" | |
| 69 #include "content/browser/web_contents/web_contents_impl.h" | |
| 70 #include "content/port/browser/render_widget_host_view_frame_subscriber.h" | |
| 71 #include "content/port/browser/render_widget_host_view_port.h" | |
| 72 #include "content/public/browser/browser_thread.h" | |
| 73 #include "content/public/browser/notification_source.h" | |
| 74 #include "content/public/browser/notification_types.h" | |
| 75 #include "content/public/browser/render_view_host.h" | |
| 76 #include "content/public/browser/render_widget_host_view.h" | |
| 77 #include "content/public/browser/web_contents_observer.h" | |
| 78 #include "media/base/video_util.h" | |
| 79 #include "media/video/capture/video_capture_types.h" | |
| 80 #include "skia/ext/image_operations.h" | |
| 81 #include "third_party/skia/include/core/SkBitmap.h" | |
| 82 #include "third_party/skia/include/core/SkColor.h" | |
| 83 | |
| 84 namespace content { | |
| 85 | |
| 86 namespace { | |
| 87 | |
| 88 // Compute a letterbox region, aligned to even coordinates. | |
| 89 gfx::Rect ComputeYV12LetterboxRegion(const gfx::Size& frame_size, | |
| 90 const gfx::Size& content_size) { | |
| 91 | |
| 92 gfx::Rect result = media::ComputeLetterboxRegion(gfx::Rect(frame_size), | |
| 93 content_size); | |
| 94 | |
| 95 result.set_x(MakeEven(result.x())); | |
| 96 result.set_y(MakeEven(result.y())); | |
| 97 result.set_width(std::max(kMinFrameWidth, MakeEven(result.width()))); | |
| 98 result.set_height(std::max(kMinFrameHeight, MakeEven(result.height()))); | |
| 99 | |
| 100 return result; | |
| 101 } | |
| 102 | |
| 103 // Wrapper function to invoke ThreadSafeCaptureOracle::CaptureFrameCallback, is | |
| 104 // compatible with RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback. | |
| 105 void InvokeCaptureFrameCallback( | |
| 106 const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb, | |
| 107 base::TimeTicks timestamp, | |
| 108 bool frame_captured) { | |
| 109 capture_frame_cb.Run(timestamp, frame_captured); | |
| 110 } | |
| 111 | |
| 112 void DeleteOnWorkerThread(scoped_ptr<base::Thread> render_thread, | |
| 113 const base::Closure& callback) { | |
| 114 render_thread.reset(); | |
| 115 | |
| 116 // After thread join call the callback on UI thread. | |
| 117 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, callback); | |
| 118 } | |
| 119 | |
| 120 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible | |
| 121 // with RenderWidgetHostViewFrameSubscriber. We create one per event type. | |
| 122 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber { | |
| 123 public: | |
| 124 FrameSubscriber(VideoCaptureOracle::Event event_type, | |
| 125 const scoped_refptr<ThreadSafeCaptureOracle>& oracle) | |
| 126 : event_type_(event_type), | |
| 127 oracle_proxy_(oracle) {} | |
| 128 | |
| 129 virtual bool ShouldCaptureFrame( | |
| 130 base::TimeTicks present_time, | |
| 131 scoped_refptr<media::VideoFrame>* storage, | |
| 132 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback* | |
| 133 deliver_frame_cb) OVERRIDE; | |
| 134 | |
| 135 private: | |
| 136 const VideoCaptureOracle::Event event_type_; | |
| 137 scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_; | |
| 138 }; | |
| 139 | |
| 140 // ContentCaptureSubscription is the relationship between a RenderWidgetHost | |
| 141 // whose content is updating, a subscriber that is deciding which of these | |
| 142 // updates to capture (and where to deliver them to), and a callback that | |
| 143 // knows how to do the capture and prepare the result for delivery. | |
| 144 // | |
| 145 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in | |
| 146 // the RenderWidgetHostView, to process updates that occur via accelerated | |
| 147 // compositing, (b) installing itself as an observer of updates to the | |
| 148 // RenderWidgetHost's backing store, to hook updates that occur via software | |
| 149 // rendering, and (c) running a timer to possibly initiate non-event-driven | |
| 150 // captures that the subscriber might request. | |
| 151 // | |
| 152 // All of this happens on the UI thread, although the | |
| 153 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates | |
| 154 // autonomously on some other thread. | |
| 155 class ContentCaptureSubscription : public content::NotificationObserver { | |
| 156 public: | |
| 157 typedef base::Callback< | |
| 158 void(const base::TimeTicks&, | |
| 159 const scoped_refptr<media::VideoFrame>&, | |
| 160 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&)> | |
| 161 CaptureCallback; | |
| 162 | |
| 163 // Create a subscription. Whenever a manual capture is required, the | |
| 164 // subscription will invoke |capture_callback| on the UI thread to do the | |
| 165 // work. | |
| 166 ContentCaptureSubscription( | |
| 167 const RenderWidgetHost& source, | |
| 168 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy, | |
| 169 const CaptureCallback& capture_callback); | |
| 170 virtual ~ContentCaptureSubscription(); | |
| 171 | |
| 172 // content::NotificationObserver implementation. | |
| 173 virtual void Observe(int type, | |
| 174 const content::NotificationSource& source, | |
| 175 const content::NotificationDetails& details) OVERRIDE; | |
| 176 | |
| 177 private: | |
| 178 void OnTimer(); | |
| 179 | |
| 180 const int render_process_id_; | |
| 181 const int render_view_id_; | |
| 182 | |
| 183 FrameSubscriber paint_subscriber_; | |
| 184 FrameSubscriber timer_subscriber_; | |
| 185 content::NotificationRegistrar registrar_; | |
| 186 CaptureCallback capture_callback_; | |
| 187 base::Timer timer_; | |
| 188 | |
| 189 DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription); | |
| 190 }; | |
| 191 | |
| 192 // Render the SkBitmap |input| into the given VideoFrame buffer |output|, then | |
| 193 // invoke |done_cb| to indicate success or failure. |input| is expected to be | |
| 194 // ARGB. |output| must be YV12 or I420. Colorspace conversion is always done. | |
| 195 // Scaling and letterboxing will be done as needed. | |
| 196 // | |
| 197 // This software implementation should be used only when GPU acceleration of | |
| 198 // these activities is not possible. This operation may be expensive (tens to | |
| 199 // hundreds of milliseconds), so the caller should ensure that it runs on a | |
| 200 // thread where such a pause would cause UI jank. | |
| 201 void RenderVideoFrame(const SkBitmap& input, | |
| 202 const scoped_refptr<media::VideoFrame>& output, | |
| 203 const base::Callback<void(bool)>& done_cb); | |
| 204 | |
| 205 // Keeps track of the RenderView to be sourced, and executes copying of the | |
| 206 // backing store on the UI BrowserThread. | |
| 207 // | |
| 208 // TODO(nick): It would be nice to merge this with WebContentsTracker, but its | |
| 209 // implementation is currently asynchronous -- in our case, the "rvh changed" | |
| 210 // notification would get posted back to the UI thread and processed later, and | |
| 211 // this seems disadvantageous. | |
| 212 class WebContentsCaptureMachine | |
| 213 : public VideoCaptureMachine, | |
| 214 public WebContentsObserver { | |
| 215 public: | |
| 216 WebContentsCaptureMachine(int render_process_id, int render_view_id); | |
| 217 virtual ~WebContentsCaptureMachine(); | |
| 218 | |
| 219 // VideoCaptureMachine overrides. | |
| 220 virtual bool Start( | |
| 221 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy) OVERRIDE; | |
| 222 virtual void Stop(const base::Closure& callback) OVERRIDE; | |
| 223 | |
| 224 // Starts a copy from the backing store or the composited surface. Must be run | |
| 225 // on the UI BrowserThread. |deliver_frame_cb| will be run when the operation | |
| 226 // completes. The copy will occur to |target|. | |
| 227 // | |
| 228 // This may be used as a ContentCaptureSubscription::CaptureCallback. | |
| 229 void Capture(const base::TimeTicks& start_time, | |
| 230 const scoped_refptr<media::VideoFrame>& target, | |
| 231 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | |
| 232 deliver_frame_cb); | |
| 233 | |
| 234 // content::WebContentsObserver implementation. | |
| 235 virtual void DidShowFullscreenWidget(int routing_id) OVERRIDE { | |
| 236 fullscreen_widget_id_ = routing_id; | |
| 237 RenewFrameSubscription(); | |
| 238 } | |
| 239 | |
| 240 virtual void DidDestroyFullscreenWidget(int routing_id) OVERRIDE { | |
| 241 DCHECK_EQ(fullscreen_widget_id_, routing_id); | |
| 242 fullscreen_widget_id_ = MSG_ROUTING_NONE; | |
| 243 RenewFrameSubscription(); | |
| 244 } | |
| 245 | |
| 246 virtual void RenderViewReady() OVERRIDE { | |
| 247 RenewFrameSubscription(); | |
| 248 } | |
| 249 | |
| 250 virtual void AboutToNavigateRenderView(RenderViewHost* rvh) OVERRIDE { | |
| 251 RenewFrameSubscription(); | |
| 252 } | |
| 253 | |
| 254 virtual void DidNavigateMainFrame( | |
| 255 const LoadCommittedDetails& details, | |
| 256 const FrameNavigateParams& params) OVERRIDE { | |
| 257 RenewFrameSubscription(); | |
| 258 } | |
| 259 | |
| 260 virtual void WebContentsDestroyed(WebContents* web_contents) OVERRIDE; | |
| 261 | |
| 262 private: | |
| 263 // Starts observing the web contents, returning false if lookup fails. | |
| 264 bool StartObservingWebContents(); | |
| 265 | |
| 266 // Helper function to determine the view that we are currently tracking. | |
| 267 RenderWidgetHost* GetTarget(); | |
| 268 | |
| 269 // Response callback for RenderWidgetHost::CopyFromBackingStore(). | |
| 270 void DidCopyFromBackingStore( | |
| 271 const base::TimeTicks& start_time, | |
| 272 const scoped_refptr<media::VideoFrame>& target, | |
| 273 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | |
| 274 deliver_frame_cb, | |
| 275 bool success, | |
| 276 const SkBitmap& bitmap); | |
| 277 | |
| 278 // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame(). | |
| 279 void DidCopyFromCompositingSurfaceToVideoFrame( | |
| 280 const base::TimeTicks& start_time, | |
| 281 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | |
| 282 deliver_frame_cb, | |
| 283 bool success); | |
| 284 | |
| 285 // Remove the old subscription, and start a new one. This should be called | |
| 286 // after any change to the WebContents that affects the RenderWidgetHost or | |
| 287 // attached views. | |
| 288 void RenewFrameSubscription(); | |
| 289 | |
| 290 // Parameters saved in constructor. | |
| 291 const int initial_render_process_id_; | |
| 292 const int initial_render_view_id_; | |
| 293 | |
| 294 // A dedicated worker thread on which SkBitmap->VideoFrame conversion will | |
| 295 // occur. Only used when this activity cannot be done on the GPU. | |
| 296 scoped_ptr<base::Thread> render_thread_; | |
| 297 | |
| 298 // Makes all the decisions about which frames to copy, and how. | |
| 299 scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_; | |
| 300 | |
| 301 // Routing ID of any active fullscreen render widget or MSG_ROUTING_NONE | |
| 302 // otherwise. | |
| 303 int fullscreen_widget_id_; | |
| 304 | |
| 305 // Last known RenderView size. | |
| 306 gfx::Size last_view_size_; | |
| 307 | |
| 308 // Responsible for forwarding events from the active RenderWidgetHost to the | |
| 309 // oracle, and initiating captures accordingly. | |
| 310 scoped_ptr<ContentCaptureSubscription> subscription_; | |
| 311 | |
| 312 // Weak pointer factory used to invalidate callbacks. | |
| 313 base::WeakPtrFactory<WebContentsCaptureMachine> weak_ptr_factory_; | |
| 314 | |
| 315 DISALLOW_COPY_AND_ASSIGN(WebContentsCaptureMachine); | |
| 316 }; | |
| 317 | |
| 318 // Responsible for logging the effective frame rate. | |
| 319 // TODO(nick): Make this compatible with the push model and hook it back up. | |
| 320 class VideoFrameDeliveryLog { | |
| 321 public: | |
| 322 VideoFrameDeliveryLog(); | |
| 323 | |
| 324 // Treat |frame_number| as having been delivered, and update the | |
| 325 // frame rate statistics accordingly. | |
| 326 void ChronicleFrameDelivery(int frame_number); | |
| 327 | |
| 328 private: | |
| 329 // The following keep track of and log the effective frame rate whenever | |
| 330 // verbose logging is turned on. | |
| 331 base::TimeTicks last_frame_rate_log_time_; | |
| 332 int count_frames_rendered_; | |
| 333 int last_frame_number_; | |
| 334 | |
| 335 DISALLOW_COPY_AND_ASSIGN(VideoFrameDeliveryLog); | |
| 336 }; | |
| 337 | |
| 338 bool FrameSubscriber::ShouldCaptureFrame( | |
| 339 base::TimeTicks present_time, | |
| 340 scoped_refptr<media::VideoFrame>* storage, | |
| 341 DeliverFrameCallback* deliver_frame_cb) { | |
| 342 TRACE_EVENT1("mirroring", "FrameSubscriber::ShouldCaptureFrame", | |
| 343 "instance", this); | |
| 344 | |
| 345 ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb; | |
| 346 bool oracle_decision = oracle_proxy_->ObserveEventAndDecideCapture( | |
| 347 event_type_, present_time, storage, &capture_frame_cb); | |
| 348 | |
| 349 *deliver_frame_cb = base::Bind(&InvokeCaptureFrameCallback, capture_frame_cb); | |
| 350 return oracle_decision; | |
| 351 } | |
| 352 | |
| 353 ContentCaptureSubscription::ContentCaptureSubscription( | |
| 354 const RenderWidgetHost& source, | |
| 355 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy, | |
| 356 const CaptureCallback& capture_callback) | |
| 357 : render_process_id_(source.GetProcess()->GetID()), | |
| 358 render_view_id_(source.GetRoutingID()), | |
| 359 paint_subscriber_(VideoCaptureOracle::kSoftwarePaint, oracle_proxy), | |
| 360 timer_subscriber_(VideoCaptureOracle::kTimerPoll, oracle_proxy), | |
| 361 capture_callback_(capture_callback), | |
| 362 timer_(true, true) { | |
| 363 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
| 364 | |
| 365 RenderWidgetHostViewPort* view = | |
| 366 RenderWidgetHostViewPort::FromRWHV(source.GetView()); | |
| 367 | |
| 368 // Subscribe to accelerated presents. These will be serviced directly by the | |
| 369 // oracle. | |
| 370 if (view && kAcceleratedSubscriberIsSupported) { | |
| 371 scoped_ptr<RenderWidgetHostViewFrameSubscriber> subscriber( | |
| 372 new FrameSubscriber(VideoCaptureOracle::kCompositorUpdate, | |
| 373 oracle_proxy)); | |
| 374 view->BeginFrameSubscription(subscriber.Pass()); | |
| 375 } | |
| 376 | |
| 377 // Subscribe to software paint events. This instance will service these by | |
| 378 // reflecting them back to the WebContentsCaptureMachine via | |
| 379 // |capture_callback|. | |
| 380 registrar_.Add( | |
| 381 this, content::NOTIFICATION_RENDER_WIDGET_HOST_DID_UPDATE_BACKING_STORE, | |
| 382 Source<RenderWidgetHost>(&source)); | |
| 383 | |
| 384 // Subscribe to timer events. This instance will service these as well. | |
| 385 timer_.Start(FROM_HERE, oracle_proxy->capture_period(), | |
| 386 base::Bind(&ContentCaptureSubscription::OnTimer, | |
| 387 base::Unretained(this))); | |
| 388 } | |
| 389 | |
| 390 ContentCaptureSubscription::~ContentCaptureSubscription() { | |
| 391 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
| 392 if (kAcceleratedSubscriberIsSupported) { | |
| 393 RenderViewHost* source = RenderViewHost::FromID(render_process_id_, | |
| 394 render_view_id_); | |
| 395 if (source) { | |
| 396 RenderWidgetHostViewPort* view = | |
| 397 RenderWidgetHostViewPort::FromRWHV(source->GetView()); | |
| 398 if (view) | |
| 399 view->EndFrameSubscription(); | |
| 400 } | |
| 401 } | |
| 402 } | |
| 403 | |
| 404 void ContentCaptureSubscription::Observe( | |
| 405 int type, | |
| 406 const content::NotificationSource& source, | |
| 407 const content::NotificationDetails& details) { | |
| 408 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
| 409 DCHECK_EQ(NOTIFICATION_RENDER_WIDGET_HOST_DID_UPDATE_BACKING_STORE, type); | |
| 410 | |
| 411 RenderWidgetHostImpl* rwh = | |
| 412 RenderWidgetHostImpl::From(Source<RenderWidgetHost>(source).ptr()); | |
| 413 | |
| 414 // This message occurs on window resizes and visibility changes even when | |
| 415 // accelerated compositing is active, so we need to filter out these cases. | |
| 416 if (!rwh || !rwh->GetView() || (rwh->is_accelerated_compositing_active() && | |
| 417 rwh->GetView()->IsSurfaceAvailableForCopy())) | |
| 418 return; | |
| 419 | |
| 420 TRACE_EVENT1("mirroring", "ContentCaptureSubscription::Observe", | |
| 421 "instance", this); | |
| 422 | |
| 423 base::Closure copy_done_callback; | |
| 424 scoped_refptr<media::VideoFrame> frame; | |
| 425 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb; | |
| 426 const base::TimeTicks start_time = base::TimeTicks::Now(); | |
| 427 if (paint_subscriber_.ShouldCaptureFrame(start_time, | |
| 428 &frame, | |
| 429 &deliver_frame_cb)) { | |
| 430 // This message happens just before paint. If we post a task to do the copy, | |
| 431 // it should run soon after the paint. | |
| 432 BrowserThread::PostTask( | |
| 433 BrowserThread::UI, FROM_HERE, | |
| 434 base::Bind(capture_callback_, start_time, frame, deliver_frame_cb)); | |
| 435 } | |
| 436 } | |
| 437 | |
| 438 void ContentCaptureSubscription::OnTimer() { | |
| 439 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
| 440 TRACE_EVENT0("mirroring", "ContentCaptureSubscription::OnTimer"); | |
| 441 | |
| 442 scoped_refptr<media::VideoFrame> frame; | |
| 443 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb; | |
| 444 | |
| 445 const base::TimeTicks start_time = base::TimeTicks::Now(); | |
| 446 if (timer_subscriber_.ShouldCaptureFrame(start_time, | |
| 447 &frame, | |
| 448 &deliver_frame_cb)) { | |
| 449 capture_callback_.Run(start_time, frame, deliver_frame_cb); | |
| 450 } | |
| 451 } | |
| 452 | |
| 453 void RenderVideoFrame(const SkBitmap& input, | |
| 454 const scoped_refptr<media::VideoFrame>& output, | |
| 455 const base::Callback<void(bool)>& done_cb) { | |
| 456 base::ScopedClosureRunner failure_handler(base::Bind(done_cb, false)); | |
| 457 | |
| 458 SkAutoLockPixels locker(input); | |
| 459 | |
| 460 // Sanity-check the captured bitmap. | |
| 461 if (input.empty() || | |
| 462 !input.readyToDraw() || | |
| 463 input.config() != SkBitmap::kARGB_8888_Config || | |
| 464 input.width() < 2 || input.height() < 2) { | |
| 465 DVLOG(1) << "input unacceptable (size=" | |
| 466 << input.getSize() | |
| 467 << ", ready=" << input.readyToDraw() | |
| 468 << ", config=" << input.config() << ')'; | |
| 469 return; | |
| 470 } | |
| 471 | |
| 472 // Sanity-check the output buffer. | |
| 473 if (output->format() != media::VideoFrame::I420) { | |
| 474 NOTREACHED(); | |
| 475 return; | |
| 476 } | |
| 477 | |
| 478 // Calculate the width and height of the content region in the |output|, based | |
| 479 // on the aspect ratio of |input|. | |
| 480 gfx::Rect region_in_frame = ComputeYV12LetterboxRegion( | |
| 481 output->coded_size(), gfx::Size(input.width(), input.height())); | |
| 482 | |
| 483 // Scale the bitmap to the required size, if necessary. | |
| 484 SkBitmap scaled_bitmap; | |
| 485 if (input.width() != region_in_frame.width() || | |
| 486 input.height() != region_in_frame.height()) { | |
| 487 | |
| 488 skia::ImageOperations::ResizeMethod method; | |
| 489 if (input.width() < region_in_frame.width() || | |
| 490 input.height() < region_in_frame.height()) { | |
| 491 // Avoid box filtering when magnifying, because it's actually | |
| 492 // nearest-neighbor. | |
| 493 method = skia::ImageOperations::RESIZE_HAMMING1; | |
| 494 } else { | |
| 495 method = skia::ImageOperations::RESIZE_BOX; | |
| 496 } | |
| 497 | |
| 498 TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", output.get(), "Scale"); | |
| 499 scaled_bitmap = skia::ImageOperations::Resize(input, method, | |
| 500 region_in_frame.width(), | |
| 501 region_in_frame.height()); | |
| 502 } else { | |
| 503 scaled_bitmap = input; | |
| 504 } | |
| 505 | |
| 506 TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", output.get(), "YUV"); | |
| 507 { | |
| 508 SkAutoLockPixels scaled_bitmap_locker(scaled_bitmap); | |
| 509 | |
| 510 media::CopyRGBToVideoFrame( | |
| 511 reinterpret_cast<uint8*>(scaled_bitmap.getPixels()), | |
| 512 scaled_bitmap.rowBytes(), | |
| 513 region_in_frame, | |
| 514 output.get()); | |
| 515 } | |
| 516 | |
| 517 // The result is now ready. | |
| 518 ignore_result(failure_handler.Release()); | |
| 519 done_cb.Run(true); | |
| 520 } | |
| 521 | |
| 522 VideoFrameDeliveryLog::VideoFrameDeliveryLog() | |
| 523 : last_frame_rate_log_time_(), | |
| 524 count_frames_rendered_(0), | |
| 525 last_frame_number_(0) { | |
| 526 } | |
| 527 | |
| 528 void VideoFrameDeliveryLog::ChronicleFrameDelivery(int frame_number) { | |
| 529 // Log frame rate, if verbose logging is turned on. | |
| 530 static const base::TimeDelta kFrameRateLogInterval = | |
| 531 base::TimeDelta::FromSeconds(10); | |
| 532 const base::TimeTicks now = base::TimeTicks::Now(); | |
| 533 if (last_frame_rate_log_time_.is_null()) { | |
| 534 last_frame_rate_log_time_ = now; | |
| 535 count_frames_rendered_ = 0; | |
| 536 last_frame_number_ = frame_number; | |
| 537 } else { | |
| 538 ++count_frames_rendered_; | |
| 539 const base::TimeDelta elapsed = now - last_frame_rate_log_time_; | |
| 540 if (elapsed >= kFrameRateLogInterval) { | |
| 541 const double measured_fps = | |
| 542 count_frames_rendered_ / elapsed.InSecondsF(); | |
| 543 const int frames_elapsed = frame_number - last_frame_number_; | |
| 544 const int count_frames_dropped = frames_elapsed - count_frames_rendered_; | |
| 545 DCHECK_LE(0, count_frames_dropped); | |
| 546 UMA_HISTOGRAM_PERCENTAGE( | |
| 547 "TabCapture.FrameDropPercentage", | |
| 548 (count_frames_dropped * 100 + frames_elapsed / 2) / frames_elapsed); | |
| 549 UMA_HISTOGRAM_COUNTS( | |
| 550 "TabCapture.FrameRate", | |
| 551 static_cast<int>(measured_fps)); | |
| 552 VLOG(1) << "Current measured frame rate for " | |
| 553 << "WebContentsVideoCaptureDevice is " << measured_fps << " FPS."; | |
| 554 last_frame_rate_log_time_ = now; | |
| 555 count_frames_rendered_ = 0; | |
| 556 last_frame_number_ = frame_number; | |
| 557 } | |
| 558 } | |
| 559 } | |
| 560 | |
| 561 WebContentsCaptureMachine::WebContentsCaptureMachine(int render_process_id, | |
| 562 int render_view_id) | |
| 563 : initial_render_process_id_(render_process_id), | |
| 564 initial_render_view_id_(render_view_id), | |
| 565 fullscreen_widget_id_(MSG_ROUTING_NONE), | |
| 566 weak_ptr_factory_(this) {} | |
| 567 | |
| 568 WebContentsCaptureMachine::~WebContentsCaptureMachine() { | |
| 569 BrowserThread::PostBlockingPoolTask( | |
| 570 FROM_HERE, | |
| 571 base::Bind(&DeleteOnWorkerThread, base::Passed(&render_thread_), | |
| 572 base::Bind(&base::DoNothing))); | |
| 573 } | |
| 574 | |
| 575 bool WebContentsCaptureMachine::Start( | |
| 576 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy) { | |
| 577 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
| 578 DCHECK(!started_); | |
| 579 | |
| 580 DCHECK(oracle_proxy.get()); | |
| 581 oracle_proxy_ = oracle_proxy; | |
| 582 | |
| 583 render_thread_.reset(new base::Thread("WebContentsVideo_RenderThread")); | |
| 584 if (!render_thread_->Start()) { | |
| 585 DVLOG(1) << "Failed to spawn render thread."; | |
| 586 render_thread_.reset(); | |
| 587 return false; | |
| 588 } | |
| 589 | |
| 590 if (!StartObservingWebContents()) { | |
| 591 DVLOG(1) << "Failed to observe web contents."; | |
| 592 render_thread_.reset(); | |
| 593 return false; | |
| 594 } | |
| 595 | |
| 596 started_ = true; | |
| 597 return true; | |
| 598 } | |
| 599 | |
| 600 void WebContentsCaptureMachine::Stop(const base::Closure& callback) { | |
| 601 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
| 602 subscription_.reset(); | |
| 603 if (web_contents()) { | |
| 604 web_contents()->DecrementCapturerCount(); | |
| 605 Observe(NULL); | |
| 606 } | |
| 607 | |
| 608 // Any callback that intend to use render_thread_ will not work after it is | |
| 609 // passed. | |
| 610 weak_ptr_factory_.InvalidateWeakPtrs(); | |
| 611 | |
| 612 // The render thread cannot be stopped on the UI thread, so post a message | |
| 613 // to the thread pool used for blocking operations. | |
| 614 BrowserThread::PostBlockingPoolTask( | |
| 615 FROM_HERE, | |
| 616 base::Bind(&DeleteOnWorkerThread, base::Passed(&render_thread_), | |
| 617 callback)); | |
| 618 | |
| 619 started_ = false; | |
| 620 } | |
| 621 | |
| 622 void WebContentsCaptureMachine::Capture( | |
| 623 const base::TimeTicks& start_time, | |
| 624 const scoped_refptr<media::VideoFrame>& target, | |
| 625 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | |
| 626 deliver_frame_cb) { | |
| 627 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
| 628 | |
| 629 RenderWidgetHost* rwh = GetTarget(); | |
| 630 RenderWidgetHostViewPort* view = | |
| 631 rwh ? RenderWidgetHostViewPort::FromRWHV(rwh->GetView()) : NULL; | |
| 632 if (!view || !rwh) { | |
| 633 deliver_frame_cb.Run(base::TimeTicks(), false); | |
| 634 return; | |
| 635 } | |
| 636 | |
| 637 gfx::Size video_size = target->coded_size(); | |
| 638 gfx::Size view_size = view->GetViewBounds().size(); | |
| 639 gfx::Size fitted_size; | |
| 640 if (!view_size.IsEmpty()) { | |
| 641 fitted_size = ComputeYV12LetterboxRegion(video_size, view_size).size(); | |
| 642 } | |
| 643 if (view_size != last_view_size_) { | |
| 644 last_view_size_ = view_size; | |
| 645 | |
| 646 // Measure the number of kilopixels. | |
| 647 UMA_HISTOGRAM_COUNTS_10000( | |
| 648 "TabCapture.ViewChangeKiloPixels", | |
| 649 view_size.width() * view_size.height() / 1024); | |
| 650 } | |
| 651 | |
| 652 if (!view->IsSurfaceAvailableForCopy()) { | |
| 653 // Fallback to the more expensive renderer-side copy if the surface and | |
| 654 // backing store are not accessible. | |
| 655 rwh->GetSnapshotFromRenderer( | |
| 656 gfx::Rect(), | |
| 657 base::Bind(&WebContentsCaptureMachine::DidCopyFromBackingStore, | |
| 658 weak_ptr_factory_.GetWeakPtr(), | |
| 659 start_time, target, deliver_frame_cb)); | |
| 660 } else if (view->CanCopyToVideoFrame()) { | |
| 661 view->CopyFromCompositingSurfaceToVideoFrame( | |
| 662 gfx::Rect(view_size), | |
| 663 target, | |
| 664 base::Bind(&WebContentsCaptureMachine:: | |
| 665 DidCopyFromCompositingSurfaceToVideoFrame, | |
| 666 weak_ptr_factory_.GetWeakPtr(), | |
| 667 start_time, deliver_frame_cb)); | |
| 668 } else { | |
| 669 rwh->CopyFromBackingStore( | |
| 670 gfx::Rect(), | |
| 671 fitted_size, // Size here is a request not always honored. | |
| 672 base::Bind(&WebContentsCaptureMachine::DidCopyFromBackingStore, | |
| 673 weak_ptr_factory_.GetWeakPtr(), | |
| 674 start_time, target, deliver_frame_cb)); | |
| 675 } | |
| 676 } | |
| 677 | |
| 678 bool WebContentsCaptureMachine::StartObservingWebContents() { | |
| 679 // Look-up the RenderViewHost and, from that, the WebContents that wraps it. | |
| 680 // If successful, begin observing the WebContents instance. | |
| 681 // | |
| 682 // Why this can be unsuccessful: The request for mirroring originates in a | |
| 683 // render process, and this request is based on the current RenderView | |
| 684 // associated with a tab. However, by the time we get up-and-running here, | |
| 685 // there have been multiple back-and-forth IPCs between processes, as well as | |
| 686 // a bit of indirection across threads. It's easily possible that, in the | |
| 687 // meantime, the original RenderView may have gone away. | |
| 688 RenderViewHost* const rvh = | |
| 689 RenderViewHost::FromID(initial_render_process_id_, | |
| 690 initial_render_view_id_); | |
| 691 DVLOG_IF(1, !rvh) << "RenderViewHost::FromID(" | |
| 692 << initial_render_process_id_ << ", " | |
| 693 << initial_render_view_id_ << ") returned NULL."; | |
| 694 Observe(rvh ? WebContents::FromRenderViewHost(rvh) : NULL); | |
| 695 | |
| 696 WebContentsImpl* contents = static_cast<WebContentsImpl*>(web_contents()); | |
| 697 if (contents) { | |
| 698 contents->IncrementCapturerCount(oracle_proxy_->GetCaptureSize()); | |
| 699 fullscreen_widget_id_ = contents->GetFullscreenWidgetRoutingID(); | |
| 700 RenewFrameSubscription(); | |
| 701 return true; | |
| 702 } | |
| 703 | |
| 704 DVLOG(1) << "WebContents::FromRenderViewHost(" << rvh << ") returned NULL."; | |
| 705 return false; | |
| 706 } | |
| 707 | |
| 708 void WebContentsCaptureMachine::WebContentsDestroyed( | |
| 709 WebContents* web_contents) { | |
| 710 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
| 711 | |
| 712 subscription_.reset(); | |
| 713 web_contents->DecrementCapturerCount(); | |
| 714 oracle_proxy_->ReportError("WebContentsDestroyed()"); | |
| 715 } | |
| 716 | |
| 717 RenderWidgetHost* WebContentsCaptureMachine::GetTarget() { | |
| 718 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
| 719 if (!web_contents()) | |
| 720 return NULL; | |
| 721 | |
| 722 RenderWidgetHost* rwh = NULL; | |
| 723 if (fullscreen_widget_id_ != MSG_ROUTING_NONE) { | |
| 724 RenderProcessHost* process = web_contents()->GetRenderProcessHost(); | |
| 725 if (process) | |
| 726 rwh = RenderWidgetHost::FromID(process->GetID(), fullscreen_widget_id_); | |
| 727 } else { | |
| 728 rwh = web_contents()->GetRenderViewHost(); | |
| 729 } | |
| 730 | |
| 731 return rwh; | |
| 732 } | |
| 733 | |
| 734 void WebContentsCaptureMachine::DidCopyFromBackingStore( | |
| 735 const base::TimeTicks& start_time, | |
| 736 const scoped_refptr<media::VideoFrame>& target, | |
| 737 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | |
| 738 deliver_frame_cb, | |
| 739 bool success, | |
| 740 const SkBitmap& bitmap) { | |
| 741 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
| 742 | |
| 743 base::TimeTicks now = base::TimeTicks::Now(); | |
| 744 DCHECK(render_thread_.get()); | |
| 745 if (success) { | |
| 746 UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeBitmap", now - start_time); | |
| 747 TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", target.get(), | |
| 748 "Render"); | |
| 749 render_thread_->message_loop_proxy()->PostTask(FROM_HERE, base::Bind( | |
| 750 &RenderVideoFrame, bitmap, target, | |
| 751 base::Bind(deliver_frame_cb, start_time))); | |
| 752 } else { | |
| 753 // Capture can fail due to transient issues, so just skip this frame. | |
| 754 DVLOG(1) << "CopyFromBackingStore failed; skipping frame."; | |
| 755 deliver_frame_cb.Run(start_time, false); | |
| 756 } | |
| 757 } | |
| 758 | |
| 759 void WebContentsCaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame( | |
| 760 const base::TimeTicks& start_time, | |
| 761 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback& | |
| 762 deliver_frame_cb, | |
| 763 bool success) { | |
| 764 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
| 765 base::TimeTicks now = base::TimeTicks::Now(); | |
| 766 | |
| 767 if (success) { | |
| 768 UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeVideoFrame", now - start_time); | |
| 769 } else { | |
| 770 // Capture can fail due to transient issues, so just skip this frame. | |
| 771 DVLOG(1) << "CopyFromCompositingSurface failed; skipping frame."; | |
| 772 } | |
| 773 deliver_frame_cb.Run(start_time, success); | |
| 774 } | |
| 775 | |
| 776 void WebContentsCaptureMachine::RenewFrameSubscription() { | |
| 777 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
| 778 | |
| 779 // Always destroy the old subscription before creating a new one. | |
| 780 subscription_.reset(); | |
| 781 | |
| 782 RenderWidgetHost* rwh = GetTarget(); | |
| 783 if (!rwh || !rwh->GetView()) | |
| 784 return; | |
| 785 | |
| 786 subscription_.reset(new ContentCaptureSubscription(*rwh, oracle_proxy_, | |
| 787 base::Bind(&WebContentsCaptureMachine::Capture, | |
| 788 weak_ptr_factory_.GetWeakPtr()))); | |
| 789 } | |
| 790 | |
| 791 } // namespace | |
| 792 | |
| 793 WebContentsVideoCaptureDevice::WebContentsVideoCaptureDevice( | |
| 794 int render_process_id, int render_view_id) | |
| 795 : core_(new ContentVideoCaptureDeviceCore(scoped_ptr<VideoCaptureMachine>( | |
| 796 new WebContentsCaptureMachine(render_process_id, render_view_id)))) {} | |
| 797 | |
| 798 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() { | |
| 799 DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying."; | |
| 800 } | |
| 801 | |
| 802 // static | |
| 803 media::VideoCaptureDevice* WebContentsVideoCaptureDevice::Create( | |
| 804 const std::string& device_id) { | |
| 805 // Parse device_id into render_process_id and render_view_id. | |
| 806 int render_process_id = -1; | |
| 807 int render_view_id = -1; | |
| 808 if (!WebContentsCaptureUtil::ExtractTabCaptureTarget( | |
| 809 device_id, &render_process_id, &render_view_id)) { | |
| 810 return NULL; | |
| 811 } | |
| 812 | |
| 813 return new WebContentsVideoCaptureDevice(render_process_id, render_view_id); | |
| 814 } | |
| 815 | |
| 816 void WebContentsVideoCaptureDevice::AllocateAndStart( | |
| 817 const media::VideoCaptureParams& params, | |
| 818 scoped_ptr<Client> client) { | |
| 819 DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString(); | |
| 820 core_->AllocateAndStart(params, client.Pass()); | |
| 821 } | |
| 822 | |
| 823 void WebContentsVideoCaptureDevice::StopAndDeAllocate() { | |
| 824 core_->StopAndDeAllocate(); | |
| 825 } | |
| 826 | |
| 827 } // namespace content | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 172003004:
Fixit: Move tab and desktop capture code to new location. (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/src