All together now

content/browser/renderer_host/media/desktop_capture_device_aura.cc

+// Copyright (c) 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/browser/renderer_host/media/desktop_capture_device_aura.h"
+
+#include <map>
+
+#include "base/bind.h"
+#include "base/debug/trace_event.h"
+#include "base/message_loop/message_loop_proxy.h"
+#include "base/synchronization/lock.h"
+#include "content/browser/aura/image_transport_factory.h"
+#include "content/browser/renderer_host/media/video_capture_oracle.h"
+#include "content/common/gpu/surface_capturer.h"
+#include "media/base/bitstream_buffer.h"
+#include "media/base/video_decoder_config.h"
+#include "ui/aura/root_window.h"
+#include "ui/compositor/compositor.h"
+#include "ui/compositor/compositor_retriever_delegate.h"
+#include "ui/gfx/screen.h"
+#include "ui/gfx/size.h"
+
+namespace content {
+
+namespace {
+
+const int kMinFrameWidth = 2;
+const int kMinFrameHeight = 2;
+
+const int kDefaultBitrate = 64;
+
+// Returns the nearest even integer closer to zero.
+template<typename IntType>
+IntType MakeEven(IntType x) {
+  return x & static_cast<IntType>(-2);
+}
+
+}  // anonymous namespace
+
+// The implementation class that actually handles screen capture.  The capture
+// callback for the output surface is bound with a scoped_refptr to this object
+// to allow for thread-safe destruction.
+//
+// This class uses manual locking for thread-safety since it needs to be able
+// to be used from thwo different threads:
+// * The media manager thread (Allocate(), Start(), Stop(), DeAllocate())
+// * The output surface thread (DoFrameCapture())
+
+class DesktopCaptureDeviceAura::Impl
+    : public SurfaceCapturer::Client,
+      public ui::CompositorObserver,
+      public base::RefCountedThreadSafe<Impl> {
+ public:
+  Impl(const base::WeakPtr<DesktopCaptureDeviceAura>& weak_client,
+       const scoped_refptr<base::MessageLoopProxy>& client_loop,
+       scoped_ptr<VideoCaptureOracle> oracle,
+       media::VideoCaptureDevice::EventHandler* consumer,
+       const gfx::Size& requested_size,
+       int requested_frame_rate);
+  virtual ~Impl();
+
+  void CreateCapturerOnUIThread();
+
+  void Start();
+  void Stop();
+  void Destroy();
+
+  // Implements SurfaceCapturer::Client.
+  virtual void NotifyCaptureParameters(const gfx::Size& buffer_size,
+                                       const gfx::Rect& visible_rect) OVERRIDE;
+  virtual void NotifyCopyCaptureDone(
+      const scoped_refptr<media::VideoFrame>& frame) OVERRIDE;
+  virtual void NotifyError(SurfaceCapturer::Error error) OVERRIDE;
+
+  // Implements ui::CompositorObserver.
+  virtual void OnCompositingDidCommit(ui::Compositor* compositor) OVERRIDE {}
+  virtual void OnCompositingStarted(ui::Compositor* compositor,
+                                    base::TimeTicks start_time) OVERRIDE {}
+  virtual void OnCompositingEnded(ui::Compositor* compositor) OVERRIDE;
+  virtual void OnCompositingAborted(ui::Compositor* compositor) OVERRIDE {}
+  virtual void OnCompositingLockStateChanged(
+      ui::Compositor* compositor) OVERRIDE {}
+  virtual void OnUpdateVSyncParameters(ui::Compositor* compositor,
+                                       base::TimeTicks timebase,
+                                       base::TimeDelta interval) OVERRIDE {}
+ private:
+  static const size_t kMaxPendingFrames = 2;
+
+  void ReportError();
+  void DoFrameCapture(bool is_compositor_update);
+
+  // The timer should be started and stopped on UI thread because this is the
+  // thread in which the surface capturer is initialized.
+  void StartCaptureOnUIThread();
+  void StopCaptureOnUIThread();
+
+  static void DestroyCapturerOnUIThread(scoped_ptr<SurfaceCapturer> capturer);
+
+  const base::ThreadChecker thread_checker_;
+
+  const base::WeakPtr<DesktopCaptureDeviceAura> weak_client_;
+  const scoped_refptr<base::MessageLoopProxy> client_loop_;
+  const gfx::Size requested_size_;
+  const int requested_frame_rate_;
+  base::RepeatingTimer<DesktopCaptureDeviceAura::Impl> timer_;
+
+  // This lock protects everything under it.
+  base::Lock lock_;
+  scoped_ptr<VideoCaptureOracle> oracle_;
+  scoped_ptr<SurfaceCapturer> capturer_;
+  media::VideoCaptureDevice::EventHandler* consumer_;
+  ui::Compositor* compositor_;
+  bool is_started_;
+
+  // Map from media::VideoFrame* to frame number.
+  typedef std::map<media::VideoFrame*, int> FrameNumberMap;
+  FrameNumberMap pending_frames_;
+};
+
+DesktopCaptureDeviceAura::Impl::Impl(
+    const base::WeakPtr<DesktopCaptureDeviceAura>& weak_client,
+    const scoped_refptr<base::MessageLoopProxy>& client_loop,
+    scoped_ptr<VideoCaptureOracle> oracle,
+    media::VideoCaptureDevice::EventHandler* consumer,
+    const gfx::Size& requested_size,
+    int requested_frame_rate)
+    : thread_checker_(),
+      weak_client_(weak_client),
+      client_loop_(client_loop),
+      requested_size_(requested_size),
+      requested_frame_rate_(requested_frame_rate),
+      oracle_(oracle.Pass()),
+      consumer_(consumer),
+      compositor_(NULL),
+      is_started_(false) {
+}
+
+DesktopCaptureDeviceAura::Impl::~Impl() {
+  DCHECK(pending_frames_.empty());
+}
+
+void DesktopCaptureDeviceAura::Impl::CreateCapturerOnUIThread() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+  base::AutoLock guard(lock_);
+  DCHECK(!capturer_);
+
+  if (!oracle_ || !consumer_) {
+    // We were destroyed sometime in the interim.
+    return;
+  }
+
+  ui::CompositorRetrieverDelegate* delegate =
+      ui::CompositorRetrieverDelegate::GetInstance();
+  if (!delegate) {
+    NOTREACHED();
+    return;
+  }
+
+  // TODO(scottmg): Native is wrong http://crbug.com/133312.
+  // TODO(hshi): support multiple displays. The display identifier should be
+  // passed to DesktopCaptureDeviceAura::Create in the DesktopMediaID.
+  const gfx::Display& display =
+      gfx::Screen::GetNativeScreen()->GetPrimaryDisplay();
+
+  ui::Compositor* compositor = delegate->GetCompositorForDisplay(display);
+  if (!compositor) {
+    ReportError();
+    return;
+  }
+
+  SurfaceCapturingContextFactory* factory =
+      ImageTransportFactory::GetInstance()->AsSurfaceCapturingContextFactory();
+  if (!factory) {
+    ReportError();
+    return;
+  }
+
+  scoped_ptr<SurfaceCapturer> capturer =
+      factory->CreateOutputSurfaceCapturer(compositor, this);
+  if (!capturer) {
+    ReportError();
+    return;
+  }
+
+  capturer->Initialize(media::VideoFrame::I420);
+  capturer_ = capturer.Pass();
+  compositor_ = compositor;
+}
+
+void DesktopCaptureDeviceAura::Impl::StartCaptureOnUIThread() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+  const base::TimeDelta capture_period = base::TimeDelta::FromMicroseconds(
+      1000000.0 / requested_frame_rate_ + 0.5);
+  timer_.Start(
+      FROM_HERE, capture_period,
+      base::Bind(&DesktopCaptureDeviceAura::Impl::DoFrameCapture, this, false));
+
+  DCHECK(compositor_);
+  compositor_->AddObserver(this);
+}
+
+void DesktopCaptureDeviceAura::Impl::StopCaptureOnUIThread() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+  timer_.Stop();
+
+  DCHECK(compositor_);
+  compositor_->RemoveObserver(this);
+}
+
+void DesktopCaptureDeviceAura::Impl::Start() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  base::AutoLock guard(lock_);
+  is_started_ = true;
+  BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
+      &DesktopCaptureDeviceAura::Impl::StartCaptureOnUIThread, this));
+}
+
+void DesktopCaptureDeviceAura::Impl::Stop() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  base::AutoLock guard(lock_);
+  is_started_ = false;
+  BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
+      &DesktopCaptureDeviceAura::Impl::StopCaptureOnUIThread, this));
+}
+
+void DesktopCaptureDeviceAura::Impl::Destroy() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  base::AutoLock guard(lock_);
+  if (oracle_)
+    oracle_.reset();
+  if (capturer_) {
+    BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
+        &DesktopCaptureDeviceAura::Impl::DestroyCapturerOnUIThread,
+        base::Passed(&capturer_)));
+  }
+  consumer_ = NULL;
+  is_started_ = false;
+}
+
+void DesktopCaptureDeviceAura::Impl::DoFrameCapture(bool is_compositor_update) {
+  base::AutoLock guard(lock_);
+  if (!oracle_ || !capturer_ || !consumer_)
+    return;
+
+  if (!is_started_)
+    return;
+
+  if (pending_frames_.size() >= kMaxPendingFrames)
+    return;
+
+  bool has_frame = true;
+  scoped_refptr<media::VideoFrame> frame = consumer_->ReserveOutputBuffer();
+  if (!frame) {
+    has_frame = false;
+  }
+
+  const base::Time now = base::Time::Now();
+  const VideoCaptureOracle::Event event =
+      is_compositor_update ? VideoCaptureOracle::kCompositorUpdate
+                           : VideoCaptureOracle::kTimerPoll;
+  const bool should_capture = oracle_->ObserveEventAndDecideCapture(event, now);
+
+  // Consider the various reasons not to initiate a capture.
+  if (should_capture && !has_frame) {
+    TRACE_EVENT_INSTANT1("mirroring", "EncodeLimited",
+                         TRACE_EVENT_SCOPE_THREAD,
+                         "trigger", "gpu");
+    return;
+  } else if (!should_capture && has_frame) {
+    // This is a normal and acceptable way to drop a frame. We've hit our
+    // capture rate limit: for example, the content is animating at 60fps but
+    // we're capturing at 30fps.
+    TRACE_EVENT_INSTANT1("mirroring", "FpsRateLimited",
+                         TRACE_EVENT_SCOPE_THREAD,
+                         "trigger", "gpu");
+    return;
+  } else if (!should_capture && !has_frame) {
+    // We decided not to capture, but we wouldn't have been able to if we wanted
+    // to because no output buffer was available.
+    TRACE_EVENT_INSTANT1("mirroring", "NearlyEncodeLimited",
+                         TRACE_EVENT_SCOPE_THREAD,
+                         "trigger", "gpu");
+    return;
+  }
+
+  // Enqueue a video frame for capture output.
+  DCHECK(pending_frames_.find(frame.get()) == pending_frames_.end());
+  pending_frames_[frame.get()] = oracle_->RecordCapture();
+  capturer_->CopyCaptureToVideoFrame(frame);
+}
+
+void DesktopCaptureDeviceAura::Impl::ReportError() {
+  LOG(ERROR) << "DesktopCaptureDeviceAura::ReportError()";
+  client_loop_->PostTask(FROM_HERE, base::Bind(
+      &DesktopCaptureDeviceAura::ReportError,
+      weak_client_));
+}
+
+void DesktopCaptureDeviceAura::Impl::NotifyCaptureParameters(
+    const gfx::Size& buffer_size, const gfx::Rect& visible_rect) {
+  base::AutoLock guard(lock_);
+  if (!consumer_)
+    return;
+
+  // Initialize capture settings which will be consistent for the duration of
+  // the capture.
+  media::VideoCaptureCapability settings;
+
+  // We'll ignore requested_size_ and just use the size as reported by the
+  // SurfaceCapturer.
+  settings.width = buffer_size.width();
+  settings.height = buffer_size.height();
+  settings.frame_rate = requested_frame_rate_;
+  // Note: the value of |settings.color| doesn't matter if we use only the
+  // VideoFrame based methods on |consumer|.
+  settings.color = media::VideoCaptureCapability::kI420;
+  settings.expected_capture_delay = 0;
+  settings.interlaced = false;
+
+  consumer_->OnFrameInfo(settings);
+}
+
+void DesktopCaptureDeviceAura::Impl::NotifyCopyCaptureDone(
+    const scoped_refptr<media::VideoFrame>& frame) {
+  if (!oracle_ || !capturer_ || !consumer_)
+    return;
+
+  FrameNumberMap::const_iterator it = pending_frames_.find(frame.get());
+  if (it == pending_frames_.end()) {
+    NOTREACHED();
+    return;
+  }
+
+  // Notify oracle that capture is complete.
+  int frame_number = it->second;
+  const base::Time now = base::Time::Now();
+  if (oracle_->CompleteCapture(frame_number, now)) {
+    // Deliver the captured frame to the consumer.
+    consumer_->OnIncomingCapturedVideoFrame(frame, now);
+  }
+  pending_frames_.erase(frame.get());
+}
+
+void DesktopCaptureDeviceAura::Impl::NotifyError(
+    SurfaceCapturer::Error error) {
+  ReportError();
+}
+
+void DesktopCaptureDeviceAura::Impl::OnCompositingEnded(
+    ui::Compositor* compositor) {
+  BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
+      &DesktopCaptureDeviceAura::Impl::DoFrameCapture, this, true));
+}
+
+// static
+void DesktopCaptureDeviceAura::Impl::DestroyCapturerOnUIThread(
+    scoped_ptr<SurfaceCapturer> capturer) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+  capturer.release()->Destroy();
+}
+
+DesktopCaptureDeviceAura::DesktopCaptureDeviceAura(const Name& device_name)
+    : thread_checker_(),
+      device_name_(device_name),
+      state_(kIdle),
+      weak_ptr_factory_(this) {
+}
+
+DesktopCaptureDeviceAura::~DesktopCaptureDeviceAura() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(!impl_);
+}
+
+// static
+media::VideoCaptureDevice* DesktopCaptureDeviceAura::Create(
+    const DesktopMediaID& source) {
+  media::VideoCaptureDevice::Name name("AuraScreen", source.ToString());
+  return new DesktopCaptureDeviceAura(name);
+}
+
+void DesktopCaptureDeviceAura::Allocate(
+    const media::VideoCaptureCapability& capture_format,
+    VideoCaptureDevice::EventHandler* consumer) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(!impl_);
+
+  if (state_ != kIdle) {
+    DVLOG(1) << "Allocate() invoked when not in state idle.";
+    return;
+  }
+
+  // Frame dimensions must each be a positive, even integer, since the consumer
+  // wants (or will convert to) YUV420.
+  int width = MakeEven(capture_format.width);
+  int height = MakeEven(capture_format.height);
+  if (width < kMinFrameWidth || height < kMinFrameHeight) {
+    DVLOG(1) << "invalid width (" << width << ") and/or height ("
+             << height << ")";
+    consumer->OnError();
+    return;
+  }
+
+  base::TimeDelta capture_period = base::TimeDelta::FromMicroseconds(
+      1000000.0 / capture_format.frame_rate + 0.5);
+
+  scoped_ptr<VideoCaptureOracle> oracle(new VideoCaptureOracle(
+      capture_period, true));
+
+  impl_ = new Impl(weak_ptr_factory_.GetWeakPtr(),
+                   base::MessageLoopProxy::current(),
+                   oracle.Pass(),
+                   consumer,
+                   gfx::Size(width, height),
+                   capture_format.frame_rate);
+
+  BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
+      &DesktopCaptureDeviceAura::Impl::CreateCapturerOnUIThread,
+      impl_));
+
+  state_ = kAllocated;
+}
+
+void DesktopCaptureDeviceAura::Start() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  if (state_ != kAllocated)
+    return;
+
+  impl_->Start();
+  state_ = kCapturing;
+}
+
+void DesktopCaptureDeviceAura::Stop() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  if (state_ != kCapturing)
+    return;
+
+  impl_->Stop();
+  state_ = kAllocated;
+}
+
+void DesktopCaptureDeviceAura::DeAllocate() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  Stop();
+
+  weak_ptr_factory_.InvalidateWeakPtrs();
+  if (impl_) {
+    impl_->Destroy();
+    impl_ = NULL;
+  }
+  state_ = kIdle;
+}
+
+const media::VideoCaptureDevice::Name&
+DesktopCaptureDeviceAura::device_name() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  return device_name_;
+}
+
+void DesktopCaptureDeviceAura::ReportError() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  Stop();
+  state_ = kError;
+}
+
+}  // namespace content