Interface with webrtc through encoded frames

remoting/protocol/webrtc_frame_scheduler.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "remoting/protocol/webrtc_frame_scheduler.h"
+
+#include "base/logging.h"
+#include "remoting/proto/video.pb.h"
+
+namespace remoting {
+namespace protocol {
+
+// The frame scheduler currently uses a simple polling technique
+// at 30 FPS to capture, encode and send frames over webrtc transport.
+// An improved solution will use target bitrate feedback to pace out
+// the capture rate.
+WebRtcFrameScheduler::WebRtcFrameScheduler(
+    scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner,
+    scoped_ptr<webrtc::DesktopCapturer> capturer,
+    WebrtcTransport* webrtc_transport,
+    scoped_ptr<VideoEncoder> encoder)
+    : key_frame_request_(false),

[Sergey Ulanov, 2016/03/31 22:06:19]

initialize boolean values in the header please.

[Irfan, 2016/04/05 21:23:27]

Done.

+      capture_pending_(false),
+      encode_pending_(false),
+      encode_task_runner_(encode_task_runner),
+      capturer_(std::move(capturer)),
+      webrtc_transport_(webrtc_transport),
+      encoder_(std::move(encoder)),
+      weak_factory_(this) {
+  DCHECK(encode_task_runner_);
+  DCHECK(capturer_);
+  DCHECK(webrtc_transport_);
+  DCHECK(encoder_);
+  // Does not really start anything. Registers callback on this class.
+  capturer_->Start(this);
+  capture_timer_.reset(new base::RepeatingTimer());
+
+  // Register for PLI requests.
+  webrtc_transport_->video_encoder_factory()->SetKeyFrameRequestCallback(
+      base::Bind(&WebRtcFrameScheduler::SetKeyFrameRequest,
+                 base::Unretained(this)));
+}
+
+WebRtcFrameScheduler::~WebRtcFrameScheduler() {
+  encode_task_runner_->DeleteSoon(FROM_HERE, encoder_.release());
+}
+
+void WebRtcFrameScheduler::Start() {
+  capture_timer_->Start(FROM_HERE, base::TimeDelta::FromSeconds(1) / 30, this,
+                        &WebRtcFrameScheduler::CaptureNextFrame);
+}
+
+void WebRtcFrameScheduler::Stop() {
+  // Cancel any pending encode.
+  task_tracker_.TryCancelAll();
+  capture_timer_->Stop();
+}
+
+void WebRtcFrameScheduler::SetSizeCallback(const SizeCallback& callback) {
+  size_callback_ = callback;
+}
+
+void WebRtcFrameScheduler::SetKeyFrameRequest() {
+  VLOG(1) << "Request key frame";
+  base::AutoLock lock(lock_);
+  key_frame_request_ = true;
+}
+
+bool WebRtcFrameScheduler::ClearAndGetKeyFrameRequest() {
+  base::AutoLock lock(lock_);
+  bool key_frame_request = key_frame_request_;
+  key_frame_request_ = false;
+  return key_frame_request;
+}
+
+webrtc::SharedMemory* WebRtcFrameScheduler::CreateSharedMemory(size_t size) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  return nullptr;
+}
+
+void WebRtcFrameScheduler::OnCaptureCompleted(webrtc::DesktopFrame* frame) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  VLOG(1) << "Capture overhead "
+          << (base::TimeTicks::Now() - last_capture_ticks_).InMilliseconds();
+  capture_pending_ = false;
+
+  scoped_ptr<webrtc::DesktopFrame> owned_frame(frame);
+
+  if (encode_pending_) {
+    // TODO(isheriff): consider queuing here
+    VLOG(1) << "Dropping captured frame since encoder is still busy";
+    return;
+  }
+  if (!frame || frame->updated_region().is_empty())
+    return;
+
+  if (!frame_size_.equals(frame->size())) {
+    frame_size_ = frame->size();
+    size_callback_.Run(frame_size_);
+  }
+  encode_pending_ = true;
+  task_tracker_.PostTaskAndReply(
+      encode_task_runner_.get(), FROM_HERE,
+      base::Bind(&WebRtcFrameScheduler::EncodeFrame, base::Unretained(this),

[Sergey Ulanov, 2016/03/31 22:06:19]

I don't think base::Unretained() is safe here. The scheduler may be deleted
before EncodeFrame() finished executing. As far as I can tell
CancellableTaskTracker doesn't help here (it cannot cancel the task once it
started running). You can make is safe by making EncodeFrame() static, similar
to VideoFramePump.

[Irfan, 2016/04/05 21:23:27]

Done.

+                 base::Passed(std::move(owned_frame))),
+      base::Bind(&WebRtcFrameScheduler::OnFrameEncoded,
+                 weak_factory_.GetWeakPtr()));
+}
+
+void WebRtcFrameScheduler::CaptureNextFrame() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  if (capture_pending_ || encode_pending_) {
+    VLOG(1) << "Capture/encode still pending..";
+    return;
+  }
+  capture_pending_ = true;
+  VLOG(1) << "Capture next frame after "
+          << (base::TimeTicks::Now() - last_capture_ticks_).InMilliseconds();
+  last_capture_ticks_ = base::TimeTicks::Now();
+  capturer_->Capture(webrtc::DesktopRegion());
+}
+
+void WebRtcFrameScheduler::EncodeFrame(scoped_ptr<webrtc::DesktopFrame> frame) {
+  base::TimeTicks current = base::TimeTicks::Now();
+  uint32_t flags = 0;
+  if (ClearAndGetKeyFrameRequest())
+    flags |= VideoEncoder::kRequestKeyFrame;
+
+  scoped_ptr<VideoPacket> packet = encoder_->Encode(*frame, flags);
+  if (!packet) {

[Sergey Ulanov, 2016/03/31 22:06:19]

nit: remove brackets for consistency with other single-line if statements in
this file

[Irfan, 2016/04/05 21:23:27]

Done.

+    return;
+  }
+  VLOG(1) << "Encode duration "
+          << (base::TimeTicks::Now() - current).InMilliseconds()
+          << " payload size " << packet->data().size();
+  int64_t capture_timestamp_ms =
+      (base::TimeTicks::Now() - base::TimeTicks()).InMilliseconds();
+  current = base::TimeTicks::Now();
+
+  // TODO(isheriff): Investigate why first frame fails to send at times.
+  // This gets resolved through a PLI request.
+  webrtc_transport_->video_encoder_factory()->SendEncodedFrame(
+      capture_timestamp_ms,
+      reinterpret_cast<uint8_t*>(const_cast<char*>(packet->data().data())),
+      packet->data().size(), packet->format().screen_width(),
+      packet->format().screen_height(), packet->key_frame());
+  VLOG(1) << "Send duration "
+          << (base::TimeTicks::Now() - current).InMilliseconds();
+}
+
+void WebRtcFrameScheduler::OnFrameEncoded() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  encode_pending_ = false;
+}
+
+}  // namespace protocol
+}  // namespace remoting