Propagate MediaStreamTrack video content hints.

content/renderer/media/webrtc/media_stream_video_webrtc_sink.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/webrtc/media_stream_video_webrtc_sink.h"
 
 #include "base/location.h"
 #include "base/numerics/safe_conversions.h"
 #include "base/single_thread_task_runner.h"
 #include "base/strings/utf_string_conversions.h"
@@ skipping 41 matching lines @@
 
 // The default number of microseconds that should elapse since the last video
 // frame was received, before requesting a refresh frame.
 const int64_t kDefaultRefreshIntervalMicros =
     base::Time::kMicrosecondsPerSecond;
 
 // A lower-bound for the refresh interval.
 const int64_t kLowerBoundRefreshIntervalMicros =
     base::Time::kMicrosecondsPerSecond / media::limits::kMaxFramesPerSecond;
 
+webrtc::VideoTrackInterface::ContentHint TranslateContentHint(
+    blink::WebMediaStreamTrack::ContentHintType content_hint) {
+  switch (content_hint) {
+    case blink::WebMediaStreamTrack::ContentHintType::None:
+      return webrtc::VideoTrackInterface::ContentHint::kNone;
+    case blink::WebMediaStreamTrack::ContentHintType::AudioSpeech:
+    case blink::WebMediaStreamTrack::ContentHintType::AudioMusic:
+      NOTREACHED();

[mcasas, 2016/12/19 19:32:05]

Why are these hints causing a NOTREACHED() ?
IOW why are they a bad thing?

[pbos, 2016/12/19 19:42:09]

They should never be settable on a video track. This is filtered out in blink's
javascript layer, so audio hints should never be set on a video track.

+      break;
+    case blink::WebMediaStreamTrack::ContentHintType::VideoFluid:
+      return webrtc::VideoTrackInterface::ContentHint::kFluid;
+    case blink::WebMediaStreamTrack::ContentHintType::VideoDetailed:
+      return webrtc::VideoTrackInterface::ContentHint::kDetailed;
+  }
+  NOTREACHED();
+  return webrtc::VideoTrackInterface::ContentHint::kNone;
+}
+
 }  // namespace
 
 // Simple help class used for receiving video frames on the IO-thread from a
 // MediaStreamVideoTrack and forward the frames to a WebRtcVideoCapturerAdapter
 // on libjingle's worker thread. WebRtcVideoCapturerAdapter implements a video
 // capturer for libjingle.
 class MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter
     : public base::RefCountedThreadSafe<WebRtcVideoSourceAdapter> {
  public:
   WebRtcVideoSourceAdapter(const scoped_refptr<base::SingleThreadTaskRunner>&
                                libjingle_worker_thread,
                            const scoped_refptr<WebRtcVideoSource>& source,
                            base::TimeDelta refresh_interval,
                            const base::Closure& refresh_callback);
 
   // MediaStreamVideoWebRtcSink can be destroyed on the main render thread or
   // libjingles worker thread since it posts video frames on that thread. But
   // |video_source_| must be released on the main render thread before the
   // PeerConnectionFactory has been destroyed. The only way to ensure that is to
   // make sure |video_source_| is released when MediaStreamVideoWebRtcSink() is
   // destroyed.
   void ReleaseSourceOnMainThread();
 
+  void SetContentHint(blink::WebMediaStreamTrack::ContentHintType content_hint);
+
   void OnVideoFrameOnIO(const scoped_refptr<media::VideoFrame>& frame,
                         base::TimeTicks estimated_capture_time);
 
  private:
+  friend class base::RefCountedThreadSafe<WebRtcVideoSourceAdapter>;
+
   void OnVideoFrameOnWorkerThread(
       const scoped_refptr<media::VideoFrame>& frame);
-  friend class base::RefCountedThreadSafe<WebRtcVideoSourceAdapter>;
+
+  void SetContentHintOnWorkerThread(
+      blink::WebMediaStreamTrack::ContentHintType content_hint);
+
   virtual ~WebRtcVideoSourceAdapter();
 
   // Called whenever a video frame was just delivered on the IO thread. This
   // restarts the delay period before the |refresh_timer_| will fire the next
   // time.
   void ResetRefreshTimerOnMainThread();
 
   scoped_refptr<base::SingleThreadTaskRunner> render_thread_task_runner_;
 
   // |render_thread_checker_| is bound to the main render thread.
@@ skipping 52 matching lines @@
   DVLOG(3) << "~WebRtcVideoSourceAdapter()";
   DCHECK(!capture_adapter_);
   // This object can be destroyed on the main render thread or libjingles worker
   // thread since it posts video frames on that thread. But |video_source_| must
   // be released on the main render thread before the PeerConnectionFactory has
   // been destroyed. The only way to ensure that is to make sure |video_source_|
   // is released when MediaStreamVideoWebRtcSink() is destroyed.
 }
 
 void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
-ResetRefreshTimerOnMainThread() {
+    ResetRefreshTimerOnMainThread() {
   DCHECK(render_thread_checker_.CalledOnValidThread());
   if (refresh_timer_.IsRunning())
     refresh_timer_.Reset();
 }
 
 void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
-ReleaseSourceOnMainThread() {
+    ReleaseSourceOnMainThread() {
   DCHECK(render_thread_checker_.CalledOnValidThread());
   // Since frames are posted to the worker thread, this object might be deleted
   // on that thread. However, since |video_source_| was created on the render
   // thread, it should be released on the render thread.
   base::AutoLock auto_lock(capture_adapter_stop_lock_);
   // |video_source| owns |capture_adapter_|.
   capture_adapter_ = NULL;
   video_source_ = NULL;
 }
 
+void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::SetContentHint(
+    blink::WebMediaStreamTrack::ContentHintType content_hint) {
+  DCHECK(render_thread_checker_.CalledOnValidThread());
+  libjingle_worker_thread_->PostTask(
+      FROM_HERE,
+      base::Bind(&WebRtcVideoSourceAdapter::SetContentHintOnWorkerThread, this,
+                 content_hint));
+}
+
+void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
+    SetContentHintOnWorkerThread(
+        blink::WebMediaStreamTrack::ContentHintType content_hint) {
+  DCHECK(libjingle_worker_thread_->BelongsToCurrentThread());
+  base::AutoLock auto_lock(capture_adapter_stop_lock_);
+  if (capture_adapter_)
+    capture_adapter_->SetContentHint(content_hint);
+}
+
 void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::OnVideoFrameOnIO(
     const scoped_refptr<media::VideoFrame>& frame,
     base::TimeTicks estimated_capture_time) {
   DCHECK(io_thread_checker_.CalledOnValidThread());
   render_thread_task_runner_->PostTask(
       FROM_HERE,
       base::Bind(&WebRtcVideoSourceAdapter::ResetRefreshTimerOnMainThread,
                  this));
   libjingle_worker_thread_->PostTask(
       FROM_HERE,
@@ skipping 68 matching lines @@
     if (refresh_interval.InMicroseconds() < kLowerBoundRefreshIntervalMicros) {
       refresh_interval =
           base::TimeDelta::FromMicroseconds(kLowerBoundRefreshIntervalMicros);
     }
   }
 
   // TODO(pbos): Consolidate WebRtcVideoCapturerAdapter into WebRtcVideoSource
   // by removing the need for and dependency on a cricket::VideoCapturer.
   video_source_ = scoped_refptr<WebRtcVideoSource>(
       new rtc::RefCountedObject<WebRtcVideoSource>(
-          new WebRtcVideoCapturerAdapter(is_screencast), is_screencast,
-          needs_denoising));
+          new WebRtcVideoCapturerAdapter(is_screencast, track.contentHint()),
+          is_screencast, needs_denoising));
 
   // TODO(pbos): Consolidate the local video track with the source proxy and
   // move into PeerConnectionDependencyFactory. This now separately holds on a
   // reference to the proxy object because
   // PeerConnectionFactory::CreateVideoTrack doesn't do reference counting.
   video_source_proxy_ =
       factory->CreateVideoTrackSourceProxy(video_source_.get());
   video_track_ = factory->CreateLocalVideoTrack(track.id().utf8(),
                                                 video_source_proxy_.get());
 
+  video_track_->set_content_hint(TranslateContentHint(track.contentHint()));

[mcasas, 2016/12/19 19:32:05]

Isn't this repeating what's done in l.313?

[pbos, 2016/12/19 19:42:09]

Nope, l.313 is for the adapter.

   video_track_->set_enabled(track.isEnabled());
 
   source_adapter_ = new WebRtcVideoSourceAdapter(
       factory->GetWebRtcWorkerThread(), video_source_.get(), refresh_interval,
       base::Bind(&MediaStreamVideoWebRtcSink::RequestRefreshFrame,
                  weak_factory_.GetWeakPtr()));
 
   MediaStreamVideoSink::ConnectToTrack(
       track,
       base::Bind(&WebRtcVideoSourceAdapter::OnVideoFrameOnIO, source_adapter_),
       false);
 
   DVLOG(3) << "MediaStreamVideoWebRtcSink ctor() : is_screencast "
            << is_screencast;
 }
 
 MediaStreamVideoWebRtcSink::~MediaStreamVideoWebRtcSink() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DVLOG(3) << "MediaStreamVideoWebRtcSink dtor().";
   weak_factory_.InvalidateWeakPtrs();
   MediaStreamVideoSink::DisconnectFromTrack();
   source_adapter_->ReleaseSourceOnMainThread();
 }
 
 void MediaStreamVideoWebRtcSink::OnEnabledChanged(bool enabled) {
   DCHECK(thread_checker_.CalledOnValidThread());
   video_track_->set_enabled(enabled);
 }
 
+void MediaStreamVideoWebRtcSink::OnContentHintChanged(
+    blink::WebMediaStreamTrack::ContentHintType content_hint) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  source_adapter_->SetContentHint(content_hint);

[mcasas, 2016/12/19 19:32:05]

IIUC,WebRtcVideoCapturerAdapter::SetContentHint() checks
that the update happens in jingle's worker thread, correct?
Where are we here? (if this is the worker thread, can we
s/thread_checker_/worker_thread_checker_/ ?

[pbos, 2016/12/19 19:42:09]

We are on the renderer (signalling) thread here. SetContentHint does a thread
hop to SetContentHintOnWorker.

+  video_track_->set_content_hint(TranslateContentHint(content_hint));
+}
+
 void MediaStreamVideoWebRtcSink::RequestRefreshFrame() {
   DCHECK(thread_checker_.CalledOnValidThread());
   content::RequestRefreshFrameFromVideoTrack(connected_track());
 }
 
 rtc::Optional<bool> MediaStreamVideoWebRtcSink::SourceNeedsDenoisingForTesting()
     const {
   return video_source_->needs_denoising();
 }
 
 }  // namespace content