Change MediaStreamVideoSource to output different resolutions to different tracks depending on the …

content/renderer/media/video_track_adapter.cc

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/renderer/media/video_track_adapter.h"
+
+#include <algorithm>
+#include <limits>
+#include <utility>
+
+#include "base/bind.h"
+#include "base/debug/trace_event.h"
+#include "base/location.h"
+#include "media/base/video_util.h"
+
+namespace content {
+
+namespace {
+
+// Empty method used for keeping a reference to the original media::VideoFrame
+// in VideoFrameResolutionAdapter::DeliverFrame if cropping is needed.
+// The reference to |frame| is kept in the closure that calls this method.
+void ReleaseOriginalFrame(
+    const scoped_refptr<media::VideoFrame>& frame) {
+}
+
+void ResetCallbackOnMainRenderThread(
+    scoped_ptr<VideoCaptureDeliverFrameCB> callback) {
+    // |callback| will be deleted when this exits.

[tommi (sloooow) - chröme, 2014/05/14 09:43:38]

indent

[perkj_chrome, 2014/05/14 13:29:44]

Done.

+}
+
+}  // anonymous namespace
+
+// VideoFrameResolutionAdapter is created on and lives on
+// on the IO-thread. It does the resolution adaptation and delivers frames to
+// all registered tracks on the IO-thread.
+// All method calls must be on the IO-thread.
+class VideoTrackAdapter::VideoFrameResolutionAdapter
+    : public base::RefCountedThreadSafe<VideoFrameResolutionAdapter> {
+ public:
+  VideoFrameResolutionAdapter(
+      scoped_refptr<base::MessageLoopProxy> render_message_loop,
+      int max_width,
+      int max_height,
+      double min_aspect_ratio,
+      double max_aspect_ratio);
+
+  // Add |callback| to receive video frames on the IO-thread.
+  // |callback| will however be released on thee main render thread.
+  void AddCallback(const MediaStreamVideoTrack* track,
+                   const VideoCaptureDeliverFrameCB& callback);
+
+  // Removes |callback| associated with |track| from receiving video frames if
+  // |track| has been added. It is ok to call RemoveCallback even if the |track|
+  // has not been added. The |callback| is released on the main render thread.
+  void RemoveCallback(const MediaStreamVideoTrack* track);
+
+  void DeliverFrame(const scoped_refptr<media::VideoFrame>& frame,
+                    const media::VideoCaptureFormat& format);
+
+  // Returns true if all arguments match with the output of this adapter.
+  bool ConstraintsMatch(int max_width,
+                        int max_height,
+                        double min_aspect_ratio,
+                        double max_aspect_ratio) const;
+
+  bool IsEmpty() const;
+
+ private:
+  virtual ~VideoFrameResolutionAdapter();
+  friend class base::RefCountedThreadSafe<VideoFrameResolutionAdapter>;
+
+  virtual void DoDeliverFrame(
+      const scoped_refptr<media::VideoFrame>& frame,
+      const media::VideoCaptureFormat& format);
+
+  // Bound to the IO-thread.
+  base::ThreadChecker io_thread_checker_;
+
+  // The message loop where we will release VideoCaptureDeliverFrameCB
+  // registered in AddCallback.
+  scoped_refptr<base::MessageLoopProxy> render_message_loop_;
+
+  gfx::Size max_frame_size_;
+  double min_aspect_ratio_;
+  double max_aspect_ratio_;
+
+  typedef std::pair<const void*, VideoCaptureDeliverFrameCB>
+      VideoIdCallbackPair;
+  std::vector<VideoIdCallbackPair> callbacks_;
+
+  DISALLOW_COPY_AND_ASSIGN(VideoFrameResolutionAdapter);
+};
+
+VideoTrackAdapter::
+VideoFrameResolutionAdapter::VideoFrameResolutionAdapter(
+    scoped_refptr<base::MessageLoopProxy> render_message_loop,
+    int max_width,
+    int max_height,
+    double min_aspect_ratio,
+    double max_aspect_ratio)
+    : render_message_loop_(render_message_loop),
+      max_frame_size_(max_width, max_height),
+      min_aspect_ratio_(min_aspect_ratio),
+      max_aspect_ratio_(max_aspect_ratio) {
+  DCHECK(render_message_loop_);

[tommi (sloooow) - chröme, 2014/05/14 09:43:38]

w00t! dcheck party!
(it's a good thing)

[perkj_chrome, 2014/05/14 13:29:44]

Done.

+  DCHECK(io_thread_checker_.CalledOnValidThread());
+  DCHECK_GE(max_aspect_ratio_, min_aspect_ratio_);
+  CHECK_NE(0, max_aspect_ratio_);
+  DVLOG(3) << "VideoFrameResolutionAdapter("
+          << "{ max_width =" << max_width << "}, "
+          << "{ max_height =" << max_height << "}, "
+          << "{ min_aspect_ratio =" << min_aspect_ratio << "}, "
+          << "{ max_aspect_ratio_ =" << max_aspect_ratio_ << "}) ";
+}
+
+VideoTrackAdapter::
+VideoFrameResolutionAdapter::~VideoFrameResolutionAdapter() {
+  DCHECK(io_thread_checker_.CalledOnValidThread());
+  DCHECK(callbacks_.empty());
+}
+
+void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame(
+    const scoped_refptr<media::VideoFrame>& frame,
+    const media::VideoCaptureFormat& format) {
+  DCHECK(io_thread_checker_.CalledOnValidThread());
+  // TODO(perkj): Allow cropping / scaling of textures once
+  // http://crbug/362521 is fixed.
+  if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
+    DoDeliverFrame(frame, format);
+    return;
+  }
+  scoped_refptr<media::VideoFrame> video_frame(frame);
+  double input_ratio =
+      static_cast<double>(frame->natural_size().width()) /
+      frame->natural_size().height();
+
+  // If |frame| has larger width or height than requested, or the aspect ratio
+  // does not match the requested, we want to create a wrapped version of this
+  // frame with a size that fulfills the constraints.
+  if (frame->natural_size().width() > max_frame_size_.width() ||
+      frame->natural_size().height() > max_frame_size_.height() ||
+      input_ratio > max_aspect_ratio_ ||
+      input_ratio < min_aspect_ratio_) {
+    int desired_width = std::min(max_frame_size_.width(),
+                                 frame->natural_size().width());
+    int desired_height = std::min(max_frame_size_.height(),
+                                  frame->natural_size().height());
+
+    double resulting_ratio =
+        static_cast<double>(desired_width) / desired_height;
+    double requested_ratio = resulting_ratio;
+
+    if (requested_ratio > max_aspect_ratio_)
+      requested_ratio = max_aspect_ratio_;
+    else if (requested_ratio < min_aspect_ratio_)
+      requested_ratio = min_aspect_ratio_;
+
+    if (resulting_ratio < requested_ratio) {
+      // Make sure we scale to an even height to avoid rounding errors
+      // (+ 1 & ~1).
+      desired_height = static_cast<int>((desired_height * resulting_ratio) /
+                                        requested_ratio) + 1 & ~1;
+    } else if (resulting_ratio > requested_ratio) {
+      // Make sure we scale to an even width to avoid rounding errors.
+      desired_width = static_cast<int>((desired_width * requested_ratio) /
+                                       resulting_ratio) + 1 & ~1;
+    }
+
+    gfx::Size desired_size(desired_width, desired_height);
+
+    // Get the largest centered rectangle with the same aspect ratio of
+    // |desired_size| that fits entirely inside of |frame->visible_rect()|.
+    // This will be the rect we need to crop the original frame to.
+    // From this rect, the original frame can be scaled down to |desired_size|.
+    gfx::Rect region_in_frame =
+        media::ComputeLetterboxRegion(frame->visible_rect(), desired_size);
+
+    video_frame = media::VideoFrame::WrapVideoFrame(
+        frame,
+        region_in_frame,
+        desired_size,
+        base::Bind(&ReleaseOriginalFrame, frame));
+
+    DVLOG(3) << "desired size  " << desired_size.ToString()
+             << " output natural size "
+             << video_frame->natural_size().ToString()
+             << " output visible rect  "
+             << video_frame->visible_rect().ToString();
+  }
+  DoDeliverFrame(video_frame, format);
+}
+
+void VideoTrackAdapter::
+VideoFrameResolutionAdapter::DoDeliverFrame(
+    const scoped_refptr<media::VideoFrame>& frame,
+    const media::VideoCaptureFormat& format) {
+  DCHECK(io_thread_checker_.CalledOnValidThread());
+  for (std::vector<VideoIdCallbackPair>::const_iterator it = callbacks_.begin();
+       it != callbacks_.end(); ++it) {
+    it->second.Run(frame, format);
+  }
+}
+
+void VideoTrackAdapter::VideoFrameResolutionAdapter::AddCallback(
+    const MediaStreamVideoTrack* track,
+    const VideoCaptureDeliverFrameCB& callback) {
+  DCHECK(io_thread_checker_.CalledOnValidThread());
+  callbacks_.push_back(std::make_pair(track, callback));
+}
+
+void VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveCallback(
+    const MediaStreamVideoTrack* track) {
+  DCHECK(io_thread_checker_.CalledOnValidThread());
+  std::vector<VideoIdCallbackPair>::iterator it = callbacks_.begin();
+  for (; it != callbacks_.end(); ++it) {
+    if (it->first == track) {
+      // Callback is copied to heap and then deleted on the target thread.
+      // The following code ensures that the callback is not referenced on
+      // the stack.
+      scoped_ptr<VideoCaptureDeliverFrameCB> callback;
+      {
+        callback.reset(new VideoCaptureDeliverFrameCB(it->second));

[tommi (sloooow) - chröme, 2014/05/14 09:43:38]

add a comment for why this is done inside a scope... I'm not seeing why :)

I don't see anything constructed on the stack inside the scope, so I think this
should simply be:

scoped_ptr<VideoCaptureDeliverFrameCB> callback(new
VideoCaptureDeliverFrameCB(it->second));
callbacks_.erase(it);
render_message_loop_->PostTask(
    FROM_HERE, base::Bind(&ResetCallbackOnMainRenderThread,
                          base::Passed(&callback)));


However, I'm guessing I'm just not seeing it.

[perkj_chrome, 2014/05/14 13:29:44]

You are right. Since the object is on the heap

+        callbacks_.erase(it);
+      }
+      render_message_loop_->PostTask(
+          FROM_HERE, base::Bind(&ResetCallbackOnMainRenderThread,
+                                base::Passed(&callback)));
+
+      return;
+    }
+  }
+}
+
+bool VideoTrackAdapter::VideoFrameResolutionAdapter::ConstraintsMatch(
+    int max_width,
+    int max_height,
+    double min_aspect_ratio,
+    double max_aspect_ratio) const {
+  DCHECK(io_thread_checker_.CalledOnValidThread());
+  return max_frame_size_.width() == max_width &&
+      max_frame_size_.height() == max_height &&
+      min_aspect_ratio_ == min_aspect_ratio &&
+      max_aspect_ratio_ == max_aspect_ratio;
+}
+
+bool VideoTrackAdapter::VideoFrameResolutionAdapter::IsEmpty() const {
+  DCHECK(io_thread_checker_.CalledOnValidThread());
+  return callbacks_.empty();
+}
+
+VideoTrackAdapter::VideoTrackAdapter(
+    const scoped_refptr<base::MessageLoopProxy>& io_message_loop)
+    : io_message_loop_(io_message_loop) {
+  DCHECK(io_message_loop_);
+  io_message_loop_->PostTask(
+      FROM_HERE,
+      base::Bind(&VideoTrackAdapter::SetRenderMessageLoopOnIo,
+                 this, base::MessageLoopProxy::current()));
+}
+
+VideoTrackAdapter::~VideoTrackAdapter() {
+  DCHECK(adapters_.empty());
+}
+
+void VideoTrackAdapter::SetRenderMessageLoopOnIo(
+    const scoped_refptr<base::MessageLoopProxy>& renderer_message_loop) {
+  DCHECK(io_message_loop_->BelongsToCurrentThread());
+  DCHECK(renderer_message_loop);
+  renderer_message_loop_ = renderer_message_loop;
+}
+
+void VideoTrackAdapter::AddTrack(const MediaStreamVideoTrack* track,
+                                 VideoCaptureDeliverFrameCB frame_callback,
+                                 int max_width,
+                                 int max_height,
+                                 double min_aspect_ratio,
+                                 double max_aspect_ratio) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  io_message_loop_->PostTask(
+      FROM_HERE,
+      base::Bind(&VideoTrackAdapter::AddTrackOnIO,
+                 this, track, frame_callback, max_width, max_height,
+                 min_aspect_ratio, max_aspect_ratio));
+}
+
+void VideoTrackAdapter::AddTrackOnIO(
+    const MediaStreamVideoTrack* track,
+    VideoCaptureDeliverFrameCB frame_callback,
+    int max_width,
+    int max_height,
+    double min_aspect_ratio,
+    double max_aspect_ratio) {
+  DCHECK(io_message_loop_->BelongsToCurrentThread());
+  scoped_refptr<VideoFrameResolutionAdapter> adapter;
+  for (FrameAdapters::const_iterator it = adapters_.begin();
+       it != adapters_.end(); ++it) {
+    if ((*it)->ConstraintsMatch(max_width, max_height, min_aspect_ratio,
+                                max_aspect_ratio)) {
+      adapter = it->get();
+      break;
+    }
+  }
+  if (!adapter) {
+    adapter = new VideoFrameResolutionAdapter(renderer_message_loop_,
+                                              max_width,
+                                              max_height,
+                                              min_aspect_ratio,
+                                              max_aspect_ratio);
+    adapters_.push_back(adapter);
+  }
+
+  adapter->AddCallback(track, frame_callback);
+}
+
+void VideoTrackAdapter::RemoveTrack(const MediaStreamVideoTrack* track) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  io_message_loop_->PostTask(
+      FROM_HERE,
+      base::Bind(&VideoTrackAdapter::RemoveTrackOnIO, this, track));
+}
+
+void VideoTrackAdapter::RemoveTrackOnIO(const MediaStreamVideoTrack* track) {
+  DCHECK(io_message_loop_->BelongsToCurrentThread());
+  for (FrameAdapters::iterator it = adapters_.begin();
+       it != adapters_.end(); ++it) {
+    (*it)->RemoveCallback(track);
+    if ((*it)->IsEmpty()) {
+      adapters_.erase(it);
+      break;
+    }
+  }
+}
+
+void VideoTrackAdapter::DeliverFrameOnIO(
+    const scoped_refptr<media::VideoFrame>& frame,
+    const media::VideoCaptureFormat& format) {
+  DCHECK(io_message_loop_->BelongsToCurrentThread());
+  TRACE_EVENT0("video", "VideoTrackAdapter::DeliverFrameOnIO");
+  for (FrameAdapters::iterator it = adapters_.begin();
+       it != adapters_.end(); ++it) {
+    (*it)->DeliverFrame(frame, format);
+  }
+}
+
+}  // namespace content