Change MediaStreamVideoSource to output different resolutions to different tracks depending on the …

content/renderer/media/video_track_adapter.cc

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/renderer/media/video_track_adapter.h"
+
+#include <algorithm>
+#include <limits>
+#include <utility>
+
+#include "base/bind.h"
+#include "base/debug/trace_event.h"
+#include "base/location.h"
+#include "media/base/video_util.h"
+
+namespace {
+
+// Empty method used for keeping a reference to the original media::VideoFrame
+// in VideoFrameResolutionAdapter::DeliverFrame if cropping is needed.
+// The reference to |frame| is kept in the closure that calls this method.
+void ReleaseOriginalFrame(
+    const scoped_refptr<media::VideoFrame>& frame) {
+}
+
+}  // anonymous namespace
+
+namespace content {
+
+// VideoFrameResolutionAdapter is created on the main
+// render thread but operates on the IO-thread. It does the resolution
+// adaptation and delivers frames to all registered tracks on the IO-thread.
+// All method calls except creation, must be on the IO-thread.
+class VideoTrackAdapter::VideoFrameResolutionAdapter
+    : public base::RefCountedThreadSafe<VideoFrameResolutionAdapter> {
+ public:
+  VideoFrameResolutionAdapter(
+      int max_width,
+      int max_height,
+      double min_aspect_ratio,
+      double max_aspect_ratio);
+
+  // Add |callback| to receive video frames on the IO-thread.
+  void AddCallback(const MediaStreamVideoTrack* track,
+                   const VideoCaptureDeliverFrameCB& callback);
+
+  // Removes |callback| associated with |id| from receiving video frames if

[perkj_chrome, 2014/05/08 11:59:02]

|track|

[perkj_chrome, 2014/05/08 13:36:25]

Done.

+  // |track| has been added. It is ok to call RemoveCallback even if the |id|
+  // has not been added.
+  void RemoveCallback(const MediaStreamVideoTrack* track);
+
+  void DeliverFrame(const scoped_refptr<media::VideoFrame>& frame,
+                    const media::VideoCaptureFormat& format);
+
+  // Returns true if all arguments match with the output of this adapter.
+  bool ConstraintsMatch(int max_width,
+                        int max_height,
+                        double min_aspect_ratio,
+                        double max_aspect_ratio) const;
+
+  bool IsEmpty() const { return callbacks_.empty(); }

[tommi (sloooow) - chröme, 2014/05/08 11:57:58]

would like to have a thread check here

[perkj_chrome, 2014/05/08 13:36:25]

Done.

+
+ private:
+  virtual ~VideoFrameResolutionAdapter();
+  friend class base::RefCountedThreadSafe<VideoFrameResolutionAdapter>;
+
+  virtual void DoDeliverFrame(
+      const scoped_refptr<media::VideoFrame>& frame,
+      const media::VideoCaptureFormat& format);
+
+  // Bound to the IO-thread.
+  base::ThreadChecker io_thread_checker_;
+
+  gfx::Size max_frame_size_;
+  double min_aspect_ratio_;
+  double max_aspect_ratio_;
+
+  typedef std::pair<const void*, VideoCaptureDeliverFrameCB>
+      VideoIdCallbackPair;
+  std::vector<VideoIdCallbackPair> callbacks_;
+
+  DISALLOW_COPY_AND_ASSIGN(VideoFrameResolutionAdapter);
+};
+
+VideoTrackAdapter::
+VideoFrameResolutionAdapter::VideoFrameResolutionAdapter(
+    int max_width,

[perkj_chrome, 2014/05/08 11:59:02]

why separate arguments?

+    int max_height,
+    double min_aspect_ratio,
+    double max_aspect_ratio)
+    : max_frame_size_(max_width, max_height),
+      min_aspect_ratio_(min_aspect_ratio),
+      max_aspect_ratio_(max_aspect_ratio) {
+  DCHECK_GE(max_aspect_ratio_, min_aspect_ratio_);
+  DVLOG(3) << "VideoFrameResolutionAdapter("
+          << "{ max_width =" << max_width << "}, "
+          << "{ max_height =" << max_height << "}, "
+          << "{ min_aspect_ratio =" << min_aspect_ratio << "}, "
+          << "{ max_aspect_ratio_ =" << max_aspect_ratio_ << "}) ";
+}
+
+VideoTrackAdapter::
+VideoFrameResolutionAdapter::~VideoFrameResolutionAdapter() {
+  DCHECK(callbacks_.empty());

[tommi (sloooow) - chröme, 2014/05/08 11:57:58]

thread check

[perkj_chrome, 2014/05/08 13:36:25]

Done.

+}
+
+void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame(
+    const scoped_refptr<media::VideoFrame>& frame,
+    const media::VideoCaptureFormat& format) {
+  DCHECK(io_thread_checker_.CalledOnValidThread());
+  // TODO(perkj): Allow cropping / scaling of textures once
+  // http://crbug/362521 is fixed.
+  if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
+    DoDeliverFrame(frame, format);
+    return;
+  }
+  scoped_refptr<media::VideoFrame> video_frame(frame);
+  double input_ratio =
+      static_cast<double>(frame->natural_size().width()) /
+      frame->natural_size().height();
+
+  // If |frame| has larger width or height than requested, or the aspect ratio
+  // does not match the requested, we want to create a wrapped version of this
+  // frame with a size that fulfills the constraints.
+  if (frame->natural_size().width() > max_frame_size_.width() ||
+      frame->natural_size().height() > max_frame_size_.height() ||
+      input_ratio > max_aspect_ratio_ ||
+      input_ratio < min_aspect_ratio_) {
+    int desired_width = std::min(max_frame_size_.width(),
+                                 frame->natural_size().width());
+    int desired_height = std::min(max_frame_size_.height(),
+                                  frame->natural_size().height());
+
+    double resulting_ratio =
+        static_cast<double>(desired_width) / desired_height;
+    double requested_ratio = resulting_ratio;
+
+    if (requested_ratio > max_aspect_ratio_)
+      requested_ratio = max_aspect_ratio_;
+    else if (requested_ratio < min_aspect_ratio_)
+      requested_ratio = min_aspect_ratio_;
+
+    if (resulting_ratio < requested_ratio) {
+      desired_height = static_cast<int>((desired_height * resulting_ratio) /
+                                        requested_ratio) + 1 & ~1;

[tommi (sloooow) - chröme, 2014/05/08 11:57:58]

why & ~1?
can you add documentation?

[perkj_chrome, 2014/05/08 13:36:25]

That is kind of a good question- I did it to avoid rounding errors - but I am
not entirely sure its needed.
Done.

+    } else if (resulting_ratio > requested_ratio) {
+      desired_width = static_cast<int>((desired_width * requested_ratio) /
+                                       resulting_ratio) + 1 & ~1;
+    }
+
+    gfx::Size desired_size(desired_width, desired_height);
+
+    // Get the largest centered rectangle with the same aspect ratio of
+    // |desired_size| that fits entirely inside of |frame->visible_rect()|.
+    // This will be the rect we need to crop the original frame to.
+    // From this rect, the original frame can be scaled down to |desired_size|.
+    gfx::Rect region_in_frame =
+          media::ComputeLetterboxRegion(frame->visible_rect(), desired_size);

[tommi (sloooow) - chröme, 2014/05/08 11:57:58]

indent

[perkj_chrome, 2014/05/08 13:36:25]

Done.

+
+    video_frame = media::VideoFrame::WrapVideoFrame(
+        frame,
+        region_in_frame,
+        desired_size,
+        base::Bind(&ReleaseOriginalFrame, frame));
+
+    DVLOG(3) << "desired size  " << desired_size.ToString()
+             << " output natural size "
+             << video_frame->natural_size().ToString()
+             << " output visible rect  "
+             << video_frame->visible_rect().ToString();
+  }
+  DoDeliverFrame(video_frame, format);
+}
+
+void VideoTrackAdapter::
+VideoFrameResolutionAdapter::DoDeliverFrame(
+    const scoped_refptr<media::VideoFrame>& frame,
+    const media::VideoCaptureFormat& format) {
+  for (std::vector<VideoIdCallbackPair>::const_iterator it = callbacks_.begin();

[tommi (sloooow) - chröme, 2014/05/08 11:57:58]

thread check here or at least a comment that explains where we are?

[perkj_chrome, 2014/05/08 13:36:25]

Done.

+       it != callbacks_.end(); ++it) {
+    it->second.Run(frame, format);
+  }
+}
+
+void VideoTrackAdapter::VideoFrameResolutionAdapter::AddCallback(
+    const MediaStreamVideoTrack* track,
+    const VideoCaptureDeliverFrameCB& callback) {
+  DCHECK(io_thread_checker_.CalledOnValidThread());
+  callbacks_.push_back(std::make_pair(track, callback));
+}
+
+void VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveCallback(
+    const MediaStreamVideoTrack* track) {
+  DCHECK(io_thread_checker_.CalledOnValidThread());
+  std::vector<VideoIdCallbackPair>::iterator it = callbacks_.begin();
+  for (; it != callbacks_.end(); ++it) {
+    if (it->first == track) {
+      callbacks_.erase(it);
+      return;
+    }
+  }
+}
+
+bool VideoTrackAdapter::VideoFrameResolutionAdapter::ConstraintsMatch(
+    int max_width,
+    int max_height,
+    double min_aspect_ratio,
+    double max_aspect_ratio) const {
+  return max_frame_size_.width() == max_width &&

[tommi (sloooow) - chröme, 2014/05/08 11:57:58]

thread check

[perkj_chrome, 2014/05/08 13:36:25]

Done.

+      max_frame_size_.height() == max_height &&
+      min_aspect_ratio_ == min_aspect_ratio &&
+      max_aspect_ratio_ == max_aspect_ratio;
+}
+
+VideoTrackAdapter::VideoTrackAdapter(
+    const scoped_refptr<base::MessageLoopProxy>& io_message_loop)
+    : io_message_loop_(io_message_loop) {
+}
+
+VideoTrackAdapter::~VideoTrackAdapter() {

[tommi (sloooow) - chröme, 2014/05/08 11:57:58]

thread check

[perkj_chrome, 2014/05/08 13:36:25]

Can I do that ? its thread safe recounted and bound both to the IO and the main
render thread.

+  DCHECK(adapters_.empty());
+}
+
+void VideoTrackAdapter::AddTrack(MediaStreamVideoTrack* track,

[perkj_chrome, 2014/05/08 11:59:02]

const * track

[perkj_chrome, 2014/05/08 13:36:25]

Done.

+                                 VideoCaptureDeliverFrameCB frame_callback,
+                                 int max_width,
+                                 int max_height,
+                                 double min_aspect_ratio,
+                                 double max_aspect_ratio) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  io_message_loop_->PostTask(
+      FROM_HERE,
+      base::Bind(&VideoTrackAdapter::AddTrackOnIO,
+                 this, track, frame_callback, max_width, max_height,
+                 min_aspect_ratio, max_aspect_ratio));
+}
+
+void VideoTrackAdapter::AddTrackOnIO(MediaStreamVideoTrack* track,
+                                     VideoCaptureDeliverFrameCB frame_callback,
+                                     int max_width,
+                                     int max_height,
+                                     double min_aspect_ratio,
+                                     double max_aspect_ratio) {
+  DCHECK(io_message_loop_->BelongsToCurrentThread());
+  scoped_refptr<VideoFrameResolutionAdapter> adapter;
+  for (FrameAdapters::const_iterator it = adapters_.begin();
+       it != adapters_.end(); ++it) {
+    if ((*it)->ConstraintsMatch(max_width, max_height, min_aspect_ratio,
+                                max_aspect_ratio)) {
+      adapter = it->get();
+      break;
+    }
+  }
+  if (!adapter) {
+    adapter = new VideoFrameResolutionAdapter(max_width,
+                                              max_height,
+                                              min_aspect_ratio,
+                                              max_aspect_ratio);
+    adapters_.push_back(adapter);
+  }
+
+  adapter->AddCallback(track, frame_callback);
+}
+
+void VideoTrackAdapter::RemoveTrack(MediaStreamVideoTrack* track) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  io_message_loop_->PostTask(
+      FROM_HERE,
+      base::Bind(&VideoTrackAdapter::RemoveTrackOnIO, this, track));
+}
+
+void VideoTrackAdapter::RemoveTrackOnIO(MediaStreamVideoTrack* track) {
+  DCHECK(io_message_loop_->BelongsToCurrentThread());
+  for (FrameAdapters::iterator it = adapters_.begin();
+       it != adapters_.end(); ++it) {
+    (*it)->RemoveCallback(track);
+    if ((*it)->IsEmpty()) {
+      adapters_.erase(it);
+      break;
+    }
+  }
+}
+
+void VideoTrackAdapter::DeliverFrameOnIO(
+    const scoped_refptr<media::VideoFrame>& frame,
+    const media::VideoCaptureFormat& format) {
+  DCHECK(io_message_loop_->BelongsToCurrentThread());
+  TRACE_EVENT0("video", "VideoTrackAdapter::DeliverFrameOnIO");
+  for (FrameAdapters::iterator it = adapters_.begin();
+       it != adapters_.end(); ++it) {
+    (*it)->DeliverFrame(frame, format);
+  }
+}
+
+}  // namespace content