Change MediaStreamVideoSource to output different resolutions to different tracks depending on the …

content/renderer/media/media_stream_video_source.cc

Index: content/renderer/media/media_stream_video_source.cc
diff --git a/content/renderer/media/media_stream_video_source.cc b/content/renderer/media/media_stream_video_source.cc
index 64ab63c15e37092b1d2e869a602567d5c35afd94..d55239c9a8b294a6d96ef05a717a8fb27dffe3d5 100644
--- a/content/renderer/media/media_stream_video_source.cc
+++ b/content/renderer/media/media_stream_video_source.cc
@@ -11,10 +11,8 @@
 #include "base/debug/trace_event.h"
 #include "base/logging.h"
 #include "base/strings/string_number_conversions.h"
-#include "content/renderer/media/media_stream_dependency_factory.h"
 #include "content/renderer/media/media_stream_video_track.h"
-#include "content/renderer/media/video_frame_deliverer.h"
-#include "content/renderer/media/webrtc/webrtc_video_capturer_adapter.h"
+#include "content/renderer/media/video_track_adapter.h"
 
 namespace content {
 
@@ -89,22 +87,7 @@ bool UpdateFormatForConstraint(
 
   if (constraint_name == MediaStreamVideoSource::kMinAspectRatio ||
       constraint_name == MediaStreamVideoSource::kMaxAspectRatio) {
-    double double_value = 0;
-    base::StringToDouble(constraint_value, &double_value);
-
-    // The aspect ratio in |constraint.m_value| has been converted to a string
-    // and back to a double, so it may have a rounding error.
-    // E.g if the value 1/3 is converted to a string, the string will not have
-    // infinite length.
-    // We add a margin of 0.0005 which is high enough to detect the same aspect
-    // ratio but small enough to avoid matching wrong aspect ratios.
-    const double kRoundingTruncation = 0.0005;
-    double ratio = static_cast<double>(format->frame_size.width()) /
-        format->frame_size.height();
-    if (constraint_name == MediaStreamVideoSource::kMinAspectRatio)
-      return (double_value <= ratio + kRoundingTruncation);
-    // Subtract 0.0005 to avoid rounding problems. Same as above.
-    return (double_value >= ratio - kRoundingTruncation);
+    return true;
   }
 
   int value;
@@ -217,6 +200,18 @@ bool GetConstraintValue(const blink::WebMediaConstraints& constraints,
   return ret;
 }
 
+bool GetConstraintValue(const blink::WebMediaConstraints& constraints,
+                        bool mandatory, const blink::WebString& name,
+                        double* value) {
+  blink::WebString value_str;
+  bool ret = mandatory ?
+      constraints.getMandatoryConstraintValue(name, value_str) :
+      constraints.getOptionalConstraintValue(name, value_str);
+  if (ret)
+    base::StringToDouble(value_str.utf8(), value);
+  return ret;
+}
+
 // Returns true if |constraint| has mandatory constraints.
 bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {
   blink::WebVector<blink::WebMediaConstraint> mandatory_constraints;
@@ -224,9 +219,13 @@ bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {
   return !mandatory_constraints.isEmpty();
 }
 
-// Retrieve the desired max width and height from |constraints|.
+// Retrieve the desired max width and height from |constraints|. If not set,
+// the |desired_width| and |desired_height| is to
+// std::numeric_limits<int>::max();
 void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
                                  int* desired_width, int* desired_height) {
+  *desired_width = std::numeric_limits<int>::max();
+  *desired_height = std::numeric_limits<int>::max();;
   bool mandatory = GetConstraintValue(constraints, true,
                                       MediaStreamVideoSource::kMaxWidth,
                                       desired_width);
@@ -242,6 +241,41 @@ void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
                      desired_height);
 }
 
+void GetDesiredMinAndMaxAspectRatio(
+    const blink::WebMediaConstraints& constraints,
+    double* min_aspect_ratio,
+    double* max_aspect_ratio) {
+  *min_aspect_ratio = 0;
+  *max_aspect_ratio = std::numeric_limits<double>::max();
+
+  bool mandatory = GetConstraintValue(constraints, true,
+                                      MediaStreamVideoSource::kMinAspectRatio,
+                                      min_aspect_ratio);
+  mandatory |= GetConstraintValue(constraints, true,
+                                  MediaStreamVideoSource::kMaxAspectRatio,
+                                  max_aspect_ratio);
+  if (!mandatory &&
+      !GetConstraintValue(constraints, false,
+                          MediaStreamVideoSource::kMinAspectRatio,
+                          min_aspect_ratio) &&
+      !GetConstraintValue(constraints, false,
+                          MediaStreamVideoSource::kMaxAspectRatio,
+                          max_aspect_ratio)) {
+    return;
+  }
+  // The aspect ratio in |constraint.m_value| has been converted to a string
+  // and back to a double, so it may have a rounding error.
+  // E.g if the value 1/3 is converted to a string, the string will not have
+  // infinite length.
+  // We add a margin of 0.0005 which is high enough to detect the same aspect
+  // ratio but small enough to avoid matching wrong aspect ratios.
+  const double kRoundingTruncation = 0.0005;
+  if (*min_aspect_ratio != 0)

[mcasas, 2014/05/07 14:10:43]

|min_aspect_ratio| is a double, so this comparison is always true.

[perkj_chrome, 2014/05/08 11:29:47]

Why would it be true? I set it to 0 at line 248?

[mcasas, 2014/05/09 07:46:24]

A float cannot be 0.0 exactly, and the most likely thing is for the
compiler to upgrade your (int) 0 to (float) 0.0, hence hitting this case.

Float comparisons are usually like
if (abs(*min_aspect_ratio) < epsilon) ...

+    *min_aspect_ratio += kRoundingTruncation;
+  if (*max_aspect_ratio != std::numeric_limits<double>::max())
+    *max_aspect_ratio -= kRoundingTruncation;
+}
+
 const media::VideoCaptureFormat& GetBestFormatBasedOnArea(
     const media::VideoCaptureFormats& formats,
     int area) {
@@ -267,10 +301,8 @@ const media::VideoCaptureFormat& GetBestFormatBasedOnArea(
 void GetBestCaptureFormat(
     const media::VideoCaptureFormats& formats,
     const blink::WebMediaConstraints& constraints,
-    media::VideoCaptureFormat* capture_format,
-    gfx::Size* max_frame_output_size) {
+    media::VideoCaptureFormat* capture_format) {
   DCHECK(!formats.empty());
-  DCHECK(max_frame_output_size);
 
   int max_width = std::numeric_limits<int>::max();
   int max_height = std::numeric_limits<int>::max();;
@@ -280,96 +312,10 @@ void GetBestCaptureFormat(
       formats,
       std::min(max_width, MediaStreamVideoSource::kDefaultWidth) *
       std::min(max_height, MediaStreamVideoSource::kDefaultHeight));
-
-  max_frame_output_size->set_width(max_width);
-  max_frame_output_size->set_height(max_height);
-}
-
-// Empty method used for keeping a reference to the original media::VideoFrame
-// in MediaStreamVideoSource::FrameDeliverer::DeliverFrameOnIO if cropping is
-// needed. The reference to |frame| is kept in the closure that calls this
-// method.
-void ReleaseOriginalFrame(
-    const scoped_refptr<media::VideoFrame>& frame) {
 }
 
 }  // anonymous namespace
 
-// Helper class used for delivering video frames to all registered tracks
-// on the IO-thread.
-class MediaStreamVideoSource::FrameDeliverer : public VideoFrameDeliverer {
- public:
-  FrameDeliverer(
-      const scoped_refptr<base::MessageLoopProxy>& io_message_loop)
-      : VideoFrameDeliverer(io_message_loop)  {
-  }
-
-  // Register |callback| to receive video frames of max size
-  // |max_frame_output_size| on the IO thread.
-  // TODO(perkj): Currently |max_frame_output_size| must be the same for all
-  // |callbacks|.
-  void AddCallback(void* id,
-                   const VideoCaptureDeliverFrameCB& callback,
-                   const gfx::Size& max_frame_output_size) {
-    DCHECK(thread_checker().CalledOnValidThread());
-    io_message_loop()->PostTask(
-        FROM_HERE,
-        base::Bind(
-            &FrameDeliverer::AddCallbackWithResolutionOnIO,
-            this, id, callback, max_frame_output_size));
-  }
-
-  virtual void DeliverFrameOnIO(
-      const scoped_refptr<media::VideoFrame>& frame,
-      const media::VideoCaptureFormat& format) OVERRIDE {
-    DCHECK(io_message_loop()->BelongsToCurrentThread());
-    TRACE_EVENT0("video", "MediaStreamVideoSource::DeliverFrameOnIO");
-    if (max_output_size_.IsEmpty())
-      return;  // Frame received before the output has been decided.
-
-    scoped_refptr<media::VideoFrame> video_frame(frame);
-    const gfx::Size& visible_size = frame->visible_rect().size();
-    if (visible_size.width() > max_output_size_.width() ||
-        visible_size.height() > max_output_size_.height()) {
-      // If |frame| is not the size that is expected, we need to crop it by
-      // providing a new |visible_rect|. The new visible rect must be within the
-      // original |visible_rect|.
-      gfx::Rect output_rect = frame->visible_rect();
-      output_rect.ClampToCenteredSize(max_output_size_);
-      // TODO(perkj): Allow cropping of textures once http://crbug/362521 is
-      // fixed.
-      if (frame->format() != media::VideoFrame::NATIVE_TEXTURE) {
-        video_frame = media::VideoFrame::WrapVideoFrame(
-            frame,
-            output_rect,
-            output_rect.size(),
-            base::Bind(&ReleaseOriginalFrame, frame));
-      }
-    }
-    VideoFrameDeliverer::DeliverFrameOnIO(video_frame, format);
-  }
-
- protected:
-  virtual ~FrameDeliverer() {
-  }
-
-  void AddCallbackWithResolutionOnIO(
-      void* id,
-     const VideoCaptureDeliverFrameCB& callback,
-      const gfx::Size& max_frame_output_size) {
-    DCHECK(io_message_loop()->BelongsToCurrentThread());
-    // Currently we only support one frame output size.
-    DCHECK(!max_frame_output_size.IsEmpty() &&
-           (max_output_size_.IsEmpty() ||
-            max_output_size_ == max_frame_output_size));
-    max_output_size_ = max_frame_output_size;
-    VideoFrameDeliverer::AddCallbackOnIO(id, callback);
-  }
-
- private:
-  gfx::Size max_output_size_;
-};
-
 // static
 MediaStreamVideoSource* MediaStreamVideoSource::GetVideoSource(
     const blink::WebMediaStreamSource& source) {
@@ -388,8 +334,8 @@ bool MediaStreamVideoSource::IsConstraintSupported(const std::string& name) {
 MediaStreamVideoSource::MediaStreamVideoSource(
     const scoped_refptr<base::MessageLoopProxy>& io_message_loop)
     : state_(NEW),
-      frame_deliverer_(
-          new MediaStreamVideoSource::FrameDeliverer(io_message_loop)),
+      track_adapter_(
+          new VideoTrackAdapter(io_message_loop)),

[mcasas, 2014/05/07 14:10:43]

nit: Fits in one line.

[perkj_chrome, 2014/05/08 11:29:47]

Done.

       weak_factory_(this) {
 }
 
@@ -449,9 +395,20 @@ void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack* video_track) {
       std::find(tracks_.begin(), tracks_.end(), video_track);
   DCHECK(it != tracks_.end());
   tracks_.erase(it);
-  // Call |RemoveCallback| here even if adding the track has failed and
-  // frame_deliverer_->AddCallback has not been called.
-  frame_deliverer_->RemoveCallback(video_track);
+
+  // Check if the track is waiting for applying new constraints and remove
+  // it in that case.
+  for (std::vector<RequestedConstraints>::iterator it =
+           requested_constraints_.begin();
+       it != requested_constraints_.end(); ++it) {
+    if (it->track == video_track) {
+      requested_constraints_.erase(it);
+      break;
+    }
+  }
+  // Call |frame_adapter_->RemoveTrack| here even if adding the track has
+  // failed and |frame_adapter_->AddCallback| has not been called.
+  track_adapter_->RemoveTrack(video_track);
 
   if (tracks_.empty())
     StopSource();
@@ -459,7 +416,7 @@ void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack* video_track) {
 
 const scoped_refptr<base::MessageLoopProxy>&
 MediaStreamVideoSource::io_message_loop() const {
-  return frame_deliverer_->io_message_loop();
+  return track_adapter_->io_message_loop();

[mcasas, 2014/05/07 14:10:43]

I think nobody uses this getter anymore.

[perkj_chrome, 2014/05/08 11:29:47]

no- its used by a few of the classes that inherits from MediaStreamVideoSource.

 }
 
 void MediaStreamVideoSource::DoStopSource() {
@@ -479,9 +436,7 @@ void MediaStreamVideoSource::OnSupportedFormats(
 
   supported_formats_ = formats;
   if (!FindBestFormatWithConstraints(supported_formats_,
-                                     &current_format_,
-                                     &max_frame_output_size_,
-                                     &current_constraints_)) {
+                                     &current_format_)) {
     SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
     // This object can be deleted after calling FinalizeAddTrack. See comment
     // in the header file.
@@ -499,15 +454,12 @@ void MediaStreamVideoSource::OnSupportedFormats(
   params.requested_format = current_format_;
   StartSourceImpl(
       params,
-      base::Bind(&MediaStreamVideoSource::FrameDeliverer::DeliverFrameOnIO,
-                 frame_deliverer_));
+      base::Bind(&VideoTrackAdapter::DeliverFrameOnIO, track_adapter_));
 }
 
 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
     const media::VideoCaptureFormats& formats,
-    media::VideoCaptureFormat* best_format,
-    gfx::Size* max_frame_output_size,
-    blink::WebMediaConstraints* resulting_constraints) {
+    media::VideoCaptureFormat* best_format) {
   // Find the first constraints that we can fulfill.
   for (std::vector<RequestedConstraints>::iterator request_it =
            requested_constraints_.begin();
@@ -520,9 +472,6 @@ bool MediaStreamVideoSource::FindBestFormatWithConstraints(
     // we will start with whatever format is native to the source.
     if (formats.empty() && !HasMandatoryConstraints(requested_constraints)) {
       *best_format = media::VideoCaptureFormat();
-      *resulting_constraints = requested_constraints;
-      *max_frame_output_size = gfx::Size(std::numeric_limits<int>::max(),
-                                         std::numeric_limits<int>::max());
       return true;
     }
     media::VideoCaptureFormats filtered_formats =
@@ -531,9 +480,7 @@ bool MediaStreamVideoSource::FindBestFormatWithConstraints(
       // A request with constraints that can be fulfilled.
       GetBestCaptureFormat(filtered_formats,
                            requested_constraints,
-                           best_format,
-                           max_frame_output_size);
-      *resulting_constraints= requested_constraints;
+                           best_format);
       return true;
     }
   }
@@ -575,11 +522,23 @@ void MediaStreamVideoSource::FinalizeAddTrack() {
         ((!current_format_.IsValid() && !HasMandatoryConstraints(
             it->constraints)) ||
          !FilterFormats(it->constraints, formats).empty());
+
     if (success) {
-      frame_deliverer_->AddCallback(it->track, it->frame_callback,
-                                    max_frame_output_size_);
+      int max_width = std::numeric_limits<int>::max();
+      int max_height = std::numeric_limits<int>::max();
+      GetDesiredMaxWidthAndHeight(it->constraints, &max_width, &max_height);

[mcasas, 2014/05/07 14:10:43]

nit: Inside GetDesiredMaxWidthAndHeight(), |max_width| and |max_height| are also
initialised to max() values.

[perkj_chrome, 2014/05/08 11:29:47]

done

+      double max_aspect_ratio = std::numeric_limits<double>::max();
+      double min_aspect_ratio = 0;
+      GetDesiredMinAndMaxAspectRatio(it->constraints,
+                                     &min_aspect_ratio,
+                                     &max_aspect_ratio);
+      track_adapter_->AddTrack(it->track,it->frame_callback,
+                               max_width, max_height,
+                               min_aspect_ratio, max_aspect_ratio);
     }
+
     DVLOG(3) << "FinalizeAddTrack() success " << success;
+
     if (!it->callback.is_null())
       it->callback.Run(this, success);
   }