Change MediaStreamVideoSource to output different resolutions to different tracks depending on the …

content/renderer/media/media_stream_video_source.cc

Index: content/renderer/media/media_stream_video_source.cc
diff --git a/content/renderer/media/media_stream_video_source.cc b/content/renderer/media/media_stream_video_source.cc
index 2db0ef0e8f3cc0f3adc59dcb70a4bf9a69d37560..5093b9981ee9222b3504a08836d986e35aa14f7b 100644
--- a/content/renderer/media/media_stream_video_source.cc
+++ b/content/renderer/media/media_stream_video_source.cc
@@ -12,10 +12,8 @@
 #include "base/logging.h"
 #include "base/strings/string_number_conversions.h"
 #include "content/child/child_process.h"
-#include "content/renderer/media/media_stream_dependency_factory.h"
 #include "content/renderer/media/media_stream_video_track.h"
-#include "content/renderer/media/video_frame_deliverer.h"
-#include "content/renderer/media/webrtc/webrtc_video_capturer_adapter.h"
+#include "content/renderer/media/video_track_adapter.h"
 
 namespace content {
 
@@ -57,6 +55,98 @@ const char kGooglePrefix[] = "goog";
 // input frame height of max 360 * kMaxCropFactor pixels is accepted.
 const int kMaxCropFactor = 2;
 
+bool GetConstraintValue(const blink::WebMediaConstraints& constraints,

[tommi (sloooow) - chröme, 2014/05/08 11:57:58]

this feels like copy/paste of some other code. Can you use what's already
available? (I think Shijing added some utility functions that do this recently)

[perkj_chrome, 2014/05/08 13:36:25]

This is just a move from further down below. Shijing has not landed his cl yet. 

+                        bool mandatory, const blink::WebString& name,
+                        int* value) {
+  blink::WebString value_str;
+  bool ret = mandatory ?
+      constraints.getMandatoryConstraintValue(name, value_str) :
+      constraints.getOptionalConstraintValue(name, value_str);
+  if (ret)
+    base::StringToInt(value_str.utf8(), value);

[tommi (sloooow) - chröme, 2014/05/08 11:57:58]

you need to handle this error

[perkj_chrome, 2014/05/08 13:36:25]

See comment above +  according to base::StringToInt - value is set to best
effort that depends on the error.
This is not changed by this cl.

How about we consider a follow up cl next week where we parse the blink
constraints into a Chrome object that has normal std::vectors and then
convenience functions like this will be simple. Hopefully Shijings patch is then
also landed and we can consider how we should treat parsing errors. I guess, if
its a mandatory- we should fail the request - if its optional- maybe we should
just ignore the  constraint.

+  return ret;
+}
+
+bool GetConstraintValue(const blink::WebMediaConstraints& constraints,
+                        bool mandatory, const blink::WebString& name,
+                        double* value) {
+  blink::WebString value_str;
+  bool ret = mandatory ?
+      constraints.getMandatoryConstraintValue(name, value_str) :
+      constraints.getOptionalConstraintValue(name, value_str);
+  if (ret)
+    base::StringToDouble(value_str.utf8(), value);

[tommi (sloooow) - chröme, 2014/05/08 11:57:58]

here too

[perkj_chrome, 2014/05/08 13:36:25]

dito

+  return ret;
+}
+
+// Returns true if |constraint| has mandatory constraints.
+bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {

[tommi (sloooow) - chröme, 2014/05/08 11:57:58]

This feels like an expensive way to check if the mandatory constraints are
there.  Are we ever calling this method first and then fetching the mandatory
constraints again?  Could we do without this method?

[perkj_chrome, 2014/05/08 13:36:25]

This is also just a move and not changed in this cl. 
How about we consider a follow up cl next week where we parse the blink
constraints into a Chrome object that has normal std::vectors and then
convenience functions like this will be simple. Hopefully Shijings patch is then
also landed.

+  blink::WebVector<blink::WebMediaConstraint> mandatory_constraints;
+  constraints.getMandatoryConstraints(mandatory_constraints);
+  return !mandatory_constraints.isEmpty();
+}
+
+// Retrieve the desired max width and height from |constraints|. If not set,
+// the |desired_width| and |desired_height| are set to
+// std::numeric_limits<int>::max();

[tommi (sloooow) - chröme, 2014/05/08 11:57:58]

also document the behavior if the width and height are set in separate parts of
the constraints (optional and mandatory)

[perkj_chrome, 2014/05/08 13:36:25]

This is just a move as well. 

Done for the documentation part.

+void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
+                                 int* desired_width, int* desired_height) {
+  *desired_width = std::numeric_limits<int>::max();
+  *desired_height = std::numeric_limits<int>::max();
+
+  bool mandatory = GetConstraintValue(constraints, true,
+                                      MediaStreamVideoSource::kMaxWidth,
+                                      desired_width);
+  mandatory |= GetConstraintValue(constraints, true,
+                                  MediaStreamVideoSource::kMaxHeight,
+                                  desired_height);
+  if (mandatory)
+    return;
+
+  GetConstraintValue(constraints, false, MediaStreamVideoSource::kMaxWidth,
+                     desired_width);
+  GetConstraintValue(constraints, false, MediaStreamVideoSource::kMaxHeight,
+                     desired_height);
+}
+
+// Retrieve the desired max and min aspect ratio from |constraints|. If not set,
+// the |min_aspect_ratio| is set to 0 and |max_aspect_ratio| is set to
+// std::numeric_limits<double>::max();
+void GetDesiredMinAndMaxAspectRatio(
+    const blink::WebMediaConstraints& constraints,
+    double* min_aspect_ratio,
+    double* max_aspect_ratio) {
+  *min_aspect_ratio = 0;
+  *max_aspect_ratio = std::numeric_limits<double>::max();
+
+  bool mandatory = GetConstraintValue(constraints, true,
+                                      MediaStreamVideoSource::kMinAspectRatio,
+                                      min_aspect_ratio);
+  mandatory |= GetConstraintValue(constraints, true,
+                                  MediaStreamVideoSource::kMaxAspectRatio,
+                                  max_aspect_ratio);
+  if (!mandatory &&
+      !GetConstraintValue(constraints, false,
+                          MediaStreamVideoSource::kMinAspectRatio,
+                          min_aspect_ratio) &&
+      !GetConstraintValue(constraints, false,
+                          MediaStreamVideoSource::kMaxAspectRatio,
+                          max_aspect_ratio)) {
+    return;
+  }
+  // The aspect ratio in |constraint.m_value| has been converted to a string
+  // and back to a double, so it may have a rounding error.
+  // E.g if the value 1/3 is converted to a string, the string will not have
+  // infinite length.
+  // We add a margin of 0.0005 which is high enough to detect the same aspect
+  // ratio but small enough to avoid matching wrong aspect ratios.
+  const double kRoundingTruncation = 0.0005;
+  if (*min_aspect_ratio != 0)
+    *min_aspect_ratio += kRoundingTruncation;
+  if (*max_aspect_ratio != std::numeric_limits<double>::max())
+    *max_aspect_ratio -= kRoundingTruncation;
+}
+
 // Returns true if |constraint| is fulfilled. |format| can be changed
 // changed by a constraint. Ie - the frame rate can be changed by setting
 // maxFrameRate.
@@ -90,22 +180,9 @@ bool UpdateFormatForConstraint(
 
   if (constraint_name == MediaStreamVideoSource::kMinAspectRatio ||
       constraint_name == MediaStreamVideoSource::kMaxAspectRatio) {
-    double double_value = 0;
-    base::StringToDouble(constraint_value, &double_value);
-
-    // The aspect ratio in |constraint.m_value| has been converted to a string
-    // and back to a double, so it may have a rounding error.
-    // E.g if the value 1/3 is converted to a string, the string will not have
-    // infinite length.
-    // We add a margin of 0.0005 which is high enough to detect the same aspect
-    // ratio but small enough to avoid matching wrong aspect ratios.
-    const double kRoundingTruncation = 0.0005;
-    double ratio = static_cast<double>(format->frame_size.width()) /
-        format->frame_size.height();
-    if (constraint_name == MediaStreamVideoSource::kMinAspectRatio)
-      return (double_value <= ratio + kRoundingTruncation);
-    // Subtract 0.0005 to avoid rounding problems. Same as above.
-    return (double_value >= ratio - kRoundingTruncation);
+    // These constraints are handled by cropping if the camera output the wrong

[tommi (sloooow) - chröme, 2014/05/08 11:57:58]

s/output the/outputs a

[perkj_chrome, 2014/05/08 13:36:25]

Done.

+    // aspect ratio.
+    return true;
   }
 
   int value;
@@ -175,13 +252,35 @@ media::VideoCaptureFormats FilterFormats(
     return supported_formats;
   }
 
+  double max_aspect_ratio;
+  double min_aspect_ratio;
+  GetDesiredMinAndMaxAspectRatio(constraints,
+                                 &min_aspect_ratio,
+                                 &max_aspect_ratio);
+
+  if (min_aspect_ratio > max_aspect_ratio)
+    return media::VideoCaptureFormats();
+
+  int min_width = 0;
+  GetConstraintValue(constraints, true,
+                     MediaStreamVideoSource::kMinWidth,
+                     &min_width);
+  int min_height = 0;
+  GetConstraintValue(constraints, true,
+                     MediaStreamVideoSource::kMinHeight,
+                     &min_height);
+  int max_width;
+  int max_height;
+  GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
+
+  if (min_width > max_width || min_height > max_height)
+    return media::VideoCaptureFormats();
+
   blink::WebVector<blink::WebMediaConstraint> mandatory;
   blink::WebVector<blink::WebMediaConstraint> optional;
   constraints.getMandatoryConstraints(mandatory);
   constraints.getOptionalConstraints(optional);
-
   media::VideoCaptureFormats candidates = supported_formats;
-
   for (size_t i = 0; i < mandatory.size(); ++i)
     FilterFormatsByConstraint(mandatory[i], true, &candidates);
 
@@ -206,43 +305,6 @@ media::VideoCaptureFormats FilterFormats(
   return candidates;
 }
 
-bool GetConstraintValue(const blink::WebMediaConstraints& constraints,
-                        bool mandatory, const blink::WebString& name,
-                        int* value) {
-  blink::WebString value_str;
-  bool ret = mandatory ?
-      constraints.getMandatoryConstraintValue(name, value_str) :
-      constraints.getOptionalConstraintValue(name, value_str);
-  if (ret)
-    base::StringToInt(value_str.utf8(), value);
-  return ret;
-}
-
-// Returns true if |constraint| has mandatory constraints.
-bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {
-  blink::WebVector<blink::WebMediaConstraint> mandatory_constraints;
-  constraints.getMandatoryConstraints(mandatory_constraints);
-  return !mandatory_constraints.isEmpty();
-}
-
-// Retrieve the desired max width and height from |constraints|.
-void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
-                                 int* desired_width, int* desired_height) {
-  bool mandatory = GetConstraintValue(constraints, true,
-                                      MediaStreamVideoSource::kMaxWidth,
-                                      desired_width);
-  mandatory |= GetConstraintValue(constraints, true,
-                                  MediaStreamVideoSource::kMaxHeight,
-                                  desired_height);
-  if (mandatory)
-    return;
-
-  GetConstraintValue(constraints, false, MediaStreamVideoSource::kMaxWidth,
-                     desired_width);
-  GetConstraintValue(constraints, false, MediaStreamVideoSource::kMaxHeight,
-                     desired_height);
-}
-
 const media::VideoCaptureFormat& GetBestFormatBasedOnArea(
     const media::VideoCaptureFormats& formats,
     int area) {
@@ -268,109 +330,21 @@ const media::VideoCaptureFormat& GetBestFormatBasedOnArea(
 void GetBestCaptureFormat(
     const media::VideoCaptureFormats& formats,
     const blink::WebMediaConstraints& constraints,
-    media::VideoCaptureFormat* capture_format,
-    gfx::Size* max_frame_output_size) {
+    media::VideoCaptureFormat* capture_format) {
   DCHECK(!formats.empty());
-  DCHECK(max_frame_output_size);
 
-  int max_width = std::numeric_limits<int>::max();
-  int max_height = std::numeric_limits<int>::max();;
+  int max_width;
+  int max_height;
   GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
 
   *capture_format = GetBestFormatBasedOnArea(
       formats,
       std::min(max_width, MediaStreamVideoSource::kDefaultWidth) *
       std::min(max_height, MediaStreamVideoSource::kDefaultHeight));
-
-  max_frame_output_size->set_width(max_width);
-  max_frame_output_size->set_height(max_height);
-}
-
-// Empty method used for keeping a reference to the original media::VideoFrame
-// in MediaStreamVideoSource::FrameDeliverer::DeliverFrameOnIO if cropping is
-// needed. The reference to |frame| is kept in the closure that calls this
-// method.
-void ReleaseOriginalFrame(
-    const scoped_refptr<media::VideoFrame>& frame) {
 }
 
 }  // anonymous namespace
 
-// Helper class used for delivering video frames to all registered tracks
-// on the IO-thread.
-class MediaStreamVideoSource::FrameDeliverer : public VideoFrameDeliverer {
- public:
-  FrameDeliverer(
-      const scoped_refptr<base::MessageLoopProxy>& io_message_loop)
-      : VideoFrameDeliverer(io_message_loop)  {
-  }
-
-  // Register |callback| to receive video frames of max size
-  // |max_frame_output_size| on the IO thread.
-  // TODO(perkj): Currently |max_frame_output_size| must be the same for all
-  // |callbacks|.
-  void AddCallback(void* id,
-                   const VideoCaptureDeliverFrameCB& callback,
-                   const gfx::Size& max_frame_output_size) {
-    DCHECK(thread_checker().CalledOnValidThread());
-    io_message_loop()->PostTask(
-        FROM_HERE,
-        base::Bind(
-            &FrameDeliverer::AddCallbackWithResolutionOnIO,
-            this, id, callback, max_frame_output_size));
-  }
-
-  virtual void DeliverFrameOnIO(
-      const scoped_refptr<media::VideoFrame>& frame,
-      const media::VideoCaptureFormat& format) OVERRIDE {
-    DCHECK(io_message_loop()->BelongsToCurrentThread());
-    TRACE_EVENT0("video", "MediaStreamVideoSource::DeliverFrameOnIO");
-    if (max_output_size_.IsEmpty())
-      return;  // Frame received before the output has been decided.
-
-    scoped_refptr<media::VideoFrame> video_frame(frame);
-    const gfx::Size& visible_size = frame->visible_rect().size();
-    if (visible_size.width() > max_output_size_.width() ||
-        visible_size.height() > max_output_size_.height()) {
-      // If |frame| is not the size that is expected, we need to crop it by
-      // providing a new |visible_rect|. The new visible rect must be within the
-      // original |visible_rect|.
-      gfx::Rect output_rect = frame->visible_rect();
-      output_rect.ClampToCenteredSize(max_output_size_);
-      // TODO(perkj): Allow cropping of textures once http://crbug/362521 is
-      // fixed.
-      if (frame->format() != media::VideoFrame::NATIVE_TEXTURE) {
-        video_frame = media::VideoFrame::WrapVideoFrame(
-            frame,
-            output_rect,
-            output_rect.size(),
-            base::Bind(&ReleaseOriginalFrame, frame));
-      }
-    }
-    VideoFrameDeliverer::DeliverFrameOnIO(video_frame, format);
-  }
-
- protected:
-  virtual ~FrameDeliverer() {
-  }
-
-  void AddCallbackWithResolutionOnIO(
-      void* id,
-     const VideoCaptureDeliverFrameCB& callback,
-      const gfx::Size& max_frame_output_size) {
-    DCHECK(io_message_loop()->BelongsToCurrentThread());
-    // Currently we only support one frame output size.
-    DCHECK(!max_frame_output_size.IsEmpty() &&
-           (max_output_size_.IsEmpty() ||
-            max_output_size_ == max_frame_output_size));
-    max_output_size_ = max_frame_output_size;
-    VideoFrameDeliverer::AddCallbackOnIO(id, callback);
-  }
-
- private:
-  gfx::Size max_output_size_;
-};
-
 // static
 MediaStreamVideoSource* MediaStreamVideoSource::GetVideoSource(
     const blink::WebMediaStreamSource& source) {
@@ -388,9 +362,8 @@ bool MediaStreamVideoSource::IsConstraintSupported(const std::string& name) {
 
 MediaStreamVideoSource::MediaStreamVideoSource()
     : state_(NEW),
-      frame_deliverer_(
-          new MediaStreamVideoSource::FrameDeliverer(
-              ChildProcess::current()->io_message_loop_proxy())),
+      track_adapter_(new VideoTrackAdapter(
+          ChildProcess::current()->io_message_loop_proxy())),
       weak_factory_(this) {
 }
 
@@ -450,9 +423,20 @@ void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack* video_track) {
       std::find(tracks_.begin(), tracks_.end(), video_track);
   DCHECK(it != tracks_.end());
   tracks_.erase(it);
-  // Call |RemoveCallback| here even if adding the track has failed and
-  // frame_deliverer_->AddCallback has not been called.
-  frame_deliverer_->RemoveCallback(video_track);
+
+  // Check if the track is waiting for applying new constraints and remove
+  // it in that case.

[tommi (sloooow) - chröme, 2014/05/08 11:57:58]

remove them? (and not 'it')

[perkj_chrome, 2014/05/08 13:36:25]

Refrased

+  for (std::vector<RequestedConstraints>::iterator it =
+           requested_constraints_.begin();
+       it != requested_constraints_.end(); ++it) {
+    if (it->track == video_track) {
+      requested_constraints_.erase(it);
+      break;
+    }
+  }
+  // Call |frame_adapter_->RemoveTrack| here even if adding the track has
+  // failed and |frame_adapter_->AddCallback| has not been called.
+  track_adapter_->RemoveTrack(video_track);
 
   if (tracks_.empty())
     StopSource();
@@ -460,7 +444,7 @@ void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack* video_track) {
 
 const scoped_refptr<base::MessageLoopProxy>&
 MediaStreamVideoSource::io_message_loop() const {
-  return frame_deliverer_->io_message_loop();
+  return track_adapter_->io_message_loop();
 }
 
 void MediaStreamVideoSource::DoStopSource() {
@@ -480,9 +464,7 @@ void MediaStreamVideoSource::OnSupportedFormats(
 
   supported_formats_ = formats;
   if (!FindBestFormatWithConstraints(supported_formats_,
-                                     &current_format_,
-                                     &max_frame_output_size_,
-                                     &current_constraints_)) {
+                                     &current_format_)) {
     SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
     // This object can be deleted after calling FinalizeAddTrack. See comment
     // in the header file.
@@ -500,15 +482,12 @@ void MediaStreamVideoSource::OnSupportedFormats(
   params.requested_format = current_format_;
   StartSourceImpl(
       params,
-      base::Bind(&MediaStreamVideoSource::FrameDeliverer::DeliverFrameOnIO,
-                 frame_deliverer_));
+      base::Bind(&VideoTrackAdapter::DeliverFrameOnIO, track_adapter_));
 }
 
 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
     const media::VideoCaptureFormats& formats,
-    media::VideoCaptureFormat* best_format,
-    gfx::Size* max_frame_output_size,
-    blink::WebMediaConstraints* resulting_constraints) {
+    media::VideoCaptureFormat* best_format) {
   // Find the first constraints that we can fulfill.
   for (std::vector<RequestedConstraints>::iterator request_it =
            requested_constraints_.begin();
@@ -521,9 +500,6 @@ bool MediaStreamVideoSource::FindBestFormatWithConstraints(
     // we will start with whatever format is native to the source.
     if (formats.empty() && !HasMandatoryConstraints(requested_constraints)) {
       *best_format = media::VideoCaptureFormat();
-      *resulting_constraints = requested_constraints;
-      *max_frame_output_size = gfx::Size(std::numeric_limits<int>::max(),
-                                         std::numeric_limits<int>::max());
       return true;
     }
     media::VideoCaptureFormats filtered_formats =
@@ -532,9 +508,7 @@ bool MediaStreamVideoSource::FindBestFormatWithConstraints(
       // A request with constraints that can be fulfilled.
       GetBestCaptureFormat(filtered_formats,
                            requested_constraints,
-                           best_format,
-                           max_frame_output_size);
-      *resulting_constraints= requested_constraints;
+                           best_format);
       return true;
     }
   }
@@ -576,11 +550,23 @@ void MediaStreamVideoSource::FinalizeAddTrack() {
         ((!current_format_.IsValid() && !HasMandatoryConstraints(
             it->constraints)) ||
          !FilterFormats(it->constraints, formats).empty());
+
     if (success) {
-      frame_deliverer_->AddCallback(it->track, it->frame_callback,
-                                    max_frame_output_size_);
+      int max_width;
+      int max_height;
+      GetDesiredMaxWidthAndHeight(it->constraints, &max_width, &max_height);
+      double max_aspect_ratio;
+      double min_aspect_ratio;
+      GetDesiredMinAndMaxAspectRatio(it->constraints,
+                                     &min_aspect_ratio,
+                                     &max_aspect_ratio);
+      track_adapter_->AddTrack(it->track,it->frame_callback,
+                               max_width, max_height,
+                               min_aspect_ratio, max_aspect_ratio);
     }
+
     DVLOG(3) << "FinalizeAddTrack() success " << success;
+
     if (!it->callback.is_null())
       it->callback.Run(this, success);
   }