Change MediaStreamVideoSource to output different resolutions to different tracks depending on the …

content/renderer/media/media_stream_video_source.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/media_stream_video_source.h"
 
 #include <algorithm>
 #include <limits>
 #include <string>
 
 #include "base/debug/trace_event.h"
 #include "base/logging.h"
 #include "base/strings/string_number_conversions.h"
 #include "content/child/child_process.h"
 #include "content/renderer/media/media_stream_constraints_util.h"
 #include "content/renderer/media/media_stream_video_track.h"
-#include "content/renderer/media/video_frame_deliverer.h"
+#include "content/renderer/media/video_track_adapter.h"
 
 namespace content {
 
 // Constraint keys. Specified by draft-alvestrand-constraints-resolution-00b
 const char MediaStreamVideoSource::kMinAspectRatio[] = "minAspectRatio";
 const char MediaStreamVideoSource::kMaxAspectRatio[] = "maxAspectRatio";
 const char MediaStreamVideoSource::kMaxWidth[] = "maxWidth";
 const char MediaStreamVideoSource::kMinWidth[] = "minWidth";
 const char MediaStreamVideoSource::kMaxHeight[] = "maxHeight";
 const char MediaStreamVideoSource::kMinHeight[] = "minHeight";
@@ skipping 19 matching lines @@
 
 // Google-specific key prefix. Constraints with this prefix are ignored if they
 // are unknown.
 const char kGooglePrefix[] = "goog";
 
 // MediaStreamVideoSource supports cropping of video frames but only up to
 // kMaxCropFactor. Ie - if a constraint is set to maxHeight 360, an original
 // input frame height of max 360 * kMaxCropFactor pixels is accepted.
 const int kMaxCropFactor = 2;
 
+// Returns true if |constraint| has mandatory constraints.
+bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {
+  blink::WebVector<blink::WebMediaConstraint> mandatory_constraints;
+  constraints.getMandatoryConstraints(mandatory_constraints);
+  return !mandatory_constraints.isEmpty();
+}
+
+// Retrieve the desired max width and height from |constraints|. If not set,
+// the |desired_width| and |desired_height| are set to
+// std::numeric_limits<int>::max();
+// If either max width or height is set as a mandatory constraint, the optional
+// constraints are not checked.
+void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
+                                 int* desired_width, int* desired_height) {
+  *desired_width = std::numeric_limits<int>::max();
+  *desired_height = std::numeric_limits<int>::max();
+
+  bool mandatory = GetMandatoryConstraintValueAsInteger(
+      constraints,
+      MediaStreamVideoSource::kMaxWidth,
+      desired_width);
+  mandatory |= GetMandatoryConstraintValueAsInteger(
+      constraints,
+      MediaStreamVideoSource::kMaxHeight,
+      desired_height);
+  if (mandatory)
+    return;
+
+  GetOptionalConstraintValueAsInteger(constraints,
+                                      MediaStreamVideoSource::kMaxWidth,
+                                      desired_width);
+  GetOptionalConstraintValueAsInteger(constraints,
+                                      MediaStreamVideoSource::kMaxHeight,
+                                      desired_height);
+}
+
+// Retrieve the desired max and min aspect ratio from |constraints|. If not set,
+// the |min_aspect_ratio| is set to 0 and |max_aspect_ratio| is set to
+// std::numeric_limits<double>::max();
+// If either min or max aspect ratio is set as a mandatory constraint, the
+// optional constraints are not checked.
+void GetDesiredMinAndMaxAspectRatio(
+    const blink::WebMediaConstraints& constraints,
+    double* min_aspect_ratio,
+    double* max_aspect_ratio) {
+  *min_aspect_ratio = 0;
+  *max_aspect_ratio = std::numeric_limits<double>::max();
+
+  bool mandatory = GetMandatoryConstraintValueAsDouble(
+      constraints,
+      MediaStreamVideoSource::kMinAspectRatio,
+      min_aspect_ratio);
+  mandatory |= GetMandatoryConstraintValueAsDouble(
+      constraints,
+      MediaStreamVideoSource::kMaxAspectRatio,
+      max_aspect_ratio);
+  if (mandatory)
+    return;
+
+  GetOptionalConstraintValueAsDouble(
+      constraints,
+      MediaStreamVideoSource::kMinAspectRatio,
+      min_aspect_ratio);
+  GetOptionalConstraintValueAsDouble(
+      constraints,
+      MediaStreamVideoSource::kMaxAspectRatio,
+      max_aspect_ratio);
+}
+
 // Returns true if |constraint| is fulfilled. |format| can be changed
 // changed by a constraint. Ie - the frame rate can be changed by setting
 // maxFrameRate.
 bool UpdateFormatForConstraint(
     const blink::WebMediaConstraint& constraint,
     bool mandatory,
     media::VideoCaptureFormat* format) {
   DCHECK(format != NULL);
 
   if (!format->IsValid())
@@ skipping 13 matching lines @@
     return true;
   }
 
   // Ignore Chrome specific Tab capture constraints.
   if (constraint_name == kMediaStreamSource ||
       constraint_name == kMediaStreamSourceId)
     return true;
 
   if (constraint_name == MediaStreamVideoSource::kMinAspectRatio ||
       constraint_name == MediaStreamVideoSource::kMaxAspectRatio) {
-    double double_value = 0;
-    base::StringToDouble(constraint_value, &double_value);
-
-    // The aspect ratio in |constraint.m_value| has been converted to a string
-    // and back to a double, so it may have a rounding error.
-    // E.g if the value 1/3 is converted to a string, the string will not have
-    // infinite length.
-    // We add a margin of 0.0005 which is high enough to detect the same aspect
-    // ratio but small enough to avoid matching wrong aspect ratios.
-    const double kRoundingTruncation = 0.0005;
-    double ratio = static_cast<double>(format->frame_size.width()) /
-        format->frame_size.height();
-    if (constraint_name == MediaStreamVideoSource::kMinAspectRatio)
-      return (double_value <= ratio + kRoundingTruncation);
-    // Subtract 0.0005 to avoid rounding problems. Same as above.
-    return (double_value >= ratio - kRoundingTruncation);
+    // These constraints are handled by cropping if the camera outputs the wrong
+    // aspect ratio.
+    double value;
+    return base::StringToDouble(constraint_value, &value);
   }
 
   int value;
   if (!base::StringToInt(constraint_value, &value)) {
     DLOG(WARNING) << "Can't parse MediaStream constraint. Name:"
                   <<  constraint_name << " Value:" << constraint_value;
     return false;
   }
   if (constraint_name == MediaStreamVideoSource::kMinWidth) {
     return (value <= format->frame_size.width());
@@ skipping 49 matching lines @@
 }
 
 // Returns the media::VideoCaptureFormats that matches |constraints|.
 media::VideoCaptureFormats FilterFormats(
     const blink::WebMediaConstraints& constraints,
     const media::VideoCaptureFormats& supported_formats) {
   if (constraints.isNull()) {
     return supported_formats;
   }
 
+  double max_aspect_ratio;
+  double min_aspect_ratio;
+  GetDesiredMinAndMaxAspectRatio(constraints,
+                                 &min_aspect_ratio,
+                                 &max_aspect_ratio);
+
+  if (min_aspect_ratio > max_aspect_ratio || max_aspect_ratio < 0.05f) {
+    DLOG(WARNING) << "Wrong requested aspect ratio.";
+    return media::VideoCaptureFormats();
+  }
+
+  int min_width = 0;
+  GetMandatoryConstraintValueAsInteger(constraints,
+                                       MediaStreamVideoSource::kMinWidth,
+                                       &min_width);
+  int min_height = 0;
+  GetMandatoryConstraintValueAsInteger(constraints,
+                                       MediaStreamVideoSource::kMinHeight,
+                                       &min_height);
+  int max_width;
+  int max_height;
+  GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
+
+  if (min_width > max_width || min_height > max_height)
+    return media::VideoCaptureFormats();
+
   blink::WebVector<blink::WebMediaConstraint> mandatory;
   blink::WebVector<blink::WebMediaConstraint> optional;
   constraints.getMandatoryConstraints(mandatory);
   constraints.getOptionalConstraints(optional);
-
   media::VideoCaptureFormats candidates = supported_formats;
-
   for (size_t i = 0; i < mandatory.size(); ++i)
     FilterFormatsByConstraint(mandatory[i], true, &candidates);
 
   if (candidates.empty())
     return candidates;
 
   // Ok - all mandatory checked and we still have candidates.
   // Let's try filtering using the optional constraints. The optional
   // constraints must be filtered in the order they occur in |optional|.
   // But if a constraint produce zero candidates, the constraint is ignored and
   // the next constraint is tested.
   // http://dev.w3.org/2011/webrtc/editor/getusermedia.html#idl-def-Constraints
   for (size_t i = 0; i < optional.size(); ++i) {
     media::VideoCaptureFormats current_candidates = candidates;
     FilterFormatsByConstraint(optional[i], false, &current_candidates);
     if (!current_candidates.empty()) {
       candidates = current_candidates;
     }
   }
 
   // We have done as good as we can to filter the supported resolutions.
   return candidates;
 }
 
-// Returns true if |constraint| has mandatory constraints.
-bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {
-  blink::WebVector<blink::WebMediaConstraint> mandatory_constraints;
-  constraints.getMandatoryConstraints(mandatory_constraints);
-  return !mandatory_constraints.isEmpty();
-}
-
-// Retrieve the desired max width and height from |constraints|.
-void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
-                                 int* desired_width, int* desired_height) {
-  bool mandatory = GetMandatoryConstraintValueAsInteger(
-      constraints, MediaStreamVideoSource::kMaxWidth, desired_width);
-  mandatory |= GetMandatoryConstraintValueAsInteger(
-      constraints, MediaStreamVideoSource::kMaxHeight, desired_height);
-  // Skip the optional constraints if any of the mandatory constraint is
-  // specified.
-  if (mandatory)
-    return;
-
-  GetOptionalConstraintValueAsInteger(constraints,
-                                      MediaStreamVideoSource::kMaxWidth,
-                                      desired_width);
-  GetOptionalConstraintValueAsInteger(constraints,
-                                      MediaStreamVideoSource::kMaxHeight,
-                                      desired_height);
-}
-
 const media::VideoCaptureFormat& GetBestFormatBasedOnArea(
     const media::VideoCaptureFormats& formats,
     int area) {
   media::VideoCaptureFormats::const_iterator it = formats.begin();
   media::VideoCaptureFormats::const_iterator best_it = formats.begin();
   int best_diff = std::numeric_limits<int>::max();
   for (; it != formats.end(); ++it) {
     int diff = abs(area - it->frame_size.width() * it->frame_size.height());
     if (diff < best_diff) {
       best_diff = diff;
       best_it = it;
     }
   }
   return *best_it;
 }
 
 // Find the format that best matches the default video size.
 // This algorithm is chosen since a resolution must be picked even if no
 // constraints are provided. We don't just select the maximum supported
 // resolution since higher resolutions cost more in terms of complexity and
 // many cameras have lower frame rate and have more noise in the image at
 // their maximum supported resolution.
 void GetBestCaptureFormat(
     const media::VideoCaptureFormats& formats,
     const blink::WebMediaConstraints& constraints,
-    media::VideoCaptureFormat* capture_format,
-    gfx::Size* max_frame_output_size) {
+    media::VideoCaptureFormat* capture_format) {
   DCHECK(!formats.empty());
-  DCHECK(max_frame_output_size);
 
-  int max_width = std::numeric_limits<int>::max();
-  int max_height = std::numeric_limits<int>::max();;
+  int max_width;
+  int max_height;
   GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
 
   *capture_format = GetBestFormatBasedOnArea(
       formats,
       std::min(max_width, MediaStreamVideoSource::kDefaultWidth) *
       std::min(max_height, MediaStreamVideoSource::kDefaultHeight));
-
-  max_frame_output_size->set_width(max_width);
-  max_frame_output_size->set_height(max_height);
-}
-
-// Empty method used for keeping a reference to the original media::VideoFrame
-// in MediaStreamVideoSource::FrameDeliverer::DeliverFrameOnIO if cropping is
-// needed. The reference to |frame| is kept in the closure that calls this
-// method.
-void ReleaseOriginalFrame(
-    const scoped_refptr<media::VideoFrame>& frame) {
 }
 
 }  // anonymous namespace
 
-// Helper class used for delivering video frames to all registered tracks
-// on the IO-thread.
-class MediaStreamVideoSource::FrameDeliverer : public VideoFrameDeliverer {
- public:
-  FrameDeliverer(
-      const scoped_refptr<base::MessageLoopProxy>& io_message_loop)
-      : VideoFrameDeliverer(io_message_loop)  {
-  }
-
-  // Register |callback| to receive video frames of max size
-  // |max_frame_output_size| on the IO thread.
-  // TODO(perkj): Currently |max_frame_output_size| must be the same for all
-  // |callbacks|.
-  void AddCallback(void* id,
-                   const VideoCaptureDeliverFrameCB& callback,
-                   const gfx::Size& max_frame_output_size) {
-    DCHECK(thread_checker().CalledOnValidThread());
-    io_message_loop()->PostTask(
-        FROM_HERE,
-        base::Bind(
-            &FrameDeliverer::AddCallbackWithResolutionOnIO,
-            this, id, callback, max_frame_output_size));
-  }
-
-  virtual void DeliverFrameOnIO(
-      const scoped_refptr<media::VideoFrame>& frame,
-      const media::VideoCaptureFormat& format) OVERRIDE {
-    DCHECK(io_message_loop()->BelongsToCurrentThread());
-    TRACE_EVENT0("video", "MediaStreamVideoSource::DeliverFrameOnIO");
-    if (max_output_size_.IsEmpty())
-      return;  // Frame received before the output has been decided.
-
-    scoped_refptr<media::VideoFrame> video_frame(frame);
-    const gfx::Size& visible_size = frame->visible_rect().size();
-    if (visible_size.width() > max_output_size_.width() ||
-        visible_size.height() > max_output_size_.height()) {
-      // If |frame| is not the size that is expected, we need to crop it by
-      // providing a new |visible_rect|. The new visible rect must be within the
-      // original |visible_rect|.
-      gfx::Rect output_rect = frame->visible_rect();
-      output_rect.ClampToCenteredSize(max_output_size_);
-      // TODO(perkj): Allow cropping of textures once http://crbug/362521 is
-      // fixed.
-      if (frame->format() != media::VideoFrame::NATIVE_TEXTURE) {
-        video_frame = media::VideoFrame::WrapVideoFrame(
-            frame,
-            output_rect,
-            output_rect.size(),
-            base::Bind(&ReleaseOriginalFrame, frame));
-      }
-    }
-    VideoFrameDeliverer::DeliverFrameOnIO(video_frame, format);
-  }
-
- protected:
-  virtual ~FrameDeliverer() {
-  }
-
-  void AddCallbackWithResolutionOnIO(
-      void* id,
-     const VideoCaptureDeliverFrameCB& callback,
-      const gfx::Size& max_frame_output_size) {
-    DCHECK(io_message_loop()->BelongsToCurrentThread());
-    // Currently we only support one frame output size.
-    DCHECK(!max_frame_output_size.IsEmpty() &&
-           (max_output_size_.IsEmpty() ||
-            max_output_size_ == max_frame_output_size));
-    max_output_size_ = max_frame_output_size;
-    VideoFrameDeliverer::AddCallbackOnIO(id, callback);
-  }
-
- private:
-  gfx::Size max_output_size_;
-};
-
 // static
 MediaStreamVideoSource* MediaStreamVideoSource::GetVideoSource(
     const blink::WebMediaStreamSource& source) {
   return static_cast<MediaStreamVideoSource*>(source.extraData());
 }
 
 // static
 bool MediaStreamVideoSource::IsConstraintSupported(const std::string& name) {
   for (size_t i = 0; i < arraysize(kSupportedConstraints); ++i) {
     if (kSupportedConstraints[i] == name)
       return true;
   }
   return false;
 }
 
 MediaStreamVideoSource::MediaStreamVideoSource()
     : state_(NEW),
-      frame_deliverer_(
-          new MediaStreamVideoSource::FrameDeliverer(
-              ChildProcess::current()->io_message_loop_proxy())),
+      track_adapter_(new VideoTrackAdapter(
+          ChildProcess::current()->io_message_loop_proxy())),
       weak_factory_(this) {
 }
 
 MediaStreamVideoSource::~MediaStreamVideoSource() {
   DVLOG(3) << "~MediaStreamVideoSource()";
 }
 
 void MediaStreamVideoSource::AddTrack(
     MediaStreamVideoTrack* track,
     const VideoCaptureDeliverFrameCB& frame_callback,
@@ skipping 42 matching lines @@
     }
   }
 }
 
 void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack* video_track) {
   DCHECK(CalledOnValidThread());
   std::vector<MediaStreamVideoTrack*>::iterator it =
       std::find(tracks_.begin(), tracks_.end(), video_track);
   DCHECK(it != tracks_.end());
   tracks_.erase(it);
-  // Call |RemoveCallback| here even if adding the track has failed and
-  // frame_deliverer_->AddCallback has not been called.
-  frame_deliverer_->RemoveCallback(video_track);
+
+  // Check if |video_track| is waiting for applying new constraints and remove
+  // the request in that case.
+  for (std::vector<RequestedConstraints>::iterator it =
+           requested_constraints_.begin();
+       it != requested_constraints_.end(); ++it) {
+    if (it->track == video_track) {
+      requested_constraints_.erase(it);
+      break;
+    }
+  }
+  // Call |frame_adapter_->RemoveTrack| here even if adding the track has
+  // failed and |frame_adapter_->AddCallback| has not been called.
+  track_adapter_->RemoveTrack(video_track);
 
   if (tracks_.empty())
     StopSource();
 }
 
 const scoped_refptr<base::MessageLoopProxy>&
 MediaStreamVideoSource::io_message_loop() const {
-  return frame_deliverer_->io_message_loop();
+  DCHECK(CalledOnValidThread());
+  return track_adapter_->io_message_loop();
 }
 
 void MediaStreamVideoSource::DoStopSource() {
   DCHECK(CalledOnValidThread());
   DVLOG(3) << "DoStopSource()";
   if (state_ == ENDED)
     return;
   StopSourceImpl();
   state_ = ENDED;
   SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
 }
 
 void MediaStreamVideoSource::OnSupportedFormats(
     const media::VideoCaptureFormats& formats) {
   DCHECK(CalledOnValidThread());
   DCHECK_EQ(RETRIEVING_CAPABILITIES, state_);
 
   supported_formats_ = formats;
   if (!FindBestFormatWithConstraints(supported_formats_,
-                                     &current_format_,
-                                     &max_frame_output_size_,
-                                     &current_constraints_)) {
+                                     &current_format_)) {
     SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
     // This object can be deleted after calling FinalizeAddTrack. See comment
     // in the header file.
     FinalizeAddTrack();
     return;
   }
 
   state_ = STARTING;
   DVLOG(3) << "Starting the capturer with"
            << " width = " << current_format_.frame_size.width()
            << " height = " << current_format_.frame_size.height()
            << " frame rate = " << current_format_.frame_rate;
 
   media::VideoCaptureParams params;
   params.requested_format = current_format_;
   StartSourceImpl(
       params,
-      base::Bind(&MediaStreamVideoSource::FrameDeliverer::DeliverFrameOnIO,
-                 frame_deliverer_));
+      base::Bind(&VideoTrackAdapter::DeliverFrameOnIO, track_adapter_));
 }
 
 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
     const media::VideoCaptureFormats& formats,
-    media::VideoCaptureFormat* best_format,
-    gfx::Size* max_frame_output_size,
-    blink::WebMediaConstraints* resulting_constraints) {
+    media::VideoCaptureFormat* best_format) {
   // Find the first constraints that we can fulfill.
   for (std::vector<RequestedConstraints>::iterator request_it =
            requested_constraints_.begin();
        request_it != requested_constraints_.end(); ++request_it) {
     const blink::WebMediaConstraints& requested_constraints =
         request_it->constraints;
 
     // If the source doesn't support capability enumeration it is still ok if
     // no mandatory constraints have been specified. That just means that
     // we will start with whatever format is native to the source.
     if (formats.empty() && !HasMandatoryConstraints(requested_constraints)) {
       *best_format = media::VideoCaptureFormat();
-      *resulting_constraints = requested_constraints;
-      *max_frame_output_size = gfx::Size(std::numeric_limits<int>::max(),
-                                         std::numeric_limits<int>::max());
       return true;
     }
     media::VideoCaptureFormats filtered_formats =
         FilterFormats(requested_constraints, formats);
     if (filtered_formats.size() > 0) {
       // A request with constraints that can be fulfilled.
       GetBestCaptureFormat(filtered_formats,
                            requested_constraints,
-                           best_format,
-                           max_frame_output_size);
-      *resulting_constraints= requested_constraints;
+                           best_format);
       return true;
     }
   }
   return false;
 }
 
 void MediaStreamVideoSource::OnStartDone(bool success) {
   DCHECK(CalledOnValidThread());
   DVLOG(3) << "OnStartDone({success =" << success << "})";
   if (success) {
@@ skipping 21 matching lines @@
        it != callbacks.end(); ++it) {
     // The track has been added successfully if the source has started and
     // there are either no mandatory constraints and the source doesn't expose
     // its format capabilities, or the constraints and the format match.
     // For example, a remote source doesn't expose its format capabilities.
     bool success =
         state_ == STARTED &&
         ((!current_format_.IsValid() && !HasMandatoryConstraints(
             it->constraints)) ||
          !FilterFormats(it->constraints, formats).empty());
+
     if (success) {
-      frame_deliverer_->AddCallback(it->track, it->frame_callback,
-                                    max_frame_output_size_);
+      int max_width;
+      int max_height;
+      GetDesiredMaxWidthAndHeight(it->constraints, &max_width, &max_height);
+      double max_aspect_ratio;
+      double min_aspect_ratio;
+      GetDesiredMinAndMaxAspectRatio(it->constraints,
+                                     &min_aspect_ratio,
+                                     &max_aspect_ratio);
+      track_adapter_->AddTrack(it->track,it->frame_callback,
+                               max_width, max_height,
+                               min_aspect_ratio, max_aspect_ratio);
     }
+
     DVLOG(3) << "FinalizeAddTrack() success " << success;
+
     if (!it->callback.is_null())
       it->callback.Run(this, success);
   }
 }
 
 void MediaStreamVideoSource::SetReadyState(
     blink::WebMediaStreamSource::ReadyState state) {
   if (!owner().isNull()) {
     owner().setReadyState(state);
   }
@@ skipping 11 matching lines @@
     : track(track),
       frame_callback(frame_callback),
       constraints(constraints),
       callback(callback) {
 }
 
 MediaStreamVideoSource::RequestedConstraints::~RequestedConstraints() {
 }
 
 }  // namespace content