Change MediaStreamVideoSource to output different resolutions to different tracks depending on the …

content/renderer/media/media_stream_video_source.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/media_stream_video_source.h"
 
 #include <algorithm>
 #include <limits>
 #include <string>
 
 #include "base/debug/trace_event.h"
 #include "base/logging.h"
 #include "base/strings/string_number_conversions.h"
-#include "content/renderer/media/media_stream_dependency_factory.h"
 #include "content/renderer/media/media_stream_video_track.h"
-#include "content/renderer/media/video_frame_deliverer.h"
-#include "content/renderer/media/webrtc/webrtc_video_capturer_adapter.h"
+#include "content/renderer/media/video_track_adapter.h"
 
 namespace content {
 
 // Constraint keys. Specified by draft-alvestrand-constraints-resolution-00b
 const char MediaStreamVideoSource::kMinAspectRatio[] = "minAspectRatio";
 const char MediaStreamVideoSource::kMaxAspectRatio[] = "maxAspectRatio";
 const char MediaStreamVideoSource::kMaxWidth[] = "maxWidth";
 const char MediaStreamVideoSource::kMinWidth[] = "minWidth";
 const char MediaStreamVideoSource::kMaxHeight[] = "maxHeight";
 const char MediaStreamVideoSource::kMinHeight[] = "minHeight";
@@ skipping 54 matching lines @@
     return true;
   }
 
   // Ignore Chrome specific Tab capture constraints.
   if (constraint_name == kMediaStreamSource ||
       constraint_name == kMediaStreamSourceId)
     return true;
 
   if (constraint_name == MediaStreamVideoSource::kMinAspectRatio ||
       constraint_name == MediaStreamVideoSource::kMaxAspectRatio) {
-    double double_value = 0;
-    base::StringToDouble(constraint_value, &double_value);
-
-    // The aspect ratio in |constraint.m_value| has been converted to a string
-    // and back to a double, so it may have a rounding error.
-    // E.g if the value 1/3 is converted to a string, the string will not have
-    // infinite length.
-    // We add a margin of 0.0005 which is high enough to detect the same aspect
-    // ratio but small enough to avoid matching wrong aspect ratios.
-    const double kRoundingTruncation = 0.0005;
-    double ratio = static_cast<double>(format->frame_size.width()) /
-        format->frame_size.height();
-    if (constraint_name == MediaStreamVideoSource::kMinAspectRatio)
-      return (double_value <= ratio + kRoundingTruncation);
-    // Subtract 0.0005 to avoid rounding problems. Same as above.
-    return (double_value >= ratio - kRoundingTruncation);
+    return true;
   }
 
   int value;
   if (!base::StringToInt(constraint_value, &value)) {
     DLOG(WARNING) << "Can't parse MediaStream constraint. Name:"
                   <<  constraint_name << " Value:" << constraint_value;
     return false;
   }
   if (constraint_name == MediaStreamVideoSource::kMinWidth) {
     return (value <= format->frame_size.width());
@@ skipping 92 matching lines @@
                         int* value) {
   blink::WebString value_str;
   bool ret = mandatory ?
       constraints.getMandatoryConstraintValue(name, value_str) :
       constraints.getOptionalConstraintValue(name, value_str);
   if (ret)
     base::StringToInt(value_str.utf8(), value);
   return ret;
 }
 
+bool GetConstraintValue(const blink::WebMediaConstraints& constraints,
+                        bool mandatory, const blink::WebString& name,
+                        double* value) {
+  blink::WebString value_str;
+  bool ret = mandatory ?
+      constraints.getMandatoryConstraintValue(name, value_str) :
+      constraints.getOptionalConstraintValue(name, value_str);
+  if (ret)
+    base::StringToDouble(value_str.utf8(), value);
+  return ret;
+}
+
 // Returns true if |constraint| has mandatory constraints.
 bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {
   blink::WebVector<blink::WebMediaConstraint> mandatory_constraints;
   constraints.getMandatoryConstraints(mandatory_constraints);
   return !mandatory_constraints.isEmpty();
 }
 
-// Retrieve the desired max width and height from |constraints|.
+// Retrieve the desired max width and height from |constraints|. If not set,
+// the |desired_width| and |desired_height| is to
+// std::numeric_limits<int>::max();
 void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
                                  int* desired_width, int* desired_height) {
+  *desired_width = std::numeric_limits<int>::max();
+  *desired_height = std::numeric_limits<int>::max();;
   bool mandatory = GetConstraintValue(constraints, true,
                                       MediaStreamVideoSource::kMaxWidth,
                                       desired_width);
   mandatory |= GetConstraintValue(constraints, true,
                                   MediaStreamVideoSource::kMaxHeight,
                                   desired_height);
   if (mandatory)
     return;
 
   GetConstraintValue(constraints, false, MediaStreamVideoSource::kMaxWidth,
                      desired_width);
   GetConstraintValue(constraints, false, MediaStreamVideoSource::kMaxHeight,
                      desired_height);
 }
 
+void GetDesiredMinAndMaxAspectRatio(
+    const blink::WebMediaConstraints& constraints,
+    double* min_aspect_ratio,
+    double* max_aspect_ratio) {
+  *min_aspect_ratio = 0;
+  *max_aspect_ratio = std::numeric_limits<double>::max();
+
+  bool mandatory = GetConstraintValue(constraints, true,
+                                      MediaStreamVideoSource::kMinAspectRatio,
+                                      min_aspect_ratio);
+  mandatory |= GetConstraintValue(constraints, true,
+                                  MediaStreamVideoSource::kMaxAspectRatio,
+                                  max_aspect_ratio);
+  if (!mandatory &&
+      !GetConstraintValue(constraints, false,
+                          MediaStreamVideoSource::kMinAspectRatio,
+                          min_aspect_ratio) &&
+      !GetConstraintValue(constraints, false,
+                          MediaStreamVideoSource::kMaxAspectRatio,
+                          max_aspect_ratio)) {
+    return;
+  }
+  // The aspect ratio in |constraint.m_value| has been converted to a string
+  // and back to a double, so it may have a rounding error.
+  // E.g if the value 1/3 is converted to a string, the string will not have
+  // infinite length.
+  // We add a margin of 0.0005 which is high enough to detect the same aspect
+  // ratio but small enough to avoid matching wrong aspect ratios.
+  const double kRoundingTruncation = 0.0005;
+  if (*min_aspect_ratio != 0)

[mcasas, 2014/05/07 14:10:43]

|min_aspect_ratio| is a double, so this comparison is always true.

[perkj_chrome, 2014/05/08 11:29:47]

Why would it be true? I set it to 0 at line 248?

[mcasas, 2014/05/09 07:46:24]

A float cannot be 0.0 exactly, and the most likely thing is for the
compiler to upgrade your (int) 0 to (float) 0.0, hence hitting this case.

Float comparisons are usually like
if (abs(*min_aspect_ratio) < epsilon) ...

+    *min_aspect_ratio += kRoundingTruncation;
+  if (*max_aspect_ratio != std::numeric_limits<double>::max())
+    *max_aspect_ratio -= kRoundingTruncation;
+}
+
 const media::VideoCaptureFormat& GetBestFormatBasedOnArea(
     const media::VideoCaptureFormats& formats,
     int area) {
   media::VideoCaptureFormats::const_iterator it = formats.begin();
   media::VideoCaptureFormats::const_iterator best_it = formats.begin();
   int best_diff = std::numeric_limits<int>::max();
   for (; it != formats.end(); ++it) {
     int diff = abs(area - it->frame_size.width() * it->frame_size.height());
     if (diff < best_diff) {
       best_diff = diff;
       best_it = it;
     }
   }
   return *best_it;
 }
 
 // Find the format that best matches the default video size.
 // This algorithm is chosen since a resolution must be picked even if no
 // constraints are provided. We don't just select the maximum supported
 // resolution since higher resolutions cost more in terms of complexity and
 // many cameras have lower frame rate and have more noise in the image at
 // their maximum supported resolution.
 void GetBestCaptureFormat(
     const media::VideoCaptureFormats& formats,
     const blink::WebMediaConstraints& constraints,
-    media::VideoCaptureFormat* capture_format,
-    gfx::Size* max_frame_output_size) {
+    media::VideoCaptureFormat* capture_format) {
   DCHECK(!formats.empty());
-  DCHECK(max_frame_output_size);
 
   int max_width = std::numeric_limits<int>::max();
   int max_height = std::numeric_limits<int>::max();;
   GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
 
   *capture_format = GetBestFormatBasedOnArea(
       formats,
       std::min(max_width, MediaStreamVideoSource::kDefaultWidth) *
       std::min(max_height, MediaStreamVideoSource::kDefaultHeight));
-
-  max_frame_output_size->set_width(max_width);
-  max_frame_output_size->set_height(max_height);
-}
-
-// Empty method used for keeping a reference to the original media::VideoFrame
-// in MediaStreamVideoSource::FrameDeliverer::DeliverFrameOnIO if cropping is
-// needed. The reference to |frame| is kept in the closure that calls this
-// method.
-void ReleaseOriginalFrame(
-    const scoped_refptr<media::VideoFrame>& frame) {
 }
 
 }  // anonymous namespace
 
-// Helper class used for delivering video frames to all registered tracks
-// on the IO-thread.
-class MediaStreamVideoSource::FrameDeliverer : public VideoFrameDeliverer {
- public:
-  FrameDeliverer(
-      const scoped_refptr<base::MessageLoopProxy>& io_message_loop)
-      : VideoFrameDeliverer(io_message_loop)  {
-  }
-
-  // Register |callback| to receive video frames of max size
-  // |max_frame_output_size| on the IO thread.
-  // TODO(perkj): Currently |max_frame_output_size| must be the same for all
-  // |callbacks|.
-  void AddCallback(void* id,
-                   const VideoCaptureDeliverFrameCB& callback,
-                   const gfx::Size& max_frame_output_size) {
-    DCHECK(thread_checker().CalledOnValidThread());
-    io_message_loop()->PostTask(
-        FROM_HERE,
-        base::Bind(
-            &FrameDeliverer::AddCallbackWithResolutionOnIO,
-            this, id, callback, max_frame_output_size));
-  }
-
-  virtual void DeliverFrameOnIO(
-      const scoped_refptr<media::VideoFrame>& frame,
-      const media::VideoCaptureFormat& format) OVERRIDE {
-    DCHECK(io_message_loop()->BelongsToCurrentThread());
-    TRACE_EVENT0("video", "MediaStreamVideoSource::DeliverFrameOnIO");
-    if (max_output_size_.IsEmpty())
-      return;  // Frame received before the output has been decided.
-
-    scoped_refptr<media::VideoFrame> video_frame(frame);
-    const gfx::Size& visible_size = frame->visible_rect().size();
-    if (visible_size.width() > max_output_size_.width() ||
-        visible_size.height() > max_output_size_.height()) {
-      // If |frame| is not the size that is expected, we need to crop it by
-      // providing a new |visible_rect|. The new visible rect must be within the
-      // original |visible_rect|.
-      gfx::Rect output_rect = frame->visible_rect();
-      output_rect.ClampToCenteredSize(max_output_size_);
-      // TODO(perkj): Allow cropping of textures once http://crbug/362521 is
-      // fixed.
-      if (frame->format() != media::VideoFrame::NATIVE_TEXTURE) {
-        video_frame = media::VideoFrame::WrapVideoFrame(
-            frame,
-            output_rect,
-            output_rect.size(),
-            base::Bind(&ReleaseOriginalFrame, frame));
-      }
-    }
-    VideoFrameDeliverer::DeliverFrameOnIO(video_frame, format);
-  }
-
- protected:
-  virtual ~FrameDeliverer() {
-  }
-
-  void AddCallbackWithResolutionOnIO(
-      void* id,
-     const VideoCaptureDeliverFrameCB& callback,
-      const gfx::Size& max_frame_output_size) {
-    DCHECK(io_message_loop()->BelongsToCurrentThread());
-    // Currently we only support one frame output size.
-    DCHECK(!max_frame_output_size.IsEmpty() &&
-           (max_output_size_.IsEmpty() ||
-            max_output_size_ == max_frame_output_size));
-    max_output_size_ = max_frame_output_size;
-    VideoFrameDeliverer::AddCallbackOnIO(id, callback);
-  }
-
- private:
-  gfx::Size max_output_size_;
-};
-
 // static
 MediaStreamVideoSource* MediaStreamVideoSource::GetVideoSource(
     const blink::WebMediaStreamSource& source) {
   return static_cast<MediaStreamVideoSource*>(source.extraData());
 }
 
 // static
 bool MediaStreamVideoSource::IsConstraintSupported(const std::string& name) {
   for (size_t i = 0; i < arraysize(kSupportedConstraints); ++i) {
     if (kSupportedConstraints[i] == name)
       return true;
   }
   return false;
 }
 
 MediaStreamVideoSource::MediaStreamVideoSource(
     const scoped_refptr<base::MessageLoopProxy>& io_message_loop)
     : state_(NEW),
-      frame_deliverer_(
-          new MediaStreamVideoSource::FrameDeliverer(io_message_loop)),
+      track_adapter_(
+          new VideoTrackAdapter(io_message_loop)),

[mcasas, 2014/05/07 14:10:43]

nit: Fits in one line.

[perkj_chrome, 2014/05/08 11:29:47]

Done.

       weak_factory_(this) {
 }
 
 MediaStreamVideoSource::~MediaStreamVideoSource() {
   DVLOG(3) << "~MediaStreamVideoSource()";
 }
 
 void MediaStreamVideoSource::AddTrack(
     MediaStreamVideoTrack* track,
     const VideoCaptureDeliverFrameCB& frame_callback,
@@ skipping 39 matching lines @@
     }
   }
 }
 
 void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack* video_track) {
   DCHECK(CalledOnValidThread());
   std::vector<MediaStreamVideoTrack*>::iterator it =
       std::find(tracks_.begin(), tracks_.end(), video_track);
   DCHECK(it != tracks_.end());
   tracks_.erase(it);
-  // Call |RemoveCallback| here even if adding the track has failed and
-  // frame_deliverer_->AddCallback has not been called.
-  frame_deliverer_->RemoveCallback(video_track);
+
+  // Check if the track is waiting for applying new constraints and remove
+  // it in that case.
+  for (std::vector<RequestedConstraints>::iterator it =
+           requested_constraints_.begin();
+       it != requested_constraints_.end(); ++it) {
+    if (it->track == video_track) {
+      requested_constraints_.erase(it);
+      break;
+    }
+  }
+  // Call |frame_adapter_->RemoveTrack| here even if adding the track has
+  // failed and |frame_adapter_->AddCallback| has not been called.
+  track_adapter_->RemoveTrack(video_track);
 
   if (tracks_.empty())
     StopSource();
 }
 
 const scoped_refptr<base::MessageLoopProxy>&
 MediaStreamVideoSource::io_message_loop() const {
-  return frame_deliverer_->io_message_loop();
+  return track_adapter_->io_message_loop();

[mcasas, 2014/05/07 14:10:43]

I think nobody uses this getter anymore.

[perkj_chrome, 2014/05/08 11:29:47]

no- its used by a few of the classes that inherits from MediaStreamVideoSource.

 }
 
 void MediaStreamVideoSource::DoStopSource() {
   DCHECK(CalledOnValidThread());
   DVLOG(3) << "DoStopSource()";
   if (state_ == ENDED)
     return;
   StopSourceImpl();
   state_ = ENDED;
   SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
 }
 
 void MediaStreamVideoSource::OnSupportedFormats(
     const media::VideoCaptureFormats& formats) {
   DCHECK(CalledOnValidThread());
   DCHECK_EQ(RETRIEVING_CAPABILITIES, state_);
 
   supported_formats_ = formats;
   if (!FindBestFormatWithConstraints(supported_formats_,
-                                     &current_format_,
-                                     &max_frame_output_size_,
-                                     &current_constraints_)) {
+                                     &current_format_)) {
     SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
     // This object can be deleted after calling FinalizeAddTrack. See comment
     // in the header file.
     FinalizeAddTrack();
     return;
   }
 
   state_ = STARTING;
   DVLOG(3) << "Starting the capturer with"
            << " width = " << current_format_.frame_size.width()
            << " height = " << current_format_.frame_size.height()
            << " frame rate = " << current_format_.frame_rate;
 
   media::VideoCaptureParams params;
   params.requested_format = current_format_;
   StartSourceImpl(
       params,
-      base::Bind(&MediaStreamVideoSource::FrameDeliverer::DeliverFrameOnIO,
-                 frame_deliverer_));
+      base::Bind(&VideoTrackAdapter::DeliverFrameOnIO, track_adapter_));
 }
 
 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
     const media::VideoCaptureFormats& formats,
-    media::VideoCaptureFormat* best_format,
-    gfx::Size* max_frame_output_size,
-    blink::WebMediaConstraints* resulting_constraints) {
+    media::VideoCaptureFormat* best_format) {
   // Find the first constraints that we can fulfill.
   for (std::vector<RequestedConstraints>::iterator request_it =
            requested_constraints_.begin();
        request_it != requested_constraints_.end(); ++request_it) {
     const blink::WebMediaConstraints& requested_constraints =
         request_it->constraints;
 
     // If the source doesn't support capability enumeration it is still ok if
     // no mandatory constraints have been specified. That just means that
     // we will start with whatever format is native to the source.
     if (formats.empty() && !HasMandatoryConstraints(requested_constraints)) {
       *best_format = media::VideoCaptureFormat();
-      *resulting_constraints = requested_constraints;
-      *max_frame_output_size = gfx::Size(std::numeric_limits<int>::max(),
-                                         std::numeric_limits<int>::max());
       return true;
     }
     media::VideoCaptureFormats filtered_formats =
         FilterFormats(requested_constraints, formats);
     if (filtered_formats.size() > 0) {
       // A request with constraints that can be fulfilled.
       GetBestCaptureFormat(filtered_formats,
                            requested_constraints,
-                           best_format,
-                           max_frame_output_size);
-      *resulting_constraints= requested_constraints;
+                           best_format);
       return true;
     }
   }
   return false;
 }
 
 void MediaStreamVideoSource::OnStartDone(bool success) {
   DCHECK(CalledOnValidThread());
   DVLOG(3) << "OnStartDone({success =" << success << "})";
   if (success) {
@@ skipping 21 matching lines @@
        it != callbacks.end(); ++it) {
     // The track has been added successfully if the source has started and
     // there are either no mandatory constraints and the source doesn't expose
     // its format capabilities, or the constraints and the format match.
     // For example, a remote source doesn't expose its format capabilities.
     bool success =
         state_ == STARTED &&
         ((!current_format_.IsValid() && !HasMandatoryConstraints(
             it->constraints)) ||
          !FilterFormats(it->constraints, formats).empty());
+
     if (success) {
-      frame_deliverer_->AddCallback(it->track, it->frame_callback,
-                                    max_frame_output_size_);
+      int max_width = std::numeric_limits<int>::max();
+      int max_height = std::numeric_limits<int>::max();
+      GetDesiredMaxWidthAndHeight(it->constraints, &max_width, &max_height);

[mcasas, 2014/05/07 14:10:43]

nit: Inside GetDesiredMaxWidthAndHeight(), |max_width| and |max_height| are also
initialised to max() values.

[perkj_chrome, 2014/05/08 11:29:47]

done

+      double max_aspect_ratio = std::numeric_limits<double>::max();
+      double min_aspect_ratio = 0;
+      GetDesiredMinAndMaxAspectRatio(it->constraints,
+                                     &min_aspect_ratio,
+                                     &max_aspect_ratio);
+      track_adapter_->AddTrack(it->track,it->frame_callback,
+                               max_width, max_height,
+                               min_aspect_ratio, max_aspect_ratio);
     }
+
     DVLOG(3) << "FinalizeAddTrack() success " << success;
+
     if (!it->callback.is_null())
       it->callback.Run(this, success);
   }
 }
 
 void MediaStreamVideoSource::SetReadyState(
     blink::WebMediaStreamSource::ReadyState state) {
   if (!owner().isNull()) {
     owner().setReadyState(state);
   }
@@ skipping 11 matching lines @@
     : track(track),
       frame_callback(frame_callback),
       constraints(constraints),
       callback(callback) {
 }
 
 MediaStreamVideoSource::RequestedConstraints::~RequestedConstraints() {
 }
 
 }  // namespace content