Apply new-style constraints to video_source.

content/renderer/media/media_stream_video_source.cc

Index: content/renderer/media/media_stream_video_source.cc
diff --git a/content/renderer/media/media_stream_video_source.cc b/content/renderer/media/media_stream_video_source.cc
index 38f19b0d301357bc43d747d02187563a02cc6098..6660db6fc6cc9ed38889a7d173aee07c14b61e59 100644
--- a/content/renderer/media/media_stream_video_source.cc
+++ b/content/renderer/media/media_stream_video_source.cc
@@ -13,7 +13,6 @@
 #include "base/strings/string_number_conversions.h"
 #include "base/trace_event/trace_event.h"
 #include "content/child/child_process.h"
-#include "content/renderer/media/media_stream_constraints_util.h"
 #include "content/renderer/media/media_stream_video_track.h"
 #include "content/renderer/media/video_track_adapter.h"
 
@@ -43,15 +42,14 @@ const char* kSupportedConstraints[] = {
 
 namespace {
 
-// Google-specific key prefix. Constraints with this prefix are ignored if they
-// are unknown.
-const char kGooglePrefix[] = "goog";
+const char* kLegalVideoConstraints[] = {
+    "width",      "height",   "aspectRatio", "frameRate",
+    "facingMode", "deviceId", "groupId",     "mediaStreamSource",
+};
 
 // Returns true if |constraint| has mandatory constraints.
 bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {
-  blink::WebVector<blink::WebMediaConstraint> mandatory_constraints;
-  constraints.getMandatoryConstraints(mandatory_constraints);
-  return !mandatory_constraints.isEmpty();
+  return constraints.basic().hasMandatory();
 }
 
 // Retrieve the desired max width and height from |constraints|. If not set,
@@ -64,23 +62,21 @@ void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
   *desired_width = std::numeric_limits<int>::max();
   *desired_height = std::numeric_limits<int>::max();
 
-  bool mandatory = GetMandatoryConstraintValueAsInteger(
-      constraints,
-      MediaStreamVideoSource::kMaxWidth,
-      desired_width);
-  mandatory |= GetMandatoryConstraintValueAsInteger(
-      constraints,
-      MediaStreamVideoSource::kMaxHeight,
-      desired_height);
-  if (mandatory)
+  const auto& basic_constraints = constraints.basic();
+  if (basic_constraints.width.hasMax() || basic_constraints.height.hasMax()) {
+    if (basic_constraints.width.hasMax())
+      *desired_width = basic_constraints.width.max();
+    if (basic_constraints.height.hasMax())
+      *desired_height = basic_constraints.height.max();
     return;
+  }
 
-  GetOptionalConstraintValueAsInteger(constraints,
-                                      MediaStreamVideoSource::kMaxWidth,
-                                      desired_width);
-  GetOptionalConstraintValueAsInteger(constraints,
-                                      MediaStreamVideoSource::kMaxHeight,
-                                      desired_height);
+  for (const auto& constraint_set : constraints.advanced()) {
+    if (constraint_set.width.hasMax())
+      *desired_width = constraint_set.width.max();
+    if (constraint_set.height.hasMax())
+      *desired_height = constraint_set.height.max();
+  }
 }
 
 // Retrieve the desired max and min aspect ratio from |constraints|. If not set,
@@ -95,117 +91,95 @@ void GetDesiredMinAndMaxAspectRatio(
   *min_aspect_ratio = 0;
   *max_aspect_ratio = std::numeric_limits<double>::max();
 
-  bool mandatory = GetMandatoryConstraintValueAsDouble(
-      constraints,
-      MediaStreamVideoSource::kMinAspectRatio,
-      min_aspect_ratio);
-  mandatory |= GetMandatoryConstraintValueAsDouble(
-      constraints,
-      MediaStreamVideoSource::kMaxAspectRatio,
-      max_aspect_ratio);
-  if (mandatory)
+  if (constraints.basic().aspectRatio.hasMin() ||
+      constraints.basic().aspectRatio.hasMax()) {
+    if (constraints.basic().aspectRatio.hasMin())
+      *min_aspect_ratio = constraints.basic().aspectRatio.min();
+    if (constraints.basic().aspectRatio.hasMax())
+      *max_aspect_ratio = constraints.basic().aspectRatio.max();
     return;
-
-  GetOptionalConstraintValueAsDouble(constraints,
-                                     MediaStreamVideoSource::kMinAspectRatio,
-                                     min_aspect_ratio);
-  GetOptionalConstraintValueAsDouble(constraints,
-                                     MediaStreamVideoSource::kMaxAspectRatio,
-                                     max_aspect_ratio);
+    // Note - the code will ignore attempts at successive refinement
+    // of the aspect ratio with advanced constraint. This may be wrong.
+  }
+  // Note - the code below will potentially pick min and max from different
+  // constraint sets, some of which might have been ignored.
+  for (const auto& constraint_set : constraints.advanced()) {
+    if (constraint_set.aspectRatio.hasMin()) {
+      *min_aspect_ratio = constraint_set.aspectRatio.min();
+      break;
+    }
+  }
+  for (const auto& constraint_set : constraints.advanced()) {
+    if (constraint_set.aspectRatio.hasMax()) {
+      *max_aspect_ratio = constraint_set.aspectRatio.max();
+      break;
+    }
+  }
 }
 
-// Returns true if |constraint| is fulfilled. |format| can be changed by a
+// Returns true if |constraints| are fulfilled. |format| can be changed by a
 // constraint, e.g. the frame rate can be changed by setting maxFrameRate.
-bool UpdateFormatForConstraint(const blink::WebMediaConstraint& constraint,
-                               bool mandatory,
-                               media::VideoCaptureFormat* format) {
+bool UpdateFormatForConstraints(
+    const blink::WebMediaTrackConstraintSet& constraints,
+    media::VideoCaptureFormat* format,
+    blink::WebString* failing_constraint_name_out) {
   DCHECK(format != NULL);
 
   if (!format->IsValid())
     return false;
 
-  const std::string constraint_name = constraint.m_name.utf8();
-  const std::string constraint_value = constraint.m_value.utf8();
-
-  if (constraint_name.find(kGooglePrefix) == 0) {
-    // These are actually options, not constraints, so they can be satisfied
-    // regardless of the format.
-    return true;
-  }
-
-  if (constraint_name == MediaStreamSource::kSourceId) {
-    // This is a constraint that doesn't affect the format.
-    return true;
-  }
-
-  // Ignore Chrome specific Tab capture constraints.
-  if (constraint_name == kMediaStreamSource ||
-      constraint_name == kMediaStreamSourceId)
-    return true;
-
-  if (constraint_name == MediaStreamVideoSource::kMinAspectRatio ||
-      constraint_name == MediaStreamVideoSource::kMaxAspectRatio) {
-    // These constraints are handled by cropping if the camera outputs the wrong
-    // aspect ratio.
-    double value;
-    return base::StringToDouble(constraint_value, &value);
-  }
-
-  double value = 0.0;
-  if (!base::StringToDouble(constraint_value, &value)) {
-    DLOG(WARNING) << "Can't parse MediaStream constraint. Name:"
-                  <<  constraint_name << " Value:" << constraint_value;
-    return false;
-  }
-
-  if (constraint_name == MediaStreamVideoSource::kMinWidth) {
-    return (value <= format->frame_size.width());
-  } else if (constraint_name == MediaStreamVideoSource::kMaxWidth) {
-    return value > 0.0;
-  } else if (constraint_name == MediaStreamVideoSource::kMinHeight) {
-    return (value <= format->frame_size.height());
-  } else if (constraint_name == MediaStreamVideoSource::kMaxHeight) {
-     return value > 0.0;
-  } else if (constraint_name == MediaStreamVideoSource::kMinFrameRate) {
-    return (value > 0.0) && (value <= format->frame_rate);
-  } else if (constraint_name == MediaStreamVideoSource::kMaxFrameRate) {
-    if (value <= 0.0) {
-      // The frame rate is set by constraint.
-      // Don't allow 0 as frame rate if it is a mandatory constraint.
-      // Set the frame rate to 1 if it is not mandatory.
-      if (mandatory) {
-        return false;
-      } else {
-        value = 1.0;
+  const char* failing_constraint_name = nullptr;
+
+  // The width and height are matched based on cropping occuring later:
+  // min width/height has to be >= the size of the frame (no upscale).
+  // max width/height just has to be > 0 (we can crop anything too large).
+  if ((constraints.width.hasMin() &&
+       constraints.width.min() > format->frame_size.width()) ||
+      (constraints.width.hasMax() && constraints.width.max() <= 0))
+    failing_constraint_name = constraints.width.name();

[mcasas, 2016/01/28 01:57:57]

{} here and around l. 143

[hta - Chromium, 2016/01/28 08:51:40]

https://google.github.io/styleguide/cppguide.html#Conditionals

"However, if one part of an if-else statement uses curly braces, the other part
must too"

Done.

+  else if ((constraints.height.hasMin() &&
+            constraints.height.min() > format->frame_size.height()) ||
+           (constraints.height.hasMax() && constraints.height.max() <= 0))
+    failing_constraint_name = constraints.height.name();
+  else if (!constraints.frameRate.matches(format->frame_rate)) {
+    if (constraints.frameRate.hasMax()) {
+      const double value = constraints.frameRate.max();
+      // TODO(hta): Check if handling of max = 0.0 is relevant.
+      // (old handling was to set rate to 1.0 if 0.0 was specified)
+      if (constraints.frameRate.matches(value)) {
+        format->frame_rate =
+            (format->frame_rate > value) ? value : format->frame_rate;
+        return true;
       }
     }
-    format->frame_rate =
-        (format->frame_rate > value) ? value : format->frame_rate;
+    failing_constraint_name = constraints.frameRate.name();
+  } else
     return true;
-  } else {
-    LOG(WARNING) << "Found unknown MediaStream constraint. Name:"
-                 <<  constraint_name << " Value:" << constraint_value;
-    return false;
+
+  DCHECK(failing_constraint_name);
+  if (failing_constraint_name_out) {
+    *failing_constraint_name_out =
+        blink::WebString::fromUTF8(failing_constraint_name);
   }
+  return false;
 }
 
 // Removes media::VideoCaptureFormats from |formats| that don't meet
-// |constraint|.
-void FilterFormatsByConstraint(const blink::WebMediaConstraint& constraint,
-                               bool mandatory,
-                               media::VideoCaptureFormats* formats) {
-  DVLOG(3) << "FilterFormatsByConstraint("
-           << "{ constraint.m_name = " << constraint.m_name.utf8()
-           << "  constraint.m_value = " << constraint.m_value.utf8()
-           << "  mandatory =  " << mandatory << "})";
+// |constraints|.
+void FilterFormatsByConstraints(
+    const blink::WebMediaTrackConstraintSet& constraints,
+    media::VideoCaptureFormats* formats,
+    blink::WebString* failing_constraint_name) {
   media::VideoCaptureFormats::iterator format_it = formats->begin();
   while (format_it != formats->end()) {
     // Modify the format_it to fulfill the constraint if possible.

[mcasas, 2016/01/28 01:57:57]

Modify |format_it| to fulfill the constraints ...

[hta - Chromium, 2016/01/28 08:51:40]

Done.

     // Delete it otherwise.
-    if (!UpdateFormatForConstraint(constraint, mandatory, &(*format_it)))
+    if (!UpdateFormatForConstraints(constraints, &(*format_it),
+                                    failing_constraint_name)) {
       format_it = formats->erase(format_it);
-    else
+    } else {
       ++format_it;
+    }
   }
 }
 
@@ -217,6 +191,24 @@ media::VideoCaptureFormats FilterFormats(
   if (constraints.isNull())
     return supported_formats;
 
+  const auto& basic = constraints.basic();
+
+  // Do some checks that won't be done when filtering candidates.
+
+  if (basic.width.hasMin() && basic.width.hasMax() &&
+      basic.width.min() > basic.width.max()) {
+    if (unsatisfied_constraint)
+      *unsatisfied_constraint = blink::WebString::fromUTF8(basic.width.name());
+    return media::VideoCaptureFormats();
+  }
+
+  if (basic.height.hasMin() && basic.height.hasMax() &&
+      basic.height.min() > basic.height.max()) {
+    if (unsatisfied_constraint)
+      *unsatisfied_constraint = blink::WebString::fromUTF8(basic.height.name());
+    return media::VideoCaptureFormats();
+  }
+
   double max_aspect_ratio;
   double min_aspect_ratio;
   GetDesiredMinAndMaxAspectRatio(constraints,
@@ -224,64 +216,44 @@ media::VideoCaptureFormats FilterFormats(
                                  &max_aspect_ratio);
 
   if (min_aspect_ratio > max_aspect_ratio || max_aspect_ratio < 0.05f) {
-    DLOG(WARNING) << "Wrong requested aspect ratio.";
+    DLOG(WARNING) << "Wrong requested aspect ratio: min " << min_aspect_ratio
+                  << " max " << max_aspect_ratio;
+    if (unsatisfied_constraint) {
+      *unsatisfied_constraint =
+          blink::WebString::fromUTF8(basic.aspectRatio.name());
+    }
     return media::VideoCaptureFormats();
   }
 
-  int min_width = 0;
-  GetMandatoryConstraintValueAsInteger(constraints,
-                                       MediaStreamVideoSource::kMinWidth,
-                                       &min_width);
-  int min_height = 0;
-  GetMandatoryConstraintValueAsInteger(constraints,
-                                       MediaStreamVideoSource::kMinHeight,
-                                       &min_height);
-  int max_width;
-  int max_height;
-  GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
-
-  if (min_width > max_width || min_height > max_height)
+  std::vector<std::string> temp(
+      &kLegalVideoConstraints[0],
+      &kLegalVideoConstraints[sizeof(kLegalVideoConstraints) /
+                              sizeof(kLegalVideoConstraints[0])]);
+  std::string failing_name;
+  if (basic.hasMandatoryOutsideSet(temp, failing_name)) {
+    if (unsatisfied_constraint)
+      *unsatisfied_constraint = blink::WebString::fromUTF8(failing_name);
     return media::VideoCaptureFormats();
-
-  double min_frame_rate = 0.0f;
-  double max_frame_rate = 0.0f;
-  if (GetConstraintValueAsDouble(constraints,
-                                 MediaStreamVideoSource::kMaxFrameRate,
-                                 &max_frame_rate) &&
-      GetConstraintValueAsDouble(constraints,
-                                 MediaStreamVideoSource::kMinFrameRate,
-                                 &min_frame_rate)) {
-    if (min_frame_rate > max_frame_rate) {
-      DLOG(WARNING) << "Wrong requested frame rate.";
-      return media::VideoCaptureFormats();
-    }
   }
 
-  blink::WebVector<blink::WebMediaConstraint> mandatory;
-  blink::WebVector<blink::WebMediaConstraint> optional;
-  constraints.getMandatoryConstraints(mandatory);
-  constraints.getOptionalConstraints(optional);
   media::VideoCaptureFormats candidates = supported_formats;
-  for (const auto& constraint : mandatory) {
-    FilterFormatsByConstraint(constraint, true, &candidates);
-    if (candidates.empty()) {
-      *unsatisfied_constraint = constraint.m_name;
-      return candidates;
-    }
-  }
+  FilterFormatsByConstraints(basic, &candidates, unsatisfied_constraint);
 
   if (candidates.empty())
     return candidates;
 
   // Ok - all mandatory checked and we still have candidates.
-  // Let's try filtering using the optional constraints. The optional
-  // constraints must be filtered in the order they occur in |optional|.
+  // Let's try filtering using the advanced constraints. The advanced
+  // constraints must be filtered in the order they occur in |advanced|.
   // But if a constraint produce zero candidates, the constraint is ignored and
   // the next constraint is tested.
-  // http://dev.w3.org/2011/webrtc/editor/getusermedia.html#idl-def-Constraints
-  for (const auto& constraint : optional) {
+  // http://w3c.github.io/mediacapture-main/getusermedia.html#dfn-selectsettings
+  const blink::WebVector<blink::WebMediaTrackConstraintSet>& advanced =
+      constraints.advanced();
+
+  for (const auto& constraint_set : advanced) {

[mcasas, 2016/01/28 01:57:57]

s/advanced/constraints.advanced()/ ?

[hta - Chromium, 2016/01/28 08:51:40]

Done.

     media::VideoCaptureFormats current_candidates = candidates;
-    FilterFormatsByConstraint(constraint, false, &current_candidates);
+    FilterFormatsByConstraints(constraint_set, &current_candidates, nullptr);

[mcasas, 2016/01/28 01:57:57]

Just to be sure, no s/nullptr/unsatisfied_constraint/ here ?

[hta - Chromium, 2016/01/28 08:51:40]

It throws the information away, but it can do that using a std::string. Changed.

     if (!current_candidates.empty())
       candidates = current_candidates;
   }
@@ -303,6 +275,8 @@ media::VideoCaptureFormat GetBestFormatBasedOnArea(
       best_format = &format;
     }
   }
+  DVLOG(3) << "GetBestFormatBasedOnArea chose format "
+           << media::VideoCaptureFormat::ToString(*best_format);
   return *best_format;
 }
 
@@ -337,7 +311,7 @@ MediaStreamVideoSource* MediaStreamVideoSource::GetVideoSource(
   return static_cast<MediaStreamVideoSource*>(source.extraData());
 }
 
-// static
+// static, deprecated
 bool MediaStreamVideoSource::IsConstraintSupported(const std::string& name) {
   return std::find(kSupportedConstraints,
                    kSupportedConstraints + arraysize(kSupportedConstraints),
@@ -374,16 +348,16 @@ void MediaStreamVideoSource::AddTrack(
       // Tab capture and Screen capture needs the maximum requested height
       // and width to decide on the resolution.
       int max_requested_width = 0;
-      GetMandatoryConstraintValueAsInteger(constraints, kMaxWidth,
-                                           &max_requested_width);
+      if (constraints.basic().width.hasMax())
+        max_requested_width = constraints.basic().width.max();
 
       int max_requested_height = 0;
-      GetMandatoryConstraintValueAsInteger(constraints, kMaxHeight,
-                                           &max_requested_height);
+      if (constraints.basic().height.hasMax())
+        max_requested_height = constraints.basic().height.max();
 
       double max_requested_frame_rate = kDefaultFrameRate;
-      GetConstraintValueAsDouble(constraints, kMaxFrameRate,
-                                 &max_requested_frame_rate);
+      if (constraints.basic().frameRate.hasMax())
+        max_requested_frame_rate = constraints.basic().frameRate.max();
 
       state_ = RETRIEVING_CAPABILITIES;
       GetCurrentSupportedFormats(
@@ -459,6 +433,7 @@ void MediaStreamVideoSource::OnSupportedFormats(
                                      &current_format_,
                                      &fulfilled_constraints)) {
     SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
+    DVLOG(3) << "OnSupportedFormats failed to find an usable format";
     // This object can be deleted after calling FinalizeAddTrack. See comment
     // in the header file.
     FinalizeAddTrack();
@@ -480,7 +455,9 @@ bool MediaStreamVideoSource::FindBestFormatWithConstraints(
     media::VideoCaptureFormat* best_format,
     blink::WebMediaConstraints* fulfilled_constraints) {
   DCHECK(CalledOnValidThread());
-  // Find the first constraints that we can fulfill.
+  DVLOG(3) << "MediaStreamVideoSource::FindBestFormatWithConstraints "
+           << "with " << formats.size() << " formats";
+  // Find the first track descriptor that can fulfil the constraints.
   for (const auto& track : track_descriptors_) {
     const blink::WebMediaConstraints& track_constraints = track.constraints;
 
@@ -488,6 +465,7 @@ bool MediaStreamVideoSource::FindBestFormatWithConstraints(
     // no mandatory constraints have been specified. That just means that
     // we will start with whatever format is native to the source.
     if (formats.empty() && !HasMandatoryConstraints(track_constraints)) {
+      DVLOG(3) << "No mandatory constraints and no formats";
       *fulfilled_constraints = track_constraints;
       *best_format = media::VideoCaptureFormat();
       return true;
@@ -501,8 +479,10 @@ bool MediaStreamVideoSource::FindBestFormatWithConstraints(
     // A request with constraints that can be fulfilled.
     *fulfilled_constraints = track_constraints;
     *best_format = GetBestCaptureFormat(filtered_formats, track_constraints);
+    DVLOG(3) << "Found a track that matches the constraints";
     return true;
   }
+  DVLOG(3) << "No usable format found";
   return false;
 }
 
@@ -542,6 +522,8 @@ void MediaStreamVideoSource::FinalizeAddTrack() {
         FilterFormats(track.constraints, formats, &unsatisfied_constraint)
             .empty()) {
       result = MEDIA_DEVICE_CONSTRAINT_NOT_SATISFIED;
+      DVLOG(3) << "FinalizeAddTrack() ignoring device on constraint "
+               << unsatisfied_constraint.utf8();
     }
 
     if (state_ != STARTED && result == MEDIA_DEVICE_OK)
@@ -557,8 +539,8 @@ void MediaStreamVideoSource::FinalizeAddTrack() {
                                      &min_aspect_ratio,
                                      &max_aspect_ratio);
       double max_frame_rate = 0.0f;
-      GetConstraintValueAsDouble(track.constraints,
-                                 kMaxFrameRate, &max_frame_rate);
+      if (track.constraints.basic().frameRate.hasMax())
+        max_frame_rate = track.constraints.basic().frameRate.max();
 
       track_adapter_->AddTrack(track.track, track.frame_callback, max_width,
                                max_height, min_aspect_ratio, max_aspect_ratio,