Patch constraints algorithm for device video-capture sources.

content/renderer/media/media_stream_constraints_util_video_device.cc

Index: content/renderer/media/media_stream_constraints_util_video_device.cc
diff --git a/content/renderer/media/media_stream_constraints_util_video_device.cc b/content/renderer/media/media_stream_constraints_util_video_device.cc
index 84d415f58de120bdbc4d0f0653a7c47d30eca990..d93c743eff2ed612098957ce82b5e2947e098ed2 100644
--- a/content/renderer/media/media_stream_constraints_util_video_device.cc
+++ b/content/renderer/media/media_stream_constraints_util_video_device.cc
@@ -52,10 +52,6 @@ blink::WebString ToWebString(::mojom::FacingMode facing_mode) {
 
 struct VideoDeviceCaptureSourceSettings {
  public:
-  VideoDeviceCaptureSourceSettings()
-      : facing_mode_(::mojom::FacingMode::NONE),
-        power_line_frequency_(media::PowerLineFrequency::FREQUENCY_DEFAULT) {}
-
   VideoDeviceCaptureSourceSettings(
       const std::string& device_id,
       const media::VideoCaptureFormat& format,
@@ -84,9 +80,6 @@ struct VideoDeviceCaptureSourceSettings {
   long GetPowerLineFrequency() const {
     return static_cast<long>(power_line_frequency_);
   }
-  long GetWidth() const { return format_.frame_size.width(); }
-  long GetHeight() const { return format_.frame_size.height(); }
-  double GetFrameRate() const { return format_.frame_rate; }
   blink::WebString GetDeviceId() const {
     return blink::WebString::fromASCII(device_id_.data());
   }
@@ -113,6 +106,64 @@ struct VideoDeviceCaptureSourceSettings {
   rtc::Optional<bool> noise_reduction_;
 };
 
+// This type is used to keep track of how constraints affect format variables
+// that support a range of values (height, width and frame rate).

[hbos_chromium, 2017/02/28 11:41:53]

I think it is not clear what this is doing considering we brute-force iterate
each format. We look at each case, e.g. a resolution, so how does the ranges
come into play? A resolution either is or is not supported by a given
constraint, one might think. Can you clarify this comment by adding an example,

e.g. camera: w = 1024
adv1: w <= 640
adv2: w >= 800

Here, the camera width can be cropped to support adv1 (with a penalty for
cropping), or it could satisfy adv2 because 1024 >= 800. On a set-by-set basis
the algorithm would say both are satisfied, but it can't do both at the same
time. Because constraint sets are applied in order, adv2 should be rejected
(INF) when adv1 is applied. As such this class is needed to keep track of the
so-far constrained format (range) when iterating advanced sets.

[Guido Urdaneta, 2017/02/28 16:44:00]

Done.

+class ConstrainedFormat {
+ public:
+  explicit ConstrainedFormat(const media::VideoCaptureFormat& format)
+      : native_height_(format.frame_size.height()),
+        min_height_(1),
+        max_height_(format.frame_size.height()),
+        native_width_(format.frame_size.width()),
+        min_width_(1),
+        max_width_(format.frame_size.width()),
+        native_frame_rate_(format.frame_rate),
+        min_frame_rate_(1),
+        max_frame_rate_(format.frame_rate) {}
+
+  long native_height() const { return native_height_; }
+  long min_height() const { return min_height_; }
+  long max_height() const { return max_height_; }
+  long native_width() const { return native_width_; }
+  long min_width() const { return min_width_; }
+  long max_width() const { return max_width_; }
+  long native_frame_rate() const { return native_frame_rate_; }
+  long min_frame_rate() const { return min_frame_rate_; }
+  long max_frame_rate() const { return max_frame_rate_; }
+
+  void ApplyConstraintSet(
+      const blink::WebMediaTrackConstraintSet& constraint_set) {
+    if (ConstraintHasMin(constraint_set.width))
+      min_width_ = std::max(min_width_, ConstraintMin(constraint_set.width));
+    if (ConstraintHasMax(constraint_set.width))
+      max_width_ = std::min(max_width_, ConstraintMax(constraint_set.width));
+
+    if (ConstraintHasMin(constraint_set.height))
+      min_height_ = std::max(min_height_, ConstraintMin(constraint_set.height));
+    if (ConstraintHasMax(constraint_set.height))
+      max_height_ = std::min(max_height_, ConstraintMax(constraint_set.height));
+
+    if (ConstraintHasMin(constraint_set.frameRate))
+      min_frame_rate_ =
+          std::max(min_frame_rate_, ConstraintMin(constraint_set.frameRate));
+    if (ConstraintHasMax(constraint_set.frameRate))
+      max_frame_rate_ =
+          std::min(max_frame_rate_, ConstraintMax(constraint_set.frameRate));
+  }
+
+ private:
+  // Using long for compatibility with Blink constraint classes.
+  long native_height_;
+  long min_height_;
+  long max_height_;
+  long native_width_;
+  long min_width_;
+  long max_width_;
+  double native_frame_rate_;
+  double min_frame_rate_;
+  double max_frame_rate_;
+};
+
 VideoDeviceCaptureSourceSelectionResult ResultFromSettings(
     const VideoDeviceCaptureSourceSettings& settings) {
   VideoDeviceCaptureSourceSelectionResult result;
@@ -138,29 +189,26 @@ double Distance(double value1, double value2) {
 }
 
 // Returns a pair with the minimum and maximum aspect ratios supported by the
-// source resolution settings |source_height| and |source_width|, subject to
-// given width and height constraints.
-void GetSourceAspectRatioRange(int source_height,
-                               int source_width,
+// candidate format |constrained_format|, subject to given width and height
+// constraints.
+void GetSourceAspectRatioRange(const ConstrainedFormat& constrained_format,
                                const blink::LongConstraint& height_constraint,
                                const blink::LongConstraint& width_constraint,
                                double* min_source_aspect_ratio,
                                double* max_source_aspect_ratio) {
-  DCHECK_GE(source_height, 1);
-  DCHECK_GE(source_width, 1);
-  long min_height = 1;
+  long min_height = constrained_format.min_height();
   if (ConstraintHasMin(height_constraint))
     min_height = std::max(min_height, ConstraintMin(height_constraint));
 
-  long max_height = source_height;
+  long max_height = constrained_format.max_height();
   if (ConstraintHasMax(height_constraint))
     max_height = std::min(max_height, ConstraintMax(height_constraint));
 
-  long min_width = 1;
+  long min_width = constrained_format.min_width();
   if (ConstraintHasMin(width_constraint))
     min_width = std::max(min_width, ConstraintMin(width_constraint));
 
-  long max_width = source_width;
+  long max_width = constrained_format.max_width();
   if (ConstraintHasMax(width_constraint))
     max_width = std::min(max_width, ConstraintMax(width_constraint));
 
@@ -195,10 +243,11 @@ double StringConstraintSourceDistance(const blink::WebString& value,
 // among sources that satisfy the constraint, those that have lower resource
 // usage. Otherwise, return zero.
 double ResolutionConstraintSourceDistance(
+    int native_source_value,
     int source_value,
     const blink::LongConstraint& constraint,
     const char** failed_constraint_name) {
-  DCHECK_GE(source_value, 1);
+  DCHECK_GE(native_source_value, 1);
   bool constraint_has_max = ConstraintHasMax(constraint);
   long constraint_max = constraint_has_max ? ConstraintMax(constraint) : -1;
 
@@ -213,47 +262,53 @@ double ResolutionConstraintSourceDistance(
   }
 
   // If the source value exceeds the maximum requested, penalize.
-  if (constraint_has_max && source_value > constraint_max)
-    return Distance(source_value, constraint_max);
+  if (constraint_has_max && native_source_value > constraint_max)
+    return Distance(native_source_value, constraint_max);
 
   return 0.0;
 }
 
 // Returns a custom distance function suitable for frame rate, given
-// a |constraint| and a candidate value.
-// A source can support track frame rates in the interval (0.0, |source_value|],
-// using frame-rate adjustments if necessary.
-// If the source range and the constraint range are disjoint, return HUGE_VAL.
-// If the constraint has maximum, penalize source frame rates that exceed the
-// maximum by returning Distance(|source_value|, maximum). This is intended to
-// prefer, among sources that satisfy the constraint, those that have lower
+// a |constraint| and a candidate format |constrained_format|.
+// A source can support track frame rates in the interval
+// [min_frame_rate, max_frame_rate], using frame-rate adjustments if
+// necessary. If the candidate range and the constraint range are disjoint,
+// return HUGE_VAL.
+// If the constraint has maximum, penalize native frame rates that exceed the
+// maximum by returning Distance(native_frame_rate, maximum). This is intended
+// to prefer, among sources that satisfy the constraint, those that have lower
 // resource usage. Otherwise, return zero.
 double FrameRateConstraintSourceDistance(
-    double source_value,
+    const ConstrainedFormat& constrained_format,
     const blink::DoubleConstraint& constraint,
     const char** failed_constraint_name) {
-  DCHECK_GT(source_value, 0.0);
+  bool constraint_has_min = ConstraintHasMin(constraint);
+  double constraint_min = constraint_has_min ? ConstraintMin(constraint) : -1.0;
   bool constraint_has_max = ConstraintHasMax(constraint);
   double constraint_max = constraint_has_max ? ConstraintMax(constraint) : -1.0;
 
-  if ((constraint_has_max && constraint_max <= 0.0) ||
-      (ConstraintHasMin(constraint) &&
-       source_value < ConstraintMin(constraint) -
-                          blink::DoubleConstraint::kConstraintEpsilon)) {
+  if ((constraint_has_max &&
+       constrained_format.min_frame_rate() >
+           constraint_max + blink::DoubleConstraint::kConstraintEpsilon) ||
+      (constraint_has_min &&
+       constrained_format.max_frame_rate() <
+           constraint_min - blink::DoubleConstraint::kConstraintEpsilon)) {
     if (failed_constraint_name)
       *failed_constraint_name = constraint.name();
     return HUGE_VAL;
   }
 
-  if (constraint_has_max && source_value > constraint_max)
-    return Distance(source_value, constraint_max);
+  // Compute the cost using the native rate.
+  if (constraint_has_max &&
+      constrained_format.native_frame_rate() > constraint_max)
+    return Distance(constrained_format.native_frame_rate(), constraint_max);
 
   return 0.0;
 }
 
 // Returns a custom distance function suitable for aspect ratio, given
-// the values for the aspect_ratio, width and height constraints, and candidate
-// source values for width and height.
+// the values for the aspect_ratio, width and height constraints, and a
+// candidate format |constrained_format|.
 // A source can support track resolutions that range from
 //    min_width x min_height  to  max_width x max_height
 // where
@@ -269,15 +324,11 @@ double FrameRateConstraintSourceDistance(
 // If the supported range [min_ar, max_ar] and the range specified by the
 // aspectRatio constraint are disjoint, return HUGE_VAL. Otherwise, return zero.
 double AspectRatioConstraintSourceDistance(
-    int source_height,
-    int source_width,
+    const ConstrainedFormat& constrained_format,
     const blink::LongConstraint& height_constraint,
     const blink::LongConstraint& width_constraint,
     const blink::DoubleConstraint& aspect_ratio_constraint,
     const char** failed_constraint_name) {
-  DCHECK_GT(source_height, 1);
-  DCHECK_GT(source_width, 1);
-
   bool ar_constraint_has_min = ConstraintHasMin(aspect_ratio_constraint);
   double ar_constraint_min =
       ar_constraint_has_min ? ConstraintMin(aspect_ratio_constraint) : -1.0;
@@ -287,7 +338,7 @@ double AspectRatioConstraintSourceDistance(
 
   double min_source_aspect_ratio;
   double max_source_aspect_ratio;
-  GetSourceAspectRatioRange(source_height, source_width, height_constraint,
+  GetSourceAspectRatioRange(constrained_format, height_constraint,
                             width_constraint, &min_source_aspect_ratio,
                             &max_source_aspect_ratio);
 
@@ -369,19 +420,20 @@ double DeviceSourceDistance(
 // format.
 double FormatSourceDistance(
     const media::VideoCaptureFormat& format,
+    const ConstrainedFormat& constrained_format,
     const blink::WebMediaTrackConstraintSet& constraint_set,
     const char** failed_constraint_name) {
-  return ResolutionConstraintSourceDistance(format.frame_size.height(),
+  return ResolutionConstraintSourceDistance(constrained_format.native_height(),
+                                            constrained_format.max_height(),
                                             constraint_set.height,
                                             failed_constraint_name) +
-         ResolutionConstraintSourceDistance(format.frame_size.width(),
-                                            constraint_set.width,
-                                            failed_constraint_name) +
+         ResolutionConstraintSourceDistance(
+             constrained_format.native_width(), constrained_format.max_width(),
+             constraint_set.width, failed_constraint_name) +
          AspectRatioConstraintSourceDistance(
-             format.frame_size.height(), format.frame_size.width(),
-             constraint_set.height, constraint_set.width,
+             constrained_format, constraint_set.height, constraint_set.width,
              constraint_set.aspectRatio, failed_constraint_name) +
-         FrameRateConstraintSourceDistance(format.frame_rate,
+         FrameRateConstraintSourceDistance(constrained_format,
                                            constraint_set.frameRate,
                                            failed_constraint_name) +
          StringConstraintSourceDistance(GetVideoKindForFormat(format),
@@ -389,20 +441,22 @@ double FormatSourceDistance(
                                         failed_constraint_name);
 }
 
-// Returns a custom distance between a set of candidate settings and a
-// constraint set. It is simply the sum of the distances for each individual
-// setting in |candidate|.
-// If |candidate| cannot satisfy constraint, the distance is HUGE_VAL.
+// Returns a custom distance between a constraint set |constraint_set| and the
+// settings of |candidate| and |constrained_format|.
+// It is simply the sum of the distances for each individual setting in
+// |candidate| and |constrained_format|.
+// If |constraint_set| cannot be satisfied, the distance is HUGE_VAL.
 // Otherwise the distance is a finite value. Candidates with lower distance
 // satisfy |constraint_set| in a "better" way.
 double CandidateSourceDistance(
     const VideoDeviceCaptureSourceSettings& candidate,
+    const ConstrainedFormat& constrained_format,
     const blink::WebMediaTrackConstraintSet& constraint_set,
     const char** failed_constraint_name) {
   return DeviceSourceDistance(candidate.device_id(), candidate.facing_mode(),
                               constraint_set, failed_constraint_name) +
-         FormatSourceDistance(candidate.format(), constraint_set,
-                              failed_constraint_name) +
+         FormatSourceDistance(candidate.format(), constrained_format,
+                              constraint_set, failed_constraint_name) +
          PowerLineFrequencyConstraintSourceDistance(
              constraint_set.googPowerLineFrequency,
              candidate.power_line_frequency(), failed_constraint_name) +
@@ -459,20 +513,16 @@ double ResolutionConstraintNativeFitnessDistance(
 // on the source imposed by the width and height constraints.
 // Based on https://w3c.github.io/mediacapture-main/#dfn-fitness-distance.
 double AspectRatioConstraintFitnessDistance(
-    long source_height,
-    long source_width,
+    const ConstrainedFormat& constrained_format,
     const blink::LongConstraint& height_constraint,
     const blink::LongConstraint& width_constraint,
     const blink::DoubleConstraint& aspect_ratio_constraint) {
-  DCHECK_GT(source_height, 1);
-  DCHECK_GT(source_width, 1);
-
   if (!aspect_ratio_constraint.hasIdeal())
     return 0.0;
 
   double min_source_aspect_ratio;
   double max_source_aspect_ratio;
-  GetSourceAspectRatioRange(source_height, source_width, height_constraint,
+  GetSourceAspectRatioRange(constrained_format, height_constraint,
                             width_constraint, &min_source_aspect_ratio,
                             &max_source_aspect_ratio);
 
@@ -555,56 +605,58 @@ double NoiseReductionConstraintFitnessDistance(
   return 1.0;
 }
 
-// Returns the fitness distance between a settings candidate and a constraint
-// set. The returned value is the sum of the fitness distances between each
-// setting in |candidate| and the corresponding constraint in |constraint_set|.
+// Returns the fitness distance between |constraint_set| and the settings of
+// |candidate| and |constrained_format|. The returned value is the sum of the
+// fitness distances between each setting in |candidate| or |constrained_format|
+// and the corresponding constraint in |constraint_set|.
 // Based on https://w3c.github.io/mediacapture-main/#dfn-fitness-distance.
 double CandidateFitnessDistance(
     const VideoDeviceCaptureSourceSettings& candidate,
+    const ConstrainedFormat& constrained_format,
     const blink::WebMediaTrackConstraintSet& constraint_set) {
-  DCHECK(std::isfinite(
-      CandidateSourceDistance(candidate, constraint_set, nullptr)));
+  DCHECK(std::isfinite(CandidateSourceDistance(candidate, constrained_format,
+                                               constraint_set, nullptr)));
   double fitness = 0.0;
   fitness += AspectRatioConstraintFitnessDistance(
-      candidate.GetHeight(), candidate.GetWidth(), constraint_set.height,
-      constraint_set.width, constraint_set.aspectRatio);
+      constrained_format, constraint_set.height, constraint_set.width,
+      constraint_set.aspectRatio);
   fitness += StringConstraintFitnessDistance(candidate.GetDeviceId(),
                                              constraint_set.deviceId);
   fitness += StringConstraintFitnessDistance(candidate.GetFacingMode(),
                                              constraint_set.facingMode);
-  fitness += FrameRateConstraintFitnessDistance(candidate.GetFrameRate(),
-                                                constraint_set.frameRate);
+  fitness += FrameRateConstraintFitnessDistance(
+      constrained_format.max_frame_rate(), constraint_set.frameRate);
   fitness += StringConstraintFitnessDistance(candidate.GetVideoKind(),
                                              constraint_set.videoKind);
   fitness += PowerLineFrequencyConstraintFitnessDistance(
       candidate.GetPowerLineFrequency(), constraint_set.googPowerLineFrequency);
   fitness += NoiseReductionConstraintFitnessDistance(
       candidate.noise_reduction(), constraint_set.googNoiseReduction);
-  fitness += ResolutionConstraintFitnessDistance(candidate.GetHeight(),
-                                                 constraint_set.height);
-  fitness += ResolutionConstraintFitnessDistance(candidate.GetWidth(),
+  fitness += ResolutionConstraintFitnessDistance(
+      constrained_format.max_height(), constraint_set.height);
+  fitness += ResolutionConstraintFitnessDistance(constrained_format.max_width(),
                                                  constraint_set.width);
 
   return fitness;
 }
 
-// Returns the native fitness distance between a settings candidate and a
+// Returns the native fitness distance between a candidate format and a
 // constraint set. The returned value is the sum of the fitness distances for
 // the native values of settings that support a range of values (i.e., width,
 // height and frame rate).
 // Based on https://w3c.github.io/mediacapture-main/#dfn-fitness-distance.
 double CandidateNativeFitnessDistance(
-    const VideoDeviceCaptureSourceSettings& candidate,
+    const ConstrainedFormat& constrained_format,
     const blink::WebMediaTrackConstraintSet& constraint_set) {
-  DCHECK(std::isfinite(
-      CandidateSourceDistance(candidate, constraint_set, nullptr)));
+  //  DCHECK(std::isfinite(
+  //      FormatSourceDistance(candidate, constraint_set, nullptr)));

[hbos_chromium, 2017/02/28 11:41:53]

Why is this commented-out?

[Guido Urdaneta, 2017/02/28 16:44:00]

Removed. |candidate| is no longer available here.

   double fitness = 0.0;
-  fitness += FrameRateConstraintNativeFitnessDistance(candidate.GetFrameRate(),
-                                                      constraint_set.frameRate);
-  fitness += ResolutionConstraintNativeFitnessDistance(candidate.GetHeight(),
-                                                       constraint_set.height);
-  fitness += ResolutionConstraintNativeFitnessDistance(candidate.GetWidth(),
-                                                       constraint_set.width);
+  fitness += FrameRateConstraintNativeFitnessDistance(
+      constrained_format.native_frame_rate(), constraint_set.frameRate);
+  fitness += ResolutionConstraintNativeFitnessDistance(
+      constrained_format.native_height(), constraint_set.height);
+  fitness += ResolutionConstraintNativeFitnessDistance(
+      constrained_format.native_width(), constraint_set.width);
 
   return fitness;
 }
@@ -731,10 +783,13 @@ VideoDeviceCaptureSourceSelectionResult SelectVideoDeviceCaptureSourceSettings(
       continue;
 
     for (auto& format : device->formats) {
-      double basic_format_distance = FormatSourceDistance(
-          format, constraints.basic(), &failed_constraint_name);
+      ConstrainedFormat constrained_format(format);
+      double basic_format_distance =
+          FormatSourceDistance(format, constrained_format, constraints.basic(),
+                               &failed_constraint_name);
       if (!std::isfinite(basic_format_distance))
         continue;
+      constrained_format.ApplyConstraintSet(constraints.basic());
 
       for (auto& power_line_frequency : capabilities.power_line_capabilities) {
         double basic_power_line_frequency_distance =
@@ -770,17 +825,19 @@ VideoDeviceCaptureSourceSelectionResult SelectVideoDeviceCaptureSourceSettings(
           DistanceVector candidate_distance_vector;
           // First criteria for valid candidates is satisfaction of advanced
           // constraint sets.
-          for (const auto& advanced : constraints.advanced()) {
-            double custom_distance =
-                CandidateSourceDistance(candidate, advanced, nullptr);
+          for (const auto& advanced_set : constraints.advanced()) {
+            double custom_distance = CandidateSourceDistance(
+                candidate, constrained_format, advanced_set, nullptr);
             advanced_custom_distance_vector.push_back(custom_distance);
             double spec_distance = std::isfinite(custom_distance) ? 0 : 1;
             candidate_distance_vector.push_back(spec_distance);
+            if (std::isfinite(custom_distance))
+              constrained_format.ApplyConstraintSet(advanced_set);
           }
 
           // Second criterion is fitness distance.
-          candidate_distance_vector.push_back(
-              CandidateFitnessDistance(candidate, constraints.basic()));
+          candidate_distance_vector.push_back(CandidateFitnessDistance(
+              candidate, constrained_format, constraints.basic()));
 
           // Third criteria are custom distances to constraint sets.
           candidate_distance_vector.push_back(candidate_basic_custom_distance);
@@ -789,8 +846,8 @@ VideoDeviceCaptureSourceSelectionResult SelectVideoDeviceCaptureSourceSettings(
                     std::back_inserter(candidate_distance_vector));
 
           // Fourth criteria is native fitness distance.
-          candidate_distance_vector.push_back(
-              CandidateNativeFitnessDistance(candidate, constraints.basic()));
+          candidate_distance_vector.push_back(CandidateNativeFitnessDistance(
+              constrained_format, constraints.basic()));
 
           // Final criteria are custom distances to default settings.
           AppendDistanceFromDefault(candidate, capabilities,