Add algorithm for processing constraints for video content capture.

content/renderer/media/media_stream_constraints_util_video_content.cc

Index: content/renderer/media/media_stream_constraints_util_video_content.cc
diff --git a/content/renderer/media/media_stream_constraints_util_video_content.cc b/content/renderer/media/media_stream_constraints_util_video_content.cc
new file mode 100644
index 0000000000000000000000000000000000000000..b0082d66fa40d15ecb447b9462e2bcc7a6b32bf6
--- /dev/null
+++ b/content/renderer/media/media_stream_constraints_util_video_content.cc
@@ -0,0 +1,349 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/renderer/media/media_stream_constraints_util_video_content.h"
+
+#include <cmath>
+#include <utility>
+#include <vector>
+
+#include "content/renderer/media/media_stream_constraints_util_sets.h"
+#include "content/renderer/media/media_stream_video_source.h"
+#include "media/base/limits.h"
+#include "third_party/WebKit/public/platform/WebMediaConstraints.h"
+#include "third_party/WebKit/public/platform/WebString.h"
+
+namespace content {
+
+namespace {
+
+using Point = ResolutionSet::Point;
+
+// Hard upper and lower bound frame rates for tab/desktop capture.
+constexpr double kMaxScreenCastFrameRate = 120.0;
+constexpr double kMinScreenCastFrameRate = 1.0 / 60.0;

[hbos_chromium, 2017/03/09 16:51:10]

This is what I call moving pictures! :)

[Guido Urdaneta, 2017/03/14 12:29:05]

Acknowledged.

+
+constexpr double kDefaultFrameRate = MediaStreamVideoSource::kDefaultFrameRate;
+
+constexpr int kMinScreenCastDimension = 1;
+constexpr int kMaxScreenCastDimension = media::limits::kMaxDimension - 1;
+constexpr double kMinScreenCastAspectRatio =
+    static_cast<double>(kMinScreenCastDimension) /
+    static_cast<double>(kMaxScreenCastDimension);
+constexpr double kMaxScreenCastAspectRatio =
+    static_cast<double>(kMaxScreenCastDimension) /
+    static_cast<double>(kMinScreenCastDimension);
+
+using StringSet = DiscreteSet<std::string>;

[hbos_chromium, 2017/03/09 16:51:09]

Move all using statements to before functions are being defined.

[Guido Urdaneta, 2017/03/14 12:29:06]

Done.

+StringSet StringSetFromConstraint(const blink::StringConstraint& constraint) {
+  if (!constraint.hasExact())
+    return StringSet::UniversalSet();
+
+  std::vector<std::string> elements;
+  for (const auto& entry : constraint.exact())
+    elements.push_back(entry.ascii());
+
+  return StringSet(std::move(elements));
+}
+
+using BoolSet = DiscreteSet<bool>;
+BoolSet BoolSetFromConstraint(const blink::BooleanConstraint& constraint) {
+  if (!constraint.hasExact())
+    return BoolSet::UniversalSet();
+
+  return BoolSet({constraint.exact()});
+}
+
+using DoubleRangeSet = NumericRangeSet<double>;
+
+class VideoContentCaptureCandidates {
+ public:
+  VideoContentCaptureCandidates()
+      : device_id_set(StringSet::UniversalSet()),
+        noise_reduction_set(BoolSet::UniversalSet()) {}
+  explicit VideoContentCaptureCandidates(
+      const blink::WebMediaTrackConstraintSet& constraint_set)
+      : resolution_set(ResolutionSet::FromConstraintSet(constraint_set)),
+        frame_rate_set(
+            DoubleRangeSet::FromConstraint(constraint_set.frameRate)),
+        device_id_set(StringSetFromConstraint(constraint_set.deviceId)),
+        noise_reduction_set(
+            BoolSetFromConstraint(constraint_set.googNoiseReduction)) {}
+
+  VideoContentCaptureCandidates(VideoContentCaptureCandidates&& other) =
+      default;
+  VideoContentCaptureCandidates& operator=(
+      VideoContentCaptureCandidates&& other) = default;
+
+  bool IsEmpty() const {
+    return resolution_set.IsEmpty() || frame_rate_set.IsEmpty() ||
+           device_id_set.IsEmpty() || noise_reduction_set.IsEmpty();
+  }
+
+  VideoContentCaptureCandidates Intersection(
+      const VideoContentCaptureCandidates& other) {
+    VideoContentCaptureCandidates intersection;
+    intersection.resolution_set =
+        resolution_set.Intersection(other.resolution_set);
+    intersection.frame_rate_set =
+        frame_rate_set.Intersection(other.frame_rate_set);
+    intersection.device_id_set =
+        device_id_set.Intersection(other.device_id_set);
+    intersection.noise_reduction_set =
+        noise_reduction_set.Intersection(other.noise_reduction_set);
+    return intersection;
+  }
+
+  ResolutionSet resolution_set;
+  DoubleRangeSet frame_rate_set;
+  StringSet device_id_set;
+  BoolSet noise_reduction_set;
+};
+
+media::ResolutionChangePolicy SelectResolutionPolicyFromCandidates(

[hbos_chromium, 2017/03/09 16:51:09]

Can you add a comment? I'm not sure I understand the purpose of this function or
what the resulting policy means.

[Guido Urdaneta, 2017/03/14 12:29:06]

Changed the policy to be simpler and added a TODO for what I tried in this
patchset.

+    const ResolutionSet& resolution_set,
+    int chosen_height,
+    int chosen_width) {
+  // TODO(guidou): Most of this code is copied from
+  // WebContentsCaptureMachine::ComputeOptimalViewSize(). Find a way to more
+  // easily share this code.
+  const auto HasIntendedAspectRatio = [chosen_height, chosen_width](
+      int width_units, int height_units) {
+    const int a = height_units * chosen_width;
+    const int b = width_units * chosen_height;
+    const int percentage_diff = 100 * std::abs((a - b)) / b;
+    return percentage_diff <= 1;  // Effectively, anything strictly <2%.

[hbos_chromium, 2017/03/09 16:51:09]

Why <2%? Why not exactly (within numerical tolerance), or if not exactly why
such a small percentage and not a larger tolerance?

[Guido Urdaneta, 2017/03/14 12:29:06]

See previous comment.

+  };
+  const auto RoundToExactAspectRatio = [chosen_height, chosen_width](
+      int width_step, int height_step) {
+    const int adjusted_height =
+        std::max(chosen_height - (chosen_height % height_step), height_step);
+    DCHECK_EQ((adjusted_height * width_step) % height_step, 0);
+    return gfx::Size(adjusted_height * width_step / height_step,
+                     adjusted_height);
+  };
+
+  gfx::Size adjusted_size;
+  if (HasIntendedAspectRatio(16, 9)) {
+    adjusted_size = RoundToExactAspectRatio(160, 90);
+  } else if (HasIntendedAspectRatio(4, 3)) {
+    adjusted_size = RoundToExactAspectRatio(64, 48);

[hbos_chromium, 2017/03/09 16:51:09]

Why these step sizes and not the smallest possible integer steps? 16x9 and 4x3?

[Guido Urdaneta, 2017/03/14 12:29:06]

See previous comment.

+  } else {
+    // There will be no adjustment, so any policy value will do.
+    return media::RESOLUTION_POLICY_FIXED_RESOLUTION;
+  }
+
+  double adjusted_aspect_ratio = static_cast<double>(adjusted_size.width()) /
+                                 static_cast<double>(adjusted_size.height());
+  if (adjusted_aspect_ratio >= resolution_set.min_aspect_ratio() &&
+      adjusted_aspect_ratio <= resolution_set.max_aspect_ratio() &&
+      adjusted_size.width() >= resolution_set.min_width() &&
+      adjusted_size.width() <= resolution_set.max_width() &&
+      adjusted_size.height() >= resolution_set.min_height() &&
+      adjusted_size.height() <= resolution_set.max_height()) {
+    return media::RESOLUTION_POLICY_ANY_WITHIN_LIMIT;
+  }
+  return media::RESOLUTION_POLICY_FIXED_RESOLUTION;
+}
+
+int RoundToInt(double d) {
+  return static_cast<int>(std::round(d));
+}
+
+gfx::Size ToGfxSize(const Point& point) {
+  return gfx::Size(RoundToInt(point.width()), RoundToInt(point.height()));
+}
+
+double SelectFrameRateFromCandidates(
+    const DoubleRangeSet& candidate_set,
+    const blink::WebMediaTrackConstraintSet& basic_constraint_set) {
+  double frame_rate = basic_constraint_set.frameRate.hasIdeal()
+                          ? basic_constraint_set.frameRate.ideal()
+                          : kDefaultFrameRate;
+  if (frame_rate > candidate_set.Max())
+    frame_rate = candidate_set.Max();
+  else if (frame_rate < candidate_set.Min())
+    frame_rate = candidate_set.Min();
+
+  return frame_rate;
+}
+
+media::VideoCaptureParams SelectVideoCaptureParamsFromCandidates(
+    const VideoContentCaptureCandidates& candidates,
+    const blink::WebMediaTrackConstraintSet& basic_constraint_set) {
+  double requested_frame_rate = SelectFrameRateFromCandidates(
+      candidates.frame_rate_set, basic_constraint_set);
+  Point requested_resolution =
+      candidates.resolution_set.SelectClosestPointToIdeal(basic_constraint_set);
+  media::VideoCaptureParams params;
+  params.requested_format = media::VideoCaptureFormat(
+      ToGfxSize(requested_resolution), static_cast<float>(requested_frame_rate),
+      media::PIXEL_FORMAT_I420);
+  params.resolution_change_policy = SelectResolutionPolicyFromCandidates(
+      candidates.resolution_set, params.requested_format.frame_size.height(),
+      params.requested_format.frame_size.width());
+  DCHECK(params.IsValid());

[hbos_chromium, 2017/03/09 16:51:10]

Note: Will always use default PowerLineFrequency. That is on purpose?

[Guido Urdaneta, 2017/03/14 12:29:05]

Yes. PowerLineFrequency is ignored for content capture. Added comment.

+
+  return params;
+}
+
+std::string SelectDeviceIDFromCandidates(
+    const StringSet& candidates,
+    const blink::WebMediaTrackConstraintSet& basic_constraint_set) {
+  DCHECK(!candidates.IsEmpty());
+  if (basic_constraint_set.deviceId.hasIdeal()) {
+    for (const auto& ideal_entry : basic_constraint_set.deviceId.ideal()) {
+      std::string ideal_value = ideal_entry.ascii();
+      if (candidates.Contains(ideal_value)) {
+        return ideal_value;
+      }
+    }
+  }
+
+  if (candidates.is_universal()) {
+    return std::string();

[hbos_chromium, 2017/03/09 16:51:10]

Should this return Optional<std::string> instead?

[Guido Urdaneta, 2017/03/14 12:29:05]

No. The default value for device_id is the empty string.

+  }
+
+  return candidates.FirstElement();

[hbos_chromium, 2017/03/09 16:51:09]

Is the preference for the first device id intentional or arbitrarily "any will
do"? Add comment.
Similar question for other return FirstElement places.

[Guido Urdaneta, 2017/03/14 12:29:06]

Added comment here and in the other call to FirstElement.

+}
+
+rtc::Optional<bool> SelectNoiseReductionFromCandidates(
+    const BoolSet& candidates,
+    const blink::WebMediaTrackConstraintSet& basic_constraint_set) {
+  DCHECK(!candidates.IsEmpty());
+  if (basic_constraint_set.googNoiseReduction.hasIdeal() &&
+      candidates.Contains(basic_constraint_set.googNoiseReduction.ideal())) {
+    return rtc::Optional<bool>(basic_constraint_set.googNoiseReduction.ideal());
+  }
+
+  if (candidates.is_universal())
+    return rtc::Optional<bool>();
+
+  return rtc::Optional<bool>(candidates.FirstElement());
+}
+
+VideoContentCaptureSourceSelectionResult SelectResultFromCandidates(
+    const VideoContentCaptureCandidates& candidates,
+    const blink::WebMediaTrackConstraintSet& basic_constraint_set) {
+  std::string device_id = SelectDeviceIDFromCandidates(candidates.device_id_set,
+                                                       basic_constraint_set);
+  media::VideoCaptureParams capture_params =
+      SelectVideoCaptureParamsFromCandidates(candidates, basic_constraint_set);
+
+  rtc::Optional<bool> noise_reduction = SelectNoiseReductionFromCandidates(
+      candidates.noise_reduction_set, basic_constraint_set);
+
+  return VideoContentCaptureSourceSelectionResult(
+      std::move(device_id), noise_reduction, capture_params);

[hbos_chromium, 2017/03/09 16:51:09]

What if device_id is the empty string? Shouldn't that be failed constraint
deviceId? Otherwise document the meaning of empty string.

[Guido Urdaneta, 2017/03/14 12:29:05]

Empty string is treated like any other ID, just like in the old algorithm. 
I don't think special documentation for it is needed in this particular part of
the code, but added a comment in SelectDeviceIDFromCandidates that says the
empty string is treated as a default value for device ID.

+}
+
+VideoContentCaptureSourceSelectionResult UnsatisfiedConstraintsResult(
+    const VideoContentCaptureCandidates& candidates,
+    const blink::WebMediaTrackConstraintSet& constraint_set) {
+  DCHECK(candidates.IsEmpty());
+  if (candidates.resolution_set.IsHeightEmpty()) {
+    return VideoContentCaptureSourceSelectionResult(
+        constraint_set.height.name());
+  } else if (candidates.resolution_set.IsWidthEmpty()) {
+    return VideoContentCaptureSourceSelectionResult(
+        constraint_set.width.name());
+  } else if (candidates.resolution_set.IsAspectRatioEmpty()) {
+    return VideoContentCaptureSourceSelectionResult(
+        constraint_set.aspectRatio.name());
+  } else if (candidates.frame_rate_set.IsEmpty()) {
+    return VideoContentCaptureSourceSelectionResult(
+        constraint_set.frameRate.name());
+  } else if (candidates.noise_reduction_set.IsEmpty()) {
+    return VideoContentCaptureSourceSelectionResult(
+        constraint_set.googNoiseReduction.name());
+  } else {
+    DCHECK(candidates.device_id_set.IsEmpty());
+    return VideoContentCaptureSourceSelectionResult(
+        constraint_set.deviceId.name());
+  }
+}
+
+}  // namespace
+
+VideoContentCaptureSourceSelectionResult::
+    VideoContentCaptureSourceSelectionResult(const char* failed_constraint_name)
+    : failed_constraint_name_(failed_constraint_name) {}
+
+VideoContentCaptureSourceSelectionResult::
+    VideoContentCaptureSourceSelectionResult(
+        std::string device_id,
+        const rtc::Optional<bool>& noise_reduction,
+        media::VideoCaptureParams capture_params)
+    : failed_constraint_name_(nullptr),
+      device_id_(std::move(device_id)),
+      noise_reduction_(noise_reduction),
+      capture_params_(capture_params) {}
+
+VideoContentCaptureSourceSelectionResult::
+    VideoContentCaptureSourceSelectionResult(
+        const VideoContentCaptureSourceSelectionResult& other) = default;
+VideoContentCaptureSourceSelectionResult::
+    VideoContentCaptureSourceSelectionResult(
+        VideoContentCaptureSourceSelectionResult&& other) = default;
+VideoContentCaptureSourceSelectionResult::
+    ~VideoContentCaptureSourceSelectionResult() = default;
+VideoContentCaptureSourceSelectionResult&
+VideoContentCaptureSourceSelectionResult::operator=(
+    const VideoContentCaptureSourceSelectionResult& other) = default;
+VideoContentCaptureSourceSelectionResult&
+VideoContentCaptureSourceSelectionResult::operator=(
+    VideoContentCaptureSourceSelectionResult&& other) = default;
+
+int VideoContentCaptureSourceSelectionResult::Height() const {
+  DCHECK(HasValue());
+  return capture_params_.requested_format.frame_size.height();
+}
+
+int VideoContentCaptureSourceSelectionResult::Width() const {
+  DCHECK(HasValue());
+  return capture_params_.requested_format.frame_size.width();
+}
+
+float VideoContentCaptureSourceSelectionResult::FrameRate() const {
+  DCHECK(HasValue());
+  return capture_params_.requested_format.frame_rate;
+}
+
+media::ResolutionChangePolicy
+VideoContentCaptureSourceSelectionResult::ResolutionChangePolicy() const {
+  DCHECK(HasValue());
+  return capture_params_.resolution_change_policy;
+}
+
+VideoContentCaptureSourceSelectionResult
+SelectVideoContentCaptureSourceSettings(
+    const blink::WebMediaConstraints& constraints) {
+  VideoContentCaptureCandidates candidates;
+  candidates.resolution_set =
+      ResolutionSet(kMinScreenCastDimension, kMaxScreenCastDimension,
+                    kMinScreenCastDimension, kMaxScreenCastDimension,
+                    kMinScreenCastAspectRatio, kMaxScreenCastAspectRatio);
+  candidates.frame_rate_set =
+      DoubleRangeSet(kMinScreenCastFrameRate, kMaxScreenCastFrameRate);
+  // candidates.device_id_set and candidates.noise_reduction_set are
+  // automatically initialized with the universal set.
+
+  candidates = candidates.Intersection(
+      VideoContentCaptureCandidates(constraints.basic()));
+  if (candidates.IsEmpty())
+    return UnsatisfiedConstraintsResult(candidates, constraints.basic());
+
+  for (const auto& advanced_set : constraints.advanced()) {
+    VideoContentCaptureCandidates advanced_candidates(advanced_set);
+    VideoContentCaptureCandidates intersection =
+        candidates.Intersection(advanced_candidates);
+    if (!intersection.IsEmpty())
+      candidates = std::move(intersection);
+  }
+
+  DCHECK(!candidates.IsEmpty());
+  return SelectResultFromCandidates(candidates, constraints.basic());
+}
+
+}  // namespace content