Spec compliant video constraints for getUserMedia behind flag.

content/renderer/media/media_stream_video_capturer_source.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/media_stream_video_capturer_source.h"
 
+#include <algorithm>
 #include <utility>
 
 #include "base/bind.h"
 #include "base/callback_helpers.h"
 #include "base/debug/stack_trace.h"
 #include "base/location.h"
 #include "base/macros.h"
 #include "base/strings/utf_string_conversions.h"
 #include "content/common/media/media_stream_messages.h"
 #include "content/public/common/media_stream_request.h"
@@ skipping 170 matching lines @@
           constraints,
           &blink::WebMediaTrackConstraintSet::googPowerLineFrequency, &freq)) {
     return;
   }
   if (freq == static_cast<int>(media::PowerLineFrequency::FREQUENCY_50HZ))
     params->power_line_frequency = media::PowerLineFrequency::FREQUENCY_50HZ;
   else if (freq == static_cast<int>(media::PowerLineFrequency::FREQUENCY_60HZ))
     params->power_line_frequency = media::PowerLineFrequency::FREQUENCY_60HZ;
 }
 
 // LocalVideoCapturerSource is a delegate used by MediaStreamVideoCapturerSource

[hbos_chromium, 2017/04/05 12:37:28]

nit: LegacyLocalVideoCapturerSource

[Guido Urdaneta, 2017/04/05 16:17:16]

Done.

 // for local video capture. It uses the Render singleton VideoCaptureImplManager
 // to start / stop and receive I420 frames from Chrome's video capture
 // implementation. This is a main Render thread only object.
-class LocalVideoCapturerSource final : public media::VideoCapturerSource {
+// TODO(guidou): Remove this class. http://crbug.com/706408
+class LegacyLocalVideoCapturerSource final : public media::VideoCapturerSource {
  public:
-  explicit LocalVideoCapturerSource(const StreamDeviceInfo& device_info);
-  ~LocalVideoCapturerSource() override;
+  explicit LegacyLocalVideoCapturerSource(const StreamDeviceInfo& device_info);
+  ~LegacyLocalVideoCapturerSource() override;
 
   // VideoCaptureDelegate Implementation.
   void GetCurrentSupportedFormats(
       int max_requested_width,
       int max_requested_height,
       double max_requested_frame_rate,
       const VideoCaptureDeviceFormatsCB& callback) override;
   media::VideoCaptureFormats GetPreferredFormats() override;
   void StartCapture(const media::VideoCaptureParams& params,
                     const VideoCaptureDeliverFrameCB& new_frame_callback,
@@ skipping 25 matching lines @@
   RunningCallback running_callback_;
   base::Closure stop_capture_cb_;
 
   // Placeholder keeping the callback between asynchronous device enumeration
   // calls.
   VideoCaptureDeviceFormatsCB formats_enumerated_callback_;
 
   // Bound to the main render thread.
   base::ThreadChecker thread_checker_;
 
-  base::WeakPtrFactory<LocalVideoCapturerSource> weak_factory_;
+  base::WeakPtrFactory<LegacyLocalVideoCapturerSource> weak_factory_;
 
-  DISALLOW_COPY_AND_ASSIGN(LocalVideoCapturerSource);
+  DISALLOW_COPY_AND_ASSIGN(LegacyLocalVideoCapturerSource);
 };
 
-}  // namespace
-
-LocalVideoCapturerSource::LocalVideoCapturerSource(
+LegacyLocalVideoCapturerSource::LegacyLocalVideoCapturerSource(
     const StreamDeviceInfo& device_info)
     : session_id_(device_info.session_id),
       manager_(RenderThreadImpl::current()->video_capture_impl_manager()),
       release_device_cb_(manager_->UseDevice(session_id_)),
       is_content_capture_(IsContentVideoCaptureDevice(device_info)),
       weak_factory_(this) {
   DCHECK(RenderThreadImpl::current());
 }
 
-LocalVideoCapturerSource::~LocalVideoCapturerSource() {
+LegacyLocalVideoCapturerSource::~LegacyLocalVideoCapturerSource() {
   DCHECK(thread_checker_.CalledOnValidThread());
   release_device_cb_.Run();
 }
 
-void LocalVideoCapturerSource::GetCurrentSupportedFormats(
+void LegacyLocalVideoCapturerSource::GetCurrentSupportedFormats(
     int max_requested_width,
     int max_requested_height,
     double max_requested_frame_rate,
     const VideoCaptureDeviceFormatsCB& callback) {
   DVLOG(3) << "GetCurrentSupportedFormats({ max_requested_height = "
            << max_requested_height << "}) { max_requested_width = "
            << max_requested_width << "}) { max_requested_frame_rate = "
            << max_requested_frame_rate << "})";
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (is_content_capture_) {
     const int width = max_requested_width ?
         max_requested_width : MediaStreamVideoSource::kDefaultWidth;
     const int height = max_requested_height ?
         max_requested_height : MediaStreamVideoSource::kDefaultHeight;
     callback.Run(media::VideoCaptureFormats(
         1, media::VideoCaptureFormat(
                gfx::Size(width, height),
                static_cast<float>(
                    std::min(kMaxScreenCastFrameRate, max_requested_frame_rate)),
                media::PIXEL_FORMAT_I420)));
     return;
   }
 
   DCHECK(formats_enumerated_callback_.is_null());
   formats_enumerated_callback_ = callback;
   manager_->GetDeviceFormatsInUse(
-      session_id_, media::BindToCurrentLoop(base::Bind(
-                       &LocalVideoCapturerSource::OnDeviceFormatsInUseReceived,
-                       weak_factory_.GetWeakPtr())));
+      session_id_,
+      media::BindToCurrentLoop(base::Bind(
+          &LegacyLocalVideoCapturerSource::OnDeviceFormatsInUseReceived,
+          weak_factory_.GetWeakPtr())));
 }
 
-media::VideoCaptureFormats LocalVideoCapturerSource::GetPreferredFormats() {
+media::VideoCaptureFormats
+LegacyLocalVideoCapturerSource::GetPreferredFormats() {
   return media::VideoCaptureFormats();
 }
 
-void LocalVideoCapturerSource::StartCapture(
+void LegacyLocalVideoCapturerSource::StartCapture(
     const media::VideoCaptureParams& params,
     const VideoCaptureDeliverFrameCB& new_frame_callback,
     const RunningCallback& running_callback) {
   DCHECK(params.requested_format.IsValid());
   DCHECK(thread_checker_.CalledOnValidThread());
   running_callback_ = running_callback;
 
-  stop_capture_cb_ = manager_->StartCapture(
-      session_id_, params, media::BindToCurrentLoop(base::Bind(
-                               &LocalVideoCapturerSource::OnStateUpdate,
-                               weak_factory_.GetWeakPtr())),
-      new_frame_callback);
+  stop_capture_cb_ =
+      manager_->StartCapture(session_id_, params,
+                             media::BindToCurrentLoop(base::Bind(
+                                 &LegacyLocalVideoCapturerSource::OnStateUpdate,
+                                 weak_factory_.GetWeakPtr())),
+                             new_frame_callback);
 }
 
-void LocalVideoCapturerSource::RequestRefreshFrame() {
+void LegacyLocalVideoCapturerSource::RequestRefreshFrame() {
   DVLOG(3) << __func__;
   DCHECK(thread_checker_.CalledOnValidThread());
   if (stop_capture_cb_.is_null())
     return;  // Do not request frames if the source is stopped.
   manager_->RequestRefreshFrame(session_id_);
 }
 
-void LocalVideoCapturerSource::MaybeSuspend() {
+void LegacyLocalVideoCapturerSource::MaybeSuspend() {
   DVLOG(3) << __func__;
   DCHECK(thread_checker_.CalledOnValidThread());
   manager_->Suspend(session_id_);
 }
 
-void LocalVideoCapturerSource::Resume() {
+void LegacyLocalVideoCapturerSource::Resume() {
   DVLOG(3) << __func__;
   DCHECK(thread_checker_.CalledOnValidThread());
   manager_->Resume(session_id_);
 }
 
-void LocalVideoCapturerSource::StopCapture() {
+void LegacyLocalVideoCapturerSource::StopCapture() {
   DVLOG(3) << __func__;
   DCHECK(thread_checker_.CalledOnValidThread());
   // Immediately make sure we don't provide more frames.
   if (!stop_capture_cb_.is_null())
     base::ResetAndReturn(&stop_capture_cb_).Run();
   running_callback_.Reset();
   // Invalidate any potential format enumerations going on.
   formats_enumerated_callback_.Reset();
 }
 
-void LocalVideoCapturerSource::OnStateUpdate(VideoCaptureState state) {
+void LegacyLocalVideoCapturerSource::OnStateUpdate(VideoCaptureState state) {
   DVLOG(3) << __func__ << " state = " << state;
   DCHECK(thread_checker_.CalledOnValidThread());
   if (running_callback_.is_null())
     return;
   switch (state) {
     case VIDEO_CAPTURE_STATE_STARTED:
       running_callback_.Run(true);
       break;
 
     case VIDEO_CAPTURE_STATE_STOPPING:
     case VIDEO_CAPTURE_STATE_STOPPED:
     case VIDEO_CAPTURE_STATE_ERROR:
     case VIDEO_CAPTURE_STATE_ENDED:
       base::ResetAndReturn(&running_callback_).Run(false);
       break;
 
     case VIDEO_CAPTURE_STATE_STARTING:
     case VIDEO_CAPTURE_STATE_PAUSED:
     case VIDEO_CAPTURE_STATE_RESUMED:
       // Not applicable to reporting on device starts or errors.
       break;
   }
 }
 
-void LocalVideoCapturerSource::OnDeviceFormatsInUseReceived(
+void LegacyLocalVideoCapturerSource::OnDeviceFormatsInUseReceived(
     const media::VideoCaptureFormats& formats_in_use) {
   DVLOG(3) << __func__ << ", #formats received: " << formats_in_use.size();
   DCHECK(thread_checker_.CalledOnValidThread());
   // StopCapture() might have destroyed |formats_enumerated_callback_| before
   // arriving here.
   if (formats_enumerated_callback_.is_null())
     return;
   if (formats_in_use.size()) {
     base::ResetAndReturn(&formats_enumerated_callback_).Run(formats_in_use);
     return;
   }
 
   // The device doesn't seem to have formats in use so try and retrieve the
   // whole list of supported ones.
   manager_->GetDeviceSupportedFormats(
       session_id_,
-      media::BindToCurrentLoop(
-          base::Bind(
-              &LocalVideoCapturerSource::OnDeviceSupportedFormatsEnumerated,
-              weak_factory_.GetWeakPtr())));
+      media::BindToCurrentLoop(base::Bind(
+          &LegacyLocalVideoCapturerSource::OnDeviceSupportedFormatsEnumerated,
+          weak_factory_.GetWeakPtr())));
 }
 
-void LocalVideoCapturerSource::OnDeviceSupportedFormatsEnumerated(
+void LegacyLocalVideoCapturerSource::OnDeviceSupportedFormatsEnumerated(
     const media::VideoCaptureFormats& formats) {
   DVLOG(3) << __func__ << ", #formats received: " << formats.size();
   DCHECK(thread_checker_.CalledOnValidThread());
   // StopCapture() might have destroyed |formats_enumerated_callback_| before
   // arriving here.
   if (formats_enumerated_callback_.is_null())
     return;
   if (formats.size()) {
     base::ResetAndReturn(&formats_enumerated_callback_).Run(formats);
     return;
   }
 
   // The capture device doesn't seem to support capability enumeration, compose
   // a fallback list of capabilities.
   media::VideoCaptureFormats default_formats;
   for (const auto& resolution : kVideoResolutions) {
     for (const auto frame_rate : kVideoFrameRates) {
       default_formats.push_back(media::VideoCaptureFormat(
           gfx::Size(resolution.width, resolution.height), frame_rate,
           media::PIXEL_FORMAT_I420));
     }
   }
   base::ResetAndReturn(&formats_enumerated_callback_).Run(default_formats);
 }
 
+// LocalVideoCapturerSource is a delegate used by MediaStreamVideoCapturerSource
+// for local video capture. It uses the Render singleton VideoCaptureImplManager
+// to start / stop and receive I420 frames from Chrome's video capture
+// implementation. This is a main Render thread only object.
+class LocalVideoCapturerSource final : public media::VideoCapturerSource {
+ public:
+  explicit LocalVideoCapturerSource(const StreamDeviceInfo& device_info);
+  ~LocalVideoCapturerSource() override;
+
+  // VideoCaptureDelegate Implementation.

[hbos_chromium, 2017/04/05 12:37:28]

What happened to GetCurrentSupportedFormats?

[Guido Urdaneta, 2017/04/05 16:17:16]

Using the default implementation from the base class.
That method is not needed with the new algorithm.

[hbos_chromium, 2017/04/05 16:56:31]

Acknowledged.

+  media::VideoCaptureFormats GetPreferredFormats() override;
+  void StartCapture(const media::VideoCaptureParams& params,
+                    const VideoCaptureDeliverFrameCB& new_frame_callback,
+                    const RunningCallback& running_callback) override;
+  void RequestRefreshFrame() override;
+  void MaybeSuspend() override;
+  void Resume() override;
+  void StopCapture() override;
+
+ private:
+  void OnStateUpdate(VideoCaptureState state);
+
+  // |session_id_| identifies the capture device used for this capture session.
+  const media::VideoCaptureSessionId session_id_;
+
+  VideoCaptureImplManager* const manager_;
+
+  const base::Closure release_device_cb_;
+
+  // These two are valid between StartCapture() and StopCapture().
+  // |running_call_back_| is run when capture is successfully started, and when
+  // it is stopped or error happens.
+  RunningCallback running_callback_;
+  base::Closure stop_capture_cb_;
+
+  // Bound to the main render thread.
+  base::ThreadChecker thread_checker_;
+
+  base::WeakPtrFactory<LocalVideoCapturerSource> weak_factory_;
+
+  DISALLOW_COPY_AND_ASSIGN(LocalVideoCapturerSource);
+};
+
+LocalVideoCapturerSource::LocalVideoCapturerSource(
+    const StreamDeviceInfo& device_info)
+    : session_id_(device_info.session_id),
+      manager_(RenderThreadImpl::current()->video_capture_impl_manager()),
+      release_device_cb_(manager_->UseDevice(session_id_)),
+      weak_factory_(this) {
+  DCHECK(RenderThreadImpl::current());
+}
+
+LocalVideoCapturerSource::~LocalVideoCapturerSource() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  release_device_cb_.Run();
+}
+
+media::VideoCaptureFormats LocalVideoCapturerSource::GetPreferredFormats() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  return media::VideoCaptureFormats();

[hbos_chromium, 2017/04/05 12:37:28]

No preferred formats? What does that imply? (How are preferred formats used?)

[Guido Urdaneta, 2017/04/05 16:17:16]

In this case it doesn't imply anything because the preferred formats are not
used with the LocalVideoCapturerSource.
There are two constructors for MSVCS. The first one takes a VCS (and uses the
preferred format if available), and the second one takes a device ID, a format
and a VCS.
LVCS is used only with the second one, where the format is already known from
previous execution of the constraints algorithm.

In the more general case, the implication is the same as having a
GetCurrentSupportedFormats() that returns no formats, in which case the format
would be whatever the actual source decides (implementation defined). That case
does not apply to getUserMedia.

[hbos_chromium, 2017/04/05 16:56:31]

Acknowledged.

+}
+
+void LocalVideoCapturerSource::StartCapture(
+    const media::VideoCaptureParams& params,
+    const VideoCaptureDeliverFrameCB& new_frame_callback,
+    const RunningCallback& running_callback) {
+  DCHECK(params.requested_format.IsValid());
+  DCHECK(thread_checker_.CalledOnValidThread());
+  running_callback_ = running_callback;
+
+  stop_capture_cb_ =

[hbos_chromium, 2017/04/05 12:37:28]

What if we've already stopped? Do we DCHECK, do we stop + start or is this a
NO-OP?

[Guido Urdaneta, 2017/04/05 16:17:16]

There is no change here. This is the same as LegacyLocalVideoCapturerSource.
StartCapture returns a Callback that can be used to stop the capture later.

[hbos_chromium, 2017/04/05 16:56:31]

Acknowledged.

+      manager_->StartCapture(session_id_, params,
+                             media::BindToCurrentLoop(base::Bind(
+                                 &LocalVideoCapturerSource::OnStateUpdate,
+                                 weak_factory_.GetWeakPtr())),
+                             new_frame_callback);
+}
+
+void LocalVideoCapturerSource::RequestRefreshFrame() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  if (stop_capture_cb_.is_null())
+    return;  // Do not request frames if the source is stopped.
+  manager_->RequestRefreshFrame(session_id_);
+}
+
+void LocalVideoCapturerSource::MaybeSuspend() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  manager_->Suspend(session_id_);
+}
+
+void LocalVideoCapturerSource::Resume() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  manager_->Resume(session_id_);
+}
+
+void LocalVideoCapturerSource::StopCapture() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  // Immediately make sure we don't provide more frames.
+  if (!stop_capture_cb_.is_null())
+    base::ResetAndReturn(&stop_capture_cb_).Run();
+  running_callback_.Reset();
+}
+
+void LocalVideoCapturerSource::OnStateUpdate(VideoCaptureState state) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  if (running_callback_.is_null())
+    return;
+  switch (state) {
+    case VIDEO_CAPTURE_STATE_STARTED:
+      running_callback_.Run(true);
+      break;
+
+    case VIDEO_CAPTURE_STATE_STOPPING:
+    case VIDEO_CAPTURE_STATE_STOPPED:
+    case VIDEO_CAPTURE_STATE_ERROR:
+    case VIDEO_CAPTURE_STATE_ENDED:
+      base::ResetAndReturn(&running_callback_).Run(false);
+      break;
+
+    case VIDEO_CAPTURE_STATE_STARTING:
+    case VIDEO_CAPTURE_STATE_PAUSED:
+    case VIDEO_CAPTURE_STATE_RESUMED:
+      // Not applicable to reporting on device starts or errors.
+      break;
+  }
+}
+
+}  // namespace
+
 MediaStreamVideoCapturerSource::MediaStreamVideoCapturerSource(
     const SourceStoppedCallback& stop_callback,
     std::unique_ptr<media::VideoCapturerSource> source)
     : RenderFrameObserver(nullptr), source_(std::move(source)) {
+  if (!IsOldVideoConstraints()) {
+    media::VideoCaptureFormats preferred_formats =
+        source_->GetPreferredFormats();
+    if (!preferred_formats.empty())
+      capture_params_.requested_format = preferred_formats.front();
+  }
   SetStopCallback(stop_callback);
 }
 
 MediaStreamVideoCapturerSource::MediaStreamVideoCapturerSource(
     const SourceStoppedCallback& stop_callback,
     const StreamDeviceInfo& device_info,
     RenderFrame* render_frame)
     : RenderFrameObserver(render_frame),
-      source_(new LocalVideoCapturerSource(device_info)) {
+      source_(new LegacyLocalVideoCapturerSource(device_info)) {
+  DCHECK(IsOldVideoConstraints());
   SetStopCallback(stop_callback);
   SetDeviceInfo(device_info);
 }
+
+MediaStreamVideoCapturerSource::MediaStreamVideoCapturerSource(
+    const SourceStoppedCallback& stop_callback,
+    const StreamDeviceInfo& device_info,
+    const media::VideoCaptureParams& capture_params,
+    RenderFrame* render_frame)
+    : RenderFrameObserver(render_frame),
+      source_(new LocalVideoCapturerSource(device_info)),
+      capture_params_(capture_params) {
+  DCHECK(!IsOldVideoConstraints());
+  SetStopCallback(stop_callback);
+  SetDeviceInfo(device_info);
+}
 
 MediaStreamVideoCapturerSource::~MediaStreamVideoCapturerSource() {
 }
 
 void MediaStreamVideoCapturerSource::RequestRefreshFrame() {
   source_->RequestRefreshFrame();
 }
 
 void MediaStreamVideoCapturerSource::OnHasConsumers(bool has_consumers) {
   if (has_consumers)
@@ skipping 16 matching lines @@
       max_requested_width,
       max_requested_height,
       max_requested_frame_rate,
       callback);
 }
 
 void MediaStreamVideoCapturerSource::StartSourceImpl(
     const media::VideoCaptureFormat& format,
     const blink::WebMediaConstraints& constraints,
     const VideoCaptureDeliverFrameCB& frame_callback) {
-  media::VideoCaptureParams new_params;
-  new_params.requested_format = format;
-  if (IsContentVideoCaptureDevice(device_info())) {
-    SetContentCaptureParamsFromConstraints(
-        constraints, device_info().device.type, &new_params);
-  } else if (device_info().device.type == MEDIA_DEVICE_VIDEO_CAPTURE) {
-    SetPowerLineFrequencyParamFromConstraints(constraints, &new_params);
+  if (IsOldVideoConstraints()) {
+    capture_params_.requested_format = format;
+    if (IsContentVideoCaptureDevice(device_info())) {
+      SetContentCaptureParamsFromConstraints(
+          constraints, device_info().device.type, &capture_params_);
+    } else if (device_info().device.type == MEDIA_DEVICE_VIDEO_CAPTURE) {
+      SetPowerLineFrequencyParamFromConstraints(constraints, &capture_params_);
+    }
   }
 
   is_capture_starting_ = true;
   source_->StartCapture(
-      new_params, frame_callback,
+      capture_params_, frame_callback,
       base::Bind(&MediaStreamVideoCapturerSource::OnRunStateChanged,
                  base::Unretained(this)));
 }
 
 void MediaStreamVideoCapturerSource::StopSourceImpl() {
   source_->StopCapture();
 }
 
+base::Optional<media::VideoCaptureFormat>
+MediaStreamVideoCapturerSource::GetCurrentFormatImpl() const {
+  DCHECK(!IsOldVideoConstraints());
+  return capture_params_.requested_format;
+}
+
 void MediaStreamVideoCapturerSource::OnRunStateChanged(bool is_running) {
   if (is_capture_starting_) {
     OnStartDone(is_running ? MEDIA_DEVICE_OK
                            : MEDIA_DEVICE_TRACK_START_FAILURE);
     is_capture_starting_ = false;
   } else if (!is_running) {
     StopSource();
   }
 }
 
 const char*
 MediaStreamVideoCapturerSource::GetPowerLineFrequencyForTesting() const {
   return kPowerLineFrequency;
 }
 
 }  // namespace content