Make sure MediaStreamVideoSource support cropping.

content/renderer/media/media_stream_video_source.cc

Index: content/renderer/media/media_stream_video_source.cc
diff --git a/content/renderer/media/media_stream_video_source.cc b/content/renderer/media/media_stream_video_source.cc
index 118d66e3cddc59faa72ed1fd4c7448c0bc0c377e..555730a0966fa1590ba4ddd254ce11f3d710d76b 100644
--- a/content/renderer/media/media_stream_video_source.cc
+++ b/content/renderer/media/media_stream_video_source.cc
@@ -11,6 +11,7 @@
 #include "base/strings/string_number_conversions.h"
 #include "content/renderer/media/media_stream_dependency_factory.h"
 #include "content/renderer/media/webrtc/webrtc_video_capturer_adapter.h"
+#include "third_party/libyuv/include/libyuv/convert.h"
 
 namespace content {
 
@@ -36,6 +37,10 @@ const char kSourceId[] = "sourceId";
 // are unknown.
 const char kGooglePrefix[] = "goog";
 
+// MediaStreamVideoSource support cropping of video frames but only up to

[Jói, 2014/02/27 14:52:25]

support -> supports

[perkj_chrome, 2014/03/01 12:32:59]

Done.

+// kMaxCropFactor.
+const int kMaxCropFactor = 2;
+
 // Returns true if |constraint| is fulfilled. |format| can be changed
 // changed by a constraint. Ie - the frame rate can be changed by setting
 // maxFrameRate.
@@ -96,11 +101,11 @@ bool UpdateFormatForConstraint(
   if (constraint_name == MediaStreamVideoSource::kMinWidth) {
     return (value <= format->frame_size.width());
   } else if (constraint_name == MediaStreamVideoSource::kMaxWidth) {
-    return (value >= format->frame_size.width());
+    return (value * kMaxCropFactor >= format->frame_size.width());
   } else if (constraint_name == MediaStreamVideoSource::kMinHeight) {
     return (value <= format->frame_size.height());
   } else if (constraint_name == MediaStreamVideoSource::kMaxHeight) {
-    return (value >= format->frame_size.height());
+     return (value * kMaxCropFactor >= format->frame_size.height());
   } else if (constraint_name == MediaStreamVideoSource::kMinFrameRate) {
     return (value <= format->frame_rate);
   } else if (constraint_name == MediaStreamVideoSource::kMaxFrameRate) {
@@ -185,25 +190,43 @@ media::VideoCaptureFormats FilterFormats(
   return candidates;
 }
 
-// Find the format that best matches the default video size.
-// This algorithm is chosen since a resolution must be picked even if no
-// constraints are provided. We don't just select the maximum supported
-// resolution since higher resolution cost more in terms of complexity and
-// many cameras perform worse at its maximum supported resolution.
-const media::VideoCaptureFormat& GetBestCaptureFormat(
-    const media::VideoCaptureFormats& formats) {
-  DCHECK(!formats.empty());
+// Retrieve the desired max width and height from |constraints|.
+void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
+                                 int* dw, int* dh) {
+  bool mandatory_found = false;
+  blink::WebString width;
+  if (constraints.getMandatoryConstraintValue(
+          MediaStreamVideoSource::kMaxWidth, width)) {
+    base::StringToInt(width.utf8(), dw);
+    mandatory_found = true;
+  }
+  blink::WebString height;
+  if (constraints.getMandatoryConstraintValue(
+          MediaStreamVideoSource::kMaxHeight, height)) {
+    base::StringToInt(height.utf8(), dh);
+    mandatory_found = true;
+  }
+  if (mandatory_found)
+    return;
 
-  int default_area =
-      MediaStreamVideoSource::kDefaultWidth *
-      MediaStreamVideoSource::kDefaultHeight;
+  if (constraints.getOptionalConstraintValue(
+          MediaStreamVideoSource::kMaxWidth, width)) {
+      base::StringToInt(width.utf8(), dw);
+  }
+  if (constraints.getOptionalConstraintValue(
+          MediaStreamVideoSource::kMaxHeight, height)) {
+    base::StringToInt(height.utf8(), dh);
+  }
+}
 
+const media::VideoCaptureFormat& GetBestFormatBasedOnArea(
+    const media::VideoCaptureFormats& formats,
+    int area) {
   media::VideoCaptureFormats::const_iterator it = formats.begin();
   media::VideoCaptureFormats::const_iterator best_it = formats.begin();
   int best_diff = std::numeric_limits<int>::max();
   for (; it != formats.end(); ++it) {
-    int diff = abs(default_area -
-                   it->frame_size.width() * it->frame_size.height());
+    int diff = abs(area - it->frame_size.width() * it->frame_size.height());
     if (diff < best_diff) {
       best_diff = diff;
       best_it = it;
@@ -212,6 +235,73 @@ const media::VideoCaptureFormat& GetBestCaptureFormat(
   return *best_it;
 }
 
+// Find the format that best matches the default video size.
+// This algorithm is chosen since a resolution must be picked even if no
+// constraints are provided. We don't just select the maximum supported
+// resolution since higher resolution cost more in terms of complexity and
+// many cameras perform worse at its maximum supported resolution.
+void GetBestCaptureFormat(
+    const media::VideoCaptureFormats& formats,
+    const blink::WebMediaConstraints& constraints,
+    media::VideoCaptureFormat* capture_format,
+    gfx::Size* frame_output_size) {
+  DCHECK(!formats.empty());
+  DCHECK(frame_output_size);
+
+  int max_width = std::numeric_limits<int>::max();
+  int max_height = std::numeric_limits<int>::max();;
+  GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
+
+  *capture_format = GetBestFormatBasedOnArea(

[Jói, 2014/02/27 14:52:25]

It seems this statement will never choose a capture format larger than 640x480.
Is that intentional?

[perkj_chrome, 2014/03/01 12:32:59]

Its intentional that we choose a size that we support that is as close as
posible to 640*480. 
Butbefore this function is called the supported formats has been filtered by
constraints. So if a constraint has requested a size with minHeight > 480 or
minWidth > 640 that is what we will get.

See for example the unittest MediaStreamVideoSourceTest,
MandatoryMinVgaOptional720P

[Jói, 2014/03/03 13:55:15]

OK, thanks, I see the format filtering now. Still not sure why the max width and
height are clamped at 640x480 when calling GetBestFormatBasedOnArea. But will
hold off on further review based on your last email.

+      formats,
+      std::min(max_width, MediaStreamVideoSource::kDefaultWidth) *
+      std::min(max_height, MediaStreamVideoSource::kDefaultHeight));
+
+  *frame_output_size = capture_format->frame_size;
+  if (max_width < frame_output_size->width())
+    frame_output_size->set_width(max_width);
+  if (max_height < frame_output_size->height())
+    frame_output_size->set_height(max_height);
+}
+
+scoped_refptr<media::VideoFrame> CreateCroppedFrame(
+    const scoped_refptr<media::VideoFrame>& src,
+    const scoped_refptr<media::VideoFrame>& dst) {
+  const int src_width = src->coded_size().width();
+  const int src_height = src->coded_size().height();
+  const int dst_width = dst->coded_size().width();
+  const int dst_height = dst->coded_size().height();
+  DCHECK(src_width >= dst_width);

[Jói, 2014/02/27 14:52:25]

Why not > rather than >=, to make sure we're not calling this (relatively
expensive) function unnecessarily?

[perkj_chrome, 2014/03/01 12:32:59]

Done.

+  DCHECK(src_height >= dst_height);
+
+  const int horiz_crop = ((src_width - dst_width) / 2) & ~1;
+  const int vert_crop = ((src_height - dst_height) / 2) & ~1;
+
+  const uint8* src_y = src->data(media::VideoFrame::kYPlane) +
+      (src_width * vert_crop + horiz_crop);
+  const int halfwidth = (src_width + 1) / 2;
+  const uint8* src_u = src->data(media::VideoFrame::kUPlane) +
+      (halfwidth * vert_crop + horiz_crop) / 2;
+  const uint8* src_v = src->data(media::VideoFrame::kVPlane) +
+      (halfwidth * vert_crop + horiz_crop) / 2;
+
+  libyuv::I420Copy(src_y,
+                  src->stride(media::VideoFrame::kYPlane),
+                  src_u,
+                  src->stride(media::VideoFrame::kUPlane),
+                  src_v,
+                  src->stride(media::VideoFrame::kVPlane),
+                  dst->data(media::VideoFrame::kYPlane),
+                  dst->stride(media::VideoFrame::kYPlane),
+                  dst->data(media::VideoFrame::kUPlane),
+                  dst->stride(media::VideoFrame::kUPlane),
+                  dst->data(media::VideoFrame::kVPlane),
+                  dst->stride(media::VideoFrame::kVPlane),
+                  dst_width,
+                  dst_height);
+  return dst;
+}
+
 }  // anonymous namespace
 
 MediaStreamVideoSource::MediaStreamVideoSource(
@@ -255,7 +345,7 @@ void MediaStreamVideoSource::AddTrack(
     case STARTING:
     case RETRIEVING_CAPABILITIES: {
       // The |callback| will be triggered once the delegate has started or
-      // the capabilitites has been retrieved.
+      // the capabilities has been retrieved.
       break;
     }
     case ENDED:
@@ -302,8 +392,23 @@ void MediaStreamVideoSource::DoStopSource() {
 
 void MediaStreamVideoSource::DeliverVideoFrame(
     const scoped_refptr<media::VideoFrame>& frame) {
-  if (capture_adapter_)
-    capture_adapter_->OnFrameCaptured(frame);
+  scoped_refptr<media::VideoFrame> video_frame(frame);
+
+  if (frame->format() == media::VideoFrame::I420 ||
+      frame->format() == media::VideoFrame::YV12) {
+    if (frame->coded_size() != frame_output_size_) {
+      // Cropping is needed.
+      gfx::Size size = frame_output_size_;
+      size.SetToMin(frame->coded_size());
+      scoped_refptr<media::VideoFrame> dst = frame_pool_.CreateFrame(
+          media::VideoFrame::I420, size, gfx::Rect(size), size,
+          frame->GetTimestamp());
+      video_frame = CreateCroppedFrame(frame, dst);
+    }
+
+    if (capture_adapter_)
+      capture_adapter_->OnFrameCaptured(video_frame);
+  }
 }
 
 void MediaStreamVideoSource::OnSupportedFormats(
@@ -312,8 +417,10 @@ void MediaStreamVideoSource::OnSupportedFormats(
   DCHECK_EQ(RETRIEVING_CAPABILITIES, state_);
 
   supported_formats_ = formats;
-  if (!FindBestFormatWithConstraints(supported_formats_, &current_format_,
-                      &current_constraints_)) {
+  if (!FindBestFormatWithConstraints(supported_formats_,
+                                     &current_format_,
+                                     &frame_output_size_,
+                                     &current_constraints_)) {
     FinalizeAddTrack();
     SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
     return;
@@ -328,13 +435,16 @@ void MediaStreamVideoSource::OnSupportedFormats(
   media::VideoCaptureParams params;
   params.requested_format = current_format_;
   StartSourceImpl(params);
+
+
 }
 
 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
     const media::VideoCaptureFormats& formats,
     media::VideoCaptureFormat* best_format,
+    gfx::Size* frame_output_size,
     blink::WebMediaConstraints* resulting_constraints) {
-  // Find the first constraints that we can fulfilled.
+  // Find the first constraints that we can fulfill.
   for (std::vector<RequestedConstraints>::iterator request_it =
            requested_constraints_.begin();
        request_it != requested_constraints_.end(); ++request_it) {
@@ -345,7 +455,10 @@ bool MediaStreamVideoSource::FindBestFormatWithConstraints(
         FilterFormats(requested_constraints, formats);
     if (filtered_formats.size() > 0) {
       // A request with constraints that can be fulfilled.
-      *best_format = GetBestCaptureFormat(filtered_formats);
+      GetBestCaptureFormat(filtered_formats,
+                           requested_constraints,
+                           best_format,
+                           frame_output_size);
       *resulting_constraints= requested_constraints;
       return true;
     }