[PPAPI][MediaStream] Support configure for video input.

content/renderer/pepper/pepper_media_stream_video_track_host.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/pepper/pepper_media_stream_video_track_host.h"
 
 #include "base/logging.h"
+#include "media/base/yuv_convert.h"
 #include "ppapi/c/pp_errors.h"
 #include "ppapi/c/ppb_video_frame.h"
+#include "ppapi/host/dispatch_host_message.h"
+#include "ppapi/host/host_message_context.h"
+#include "ppapi/proxy/ppapi_messages.h"
 #include "ppapi/shared_impl/media_stream_buffer.h"
+#include "third_party/libyuv/include/libyuv/scale.h"
 
 using media::VideoFrame;
+using ppapi::host::HostMessageContext;
+using ppapi::MediaStreamVideoTrackShared;
+using ppapi::proxy::SerializedHandle;

[yzshen1, 2014/02/13 19:23:26]

You are not using it.

[Peng, 2014/02/13 21:28:34]

Done.

 
 namespace {
 
-// TODO(penghuang): make it configurable.
-const int32_t kNumberOfFrames = 4;
+const int32_t kNumberOfBuffers = 4;

[yzshen1, 2014/02/13 19:23:26]

Is it k*Default*....?

[Peng, 2014/02/13 21:28:34]

Done.

+const int32_t kMaxNumberOfBuffers = 8;
 
 PP_VideoFrame_Format ToPpapiFormat(VideoFrame::Format format) {
   switch (format) {
     case VideoFrame::YV12:
       return PP_VIDEOFRAME_FORMAT_YV12;
-    case VideoFrame::YV16:
-      return PP_VIDEOFRAME_FORMAT_YV16;
     case VideoFrame::I420:
       return PP_VIDEOFRAME_FORMAT_I420;
-    case VideoFrame::YV12A:
-      return PP_VIDEOFRAME_FORMAT_YV12A;
-    case VideoFrame::YV12J:
-      return PP_VIDEOFRAME_FORMAT_YV12J;
     default:
       DVLOG(1) << "Unsupported pixel format " << format;
       return PP_VIDEOFRAME_FORMAT_UNKNOWN;
   }
 }
 
+VideoFrame::Format FromPpapiFormat(PP_VideoFrame_Format format) {
+  switch (format) {
+    case PP_VIDEOFRAME_FORMAT_YV12:
+      return VideoFrame::YV12;
+    case PP_VIDEOFRAME_FORMAT_I420:
+      return VideoFrame::I420;
+    default:
+      DVLOG(1) << "Unsupported pixel format " << format;
+      return VideoFrame::UNKNOWN;
+  }
+}
+
+// Compute size base on the size of frame received from MediaStreamVideoSink
+// and size specified by plugin.
+gfx::Size ComputeSize(const gfx::Size& frame,
+                      const gfx::Size& plugin) {
+  return gfx::Size(plugin.width() ? plugin.width() : frame.width(),
+                   plugin.height() ? plugin.height() : frame.height());
+}
+
+// Compute format base on the format of frame received from MediaStreamVideoSink
+// and format specified by plugin.
+PP_VideoFrame_Format ComputeFormat(PP_VideoFrame_Format frame,
+                                   PP_VideoFrame_Format plugin) {
+  return plugin != PP_VIDEOFRAME_FORMAT_UNKNOWN ? plugin : frame;
+}
+
+void ConvertFromMediaVideoFrame(const scoped_refptr<media::VideoFrame>& src,
+                                PP_VideoFrame_Format dst_format,
+                                const gfx::Size& dst_size,
+                                uint8_t* dst) {
+  CHECK(src->format() == VideoFrame::YV12 ||
+        src->format() == VideoFrame::I420);
+  if (dst_format == PP_VIDEOFRAME_FORMAT_BGRA) {
+    if (src->coded_size() == dst_size) {
+      media::ConvertYUVToRGB32(src->data(VideoFrame::kYPlane),
+                               src->data(VideoFrame::kUPlane),
+                               src->data(VideoFrame::kVPlane),
+                               dst,
+                               dst_size.width(),
+                               dst_size.height(),
+                               src->stride(VideoFrame::kYPlane),
+                               src->stride(VideoFrame::kUPlane),
+                               dst_size.width() * 4,
+                               media::YV12);
+    } else {
+      media::ScaleYUVToRGB32(src->data(VideoFrame::kYPlane),
+                             src->data(VideoFrame::kUPlane),
+                             src->data(VideoFrame::kVPlane),
+                             dst,
+                             src->coded_size().width(),
+                             src->coded_size().height(),
+                             dst_size.width(),
+                             dst_size.height(),
+                             src->stride(VideoFrame::kYPlane),
+                             src->stride(VideoFrame::kUPlane),
+                             dst_size.width() * 4,
+                             media::YV12,
+                             media::ROTATE_0,
+                             media::FILTER_BILINEAR);
+    }
+  } else if (dst_format == PP_VIDEOFRAME_FORMAT_YV12 ||
+             dst_format == PP_VIDEOFRAME_FORMAT_I420) {
+    static const size_t kPlanesOrder[][3] = {
+      { VideoFrame::kYPlane, VideoFrame::kVPlane, VideoFrame::kUPlane }, // YV12
+      { VideoFrame::kYPlane, VideoFrame::kUPlane, VideoFrame::kVPlane }, // I420
+    };
+    const int plane_order = (dst_format == PP_VIDEOFRAME_FORMAT_YV12) ? 0 : 1;
+    int dst_width = dst_size.width();
+    int dst_height = dst_size.height();
+    libyuv::ScalePlane(src->data(kPlanesOrder[plane_order][0]),
+                       src->stride(kPlanesOrder[plane_order][0]),
+                       src->coded_size().width(),
+                       src->coded_size().height(),
+                       dst, dst_width, dst_width, dst_height,
+                       libyuv::kFilterBox);
+    dst += dst_width * dst_height;
+    const int src_halfwidth = (src->coded_size().width() + 1) >> 1;
+    const int src_halfheight = (src->coded_size().height() + 1) >> 1;
+    const int dst_halfwidth = (dst_width + 1) >> 1;
+    const int dst_halfheight = (dst_height + 1) >> 1;
+    libyuv::ScalePlane(src->data(kPlanesOrder[plane_order][1]),
+                       src->stride(kPlanesOrder[plane_order][1]),
+                       src_halfwidth, src_halfheight,
+                       dst, dst_halfwidth, dst_halfwidth, dst_halfheight,
+                       libyuv::kFilterBox);
+    dst += dst_halfwidth * dst_halfheight;
+    libyuv::ScalePlane(src->data(kPlanesOrder[plane_order][2]),
+                       src->stride(kPlanesOrder[plane_order][2]),
+                       src_halfwidth, src_halfheight,
+                       dst, dst_halfwidth, dst_halfwidth, dst_halfheight,
+                       libyuv::kFilterBox);
+  } else {
+    NOTREACHED();
+  }
+}
+
 }  // namespace
 
 namespace content {
 
 PepperMediaStreamVideoTrackHost::PepperMediaStreamVideoTrackHost(
     RendererPpapiHost* host,
     PP_Instance instance,
     PP_Resource resource,
     const blink::WebMediaStreamTrack& track)
     : PepperMediaStreamTrackHostBase(host, instance, resource),
       track_(track),
       connected_(false),
-      frame_format_(VideoFrame::UNKNOWN),
+      buffers_(kNumberOfBuffers),
+      frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
+      plugin_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
       frame_data_size_(0) {
   DCHECK(!track_.isNull());
 }
 
 PepperMediaStreamVideoTrackHost::~PepperMediaStreamVideoTrackHost() {
   OnClose();
 }
 
+void PepperMediaStreamVideoTrackHost::InitBuffers() {
+  gfx::Size size = ComputeSize(frame_size_, plugin_frame_size_);
+  DCHECK(!size.IsEmpty());
+
+  PP_VideoFrame_Format format =
+    ComputeFormat(frame_format_, plugin_frame_format_);
+  DCHECK_NE(format, PP_VIDEOFRAME_FORMAT_UNKNOWN);
+
+  if (format == PP_VIDEOFRAME_FORMAT_BGRA) {
+    frame_data_size_ = size.width() * size.height() * 4;
+  } else {
+    frame_data_size_ = VideoFrame::AllocationSize(FromPpapiFormat(format),
+                                                  size);
+  }
+
+  DCHECK_GT(frame_data_size_, 0U);
+  int32_t buffer_size =
+      sizeof(ppapi::MediaStreamBuffer::Video) + frame_data_size_;
+  bool result = PepperMediaStreamTrackHostBase::InitBuffers(buffers_,
+                                                            buffer_size);
+  // TODO(penghuang): Send PP_ERROR_NOMEMORY to plugin.
+  CHECK(result);
+}
+
 void PepperMediaStreamVideoTrackHost::OnClose() {
   if (connected_) {
     MediaStreamVideoSink::RemoveFromVideoTrack(this, track_);
     connected_ = false;
   }
 }
 
 void PepperMediaStreamVideoTrackHost::OnVideoFrame(
     const scoped_refptr<VideoFrame>& frame) {
   DCHECK(frame);
   // TODO(penghuang): Check |frame->end_of_stream()| and close the track.
   PP_VideoFrame_Format ppformat = ToPpapiFormat(frame->format());
   if (ppformat == PP_VIDEOFRAME_FORMAT_UNKNOWN)
     return;
 
-  if (frame_size_ != frame->coded_size() || frame_format_ != frame->format()) {
+  if (frame_size_.IsEmpty()) {
     frame_size_ = frame->coded_size();
-    frame_format_ = frame->format();
-    // TODO(penghuang): Support changing |frame_size_| & |frame_format_| more
-    // than once.
-    DCHECK(!frame_data_size_);
-    frame_data_size_ = VideoFrame::AllocationSize(frame_format_, frame_size_);
-    int32_t size = sizeof(ppapi::MediaStreamBuffer::Video) + frame_data_size_;
-    bool result = InitBuffers(kNumberOfFrames, size);
-    // TODO(penghuang): Send PP_ERROR_NOMEMORY to plugin.
-    CHECK(result);
+    frame_format_ = ppformat;
+    InitBuffers();
   }
 
   int32_t index = buffer_manager()->DequeueBuffer();
   // Drop frames if the underlying buffer is full.
-  if (index < 0)
+  if (index < 0) {
+    DVLOG(1) << "A frame is dropped.";
     return;
+  }
 
-  // TODO(penghuang): support format conversion and size scaling.
+  DCHECK(frame->coded_size() == frame_size_);

[yzshen1, 2014/02/13 19:23:26]

(just to double check) So OnVideoFrame never returns frames of different size or
format? I remember you mentioned before that it is possible to change
format/size in the middle?

[Peng, 2014/02/13 21:28:34]

Currently, it will be always same. But w3c has an API to change the frame size.
But it is not supported.

[yzshen1, 2014/02/14 18:25:43]

If the backend changes to support that in the future, this code may break
silently. Maybe we should fail more explicitly, for example, discard the frame,
log an error and return?

On 2014/02/13 21:28:34, Peng wrote:

[Peng, 2014/02/14 20:04:49]

How about use CHECK()? I think it can help us locate the problem in future
easily.

+  DCHECK_EQ(ppformat, frame_format_);
+
+  gfx::Size size = ComputeSize(frame_size_, plugin_frame_size_);
+  PP_VideoFrame_Format format = ComputeFormat(frame_format_,
+                                              plugin_frame_format_);
   ppapi::MediaStreamBuffer::Video* buffer =
       &(buffer_manager()->GetBufferPointer(index)->video);
   buffer->header.size = buffer_manager()->buffer_size();
   buffer->header.type = ppapi::MediaStreamBuffer::TYPE_VIDEO;
   buffer->timestamp = frame->GetTimestamp().InSecondsF();
-  buffer->format = ppformat;
-  buffer->size.width = frame->coded_size().width();
-  buffer->size.height = frame->coded_size().height();
+  buffer->format = format;
+  buffer->size.width = size.width();
+  buffer->size.height = size.height();
   buffer->data_size = frame_data_size_;

[yzshen1, 2014/02/13 19:23:26]

I think the naming is confusing: |frame_data_size_| doesn't correspond to
|frame_size_/format_|. Instead, it corresponds to either that or
|plugin_frame_size_/format_|.

[Peng, 2014/02/13 21:28:34]

I think plugin_frame_data_size_ is not a good name as well. Currently plugin_
prefix means those value are specified by plugin, but it could be different to
the real value.

So I renamed frame_format_ and frame_size_ to source_frame_format_, and
source_frame_size_. WDYT?

Done

[yzshen1, 2014/02/14 18:25:43]

Okay.

-
-  COMPILE_ASSERT(VideoFrame::kYPlane == 0, y_plane_should_be_0);
-  COMPILE_ASSERT(VideoFrame::kUPlane == 1, u_plane_should_be_1);
-  COMPILE_ASSERT(VideoFrame::kVPlane == 2, v_plane_should_be_2);
-
-  uint8_t* dst = buffer->data;
-  size_t num_planes = VideoFrame::NumPlanes(frame->format());
-  for (size_t i = 0; i < num_planes; ++i) {
-    const uint8_t* src = frame->data(i);
-    const size_t row_bytes = frame->row_bytes(i);
-    const size_t src_stride = frame->stride(i);
-    int rows = frame->rows(i);
-    for (int j = 0; j < rows; ++j) {
-      memcpy(dst, src, row_bytes);
-      dst += row_bytes;
-      src += src_stride;
-    }
-  }
-
+  ConvertFromMediaVideoFrame(frame, format, size, buffer->data);
   SendEnqueueBufferMessageToPlugin(index);
 }
 
 void PepperMediaStreamVideoTrackHost::DidConnectPendingHostToResource() {
   if (!connected_) {
     MediaStreamVideoSink::AddToVideoTrack(this, track_);
     connected_ = true;
   }
 }
 
+int32_t PepperMediaStreamVideoTrackHost::OnResourceMessageReceived(
+    const IPC::Message& msg,
+    HostMessageContext* context) {
+  IPC_BEGIN_MESSAGE_MAP(PepperMediaStreamVideoTrackHost, msg)
+    PPAPI_DISPATCH_HOST_RESOURCE_CALL(
+        PpapiHostMsg_MediaStreamVideoTrack_Configure,
+        OnHostMsgConfigure)
+  IPC_END_MESSAGE_MAP()
+  return PepperMediaStreamTrackHostBase::OnResourceMessageReceived(msg,
+                                                                   context);
+}
+
+int32_t PepperMediaStreamVideoTrackHost::OnHostMsgConfigure(

[yzshen1, 2014/02/13 19:23:26]

The API definition: Do we allow Configure() to be called multiple times? Do we
allow it to be called after GetFrame() calls?

I am afraid of corruptions/races around reconfiguring. For example, the plugin
GetFrame(), holds it, re-configure, RecycleFrame(). An index corresponds to the
old buffers is reported.

There might be other issues.

Please think hard about this part. It is extremely easy to get into trouble.

[Peng, 2014/02/13 21:28:34]

I think we have to support calling it multiple times. Because the first frame
could be received before calling configure(). In this case, the underlying
buffers have been initialized. If we support it, we will support re-configure as
well.
Those cases will be checked in MediaStreamVideoTreackResource. If Configure() is
called with some frames hold by plugin, the configure() will return error
directly.

[yzshen1, 2014/02/14 18:25:43]

I feel a little nervous about potential corruptions / races, would you please
more thoroughly explain how you prevent all problems?

[Peng, 2014/02/14 20:04:49]

Please check the code in PepperMediaStreamVideoTrackResource::Configure().

1. If plugin holds frames, xxResource::Configure() will return error.
2. If there is pending calls of configure() or getframe(), it will return error.
3. xxResource::GetFrame() will return error, if there is a pending call of
Configure() is going.

1,2,3 will guarantee no frames will be used during a configure() call. So it
should be very safe to re-allocate underlying buffers. 

+    HostMessageContext* context,
+    const MediaStreamVideoTrackShared::Attributes& attributes) {
+  CHECK(MediaStreamVideoTrackShared::VerifyAttributes(attributes));

[yzshen1, 2014/02/13 19:23:26]

Should we reply the incoming request with failure instead of crashing ourselves?

[Peng, 2014/02/13 21:28:34]

Because MediaStreamVideoTreackResource will use this function to check
attrib_list as well. So at here, this function should always return true,
otherwise the chrome may be hacked.

+
+  bool changed = false;
+  const uint32_t kWHMask = MediaStreamVideoTrackShared::Attributes::MASK_WIDTH |
+                           MediaStreamVideoTrackShared::Attributes::MASK_HEIGHT;
+  if (attributes.mask & kWHMask) {
+    gfx::Size new_size = plugin_frame_size_;
+    if (attributes.mask & MediaStreamVideoTrackShared::Attributes::MASK_WIDTH)
+      new_size.set_width(attributes.width);
+    if (attributes.mask & MediaStreamVideoTrackShared::Attributes::MASK_HEIGHT)
+      new_size.set_height(attributes.height);
+    if (ComputeSize(frame_size_, plugin_frame_size_) !=
+        ComputeSize(frame_size_, new_size)) {
+      changed = true;
+    }
+    plugin_frame_size_ = new_size;
+  }
+
+  if (attributes.mask & MediaStreamVideoTrackShared::Attributes::MASK_BUFFERS) {
+    int32_t buffers = attributes.buffers ?

[yzshen1, 2014/02/13 19:23:26]

So it is always > 0, right?

[Peng, 2014/02/13 21:28:34]

Yes. MediaStreamVideoTrackShared::VerifyAttributes(attributes) will make sure
all attributes are valid.

+        std::min(kMaxNumberOfBuffers, attributes.buffers) : kNumberOfBuffers;
+    if (buffers != buffers_) {
+      buffers_ = buffers;
+      changed = true;
+    }
+  }
+
+  if (attributes.mask & MediaStreamVideoTrackShared::Attributes::MASK_FORMAT) {
+    if (plugin_frame_format_ != attributes.format) {
+      PP_VideoFrame_Format original_format = ComputeFormat(
+          frame_format_, plugin_frame_format_);
+      PP_VideoFrame_Format new_format = ComputeFormat(
+          frame_format_, attributes.format);
+      if (new_format != original_format)
+        changed = true;
+      plugin_frame_format_ = attributes.format;
+    }
+  }
+
+  // If the first frame has been received, we will re init buffers with

[yzshen1, 2014/02/13 19:23:26]

re init -> re-initialize

[Peng, 2014/02/13 21:28:34]

Done.

+  // new settings. Otherwise, we will initialize buffer when we receive
+  // the first frame, because plugin can only provide part of attributes
+  // which are not enough to initialize buffers.
+  if (changed && !frame_size_.IsEmpty()) {

[yzshen1, 2014/02/13 19:23:26]

nit: no need to have {}

[Peng, 2014/02/13 21:28:34]

Done.

+    InitBuffers();
+  }
+
+  context->reply_msg = PpapiPluginMsg_MediaStreamVideoTrack_ConfigureReply();
+  return PP_OK;
+}
+
 }  // namespace content