media/gpu: switch v4l2_jpeg_decode_accelerator to use multi-planar APIs

media/gpu/v4l2_jpeg_decode_accelerator.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/gpu/v4l2_jpeg_decode_accelerator.h"
 
 #include <errno.h>
 #include <linux/videodev2.h>
 #include <string.h>
 #include <sys/mman.h>
@@ skipping 90 matching lines @@
     0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66,
     0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A,
     0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x92, 0x93, 0x94,
     0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7,
     0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA,
     0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4,
     0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
     0xE8, 0xE9, 0xEA, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA};
 
 V4L2JpegDecodeAccelerator::BufferRecord::BufferRecord()
-    : address(nullptr), length(0), at_device(false) {}
+    : num_planes(0), at_device(false) {
+  memset(address, 0, sizeof(address));
+  memset(length, 0, sizeof(length));
+}
 
 V4L2JpegDecodeAccelerator::BufferRecord::~BufferRecord() {}
 
 V4L2JpegDecodeAccelerator::JobRecord::JobRecord(
     const BitstreamBuffer& bitstream_buffer,
     scoped_refptr<VideoFrame> video_frame)
     : bitstream_buffer_id(bitstream_buffer.id()),
       shm(bitstream_buffer, true),
       out_frame(video_frame) {}
 
@@ skipping 65 matching lines @@
 bool V4L2JpegDecodeAccelerator::Initialize(Client* client) {
   DCHECK(child_task_runner_->BelongsToCurrentThread());
 
   if (!device_->Open(V4L2Device::Type::kJpegDecoder, V4L2_PIX_FMT_JPEG)) {
     LOG(ERROR) << "Failed to open device";
     return false;
   }
 
   // Capabilities check.
   struct v4l2_capability caps;
-  const __u32 kCapsRequired = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_M2M;
+  const __u32 kCapsRequiredMplane =
+      V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_M2M_MPLANE;
+  const __u32 kCapsRequiredSplane = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_M2M;
   memset(&caps, 0, sizeof(caps));
   if (device_->Ioctl(VIDIOC_QUERYCAP, &caps) != 0) {
     PLOG(ERROR) << __func__ << ": ioctl() failed: VIDIOC_QUERYCAP";
     return false;
   }
-  if ((caps.capabilities & kCapsRequired) != kCapsRequired) {
+  if ((caps.capabilities & kCapsRequiredMplane) == kCapsRequiredMplane) {
+    input_buf_type_ = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+    output_buf_type_ = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+  } else if ((caps.capabilities & kCapsRequiredSplane) == kCapsRequiredSplane) {
+    input_buf_type_ = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+    output_buf_type_ = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+  } else {
     LOG(ERROR) << __func__ << ": VIDIOC_QUERYCAP, caps check failed: 0x"
                << std::hex << caps.capabilities;
     return false;
   }
 
   // Subscribe to the source change event.
   struct v4l2_event_subscription sub;
   memset(&sub, 0, sizeof(sub));
   sub.type = V4L2_EVENT_SOURCE_CHANGE;
   if (device_->Ioctl(VIDIOC_SUBSCRIBE_EVENT, &sub) != 0) {
@@ skipping 76 matching lines @@
 
 bool V4L2JpegDecodeAccelerator::ShouldRecreateInputBuffers() {
   DCHECK(decoder_task_runner_->BelongsToCurrentThread());
   if (input_jobs_.empty())
     return false;
 
   linked_ptr<JobRecord> job_record = input_jobs_.front();
   // Check input buffer size is enough
   return (input_buffer_map_.empty() ||
           (job_record->shm.size() + sizeof(kDefaultDhtSeg)) >
-              input_buffer_map_.front().length);
+              input_buffer_map_.front().length[0]);
 }
 
 bool V4L2JpegDecodeAccelerator::RecreateInputBuffers() {
   DVLOG(3) << __func__;
   DCHECK(decoder_task_runner_->BelongsToCurrentThread());
 
   // If running queue is not empty, we should wait until pending frames finish.
   if (!running_jobs_.empty())
     return true;
 
@@ skipping 26 matching lines @@
   DCHECK(decoder_task_runner_->BelongsToCurrentThread());
   DCHECK(!input_streamon_);
   DCHECK(!input_jobs_.empty());
   linked_ptr<JobRecord> job_record = input_jobs_.front();
   // The input image may miss huffman table. We didn't parse the image before,
   // so we create more to avoid the situation of not enough memory.
   // Reserve twice size to avoid recreating input buffer frequently.
   size_t reserve_size = (job_record->shm.size() + sizeof(kDefaultDhtSeg)) * 2;
   struct v4l2_format format;
   memset(&format, 0, sizeof(format));
-  format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
-  format.fmt.pix.pixelformat = V4L2_PIX_FMT_JPEG;
-  format.fmt.pix.sizeimage = reserve_size;
-  format.fmt.pix.field = V4L2_FIELD_ANY;
+  format.type = input_buf_type_;
+  if (V4L2_TYPE_IS_MULTIPLANAR(input_buf_type_)) {
+    format.fmt.pix_mp.pixelformat = V4L2_PIX_FMT_JPEG;
+    format.fmt.pix_mp.plane_fmt[0].sizeimage = reserve_size;
+    format.fmt.pix_mp.field = V4L2_FIELD_ANY;

[wuchengli, 2016/12/16 03:15:15]

format.fmt.pix_mp.num_planes = kInputPlanes;

[jcliang, 2016/12/16 04:19:53]

Done.

+  } else {
+    format.fmt.pix.pixelformat = V4L2_PIX_FMT_JPEG;
+    format.fmt.pix.sizeimage = reserve_size;
+    format.fmt.pix.field = V4L2_FIELD_ANY;
+  }
   IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_S_FMT, &format);
 
   struct v4l2_requestbuffers reqbufs;
   memset(&reqbufs, 0, sizeof(reqbufs));
   reqbufs.count = kBufferCount;
-  reqbufs.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+  reqbufs.type = input_buf_type_;
   reqbufs.memory = V4L2_MEMORY_MMAP;
   IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_REQBUFS, &reqbufs);
 
   DCHECK(input_buffer_map_.empty());
   input_buffer_map_.resize(reqbufs.count);
 
   for (size_t i = 0; i < input_buffer_map_.size(); ++i) {
     free_input_buffers_.push_back(i);
 
     struct v4l2_buffer buffer;
+    struct v4l2_plane plane;
     memset(&buffer, 0, sizeof(buffer));
+    memset(&plane, 0, sizeof(plane));
     buffer.index = i;
-    buffer.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+    buffer.type = input_buf_type_;
+    if (V4L2_TYPE_IS_MULTIPLANAR(input_buf_type_)) {
+      buffer.m.planes = &plane;
+      buffer.length = kInputPlanes;
+    }
     buffer.memory = V4L2_MEMORY_MMAP;
     IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_QUERYBUF, &buffer);
-    void* address = device_->Mmap(NULL, buffer.length, PROT_READ | PROT_WRITE,
-                                  MAP_SHARED, buffer.m.offset);
+    input_buffer_map_[i].num_planes = kInputPlanes;
+    uint32_t length, offset;
+    if (V4L2_TYPE_IS_MULTIPLANAR(input_buf_type_)) {
+      length = plane.length;
+      offset = plane.m.mem_offset;
+    } else {
+      length = buffer.length;
+      offset = buffer.m.offset;
+    }
+    void* address =
+        device_->Mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_SHARED, offset);
     if (address == MAP_FAILED) {
       PLOG(ERROR) << __func__ << ": mmap() failed";
       PostNotifyError(kInvalidBitstreamBufferId, PLATFORM_FAILURE);
       return false;
     }
-    input_buffer_map_[i].address = address;
-    input_buffer_map_[i].length = buffer.length;
+    input_buffer_map_[i].address[0] = address;
+    input_buffer_map_[i].length[0] = length;
   }
 
   return true;
 }
 
 bool V4L2JpegDecodeAccelerator::CreateOutputBuffers() {
   DVLOG(3) << __func__;
   DCHECK(decoder_task_runner_->BelongsToCurrentThread());
   DCHECK(!output_streamon_);
   DCHECK(!running_jobs_.empty());
   linked_ptr<JobRecord> job_record = running_jobs_.front();
 
   size_t frame_size = VideoFrame::AllocationSize(
       PIXEL_FORMAT_I420, job_record->out_frame->coded_size());
   struct v4l2_format format;
   memset(&format, 0, sizeof(format));
-  format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-  format.fmt.pix.width = job_record->out_frame->coded_size().width();
-  format.fmt.pix.height = job_record->out_frame->coded_size().height();
-  format.fmt.pix.sizeimage = frame_size;
-  format.fmt.pix.pixelformat = V4L2_PIX_FMT_YUV420;
-  format.fmt.pix.field = V4L2_FIELD_ANY;
+  format.type = output_buf_type_;
+  if (V4L2_TYPE_IS_MULTIPLANAR(output_buf_type_)) {
+    format.fmt.pix_mp.width = job_record->out_frame->coded_size().width();
+    format.fmt.pix_mp.height = job_record->out_frame->coded_size().height();
+    format.fmt.pix_mp.num_planes = 1;
+    format.fmt.pix_mp.pixelformat = V4L2_PIX_FMT_YUV420;
+    format.fmt.pix_mp.plane_fmt[0].sizeimage = frame_size;
+    format.fmt.pix_mp.field = V4L2_FIELD_ANY;
+  } else {
+    format.fmt.pix.width = job_record->out_frame->coded_size().width();
+    format.fmt.pix.height = job_record->out_frame->coded_size().height();
+    format.fmt.pix.pixelformat = V4L2_PIX_FMT_YUV420;
+    format.fmt.pix.sizeimage = frame_size;
+    format.fmt.pix.field = V4L2_FIELD_ANY;
+  }
   IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_S_FMT, &format);
-  output_buffer_pixelformat_ = format.fmt.pix.pixelformat;
-  output_buffer_coded_size_.SetSize(format.fmt.pix.width,
-                                    format.fmt.pix.height);
+  if (V4L2_TYPE_IS_MULTIPLANAR(output_buf_type_)) {
+    output_buffer_pixelformat_ = format.fmt.pix_mp.pixelformat;
+    output_buffer_coded_size_.SetSize(format.fmt.pix_mp.width,
+                                      format.fmt.pix_mp.height);
+  } else {
+    output_buffer_pixelformat_ = format.fmt.pix.pixelformat;
+    output_buffer_coded_size_.SetSize(format.fmt.pix.width,
+                                      format.fmt.pix.height);
+  }
 
   struct v4l2_requestbuffers reqbufs;
   memset(&reqbufs, 0, sizeof(reqbufs));
   reqbufs.count = kBufferCount;
-  reqbufs.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+  reqbufs.type = output_buf_type_;
   reqbufs.memory = V4L2_MEMORY_MMAP;
   IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_REQBUFS, &reqbufs);
 
   DCHECK(output_buffer_map_.empty());
   output_buffer_map_.resize(reqbufs.count);
 
   VideoPixelFormat output_format =
       V4L2Device::V4L2PixFmtToVideoPixelFormat(output_buffer_pixelformat_);
 
   for (size_t i = 0; i < output_buffer_map_.size(); ++i) {
     free_output_buffers_.push_back(i);
 
     struct v4l2_buffer buffer;
+    struct v4l2_plane planes[kOutputPlanes];
     memset(&buffer, 0, sizeof(buffer));
+    memset(planes, 0, sizeof(planes));
     buffer.index = i;
-    buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+    buffer.type = output_buf_type_;
     buffer.memory = V4L2_MEMORY_MMAP;
+    if (V4L2_TYPE_IS_MULTIPLANAR(output_buf_type_)) {
+      buffer.m.planes = planes;
+      buffer.length = kOutputPlanes;

[wuchengli, 2016/12/16 03:15:15]

Use format.fmt.pix_mp.num_planes returned from S_FMT. The output format can be
V4L2_PIX_FMT_YUV420 (we request it in line 434) or V4L2_PIX_FMT_YUV420M.

[jcliang, 2016/12/16 04:19:53]

Done.

+    }
     IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_QUERYBUF, &buffer);
 
-    DCHECK_GE(buffer.length,
-              VideoFrame::AllocationSize(
-                  output_format,
-                  gfx::Size(format.fmt.pix.width, format.fmt.pix.height)));
+    if (V4L2_TYPE_IS_MULTIPLANAR(output_buf_type_)) {
+      uint32_t total_length = 0;
+      for (uint32_t i = 0; i < kOutputPlanes; ++i) {

[wuchengli, 2016/12/16 03:15:15]

s/kOutputPlanes/buffer.length/

[jcliang, 2016/12/16 04:19:53]

Done.

+        total_length += planes[i].length;
+      }
+      DCHECK_GE(total_length,
+                VideoFrame::AllocationSize(

[wuchengli, 2016/12/16 03:15:15]

Use VideoFrame::PlaneSize to check each plane in the for loop.

[jcliang, 2016/12/16 04:19:53]

Done.

+                    output_format, gfx::Size(format.fmt.pix_mp.width,
+                                             format.fmt.pix_mp.height)));
+    } else {
+      DCHECK_GE(buffer.length,
+                VideoFrame::AllocationSize(
+                    output_format,
+                    gfx::Size(format.fmt.pix.width, format.fmt.pix.height)));
+    }
 
-    void* address = device_->Mmap(NULL, buffer.length, PROT_READ | PROT_WRITE,
-                                  MAP_SHARED, buffer.m.offset);
-    if (address == MAP_FAILED) {
-      PLOG(ERROR) << __func__ << ": mmap() failed";
-      PostNotifyError(kInvalidBitstreamBufferId, PLATFORM_FAILURE);
-      return false;
+    if (V4L2_TYPE_IS_MULTIPLANAR(output_buf_type_)) {

[wuchengli, 2016/12/16 03:15:15]

if and else clauses can be combined.

if (V4L2_TYPE_IS_MULTIPLANAR(output_buf_type_))
  output_buffer_map_[i].num_planes = buffer.length;
else
  output_buffer_map_[i].num_planes = 1;
for (size_t j = 0; j < output_buffer_map_[i].num_planes; ++j) {
  ...
}

[jcliang, 2016/12/16 04:19:53]

We can't do this because the fields in |buffer| that Mmap() uses are different.
For splane it needs buffer.length and buffer.m.offset; for mplane it needs
buffer.m.planes[j].length and buffer.m.planes[j].m.mem_offset.

[wuchengli, 2016/12/16 06:50:32]

You are right.

+      output_buffer_map_[i].num_planes = buffer.length;
+      for (size_t j = 0; j < buffer.length; ++j) {
+        void* address =
+            device_->Mmap(NULL, planes[j].length, PROT_READ | PROT_WRITE,
+                          MAP_SHARED, planes[j].m.mem_offset);
+        if (address == MAP_FAILED) {
+          PLOG(ERROR) << __func__ << ": mmap() failed";
+          PostNotifyError(kInvalidBitstreamBufferId, PLATFORM_FAILURE);
+          return false;
+        }
+        output_buffer_map_[i].address[j] = address;
+        output_buffer_map_[i].length[j] = planes[j].length;
+      }
+    } else {
+      output_buffer_map_[i].num_planes = 1;
+      void* address = device_->Mmap(NULL, buffer.length, PROT_READ | PROT_WRITE,
+                                    MAP_SHARED, buffer.m.offset);
+      if (address == MAP_FAILED) {
+        PLOG(ERROR) << __func__ << ": mmap() failed";
+        PostNotifyError(kInvalidBitstreamBufferId, PLATFORM_FAILURE);
+        return false;
+      }
+      output_buffer_map_[i].address[0] = address;
+      output_buffer_map_[i].length[0] = buffer.length;
     }
-    output_buffer_map_[i].address = address;
-    output_buffer_map_[i].length = buffer.length;
   }
 
   return true;
 }
 
 void V4L2JpegDecodeAccelerator::DestroyInputBuffers() {
   DCHECK(decoder_task_runner_->BelongsToCurrentThread());
 
   free_input_buffers_.clear();
 
   if (input_buffer_map_.empty())
     return;
 
   if (input_streamon_) {
-    __u32 type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+    __u32 type = input_buf_type_;
     IOCTL_OR_ERROR_RETURN(VIDIOC_STREAMOFF, &type);
     input_streamon_ = false;
   }
 
   for (size_t i = 0; i < input_buffer_map_.size(); ++i) {
     BufferRecord& input_record = input_buffer_map_[i];
-    device_->Munmap(input_record.address, input_record.length);
+    device_->Munmap(input_record.address[0], input_record.length[0]);
   }
 
   struct v4l2_requestbuffers reqbufs;
   memset(&reqbufs, 0, sizeof(reqbufs));
   reqbufs.count = 0;
-  reqbufs.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+  reqbufs.type = input_buf_type_;
   reqbufs.memory = V4L2_MEMORY_MMAP;
   IOCTL_OR_LOG_ERROR(VIDIOC_REQBUFS, &reqbufs);
 
   input_buffer_map_.clear();
 }
 
 void V4L2JpegDecodeAccelerator::DestroyOutputBuffers() {
   DCHECK(decoder_task_runner_->BelongsToCurrentThread());
 
   free_output_buffers_.clear();
 
   if (output_buffer_map_.empty())
     return;
 
   if (output_streamon_) {
-    __u32 type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+    __u32 type = output_buf_type_;
     IOCTL_OR_ERROR_RETURN(VIDIOC_STREAMOFF, &type);
     output_streamon_ = false;
   }
 
-  for (size_t i = 0; i < output_buffer_map_.size(); ++i) {
-    BufferRecord& output_record = output_buffer_map_[i];
-    device_->Munmap(output_record.address, output_record.length);
+  for (const auto& output_record : output_buffer_map_) {
+    for (size_t i = 0; i < output_record.num_planes; ++i) {
+      device_->Munmap(output_record.address[i], output_record.length[i]);
+    }
   }
 
   struct v4l2_requestbuffers reqbufs;
   memset(&reqbufs, 0, sizeof(reqbufs));
   reqbufs.count = 0;
-  reqbufs.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+  reqbufs.type = output_buf_type_;
   reqbufs.memory = V4L2_MEMORY_MMAP;
   IOCTL_OR_LOG_ERROR(VIDIOC_REQBUFS, &reqbufs);
 
   output_buffer_map_.clear();
 }
 
 void V4L2JpegDecodeAccelerator::DevicePollTask() {
   DCHECK(device_poll_task_runner_->BelongsToCurrentThread());
 
   bool event_pending;
@@ skipping 78 matching lines @@
     // If input buffers are required to re-create, do not enqueue input record
     // until all pending frames are handled by device.
     if (ShouldRecreateInputBuffers())
       break;
     if (!EnqueueInputRecord())
       return;
   }
   // Check here because we cannot STREAMON before QBUF in earlier kernel.
   // (kernel version < 3.14)
   if (!input_streamon_ && InputBufferQueuedCount()) {
-    __u32 type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+    __u32 type = input_buf_type_;
     IOCTL_OR_ERROR_RETURN(VIDIOC_STREAMON, &type);
     input_streamon_ = true;
   }
 }
 
 void V4L2JpegDecodeAccelerator::EnqueueOutput() {
   DCHECK(decoder_task_runner_->BelongsToCurrentThread());
   // Output record can be enqueued because the output coded sizes of the frames
   // currently in the pipeline are all the same.
   while (running_jobs_.size() > OutputBufferQueuedCount() &&
          !free_output_buffers_.empty()) {
     if (!EnqueueOutputRecord())
       return;
   }
   // Check here because we cannot STREAMON before QBUF in earlier kernel.
   // (kernel version < 3.14)
   if (!output_streamon_ && OutputBufferQueuedCount()) {
-    __u32 type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+    __u32 type = output_buf_type_;
     IOCTL_OR_ERROR_RETURN(VIDIOC_STREAMON, &type);
     output_streamon_ = true;
   }
 }
 
-static bool CopyOutputImage(const uint32_t src_pixelformat,
-                            const void* src_addr,
-                            const gfx::Size& src_coded_size,
-                            const scoped_refptr<VideoFrame>& dst_frame) {
-  VideoPixelFormat format =
-      V4L2Device::V4L2PixFmtToVideoPixelFormat(src_pixelformat);
-  size_t src_size = VideoFrame::AllocationSize(format, src_coded_size);
+bool V4L2JpegDecodeAccelerator::ConvertOutputImage(
+    const uint32_t src_pixelformat,
+    const BufferRecord& src_buffer,
+    const gfx::Size& src_coded_size,
+    const scoped_refptr<VideoFrame>& dst_frame) {
   uint8_t* dst_y = dst_frame->data(VideoFrame::kYPlane);
   uint8_t* dst_u = dst_frame->data(VideoFrame::kUPlane);
   uint8_t* dst_v = dst_frame->data(VideoFrame::kVPlane);
   size_t dst_y_stride = dst_frame->stride(VideoFrame::kYPlane);
   size_t dst_u_stride = dst_frame->stride(VideoFrame::kUPlane);
   size_t dst_v_stride = dst_frame->stride(VideoFrame::kVPlane);
 
-  // If the source format is I420, ConvertToI420 will simply copy the frame.
-  if (libyuv::ConvertToI420(static_cast<uint8_t*>(const_cast<void*>(src_addr)),
-                            src_size,
-                            dst_y, dst_y_stride,
-                            dst_u, dst_u_stride,
-                            dst_v, dst_v_stride,
-                            0, 0,
-                            src_coded_size.width(),
-                            src_coded_size.height(),
-                            dst_frame->coded_size().width(),
-                            dst_frame->coded_size().height(),
-                            libyuv::kRotate0,
-                            src_pixelformat)) {
-    LOG(ERROR) << "ConvertToI420 failed. Source format: " << src_pixelformat;
+  if (src_buffer.num_planes == 1) {
+    // Use ConvertToI420 to convert all splane buffers.
+    // If the source format is I420, ConvertToI420 will simply copy the frame.
+    VideoPixelFormat format =
+        V4L2Device::V4L2PixFmtToVideoPixelFormat(src_pixelformat);
+    size_t src_size = VideoFrame::AllocationSize(format, src_coded_size);
+    if (libyuv::ConvertToI420(
+            static_cast<uint8_t*>(src_buffer.address[0]), src_size, dst_y,
+            dst_y_stride, dst_u, dst_u_stride, dst_v, dst_v_stride, 0, 0,
+            src_coded_size.width(), src_coded_size.height(),
+            dst_frame->coded_size().width(), dst_frame->coded_size().height(),
+            libyuv::kRotate0, src_pixelformat)) {
+      LOG(ERROR) << "ConvertToI420 failed. Source format: " << src_pixelformat;
+      return false;
+    }
+  } else if (src_pixelformat == V4L2_PIX_FMT_YUV420M ||
+             src_pixelformat == V4L2_PIX_FMT_YUV422M) {
+    uint8_t* src_y = static_cast<uint8_t*>(src_buffer.address[0]);
+    uint8_t* src_u = static_cast<uint8_t*>(src_buffer.address[1]);
+    uint8_t* src_v = static_cast<uint8_t*>(src_buffer.address[2]);
+    size_t src_y_stride = output_buffer_coded_size_.width();
+    size_t src_u_stride = output_buffer_coded_size_.width() / 2;
+    size_t src_v_stride = output_buffer_coded_size_.width() / 2;
+    if (output_buffer_pixelformat_ == V4L2_PIX_FMT_YUV420M) {
+      if (libyuv::I420Copy(src_y, src_y_stride, src_u, src_u_stride, src_v,
+                           src_v_stride, dst_y, dst_y_stride, dst_u,
+                           dst_u_stride, dst_v, dst_v_stride,
+                           output_buffer_coded_size_.width(),
+                           output_buffer_coded_size_.height())) {
+        LOG(ERROR) << "I420Copy failed";
+        return false;
+      }
+    } else {  // output_buffer_pixelformat_ == V4L2_PIX_FMT_YUV422M
+      if (libyuv::I422ToI420(src_y, src_y_stride, src_u, src_u_stride, src_v,
+                             src_v_stride, dst_y, dst_y_stride, dst_u,
+                             dst_u_stride, dst_v, dst_v_stride,
+                             output_buffer_coded_size_.width(),
+                             output_buffer_coded_size_.height())) {
+        LOG(ERROR) << "I422ToI420 failed";
+        return false;
+      }
+    }
+  } else {
+    LOG(ERROR) << "Unsupported source buffer format: " << src_pixelformat;
     return false;
   }
   return true;
 }
 
 void V4L2JpegDecodeAccelerator::Dequeue() {
   DCHECK(decoder_task_runner_->BelongsToCurrentThread());
 
   // Dequeue completed input (VIDEO_OUTPUT) buffers,
   // and recycle to the free list.
   struct v4l2_buffer dqbuf;
+  struct v4l2_plane input_plane;

[henryhsu, 2016/12/16 03:05:27]

use input_planes[kInputPlanes];
We don't know the value of kInputPlanes here. But line 781 and 782 assume
kInputPlanes is 1.

[jcliang, 2016/12/16 04:19:53]

Done.

   while (InputBufferQueuedCount() > 0) {
     DCHECK(input_streamon_);
     memset(&dqbuf, 0, sizeof(dqbuf));
-    dqbuf.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+    memset(&input_plane, 0, sizeof(input_plane));
+    dqbuf.type = input_buf_type_;
     dqbuf.memory = V4L2_MEMORY_MMAP;
+    if (V4L2_TYPE_IS_MULTIPLANAR(input_buf_type_)) {
+      dqbuf.length = kInputPlanes;
+      dqbuf.m.planes = &input_plane;
+    }
     if (device_->Ioctl(VIDIOC_DQBUF, &dqbuf) != 0) {
       if (errno == EAGAIN) {
         // EAGAIN if we're just out of buffers to dequeue.
         break;
       }
       PLOG(ERROR) << "ioctl() failed: input buffer VIDIOC_DQBUF failed.";
       PostNotifyError(kInvalidBitstreamBufferId, PLATFORM_FAILURE);
       return;
     }
     BufferRecord& input_record = input_buffer_map_[dqbuf.index];
     DCHECK(input_record.at_device);
     input_record.at_device = false;
     free_input_buffers_.push_back(dqbuf.index);
 
     if (dqbuf.flags & V4L2_BUF_FLAG_ERROR) {
       DVLOG(1) << "Dequeue input buffer error.";
       PostNotifyError(kInvalidBitstreamBufferId, UNSUPPORTED_JPEG);
       running_jobs_.pop();
     }
   }
 
   // Dequeue completed output (VIDEO_CAPTURE) buffers, recycle to the free list.
   // Return the finished buffer to the client via the job ready callback.
   // If dequeued input buffer has an error, the error frame has removed from
   // |running_jobs_|. We only have to dequeue output buffer when we actually
   // have pending frames in |running_jobs_| and also enqueued output buffers.
+  struct v4l2_plane output_planes[kOutputPlanes];
   while (!running_jobs_.empty() && OutputBufferQueuedCount() > 0) {
     DCHECK(output_streamon_);
     memset(&dqbuf, 0, sizeof(dqbuf));
-    dqbuf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+    memset(output_planes, 0, sizeof(output_planes));
+    dqbuf.type = output_buf_type_;
     // From experiments, using MMAP and memory copy is still faster than
     // USERPTR. Also, client doesn't need to consider the buffer alignment and
     // JpegDecodeAccelerator API will be simpler.
     dqbuf.memory = V4L2_MEMORY_MMAP;
+    if (V4L2_TYPE_IS_MULTIPLANAR(output_buf_type_)) {
+      dqbuf.length = kOutputPlanes;

[wuchengli, 2016/12/16 03:15:15]

I realized the number of planes could be 1 in mplane case (line 433-434). So
kOutputPlanes should be changed to BufferRecord.num_planes. Right?

[jcliang, 2016/12/16 04:19:53]

I believe this works as long as kOutputPlanes >= kInputPlanes.

Also we can't do what you proposed here because we don't know which BufferRecord
we're going to get; we learn that by looking at the |dqbuf.index| the
VIDIOC_DQBUF ioctl returns below.

[wuchengli, 2016/12/16 06:50:32]

I see. Then we need to add a variable to store the number of planes. Maybe
|output_buffer_num_planes_|. In that case, BufferRecord.num_planes doesn't seem
to be required.

[jcliang, 2016/12/17 15:35:56]

Done.

+      dqbuf.m.planes = output_planes;
+    }
     if (device_->Ioctl(VIDIOC_DQBUF, &dqbuf) != 0) {
       if (errno == EAGAIN) {
         // EAGAIN if we're just out of buffers to dequeue.
         break;
       }
       PLOG(ERROR) << "ioctl() failed: output buffer VIDIOC_DQBUF failed.";
       PostNotifyError(kInvalidBitstreamBufferId, PLATFORM_FAILURE);
       return;
     }
     BufferRecord& output_record = output_buffer_map_[dqbuf.index];
     DCHECK(output_record.at_device);
     output_record.at_device = false;
     free_output_buffers_.push_back(dqbuf.index);
 
     // Jobs are always processed in FIFO order.
     linked_ptr<JobRecord> job_record = running_jobs_.front();
     running_jobs_.pop();
 
     if (dqbuf.flags & V4L2_BUF_FLAG_ERROR) {
       DVLOG(1) << "Dequeue output buffer error.";
       PostNotifyError(kInvalidBitstreamBufferId, UNSUPPORTED_JPEG);
     } else {
       // Copy the decoded data from output buffer to the buffer provided by the
       // client. Do format conversion when output format is not
       // V4L2_PIX_FMT_YUV420.
-      if (!CopyOutputImage(output_buffer_pixelformat_, output_record.address,
-                           output_buffer_coded_size_, job_record->out_frame)) {
+      if (!ConvertOutputImage(output_buffer_pixelformat_, output_record,
+                              output_buffer_coded_size_,
+                              job_record->out_frame)) {
         PostNotifyError(job_record->bitstream_buffer_id, PLATFORM_FAILURE);
         return;
       }
-
       DVLOG(3) << "Decoding finished, returning bitstream buffer, id="
                << job_record->bitstream_buffer_id;
 
       child_task_runner_->PostTask(
           FROM_HERE, base::Bind(&V4L2JpegDecodeAccelerator::VideoFrameReady,
                                 weak_ptr_, job_record->bitstream_buffer_id));
     }
   }
 }
 
@@ skipping 84 matching lines @@
 
   // Enqueue an input (VIDEO_OUTPUT) buffer for an input video frame.
   linked_ptr<JobRecord> job_record = input_jobs_.front();
   input_jobs_.pop();
   const int index = free_input_buffers_.back();
   BufferRecord& input_record = input_buffer_map_[index];
   DCHECK(!input_record.at_device);
 
   // It will add default huffman segment if it's missing.
   if (!AddHuffmanTable(job_record->shm.memory(), job_record->shm.size(),
-                       input_record.address, input_record.length)) {
+                       input_record.address[0], input_record.length[0])) {
     PostNotifyError(job_record->bitstream_buffer_id, PARSE_JPEG_FAILED);
     return false;
   }
 
   struct v4l2_buffer qbuf;
+  struct v4l2_plane plane;
   memset(&qbuf, 0, sizeof(qbuf));
+  memset(&plane, 0, sizeof(plane));
   qbuf.index = index;
-  qbuf.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+  qbuf.type = input_buf_type_;
   qbuf.memory = V4L2_MEMORY_MMAP;
+  if (V4L2_TYPE_IS_MULTIPLANAR(input_buf_type_)) {
+    qbuf.length = 1;
+    plane.bytesused = input_record.length[0];
+    qbuf.m.planes = &plane;
+  }

[wuchengli, 2016/12/16 03:15:15]

let's also set bytesused for splane.
} else {
  qbuf.bytesused = input_record.length[0];
}

[jcliang, 2016/12/16 04:19:53]

Done.

   IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_QBUF, &qbuf);
   input_record.at_device = true;
   running_jobs_.push(job_record);
   free_input_buffers_.pop_back();
 
   DVLOG(3) << __func__
            << ": enqueued frame id=" << job_record->bitstream_buffer_id
            << " to device.";
   return true;
 }
 
 bool V4L2JpegDecodeAccelerator::EnqueueOutputRecord() {
   DCHECK(!free_output_buffers_.empty());
 
   // Enqueue an output (VIDEO_CAPTURE) buffer.
   const int index = free_output_buffers_.back();
   BufferRecord& output_record = output_buffer_map_[index];
   DCHECK(!output_record.at_device);
   struct v4l2_buffer qbuf;
+  struct v4l2_plane planes[kOutputPlanes];
   memset(&qbuf, 0, sizeof(qbuf));
+  memset(&planes, 0, sizeof(planes));
   qbuf.index = index;
-  qbuf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+  qbuf.type = output_buf_type_;
   qbuf.memory = V4L2_MEMORY_MMAP;
+  if (V4L2_TYPE_IS_MULTIPLANAR(output_buf_type_)) {
+    qbuf.length = output_record.num_planes;
+    qbuf.m.planes = planes;
+  }
   IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_QBUF, &qbuf);
   output_record.at_device = true;
   free_output_buffers_.pop_back();
   return true;
 }
 
 void V4L2JpegDecodeAccelerator::StartDevicePoll() {
   DVLOG(3) << __func__ << ": starting device poll";
   DCHECK(decoder_task_runner_->BelongsToCurrentThread());
   DCHECK(!device_poll_thread_.IsRunning());
@@ skipping 18 matching lines @@
   device_poll_thread_.Stop();
 
   // Clear the interrupt now, to be sure.
   if (!device_->ClearDevicePollInterrupt())
     return false;
 
   return true;
 }
 
 }  // namespace media