Add support for converting I420 software frames into NV12 hardware frames

media/video/gpu_memory_buffer_video_frame_pool.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/video/gpu_memory_buffer_video_frame_pool.h"
 
 #include <GLES2/gl2.h>
 #include <GLES2/gl2ext.h>
 
 #include <algorithm>
@@ skipping 142 matching lines @@
 // output size is |kBytesPerCopyTarget| bytes and run in parallel.
 const size_t kBytesPerCopyTarget = 1024 * 1024;  // 1MB
 
 // Return the GpuMemoryBuffer format to use for a specific VideoPixelFormat
 // and plane.
 gfx::BufferFormat GpuMemoryBufferFormat(VideoPixelFormat format, size_t plane) {
   switch (format) {
     case PIXEL_FORMAT_I420:
       DCHECK_LE(plane, 2u);
       return gfx::BufferFormat::R_8;
+    case PIXEL_FORMAT_NV12:
+      DCHECK_LE(plane, 1u);
+      return gfx::BufferFormat::YUV_420_BIPLANAR;
     case PIXEL_FORMAT_UYVY:
       DCHECK_EQ(0u, plane);
       return gfx::BufferFormat::UYVY_422;
     default:
       NOTREACHED();
       return gfx::BufferFormat::BGRA_8888;
   }
 }
 
 unsigned ImageInternalFormat(VideoPixelFormat format, size_t plane) {
   switch (format) {
     case PIXEL_FORMAT_I420:
       DCHECK_LE(plane, 2u);
       return GL_R8_EXT;
+    case PIXEL_FORMAT_NV12:
+      DCHECK_LE(plane, 1u);
+      return GL_R8_EXT;

[Daniele Castagna, 2015/08/25 15:08:42]

Why is the internal format R_8?

[Andre, 2015/08/25 18:19:57]

I'm not sure actually, what do you think the format should be?

     case PIXEL_FORMAT_UYVY:
       DCHECK_EQ(0u, plane);
       return GL_RGB;
     default:
       NOTREACHED();
       return 0;
   }
 }
 
 void CopyRowsToI420Buffer(int first_row,
                           int rows,
                           int bytes_per_row,
                           const uint8* source,
                           int source_stride,
                           uint8* output,
                           int dest_stride,
                           const base::Closure& done) {
   TRACE_EVENT2("media", "CopyRowsToI420Buffer", "bytes_per_row", bytes_per_row,
                "rows", rows);
   DCHECK_NE(dest_stride, 0);
   DCHECK_LE(bytes_per_row, std::abs(dest_stride));
   DCHECK_LE(bytes_per_row, source_stride);
   for (int row = first_row; row < first_row + rows; ++row) {
     memcpy(output + dest_stride * row, source + source_stride * row,
            bytes_per_row);
   }
   done.Run();
 }
 
+void CopyRowsToNV12Buffer(int first_row,
+                          int rows,
+                          int bytes_per_row,
+                          const scoped_refptr<VideoFrame>& source_frame,
+                          uint8* dest_y,
+                          int dest_stride_y,
+                          uint8* dest_uv,
+                          int dest_stride_uv,
+                          const base::Closure& done) {
+  TRACE_EVENT2("media", "CopyRowsToNV12Buffer", "bytes_per_row", bytes_per_row,
+               "rows", rows);
+  DCHECK_NE(dest_stride_y, 0);
+  DCHECK_NE(dest_stride_uv, 0);
+  libyuv::I420ToNV12(
+      source_frame->data(VideoFrame::kYPlane) +
+          first_row * source_frame->stride(VideoFrame::kYPlane),
+      source_frame->stride(VideoFrame::kYPlane),
+      source_frame->data(VideoFrame::kUPlane) +
+          first_row * source_frame->stride(VideoFrame::kUPlane),

[Daniele Castagna, 2015/08/25 15:08:42]

Shouldn't this start at first_row/2?

[Andre, 2015/08/25 21:04:33]

You're right, fixed.
There's also still an issue where #copies is twice too big in the case of NV12,
I will need to fix that.

+      source_frame->stride(VideoFrame::kUPlane),
+      source_frame->data(VideoFrame::kVPlane) +
+          first_row * source_frame->stride(VideoFrame::kVPlane),
+      source_frame->stride(VideoFrame::kVPlane),
+      dest_y + first_row * dest_stride_y, dest_stride_y,
+      dest_uv + first_row * dest_stride_uv, dest_stride_uv,

[Daniele Castagna, 2015/08/25 15:08:42]

Same here.

+      bytes_per_row, rows);
+  done.Run();
+}
+
 void CopyRowsToUYVYBuffer(int first_row,
                           int rows,
                           int width,
                           const scoped_refptr<VideoFrame>& source_frame,
                           uint8* output,
                           int dest_stride,
                           const base::Closure& done) {
   TRACE_EVENT2("media", "CopyRowsToUYVYBuffer", "bytes_per_row", width * 2,
                "rows", rows);
   DCHECK_NE(dest_stride, 0);
@@ skipping 60 matching lines @@
 
   worker_task_runner_->PostTask(
       FROM_HERE, base::Bind(&PoolImpl::CopyVideoFrameToGpuMemoryBuffers, this,
                             video_frame, frame_resources, frame_ready_cb));
 }
 
 void GpuMemoryBufferVideoFramePool::PoolImpl::OnCopiesDone(
     const scoped_refptr<VideoFrame>& video_frame,
     FrameResources* frame_resources,
     const FrameReadyCB& frame_ready_cb) {
-  const size_t planes = VideoFrame::NumPlanes(output_format_);
-  for (size_t i = 0; i < planes; ++i) {
-    frame_resources->plane_resources[i].gpu_memory_buffer->Unmap();
+  for (const auto& plane_resource : frame_resources->plane_resources) {
+    if (plane_resource.gpu_memory_buffer)
+      plane_resource.gpu_memory_buffer->Unmap();
   }
 
   media_task_runner_->PostTask(
       FROM_HERE,
       base::Bind(&PoolImpl::BindAndCreateMailboxesHardwareFrameResources, this,
                  video_frame, frame_resources, frame_ready_cb));
 }
 
 // Copies |video_frame| into |frame_resources| asynchronously, posting n tasks
 // that will be synchronized by a barrier.
@@ skipping 13 matching lines @@
     int rows_per_copy =
         std::max<size_t>((kBytesPerCopyTarget / bytes_per_row) & ~1, 1);
     copies += rows / rows_per_copy;
     if (rows % rows_per_copy)
       ++copies;
   }
   base::Closure copies_done =
       base::Bind(&PoolImpl::OnCopiesDone, this, video_frame, frame_resources,
                  frame_ready_cb);
   base::Closure barrier = base::BarrierClosure(copies, copies_done);
+
   // Post all the async tasks.
   for (size_t i = 0; i < dest_planes; ++i) {
+    gfx::GpuMemoryBuffer* buffer =
+        frame_resources->plane_resources[i].gpu_memory_buffer.get();
+    DCHECK(buffer);
+    const size_t buffer_planes =
+        gfx::NumberOfPlanesForBufferFormat(buffer->GetFormat());
+    scoped_ptr<uint8*[]> dest_buffers(new uint8*[buffer_planes]);

[Daniele Castagna, 2015/08/25 15:08:42]

Can we avoid the allocation with an array of size kMaxPlanes?

[Andre, 2015/08/25 18:19:57]

Done.
I suppose it's ok to use VideoFrame::kMaxPlanes with a DCHECK.

+    scoped_ptr<int[]> dest_strides(new int[buffer_planes]);

[Daniele Castagna, 2015/08/25 15:08:43]

Same here.

[Andre, 2015/08/25 18:19:57]

Done.

+    bool rv = buffer->Map(reinterpret_cast<void**>(dest_buffers.get()));
+    DCHECK(rv);
+    buffer->GetStride(dest_strides.get());
+
     int rows = VideoFrame::Rows(i, output_format_, size.height());
     int bytes_per_row = VideoFrame::RowBytes(i, output_format_, size.width());
     int rows_per_copy =
         std::max<size_t>((kBytesPerCopyTarget / bytes_per_row) & ~1, 1);
 
-    void* data = nullptr;
-    DCHECK_EQ(1u, gfx::NumberOfPlanesForBufferFormat(
-                      GpuMemoryBufferFormat(output_format_, i)));
-    bool rv = frame_resources->plane_resources[i].gpu_memory_buffer->Map(&data);
-    DCHECK(rv);
-    uint8* mapped_buffer = static_cast<uint8*>(data);
-
-    int dest_stride = 0;
-    frame_resources->plane_resources[i].gpu_memory_buffer->GetStride(
-        &dest_stride);
-
     for (int row = 0; row < rows; row += rows_per_copy) {
       switch (output_format_) {
         case PIXEL_FORMAT_I420:
+          DCHECK_EQ(1u, buffer_planes);
           worker_task_runner_->PostTask(
               FROM_HERE,
               base::Bind(&CopyRowsToI420Buffer, row,
                          std::min(rows_per_copy, rows - row), bytes_per_row,
                          video_frame->data(i), video_frame->stride(i),
-                         mapped_buffer, dest_stride, barrier));
+                         dest_buffers[0], dest_strides[0], barrier));
+          break;
+        case PIXEL_FORMAT_NV12:
+          DCHECK_EQ(2u, buffer_planes);
+          worker_task_runner_->PostTask(
+              FROM_HERE,
+              base::Bind(&CopyRowsToNV12Buffer, row,
+                         std::min(rows_per_copy, rows - row), bytes_per_row,

[Daniele Castagna, 2015/08/25 15:08:43]

nit: we could move std::min(rows_per_copy, rows - row) and declare a variable
just before the switch statement. 

[Andre, 2015/08/25 18:19:57]

Done.

+                         video_frame, dest_buffers[0], dest_strides[0],
+                         dest_buffers[1], dest_strides[1], barrier));
+          ++i;  // Consume an extra plane.

[Daniele Castagna, 2015/08/25 15:08:42]

nit: I'd elaborate on the comment a little to explain that we have two planes in
one GpuMemoryBuffer.

[Andre, 2015/08/25 18:19:57]

Done.

           break;
         case PIXEL_FORMAT_UYVY:
+          DCHECK_EQ(1u, buffer_planes);
           worker_task_runner_->PostTask(
               FROM_HERE,
               base::Bind(&CopyRowsToUYVYBuffer, row,
                          std::min(rows_per_copy, rows - row), size.width(),
-                         video_frame, mapped_buffer, dest_stride, barrier));
+                         video_frame, dest_buffers[0], dest_strides[0],
+                         barrier));
           break;
         default:
           NOTREACHED();
       }
     }
   }
 }
 
 void GpuMemoryBufferVideoFramePool::PoolImpl::
     BindAndCreateMailboxesHardwareFrameResources(
@@ skipping 45 matching lines @@
       frame = VideoFrame::WrapYUV420NativeTextures(
           mailbox_holders[VideoFrame::kYPlane],
           mailbox_holders[VideoFrame::kUPlane],
           mailbox_holders[VideoFrame::kVPlane],
           base::Bind(&PoolImpl::MailboxHoldersReleased, this, frame_resources),
           size, video_frame->visible_rect(), video_frame->natural_size(),
           video_frame->timestamp());
       if (video_frame->metadata()->IsTrue(VideoFrameMetadata::ALLOW_OVERLAY))
         frame->metadata()->SetBoolean(VideoFrameMetadata::ALLOW_OVERLAY, true);
       break;
+    case PIXEL_FORMAT_NV12:
     case PIXEL_FORMAT_UYVY:
       frame = VideoFrame::WrapNativeTexture(
-          PIXEL_FORMAT_UYVY, mailbox_holders[VideoFrame::kYPlane],
+          output_format_, mailbox_holders[VideoFrame::kYPlane],
           base::Bind(&PoolImpl::MailboxHoldersReleased, this, frame_resources),
           size, video_frame->visible_rect(), video_frame->natural_size(),
           video_frame->timestamp());
       frame->metadata()->SetBoolean(VideoFrameMetadata::ALLOW_OVERLAY, true);
       break;
     default:
       NOTREACHED();
   }
   frame_ready_cb.Run(frame);
 }
@@ skipping 41 matching lines @@
   gles2->ActiveTexture(GL_TEXTURE0);
   size_t planes = VideoFrame::NumPlanes(format);
   FrameResources* frame_resources = new FrameResources(size);
   resources_pool_.push_back(frame_resources);
   for (size_t i = 0; i < planes; ++i) {
     PlaneResource& plane_resource = frame_resources->plane_resources[i];
     const size_t width = VideoFrame::Columns(i, format, size.width());
     const size_t height = VideoFrame::Rows(i, format, size.height());
     const gfx::Size plane_size(width, height);
 
+    const gfx::BufferFormat buffer_format = GpuMemoryBufferFormat(format, i);
     plane_resource.gpu_memory_buffer = gpu_factories_->AllocateGpuMemoryBuffer(
-        plane_size, GpuMemoryBufferFormat(format, i), gfx::BufferUsage::MAP);
+        plane_size, buffer_format, gfx::BufferUsage::MAP);
 
     gles2->GenTextures(1, &plane_resource.texture_id);
     gles2->BindTexture(texture_target_, plane_resource.texture_id);
     gles2->TexParameteri(texture_target_, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
     gles2->TexParameteri(texture_target_, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
     gles2->TexParameteri(texture_target_, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
     gles2->TexParameteri(texture_target_, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
     gles2->GenMailboxCHROMIUM(plane_resource.mailbox.name);
     gles2->ProduceTextureCHROMIUM(texture_target_, plane_resource.mailbox.name);
+
+    size_t buffer_planes = gfx::NumberOfPlanesForBufferFormat(buffer_format);
+    if (buffer_planes > 1) {
+      // Got a multi-planar buffer, assume all the planes fit in it.
+      DCHECK_EQ(planes, buffer_planes);
+      break;
+    }
   }
   return frame_resources;
 }
 
 // static
 void GpuMemoryBufferVideoFramePool::PoolImpl::DeleteFrameResources(
     const scoped_refptr<GpuVideoAcceleratorFactories>& gpu_factories,
     FrameResources* frame_resources) {
   // TODO(dcastagna): As soon as the context lost is dealt with in media,
   // make sure that we won't execute this callback (use a weak pointer to
@@ skipping 45 matching lines @@
 }
 
 void GpuMemoryBufferVideoFramePool::MaybeCreateHardwareFrame(
     const scoped_refptr<VideoFrame>& video_frame,
     const FrameReadyCB& frame_ready_cb) {
   DCHECK(video_frame);
   pool_impl_->CreateHardwareFrame(video_frame, frame_ready_cb);
 }
 
 }  // namespace media