16 bpp video stream capture, render and WebGL usage - Realsense R200 & SR300 support.

media/video/gpu_memory_buffer_video_frame_pool.cc

Index: media/video/gpu_memory_buffer_video_frame_pool.cc
diff --git a/media/video/gpu_memory_buffer_video_frame_pool.cc b/media/video/gpu_memory_buffer_video_frame_pool.cc
index 2b4d5d4bc0d6598e11ccb0ae5a275cd75a062399..a5cf532b3d14d8a2fdd528134caee45bc57259c5 100644
--- a/media/video/gpu_memory_buffer_video_frame_pool.cc
+++ b/media/video/gpu_memory_buffer_video_frame_pool.cc
@@ -52,8 +52,7 @@ class GpuMemoryBufferVideoFramePool::PoolImpl
            GpuVideoAcceleratorFactories* gpu_factories)
       : media_task_runner_(media_task_runner),
         worker_task_runner_(worker_task_runner),
-        gpu_factories_(gpu_factories),
-        output_format_(GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED) {
+        gpu_factories_(gpu_factories) {
     DCHECK(media_task_runner_);
     DCHECK(worker_task_runner_);
   }
@@ -86,11 +85,14 @@ class GpuMemoryBufferVideoFramePool::PoolImpl
 
   // All the resources needed to compose a frame.
   struct FrameResources {
-    explicit FrameResources(const gfx::Size& size) : size(size) {}
+    explicit FrameResources(const gfx::Size& size,
+                            GpuVideoAcceleratorFactories::OutputFormat format)
+        : size(size), format(format) {}
     void SetIsInUse(bool in_use) { in_use_ = in_use; }
     bool IsInUse() const { return in_use_; }
 
     const gfx::Size size;
+    GpuVideoAcceleratorFactories::OutputFormat format;
     PlaneResource plane_resources[VideoFrame::kMaxPlanes];
 
    private:
@@ -120,9 +122,11 @@ class GpuMemoryBufferVideoFramePool::PoolImpl
 
   // Return true if |resources| can be used to represent a frame for
   // specific |format| and |size|.
-  static bool AreFrameResourcesCompatible(const FrameResources* resources,
-                                          const gfx::Size& size) {
-    return size == resources->size;
+  static bool AreFrameResourcesCompatible(
+      const FrameResources* resources,
+      const gfx::Size& size,
+      GpuVideoAcceleratorFactories::OutputFormat format) {
+    return size == resources->size && format == resources->format;
   }
 
   // Get the resources needed for a frame out of the pool, or create them if
@@ -154,7 +158,12 @@ class GpuMemoryBufferVideoFramePool::PoolImpl
   // Pool of resources.
   std::list<FrameResources*> resources_pool_;
 
-  GpuVideoAcceleratorFactories::OutputFormat output_format_;
+  // Cache gpu_factories_->VideoFrameOutputFormat(VideoPixelFormat).
+  GpuVideoAcceleratorFactories::OutputFormat output_format_ =
+      GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED;
+
+  // Pixel format used to to calculate |output_format_|.
+  VideoPixelFormat current_pixel_format_ = PIXEL_FORMAT_UNKNOWN;
 
   DISALLOW_COPY_AND_ASSIGN(PoolImpl);
 };
@@ -165,7 +174,7 @@ namespace {
 // output size is |kBytesPerCopyTarget| bytes and run in parallel.
 const size_t kBytesPerCopyTarget = 1024 * 1024;  // 1MB
 
-// Return the GpuMemoryBuffer format to use for a specific VideoPixelFormat
+// Return the GpuMemoryBuffer format to use for a specific OutputFormat
 // and plane.
 gfx::BufferFormat GpuMemoryBufferFormat(
     media::GpuVideoAcceleratorFactories::OutputFormat format,
@@ -183,6 +192,9 @@ gfx::BufferFormat GpuMemoryBufferFormat(
     case GpuVideoAcceleratorFactories::OutputFormat::UYVY:
       DCHECK_EQ(0u, plane);
       return gfx::BufferFormat::UYVY_422;
+    case GpuVideoAcceleratorFactories::OutputFormat::Y16:
+      DCHECK_EQ(0u, plane);
+      return gfx::BufferFormat::RG_88;
     case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
       NOTREACHED();
       break;
@@ -205,6 +217,9 @@ unsigned ImageInternalFormat(GpuVideoAcceleratorFactories::OutputFormat format,
     case GpuVideoAcceleratorFactories::OutputFormat::UYVY:
       DCHECK_EQ(0u, plane);
       return GL_RGB_YCBCR_422_CHROMIUM;
+    case GpuVideoAcceleratorFactories::OutputFormat::Y16:
+      DCHECK_EQ(0u, plane);
+      return GL_RG_EXT;
     case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
       NOTREACHED();
       break;
@@ -217,6 +232,7 @@ size_t PlanesPerCopy(GpuVideoAcceleratorFactories::OutputFormat format) {
   switch (format) {
     case GpuVideoAcceleratorFactories::OutputFormat::I420:
     case GpuVideoAcceleratorFactories::OutputFormat::UYVY:
+    case GpuVideoAcceleratorFactories::OutputFormat::Y16:
       return 1;
     case GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB:
     case GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB:
@@ -238,6 +254,8 @@ VideoPixelFormat VideoFormat(
       return PIXEL_FORMAT_NV12;
     case GpuVideoAcceleratorFactories::OutputFormat::UYVY:
       return PIXEL_FORMAT_UYVY;
+    case GpuVideoAcceleratorFactories::OutputFormat::Y16:
+      return PIXEL_FORMAT_Y16;
     case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
       NOTREACHED();
       break;
@@ -354,6 +372,31 @@ void CopyRowsToUYVYBuffer(int first_row,
   done.Run();
 }
 
+// Used for pixel-order formats: e.g. Y16, Y8, etc.
+void CopyRowsToGenericSinglePlaneGPUBuffer(
+    int first_row,
+    int rows,
+    int width,
+    const scoped_refptr<VideoFrame>& source_frame,
+    uint8_t* output,
+    int dest_stride,
+    const base::Closure& done) {
+  int row_bytes = VideoFrame::RowBytes(0, source_frame->format(), width);
+  TRACE_EVENT2("media", "CopyRowsToGenericSinglePlaneGPUBuffer",
+               "bytes_per_row", row_bytes, "rows", rows);
+  if (output) {
+    DCHECK_NE(dest_stride, 0);
+    DCHECK_LE(row_bytes, std::abs(dest_stride));
+    const int source_stride = source_frame->stride(0);
+    const uint8_t* source =
+        source_frame->visible_data(0) + first_row * source_stride;
+    uint8_t* dest = output + first_row * dest_stride;
+    for (int i = 0; i < rows; ++i)
+      memcpy(dest + i * dest_stride, source + i * source_stride, row_bytes);
+  }
+  done.Run();
+}
+
 gfx::Size CodedSize(const scoped_refptr<VideoFrame>& video_frame,
                     GpuVideoAcceleratorFactories::OutputFormat output_format) {
   DCHECK(gfx::Rect(video_frame->coded_size())
@@ -372,6 +415,9 @@ gfx::Size CodedSize(const scoped_refptr<VideoFrame>& video_frame,
       output = gfx::Size((video_frame->visible_rect().width() + 1) & ~1,
                          video_frame->visible_rect().height());
       break;
+    case GpuVideoAcceleratorFactories::OutputFormat::Y16:
+      output = video_frame->visible_rect().size();
+      break;
     case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
       NOTREACHED();
   }
@@ -391,8 +437,11 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::CreateHardwareFrame(
   DCHECK(media_task_runner_->BelongsToCurrentThread());
   // Lazily initialize output_format_ since VideoFrameOutputFormat() has to be
   // called on the media_thread while this object might be instantiated on any.
-  if (output_format_ == GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED)
-    output_format_ = gpu_factories_->VideoFrameOutputFormat();
+  if (current_pixel_format_ != video_frame->format()) {
+    output_format_ =
+        gpu_factories_->VideoFrameOutputFormat(video_frame->format());
+    current_pixel_format_ = video_frame->format();
+  }
 
   if (output_format_ == GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED) {
     frame_ready_cb.Run(video_frame);
@@ -402,6 +451,7 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::CreateHardwareFrame(
     // Supported cases.
     case PIXEL_FORMAT_YV12:
     case PIXEL_FORMAT_I420:
+    case PIXEL_FORMAT_Y16:
       break;
     // Unsupported cases.
     case PIXEL_FORMAT_YV12A:
@@ -427,7 +477,6 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::CreateHardwareFrame(
     case PIXEL_FORMAT_YUV422P12:
     case PIXEL_FORMAT_YUV444P12:
     case PIXEL_FORMAT_Y8:
-    case PIXEL_FORMAT_Y16:
     case PIXEL_FORMAT_UNKNOWN:
       frame_ready_cb.Run(video_frame);
       return;
@@ -509,15 +558,16 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::CopyVideoFrameToGpuMemoryBuffers(
     FrameResources* frame_resources,
     const FrameReadyCB& frame_ready_cb) {
   // Compute the number of tasks to post and create the barrier.
-  const size_t num_planes = VideoFrame::NumPlanes(VideoFormat(output_format_));
-  const size_t planes_per_copy = PlanesPerCopy(output_format_);
-  const gfx::Size coded_size = CodedSize(video_frame, output_format_);
+  const size_t num_planes =
+      VideoFrame::NumPlanes(VideoFormat(frame_resources->format));
+  const size_t planes_per_copy = PlanesPerCopy(frame_resources->format);
+  const gfx::Size coded_size = CodedSize(video_frame, frame_resources->format);
   size_t copies = 0;
   for (size_t i = 0; i < num_planes; i += planes_per_copy) {
-    const int rows =
-        VideoFrame::Rows(i, VideoFormat(output_format_), coded_size.height());
-    const int rows_per_copy =
-        RowsPerCopy(i, VideoFormat(output_format_), coded_size.width());
+    const int rows = VideoFrame::Rows(i, VideoFormat(frame_resources->format),
+                                      coded_size.height());
+    const int rows_per_copy = RowsPerCopy(
+        i, VideoFormat(frame_resources->format), coded_size.width());
     copies += rows / rows_per_copy;
     if (rows % rows_per_copy)
       ++copies;
@@ -529,7 +579,7 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::CopyVideoFrameToGpuMemoryBuffers(
   const base::Closure barrier = base::BarrierClosure(copies, copies_done);
 
   // Map the buffers.
-  for (size_t i = 0; i < NumGpuMemoryBuffers(output_format_); i++) {
+  for (size_t i = 0; i < NumGpuMemoryBuffers(frame_resources->format); i++) {
     gfx::GpuMemoryBuffer* buffer =
         frame_resources->plane_resources[i].gpu_memory_buffer.get();
 
@@ -543,17 +593,17 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::CopyVideoFrameToGpuMemoryBuffers(
   for (size_t i = 0; i < num_planes; i += planes_per_copy) {
     gfx::GpuMemoryBuffer* buffer =
         frame_resources->plane_resources[i].gpu_memory_buffer.get();
-    const int rows =
-        VideoFrame::Rows(i, VideoFormat(output_format_), coded_size.height());
-    const int rows_per_copy =
-        RowsPerCopy(i, VideoFormat(output_format_), coded_size.width());
+    const int rows = VideoFrame::Rows(i, VideoFormat(frame_resources->format),
+                                      coded_size.height());
+    const int rows_per_copy = RowsPerCopy(
+        i, VideoFormat(frame_resources->format), coded_size.width());
 
     for (int row = 0; row < rows; row += rows_per_copy) {
       const int rows_to_copy = std::min(rows_per_copy, rows - row);
-      switch (output_format_) {
+      switch (frame_resources->format) {
         case GpuVideoAcceleratorFactories::OutputFormat::I420: {
           const int bytes_per_row = VideoFrame::RowBytes(
-              i, VideoFormat(output_format_), coded_size.width());
+              i, VideoFormat(frame_resources->format), coded_size.width());
           worker_task_runner_->PostTask(
               FROM_HERE, base::Bind(&CopyRowsToI420Buffer, row, rows_to_copy,
                                     bytes_per_row, video_frame->visible_data(i),
@@ -591,6 +641,14 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::CopyVideoFrameToGpuMemoryBuffers(
                                     static_cast<uint8_t*>(buffer->memory(0)),
                                     buffer->stride(0), barrier));
           break;
+        case GpuVideoAcceleratorFactories::OutputFormat::Y16:
+          worker_task_runner_->PostTask(
+              FROM_HERE,
+              base::Bind(&CopyRowsToGenericSinglePlaneGPUBuffer, row,
+                         rows_to_copy, coded_size.width(), video_frame,
+                         static_cast<uint8_t*>(buffer->memory(0)),
+                         buffer->stride(0), barrier));
+          break;
         case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
           NOTREACHED();
       }
@@ -611,24 +669,24 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::
   }
   gpu::gles2::GLES2Interface* gles2 = lock->ContextGL();
 
-  const gfx::Size coded_size = CodedSize(video_frame, output_format_);
+  const gfx::Size coded_size = CodedSize(video_frame, frame_resources->format);
   gpu::MailboxHolder mailbox_holders[VideoFrame::kMaxPlanes];
   // Set up the planes creating the mailboxes needed to refer to the textures.
-  for (size_t i = 0; i < NumGpuMemoryBuffers(output_format_); i++) {
+  for (size_t i = 0; i < NumGpuMemoryBuffers(frame_resources->format); i++) {
     PlaneResource& plane_resource = frame_resources->plane_resources[i];
     const gfx::BufferFormat buffer_format =
-        GpuMemoryBufferFormat(output_format_, i);
+        GpuMemoryBufferFormat(frame_resources->format, i);
     unsigned texture_target = gpu_factories_->ImageTextureTarget(buffer_format);
     // Bind the texture and create or rebind the image.
     gles2->BindTexture(texture_target, plane_resource.texture_id);
     if (plane_resource.gpu_memory_buffer && !plane_resource.image_id) {
-      const size_t width = VideoFrame::Columns(i, VideoFormat(output_format_),
-                                               coded_size.width());
-      const size_t height =
-          VideoFrame::Rows(i, VideoFormat(output_format_), coded_size.height());
+      const size_t width = VideoFrame::Columns(
+          i, VideoFormat(frame_resources->format), coded_size.width());
+      const size_t height = VideoFrame::Rows(
+          i, VideoFormat(frame_resources->format), coded_size.height());
       plane_resource.image_id = gles2->CreateImageCHROMIUM(
           plane_resource.gpu_memory_buffer->AsClientBuffer(), width, height,
-          ImageInternalFormat(output_format_, i));
+          ImageInternalFormat(frame_resources->format, i));
     } else if (plane_resource.image_id) {
       gles2->ReleaseTexImage2DCHROMIUM(texture_target, plane_resource.image_id);
     }
@@ -646,13 +704,13 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::
 
   gpu::SyncToken sync_token;
   gles2->GenUnverifiedSyncTokenCHROMIUM(fence_sync, sync_token.GetData());
-  for (size_t i = 0; i < NumGpuMemoryBuffers(output_format_); i++)
+  for (size_t i = 0; i < NumGpuMemoryBuffers(frame_resources->format); i++)
     mailbox_holders[i].sync_token = sync_token;
 
   auto release_mailbox_callback = BindToCurrentLoop(
       base::Bind(&PoolImpl::MailboxHoldersReleased, this, frame_resources));
 
-  VideoPixelFormat frame_format = FinalVideoFormat(output_format_);
+  VideoPixelFormat frame_format = FinalVideoFormat(frame_resources->format);
 
   // Create the VideoFrame backed by native textures.
   gfx::Size visible_size = video_frame->visible_rect().size();
@@ -670,7 +728,7 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::
   frame->set_color_space(video_frame->ColorSpace());
 
   bool allow_overlay = false;
-  switch (output_format_) {
+  switch (frame_resources->format) {
     case GpuVideoAcceleratorFactories::OutputFormat::I420:
       allow_overlay =
           video_frame->metadata()->IsTrue(VideoFrameMetadata::ALLOW_OVERLAY);
@@ -721,7 +779,7 @@ GpuMemoryBufferVideoFramePool::PoolImpl::GetOrCreateFrameResources(
   while (it != resources_pool_.end()) {
     FrameResources* frame_resources = *it;
     if (!frame_resources->IsInUse()) {
-      if (AreFrameResourcesCompatible(frame_resources, size)) {
+      if (AreFrameResourcesCompatible(frame_resources, size, format)) {
         frame_resources->SetIsInUse(true);
         return frame_resources;
       } else {
@@ -742,9 +800,9 @@ GpuMemoryBufferVideoFramePool::PoolImpl::GetOrCreateFrameResources(
 
   gpu::gles2::GLES2Interface* gles2 = lock->ContextGL();
   gles2->ActiveTexture(GL_TEXTURE0);
-  FrameResources* frame_resources = new FrameResources(size);
+  FrameResources* frame_resources = new FrameResources(size, format);
   resources_pool_.push_back(frame_resources);
-  for (size_t i = 0; i < NumGpuMemoryBuffers(output_format_); i++) {
+  for (size_t i = 0; i < NumGpuMemoryBuffers(format); i++) {
     PlaneResource& plane_resource = frame_resources->plane_resources[i];
     const size_t width =
         VideoFrame::Columns(i, VideoFormat(format), size.width());