media/video/gpu_memory_buffer_video_frame_pool.cc - Issue 2121043002: 16 bpp video stream capture, render and WebGL usage - Realsense R200 & SR300 support.

Unified Diff: media/video/gpu_memory_buffer_video_frame_pool.cc

Issue 2121043002: 16 bpp video stream capture, render and WebGL usage - Realsense R200 & SR300 support. Base URL: https://chromium.googlesource.com/chromium/src.git@master

Patch Set: scoping Y8 out. Created 4 years, 2 months ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

View side-by-side diff with in-line comments

Download patch

Index: media/video/gpu_memory_buffer_video_frame_pool.cc

diff --git a/media/video/gpu_memory_buffer_video_frame_pool.cc b/media/video/gpu_memory_buffer_video_frame_pool.cc

index 4b2f7a1b6b52f75453c0ed033dabe44ce23f0808..c00a8f63b641604845acc33a8293cb075a8412e7 100644

--- a/media/video/gpu_memory_buffer_video_frame_pool.cc

+++ b/media/video/gpu_memory_buffer_video_frame_pool.cc

@@ -86,11 +86,13 @@ class GpuMemoryBufferVideoFramePool::PoolImpl

// All the resources needed to compose a frame.

struct FrameResources {

- explicit FrameResources(const gfx::Size& size) : size(size) {}

+ explicit FrameResources(const gfx::Size& size, VideoPixelFormat format)

+ : size(size), format(format) {}

void SetIsInUse(bool in_use) { in_use_ = in_use; }

bool IsInUse() const { return in_use_; }

const gfx::Size size;

+ VideoPixelFormat format;

PlaneResource plane_resources[VideoFrame::kMaxPlanes];

private:

@@ -121,8 +123,9 @@ class GpuMemoryBufferVideoFramePool::PoolImpl

// Return true if |resources| can be used to represent a frame for

// specific |format| and |size|.

static bool AreFrameResourcesCompatible(const FrameResources* resources,

- const gfx::Size& size) {

- return size == resources->size;

+ const gfx::Size& size,

+ VideoPixelFormat format) {

+ return size == resources->size && format == resources->format;

}

// Get the resources needed for a frame out of the pool, or create them if

@@ -155,8 +158,13 @@ class GpuMemoryBufferVideoFramePool::PoolImpl

// TODO(dcastagna): change the following type from VideoPixelFormat to

// BufferFormat.

+ // Pixel format of the hardware video frames for I420 and YV12 video.

VideoPixelFormat output_format_;

Daniele Castagna 2016/10/10 18:42:31 This is changing slightly with crrev.com/238244300

aleksandar.stojiljkovic 2016/10/11 18:29:54 Done. CL split to: crrev.com/2410923002: GpuMemory

+ // If texture_rg is supported, gpu memory buffers can be used for R8 and RG8.

+ bool texture_rg_available_ = false;

+ bool texture_rg_available_initialized_ = false;

DISALLOW_COPY_AND_ASSIGN(PoolImpl);

};

@@ -166,25 +174,6 @@ namespace {

// output size is |kBytesPerCopyTarget| bytes and run in parallel.

const size_t kBytesPerCopyTarget = 1024 * 1024; // 1MB

-// Return the GpuMemoryBuffer format to use for a specific VideoPixelFormat

-// and plane.

-gfx::BufferFormat GpuMemoryBufferFormat(VideoPixelFormat format, size_t plane) {

- switch (format) {

- case PIXEL_FORMAT_I420:

- DCHECK_LE(plane, 2u);

- return gfx::BufferFormat::R_8;

- case PIXEL_FORMAT_NV12:

- DCHECK_LE(plane, 1u);

- return gfx::BufferFormat::YUV_420_BIPLANAR;

- case PIXEL_FORMAT_UYVY:

- DCHECK_EQ(0u, plane);

- return gfx::BufferFormat::UYVY_422;

- default:

- NOTREACHED();

- return gfx::BufferFormat::BGRA_8888;

- }

unsigned ImageInternalFormat(VideoPixelFormat format, size_t plane) {

switch (format) {

case PIXEL_FORMAT_I420:

@@ -196,6 +185,9 @@ unsigned ImageInternalFormat(VideoPixelFormat format, size_t plane) {

case PIXEL_FORMAT_UYVY:

DCHECK_EQ(0u, plane);

return GL_RGB_YCBCR_422_CHROMIUM;

+ case PIXEL_FORMAT_Y16:

+ DCHECK_EQ(0u, plane);

+ return GL_RG_EXT;

default:

NOTREACHED();

return 0;

@@ -207,6 +199,7 @@ size_t PlanesPerCopy(VideoPixelFormat format) {

switch (format) {

case PIXEL_FORMAT_I420:

case PIXEL_FORMAT_UYVY:

+ case PIXEL_FORMAT_Y16:

return 1;

case PIXEL_FORMAT_NV12:

return 2;

@@ -312,6 +305,29 @@ void CopyRowsToUYVYBuffer(int first_row,

done.Run();

}

+void CopyRowsToGPUBuffer(int first_row,

+ int rows,

+ int width,

+ const scoped_refptr<VideoFrame>& source_frame,

+ uint8_t* output,

+ int dest_stride,

+ const base::Closure& done) {

+ int row_bytes = VideoFrame::RowBytes(0, source_frame->format(), width);

+ TRACE_EVENT2("media", "CopyRowsToGPUBuffer", "bytes_per_row", row_bytes,

+ "rows", rows);

+ if (output) {

+ DCHECK_NE(dest_stride, 0);

+ DCHECK_LE(row_bytes, std::abs(dest_stride));

+ const int source_stride = source_frame->stride(0);

+ const uint8_t* source =

+ source_frame->visible_data(0) + first_row * source_stride;

+ uint8_t* dest = output + first_row * dest_stride;

+ for (int i = 0; i < rows; ++i)

+ memcpy(dest + i * dest_stride, source + i * source_stride, row_bytes);

+ }

+ done.Run();

gfx::Size CodedSize(const scoped_refptr<VideoFrame>& video_frame,

VideoPixelFormat output_format) {

DCHECK(gfx::Rect(video_frame->coded_size())

@@ -329,6 +345,9 @@ gfx::Size CodedSize(const scoped_refptr<VideoFrame>& video_frame,

output = gfx::Size((video_frame->visible_rect().width() + 1) & ~1,

video_frame->visible_rect().height());

break;

+ case PIXEL_FORMAT_Y16:

+ output = video_frame->visible_rect().size();

+ break;

default:

NOTREACHED();

}

@@ -346,19 +365,30 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::CreateHardwareFrame(

const scoped_refptr<VideoFrame>& video_frame,

const FrameReadyCB& frame_ready_cb) {

DCHECK(media_task_runner_->BelongsToCurrentThread());

- // Lazily initialize output_format_ since VideoFrameOutputFormat() has to be

- // called on the media_thread while this object might be instantiated on any.

- if (output_format_ == PIXEL_FORMAT_UNKNOWN)

- output_format_ = gpu_factories_->VideoFrameOutputFormat();

- if (output_format_ == PIXEL_FORMAT_UNKNOWN) {

- frame_ready_cb.Run(video_frame);

- return;

- }

+ media::VideoPixelFormat output_format = PIXEL_FORMAT_UNKNOWN;

switch (video_frame->format()) {

// Supported cases.

case PIXEL_FORMAT_YV12:

- case PIXEL_FORMAT_I420:

+ case PIXEL_FORMAT_I420: {

+ // Lazily initialize output_format_ since VideoFrameOutputFormat() has to

+ // be called on the media_thread while this object might be instantiated

+ // on any.

+ if (output_format_ == PIXEL_FORMAT_UNKNOWN) {

+ output_format_ =

+ gpu_factories_->VideoFrameOutputFormat(PIXEL_FORMAT_I420);

+ }

+ output_format = output_format_;

+ break;

+ }

+ case PIXEL_FORMAT_Y16:

+ // Lazily initialize texture_rg_available_.

+ if (!texture_rg_available_initialized_) {

+ texture_rg_available_initialized_ = true;

+ texture_rg_available_ = gpu_factories_->VideoFrameOutputFormat(

+ PIXEL_FORMAT_Y16) == PIXEL_FORMAT_Y16;

+ }

+ output_format =

+ texture_rg_available_ ? video_frame->format() : PIXEL_FORMAT_UNKNOWN;

break;

// Unsupported cases.

case PIXEL_FORMAT_YV12A:

@@ -384,16 +414,18 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::CreateHardwareFrame(

case PIXEL_FORMAT_YUV422P12:

case PIXEL_FORMAT_YUV444P12:

case PIXEL_FORMAT_Y8:

- case PIXEL_FORMAT_Y16:

case PIXEL_FORMAT_UNKNOWN:

- frame_ready_cb.Run(video_frame);

- return;

+ break;

+ }

+ if (output_format == PIXEL_FORMAT_UNKNOWN) {

+ frame_ready_cb.Run(video_frame);

+ return;

}

- const gfx::Size coded_size = CodedSize(video_frame, output_format_);

+ const gfx::Size coded_size = CodedSize(video_frame, output_format);

// Acquire resources. Incompatible ones will be dropped from the pool.

FrameResources* frame_resources =

- GetOrCreateFrameResources(coded_size, output_format_);

+ GetOrCreateFrameResources(coded_size, output_format);

if (!frame_resources) {

frame_ready_cb.Run(video_frame);

return;

@@ -466,14 +498,15 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::CopyVideoFrameToGpuMemoryBuffers(

FrameResources* frame_resources,

const FrameReadyCB& frame_ready_cb) {

// Compute the number of tasks to post and create the barrier.

- const size_t num_planes = VideoFrame::NumPlanes(output_format_);

- const size_t planes_per_copy = PlanesPerCopy(output_format_);

- const gfx::Size coded_size = CodedSize(video_frame, output_format_);

+ const size_t num_planes = VideoFrame::NumPlanes(frame_resources->format);

+ const size_t planes_per_copy = PlanesPerCopy(frame_resources->format);

+ const gfx::Size coded_size = CodedSize(video_frame, frame_resources->format);

size_t copies = 0;

for (size_t i = 0; i < num_planes; i += planes_per_copy) {

- const int rows = VideoFrame::Rows(i, output_format_, coded_size.height());

+ const int rows =

+ VideoFrame::Rows(i, frame_resources->format, coded_size.height());

const int rows_per_copy =

- RowsPerCopy(i, output_format_, coded_size.width());

+ RowsPerCopy(i, frame_resources->format, coded_size.width());

copies += rows / rows_per_copy;

if (rows % rows_per_copy)

++copies;

@@ -495,16 +528,17 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::CopyVideoFrameToGpuMemoryBuffers(

DCHECK_EQ(planes_per_copy,

gfx::NumberOfPlanesForBufferFormat(buffer->GetFormat()));

- const int rows = VideoFrame::Rows(i, output_format_, coded_size.height());

+ const int rows =

+ VideoFrame::Rows(i, frame_resources->format, coded_size.height());

const int rows_per_copy =

- RowsPerCopy(i, output_format_, coded_size.width());

+ RowsPerCopy(i, frame_resources->format, coded_size.width());

for (int row = 0; row < rows; row += rows_per_copy) {

const int rows_to_copy = std::min(rows_per_copy, rows - row);

- switch (output_format_) {

+ switch (frame_resources->format) {

case PIXEL_FORMAT_I420: {

- const int bytes_per_row =

- VideoFrame::RowBytes(i, output_format_, coded_size.width());

+ const int bytes_per_row = VideoFrame::RowBytes(

+ i, frame_resources->format, coded_size.width());

worker_task_runner_->PostTask(

FROM_HERE, base::Bind(&CopyRowsToI420Buffer, row, rows_to_copy,

bytes_per_row, video_frame->visible_data(i),

@@ -529,6 +563,13 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::CopyVideoFrameToGpuMemoryBuffers(

static_cast<uint8_t*>(buffer->memory(0)),

buffer->stride(0), barrier));

break;

+ case PIXEL_FORMAT_Y16:

+ worker_task_runner_->PostTask(

+ FROM_HERE, base::Bind(&CopyRowsToGPUBuffer, row, rows_to_copy,

+ coded_size.width(), video_frame,

+ static_cast<uint8_t*>(buffer->memory(0)),

+ buffer->stride(0), barrier));

+ break;

default:

NOTREACHED();

}

@@ -549,27 +590,27 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::

}

gpu::gles2::GLES2Interface* gles2 = lock->ContextGL();

- const size_t num_planes = VideoFrame::NumPlanes(output_format_);

- const size_t planes_per_copy = PlanesPerCopy(output_format_);

- const gfx::Size coded_size = CodedSize(video_frame, output_format_);

+ const size_t num_planes = VideoFrame::NumPlanes(frame_resources->format);

+ const size_t planes_per_copy = PlanesPerCopy(frame_resources->format);

+ const gfx::Size coded_size = CodedSize(video_frame, frame_resources->format);

gpu::MailboxHolder mailbox_holders[VideoFrame::kMaxPlanes];

// Set up the planes creating the mailboxes needed to refer to the textures.

for (size_t i = 0; i < num_planes; i += planes_per_copy) {

PlaneResource& plane_resource = frame_resources->plane_resources[i];

const gfx::BufferFormat buffer_format =

- GpuMemoryBufferFormat(output_format_, i);

+ VideoFrame::BufferFormat(frame_resources->format);

unsigned texture_target = gpu_factories_->ImageTextureTarget(buffer_format);

// Bind the texture and create or rebind the image.

gles2->BindTexture(texture_target, plane_resource.texture_id);

if (plane_resource.gpu_memory_buffer && !plane_resource.image_id) {

const size_t width =

- VideoFrame::Columns(i, output_format_, coded_size.width());

+ VideoFrame::Columns(i, frame_resources->format, coded_size.width());

const size_t height =

- VideoFrame::Rows(i, output_format_, coded_size.height());

+ VideoFrame::Rows(i, frame_resources->format, coded_size.height());

plane_resource.image_id = gles2->CreateImageCHROMIUM(

plane_resource.gpu_memory_buffer->AsClientBuffer(), width, height,

- ImageInternalFormat(output_format_, i));

+ ImageInternalFormat(frame_resources->format, i));

} else if (plane_resource.image_id) {

gles2->ReleaseTexImage2DCHROMIUM(texture_target, plane_resource.image_id);

}

@@ -595,8 +636,9 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::

base::Bind(&PoolImpl::MailboxHoldersReleased, this, frame_resources));

// Consumers should sample from NV12 textures as if they're XRGB.

- VideoPixelFormat frame_format =

- output_format_ == PIXEL_FORMAT_NV12 ? PIXEL_FORMAT_XRGB : output_format_;

+ VideoPixelFormat frame_format = (frame_resources->format == PIXEL_FORMAT_NV12)

+ ? PIXEL_FORMAT_XRGB

+ : frame_resources->format;

DCHECK_EQ(VideoFrame::NumPlanes(frame_format) * planes_per_copy, num_planes);

// Create the VideoFrame backed by native textures.

@@ -615,13 +657,14 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::

frame->set_color_space(video_frame->ColorSpace());

bool allow_overlay = false;

- switch (output_format_) {

+ switch (frame_resources->format) {

case PIXEL_FORMAT_I420:

allow_overlay =

video_frame->metadata()->IsTrue(VideoFrameMetadata::ALLOW_OVERLAY);

break;

case PIXEL_FORMAT_NV12:

case PIXEL_FORMAT_UYVY:

+ case PIXEL_FORMAT_Y16:

allow_overlay = true;

break;

default:

@@ -666,7 +709,7 @@ GpuMemoryBufferVideoFramePool::PoolImpl::GetOrCreateFrameResources(

while (it != resources_pool_.end()) {

FrameResources* frame_resources = *it;

if (!frame_resources->IsInUse()) {

- if (AreFrameResourcesCompatible(frame_resources, size)) {

+ if (AreFrameResourcesCompatible(frame_resources, size, format)) {

frame_resources->SetIsInUse(true);

return frame_resources;

} else {

@@ -688,7 +731,7 @@ GpuMemoryBufferVideoFramePool::PoolImpl::GetOrCreateFrameResources(

gpu::gles2::GLES2Interface* gles2 = lock->ContextGL();

gles2->ActiveTexture(GL_TEXTURE0);

size_t num_planes = VideoFrame::NumPlanes(format);

- FrameResources* frame_resources = new FrameResources(size);

+ FrameResources* frame_resources = new FrameResources(size, format);

resources_pool_.push_back(frame_resources);

for (size_t i = 0; i < num_planes; i += PlanesPerCopy(format)) {

PlaneResource& plane_resource = frame_resources->plane_resources[i];

@@ -696,7 +739,7 @@ GpuMemoryBufferVideoFramePool::PoolImpl::GetOrCreateFrameResources(

const size_t height = VideoFrame::Rows(i, format, size.height());

plane_resource.size = gfx::Size(width, height);

- const gfx::BufferFormat buffer_format = GpuMemoryBufferFormat(format, i);

+ const gfx::BufferFormat buffer_format = VideoFrame::BufferFormat(format);

plane_resource.gpu_memory_buffer = gpu_factories_->AllocateGpuMemoryBuffer(

plane_resource.size, buffer_format,

gfx::BufferUsage::GPU_READ_CPU_READ_WRITE);

@@ -704,8 +747,13 @@ GpuMemoryBufferVideoFramePool::PoolImpl::GetOrCreateFrameResources(

unsigned texture_target = gpu_factories_->ImageTextureTarget(buffer_format);

gles2->GenTextures(1, &plane_resource.texture_id);

gles2->BindTexture(texture_target, plane_resource.texture_id);

- gles2->TexParameteri(texture_target, GL_TEXTURE_MIN_FILTER, GL_LINEAR);

- gles2->TexParameteri(texture_target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);

+ if (format == PIXEL_FORMAT_Y16) {

+ gles2->TexParameteri(texture_target, GL_TEXTURE_MIN_FILTER, GL_NEAREST);

+ gles2->TexParameteri(texture_target, GL_TEXTURE_MAG_FILTER, GL_NEAREST);

+ } else {

+ gles2->TexParameteri(texture_target, GL_TEXTURE_MIN_FILTER, GL_LINEAR);

+ gles2->TexParameteri(texture_target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);

+ }

gles2->TexParameteri(texture_target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);

gles2->TexParameteri(texture_target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);

gles2->GenMailboxCHROMIUM(plane_resource.mailbox.name);

« no previous file with comments | « media/renderers/skcanvas_video_renderer_unittest.cc ('k') | third_party/WebKit/Source/core/html/HTMLVideoElement.h » ('j') | no next file with comments »