Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(252)

Unified Diff: media/renderers/skcanvas_video_renderer.cc

Issue 2121043002: 16 bpp video stream capture, render and WebGL usage - Realsense R200 & SR300 support. Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: rebase Created 4 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « media/renderers/skcanvas_video_renderer.h ('k') | media/renderers/skcanvas_video_renderer_unittest.cc » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: media/renderers/skcanvas_video_renderer.cc
diff --git a/media/renderers/skcanvas_video_renderer.cc b/media/renderers/skcanvas_video_renderer.cc
index c3387ae15db6db9f9918d5192865ca36e5827313..bf85a16945018d25598a33233c0eabd825a153f7 100644
--- a/media/renderers/skcanvas_video_renderer.cc
+++ b/media/renderers/skcanvas_video_renderer.cc
@@ -4,12 +4,14 @@
#include "media/renderers/skcanvas_video_renderer.h"
+#include <GLES3/gl3.h>
#include <limits>
#include "base/macros.h"
#include "gpu/GLES2/gl2extchromium.h"
#include "gpu/command_buffer/client/gles2_interface.h"
#include "gpu/command_buffer/common/mailbox_holder.h"
+#include "media/base/data_buffer.h"
#include "media/base/video_frame.h"
#include "media/base/yuv_convert.h"
#include "skia/ext/texture_handle.h"
@@ -165,7 +167,8 @@ sk_sp<SkImage> NewSkImageFromVideoFrameNative(VideoFrame* video_frame,
DCHECK(PIXEL_FORMAT_ARGB == video_frame->format() ||
PIXEL_FORMAT_XRGB == video_frame->format() ||
PIXEL_FORMAT_NV12 == video_frame->format() ||
- PIXEL_FORMAT_UYVY == video_frame->format());
+ PIXEL_FORMAT_UYVY == video_frame->format() ||
+ PIXEL_FORMAT_Y16 == video_frame->format());
const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(0);
DCHECK(mailbox_holder.texture_target == GL_TEXTURE_2D ||
@@ -347,6 +350,7 @@ void SkCanvasVideoRenderer::Paint(const scoped_refptr<VideoFrame>& video_frame,
// frame has an unexpected format.
if (!video_frame.get() || video_frame->natural_size().IsEmpty() ||
!(media::IsYuvPlanar(video_frame->format()) ||
+ video_frame->format() == media::PIXEL_FORMAT_Y16 ||
video_frame->HasTextures())) {
SkPaint blackWithAlphaPaint;
blackWithAlphaPaint.setAlpha(paint.getAlpha());
@@ -520,6 +524,77 @@ scoped_refptr<VideoFrame> DownShiftHighbitVideoFrame(
}
return ret;
}
+
+void ConvertY16ToARGB(const VideoFrame* video_frame,
+ void* argb_pixels,
+ size_t argb_row_bytes) {
+ const uint8_t* source =
+ reinterpret_cast<const uint8_t*>(video_frame->visible_data(0));
+ uint8_t* out = reinterpret_cast<uint8_t*>(argb_pixels);
+ const size_t stride = video_frame->stride(0);
+ for (int i = 0; i < video_frame->visible_rect().height(); ++i) {
+ const uint8_t* row = source;
+ uint32_t* rgba = reinterpret_cast<uint32_t*>(out);
+ for (const uint8_t* row_end = row + video_frame->row_bytes(0);
+ row < row_end; ++row) {
+ // We loose the precision here and take only upper 8 bits of 16 bit data.
+ // It is important not to render Y16 as RG_88. To get the full precision
+ // use float textures with WebGL1 and e.g. R16UI or R32F textures with
+ // WebGL2.
+ uint32_t green = *++row;
+ *rgba++ = SkColorSetARGB(0xFF, green, green, green);
+ }
+ out += argb_row_bytes;
+ source += stride;
+ }
+}
+
+void FlipAndConvertY16(const uint8_t* input,
+ uint8_t* output,
+ unsigned format,
+ unsigned type,
+ bool flip_y,
+ size_t row_bytes,
+ size_t stride,
+ size_t output_row_bytes,
+ size_t height) {
+ DCHECK(input != output);
+ for (size_t i = 0; i < height; ++i) {
+ const uint16_t* in = reinterpret_cast<const uint16_t*>(input + i * stride);
+ uint8_t* out = flip_y ? output + output_row_bytes * (height - i - 1)
+ : output + output_row_bytes * i;
+ if (type == GL_FLOAT) {
+ float* out_row = reinterpret_cast<float*>(out);
+ const uint16_t* in_end = in + row_bytes / 2;
+ if (format == GL_RGBA) {
+ while (in < in_end) {
+ float red = *in++ / 65536.f;
+ *out_row++ = red;
+ *out_row++ = red;
+ *out_row++ = red;
+ *out_row++ = 1.0f;
+ }
+ } else if (format == GL_RGB) {
+ while (in < in_end) {
+ float red = *in++ / 65536.f;
+ *out_row++ = red;
+ *out_row++ = red;
+ *out_row++ = red;
+ }
+ } else if (type == GL_RED) {
+ while (in < in_end)
+ *out_row++ = *in++ / 65536.f;
+ } else {
+ NOTREACHED();
+ }
+ } else if ((format == GL_RG && type == GL_UNSIGNED_BYTE) ||
+ (format == GL_RED_INTEGER && type == GL_UNSIGNED_SHORT)) {
+ memcpy(out, input + i * stride, row_bytes);
+ } else {
+ NOTREACHED();
+ }
+ }
+}
}
// static
@@ -531,10 +606,6 @@ void SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
NOTREACHED() << "Cannot extract pixels from non-CPU frame formats.";
return;
}
- if (!media::IsYuvPlanar(video_frame->format())) {
- NOTREACHED() << "Non YUV formats are not supported";
- return;
- }
switch (video_frame->format()) {
case PIXEL_FORMAT_YV12:
@@ -627,6 +698,10 @@ void SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
break;
}
+ case PIXEL_FORMAT_Y16:
+ ConvertY16ToARGB(video_frame, rgb_pixels, row_bytes);
+ break;
+
case PIXEL_FORMAT_NV12:
case PIXEL_FORMAT_NV21:
case PIXEL_FORMAT_UYVY:
@@ -637,14 +712,9 @@ void SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
case PIXEL_FORMAT_RGB32:
case PIXEL_FORMAT_MJPEG:
case PIXEL_FORMAT_MT21:
- // TODO(dshwang): Use either I400ToARGB or J400ToARGB depending if we want
- // BT.601 constrained range of 16 to 240, or JPEG full range BT.601
- // coefficients. Implement it when Y8/16 foramt is supported.
- // crbug.com/624436
case PIXEL_FORMAT_Y8:
- case PIXEL_FORMAT_Y16:
case PIXEL_FORMAT_UNKNOWN:
- NOTREACHED();
+ NOTREACHED() << "Only YUV formats and Y16 are supported.";
}
}
@@ -744,15 +814,288 @@ bool SkCanvasVideoRenderer::CopyVideoFrameTexturesToGLTexture(
SyncTokenClientImpl client(canvas_gl);
video_frame->UpdateReleaseSyncToken(&client);
+ } else if (video_frame->format() == PIXEL_FORMAT_Y16) {
+ CopyRG8ToTexture(context_3d, destination_gl, video_frame.get(), texture,
+ internal_format, type, flip_y);
} else {
CopyVideoFrameSingleTextureToGLTexture(destination_gl, video_frame.get(),
texture, internal_format, type,
premultiply_alpha, flip_y);
}
+ return true;
+}
+
+bool SkCanvasVideoRenderer::TexImageImpl(const char* functionID,
+ unsigned target,
+ gpu::gles2::GLES2Interface* gl,
+ VideoFrame* frame,
+ int level,
+ int internalformat,
+ unsigned format,
+ unsigned type,
+ int xoffset,
+ int yoffset,
+ int zoffset,
+ bool flip_y,
+ bool premultiplyAlpha) {
+ DCHECK(frame);
+ DCHECK(!frame->HasTextures());
+
+ bool has_alpha = false;
+ unsigned output_bytes_per_pixel;
+ switch (frame->format()) {
+ case PIXEL_FORMAT_Y16:
+ // Allow reinterpreting RG8 buffer here as R component in FLOAT.
+ switch (format) {
+ case GL_RGBA:
+ if (type == GL_FLOAT) {
+ output_bytes_per_pixel = 4 * sizeof(GLfloat);
+ break;
+ }
+ // Pass through.
+ case GL_RGB:
+ if (type == GL_FLOAT) {
+ output_bytes_per_pixel = 3 * sizeof(GLfloat);
+ break;
+ }
+ // Pass through.
+ default:
+ return false;
+ }
+ break;
+ default:
+ return false;
+ }
+ unsigned source_bytes_per_pixel =
+ VideoFrame::PlaneBitsPerPixel(frame->format(), 0) / 8;
+ DCHECK_EQ(VideoFrame::PlaneBitsPerPixel(frame->format(), 0) % 8, 0);
+ if (has_alpha && premultiplyAlpha) {
+ NOTREACHED() << "Premultiply alpha is not supported.";
+ return false;
+ }
+ if (xoffset || yoffset || zoffset) {
+ NOTREACHED() << "Offsets are not supported.";
+ return false;
+ }
+
+ uint8_t* data;
+ scoped_refptr<DataBuffer> temp_buffer;
+ size_t width = frame->visible_rect().width();
+ size_t height = frame->visible_rect().height();
+ size_t output_row_bytes =
+ frame->row_bytes(0) * output_bytes_per_pixel / source_bytes_per_pixel;
+ temp_buffer = new DataBuffer(output_row_bytes * height);
+ data = temp_buffer->writable_data();
+ DCHECK_EQ(frame->format(), PIXEL_FORMAT_Y16);
+ FlipAndConvertY16(frame->visible_data(0), data, format, type, flip_y,
+ frame->row_bytes(0), frame->stride(0), output_row_bytes,
+ height);
+
+ if (!strcmp(functionID, "texImage2D")) {
+ gl->TexImage2D(target, level, internalformat, width, height, 0, format,
+ type, data);
+ } else if (!strcmp(functionID, "texSubImage2D")) {
+ gl->TexSubImage2D(target, level, xoffset, yoffset, width, height, format,
+ type, data);
+ } else {
+ DCHECK(!strcmp(functionID, "texSubImage3D"));
+ gl->TexSubImage3D(target, level, xoffset, yoffset, zoffset, width, height,
+ 1, format, type, data);
+ }
return true;
}
+void SkCanvasVideoRenderer::CopyRG8ToTexture(
+ const Context3D& context_3d,
+ gpu::gles2::GLES2Interface* target_gl,
+ VideoFrame* frame,
+ unsigned texture,
+ unsigned internal_format,
+ unsigned type,
+ bool flip_y) {
+ // TODO(astojilj): After GLES 3.2, use glCopyImageSubData for RG8 to R16UI
+ // conversion. glCopyImageSubData is already available as extension on some of
+ // the Android devices (GL_EXT_COPY_IMAGE).
+ gpu::gles2::GLES2Interface* local_gl = context_3d.gl;
+
+ // Get source texture on local_gl.
+ unsigned source_texture;
+ {
+ const gpu::MailboxHolder& mailbox_holder = frame->mailbox_holder(0);
+ DCHECK(mailbox_holder.texture_target == GL_TEXTURE_2D)
+ << mailbox_holder.texture_target;
+
+ local_gl->WaitSyncTokenCHROMIUM(mailbox_holder.sync_token.GetConstData());
+ source_texture = local_gl->CreateAndConsumeTextureCHROMIUM(
+ mailbox_holder.texture_target, mailbox_holder.mailbox.name);
+ }
+
+ // Reinterpret RG8 as 16 bit normalized and then to luminance in RGBA on
+ // local_gl. Later, do the copy on target_gl.
+ unsigned intermediate_texture;
+ local_gl->GenTextures(1, &intermediate_texture);
+ DoCopyRG8ToTexture(context_3d, source_texture, intermediate_texture, GL_RGBA,
+ GL_RGBA, GL_FLOAT, frame->visible_rect().width(),
+ frame->visible_rect().height(), flip_y);
+ local_gl->DeleteTextures(1, &source_texture);
+
+ // Get intermediate_texture to target_gl.
+ {
+ gpu::MailboxHolder mailbox_holder;
+ mailbox_holder.texture_target = GL_TEXTURE_2D;
+ local_gl->GenMailboxCHROMIUM(mailbox_holder.mailbox.name);
+ local_gl->ProduceTextureDirectCHROMIUM(intermediate_texture,
+ mailbox_holder.texture_target,
+ mailbox_holder.mailbox.name);
+ // Wait for mailbox creation on local context before consuming it and
+ // copying from it on the consumer context.
+ const GLuint64 fence_sync = local_gl->InsertFenceSyncCHROMIUM();
+ local_gl->ShallowFlushCHROMIUM();
+ local_gl->GenSyncTokenCHROMIUM(fence_sync,
+ mailbox_holder.sync_token.GetData());
+
+ target_gl->WaitSyncTokenCHROMIUM(mailbox_holder.sync_token.GetConstData());
+ uint32_t target_intermediate_texture =
+ target_gl->CreateAndConsumeTextureCHROMIUM(
+ mailbox_holder.texture_target, mailbox_holder.mailbox.name);
+
+ local_gl->DeleteTextures(1, &intermediate_texture);
+ intermediate_texture = target_intermediate_texture;
+ }
+
+ // Copy intermediate_texture to texture.
+ target_gl->CopyTextureCHROMIUM(intermediate_texture, texture, internal_format,
+ type, false, false, false);
+ target_gl->DeleteTextures(1, &intermediate_texture);
+
+ // Undo gr_context state changes introduced in this function.
+ context_3d.gr_context->resetContext(
+ kMisc_GrGLBackendState | kBlend_GrGLBackendState |
+ kView_GrGLBackendState | kStencil_GrGLBackendState |
+ kVertex_GrGLBackendState | kProgram_GrGLBackendState |
+ kTextureBinding_GrGLBackendState);
+ SyncTokenClientImpl client(target_gl);
+ frame->UpdateReleaseSyncToken(&client);
+}
+
+void SkCanvasVideoRenderer::DoCopyRG8ToTexture(const Context3D& context_3d,
+ unsigned source_texture,
+ unsigned texture,
+ unsigned format,
+ unsigned internal_format,
+ unsigned type,
+ unsigned width,
+ unsigned height,
+ bool flip_y) {
+ gpu::gles2::GLES2Interface* gl = context_3d.gl;
+
+ if (cached_gl_resources_gr_context_id_ != context_3d.gr_context->uniqueID()) {
+ rg8_to_texture_program_ = 0;
+ rg8_to_texture_vertices_buffer_ = 0;
+ cached_gl_resources_gr_context_id_ = context_3d.gr_context->uniqueID();
+ }
+
+ gl->EnableVertexAttribArray(0);
+ if (!rg8_to_texture_vertices_buffer_) {
+ gl->GenBuffers(1, &rg8_to_texture_vertices_buffer_);
+ gl->BindBuffer(GL_ARRAY_BUFFER, rg8_to_texture_vertices_buffer_);
+ const GLfloat vertices[] = {-1.f, -1.f, 1.f, -1.f, -1.0f, 1.f, 1.f, 1.f};
+ gl->BufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);
+ gl->VertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 0, 0);
+ }
+
+ if (!rg8_to_texture_program_) {
+ rg8_to_texture_program_ = gl->CreateProgram();
+ GLuint vertex_shader = gl->CreateShader(GL_VERTEX_SHADER);
+ std::string source = std::string(
+ "\
+ #ifdef GL_ES\n\
+ precision mediump float;\n\
+ #define TexCoordPrecision mediump\n\
+ #else\n\
+ #define TexCoordPrecision\n\
+ #endif\n\
+ uniform vec2 coord_transform;\n\
+ attribute vec2 a_pos;\n\
+ varying TexCoordPrecision vec2 v_uv;\n\
+ void main(void) {\n\
+ gl_Position = vec4(a_pos, 0, 1.0);\n\
+ v_uv = (a_pos * coord_transform) + 0.5;\n\
+ }\n");
+ const char* vertex_source = source.c_str();
+ gl->ShaderSource(vertex_shader, 1, &vertex_source, 0);
+ gl->CompileShader(vertex_shader);
+ gl->AttachShader(rg8_to_texture_program_, vertex_shader);
+
+ GLuint fragment_shader = gl->CreateShader(GL_FRAGMENT_SHADER);
+ source = std::string(
+ "\
+ #ifdef GL_ES\n\
+ precision mediump float;\n\
+ #define TexCoordPrecision mediump\n\
+ #else\n\
+ #define TexCoordPrecision\n\
+ #endif\n\
+ uniform sampler2D u_sampler;\n\
+ varying TexCoordPrecision vec2 v_uv;\n\
+ void main(void) {\n\
+ vec4 color = texture2D(u_sampler, v_uv);\n\
+ gl_FragColor.rgb = vec3((color.r * 0.00390625) + color.g);\n\
+ }\n");
+ const char* fragment_source = source.c_str();
+ gl->ShaderSource(fragment_shader, 1, &fragment_source, 0);
+ gl->CompileShader(fragment_shader);
+
+ gl->AttachShader(rg8_to_texture_program_, fragment_shader);
+
+ gl->BindAttribLocation(rg8_to_texture_program_, 0, "a_pos");
+ gl->LinkProgram(rg8_to_texture_program_);
+ }
+
+ gl->UseProgram(rg8_to_texture_program_);
+ gl->Uniform1i(gl->GetUniformLocation(rg8_to_texture_program_, "u_sampler"),
+ 0);
+ gl->Uniform2f(
+ gl->GetUniformLocation(rg8_to_texture_program_, "coord_transform"), 0.5,
+ flip_y ? -0.5 : 0.5);
+
+ GLuint fbo;
+ gl->GenFramebuffers(1, &fbo);
+ gl->ActiveTexture(GL_TEXTURE0);
+ unsigned target = GL_TEXTURE_2D;
+ gl->BindTexture(target, texture);
+ gl->TexImage2D(GL_TEXTURE_2D, 0, internal_format, width, height, 0, format,
+ type, 0);
+
+ gl->TexParameterf(target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+ gl->TexParameterf(target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+ gl->TexParameteri(target, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+ gl->TexParameteri(target, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+ gl->BindFramebuffer(GL_FRAMEBUFFER, fbo);
+ gl->FramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, target,
+ texture, 0);
+
+ gl->BindTexture(target, source_texture);
+ gl->TexParameterf(target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+ gl->TexParameterf(target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+ gl->TexParameteri(target, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+ gl->TexParameteri(target, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+
+ gl->Disable(GL_DEPTH_TEST);
+ gl->Disable(GL_STENCIL_TEST);
+ gl->Disable(GL_CULL_FACE);
+ gl->ColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+ gl->DepthMask(GL_FALSE);
+ gl->Disable(GL_BLEND);
+ gl->Disable(GL_SCISSOR_TEST);
+ gl->Viewport(0, 0, width, height);
+ gl->DrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+ gl->Flush();
+ gl->BindFramebuffer(GL_FRAMEBUFFER, 0);
+ gl->DeleteFramebuffers(1, &fbo);
+}
+
void SkCanvasVideoRenderer::ResetCache() {
DCHECK(thread_checker_.CalledOnValidThread());
// Clear cached values.
@@ -777,6 +1120,8 @@ bool SkCanvasVideoRenderer::UpdateLastImage(
last_image_ =
NewSkImageFromVideoFrameYUVTextures(video_frame.get(), context_3d);
} else {
+ // TODO(astojilj): For Y16 on RG textures we don't want to paint 16 bit
+ // split to 8 bit R & G. Implement using CopyRG8ToTexture.
last_image_ =
NewSkImageFromVideoFrameNative(video_frame.get(), context_3d);
}
« no previous file with comments | « media/renderers/skcanvas_video_renderer.h ('k') | media/renderers/skcanvas_video_renderer_unittest.cc » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698