media: replace LUMINANCE_F16 by RG_88 for 9/10-bit h264 videos

cc/output/shader.cc

 // Copyright 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "cc/output/shader.h"
 
 #include <stddef.h>
 
 #include <algorithm>
 
@@ skipping 2005 matching lines @@
 FragmentShaderYUVVideo::FragmentShaderYUVVideo()
     : y_texture_location_(-1),
       u_texture_location_(-1),
       v_texture_location_(-1),
       uv_texture_location_(-1),
       a_texture_location_(-1),
       lut_texture_location_(-1),
       alpha_location_(-1),
       yuv_matrix_location_(-1),
       yuv_adj_location_(-1),
+      max_channel_value_location_(-1),
+      ya_size_location_(-1),
+      uv_size_location_(-1),
       ya_clamp_rect_location_(-1),
       uv_clamp_rect_location_(-1),
       resource_multiplier_location_(-1),
       resource_offset_location_(-1) {}
 
 void FragmentShaderYUVVideo::SetFeatures(bool use_alpha_texture,
                                          bool use_nv12,
-                                         bool use_color_lut) {
+                                         bool use_color_lut,
+                                         HighbitTexture highbit_texture) {
   use_alpha_texture_ = use_alpha_texture;
   use_nv12_ = use_nv12;
   use_color_lut_ = use_color_lut;
+  highbit_texture_ = highbit_texture;
 }
 
 void FragmentShaderYUVVideo::Init(GLES2Interface* context,
                                   unsigned program,
                                   int* base_uniform_index) {
   static const char* uniforms[] = {
       "y_texture",
       "u_texture",
       "v_texture",
       "uv_texture",
       "a_texture",
       "lut_texture",
+      "yuv_matrix",
+      "yuv_adj",
       "resource_multiplier",
       "resource_offset",
-      "yuv_matrix",
-      "yuv_adj",
+      "max_channel_value",
+      "ya_size",
+      "uv_size",
       "alpha",
       "ya_clamp_rect",
       "uv_clamp_rect",
   };
   int locations[arraysize(uniforms)];
 
   GetProgramUniformLocations(context,
                              program,
                              arraysize(uniforms),
                              uniforms,
                              locations,
                              base_uniform_index);
   y_texture_location_ = locations[0];
   if (!use_nv12_) {
     u_texture_location_ = locations[1];
     v_texture_location_ = locations[2];
   } else {
     uv_texture_location_ = locations[3];
   }
   if (use_alpha_texture_) {
     a_texture_location_ = locations[4];
   }
   if (use_color_lut_) {
     lut_texture_location_ = locations[5];
-    resource_multiplier_location_ = locations[6];
-    resource_offset_location_ = locations[7];
   } else {
-    yuv_matrix_location_ = locations[8];
-    yuv_adj_location_ = locations[9];
+    yuv_matrix_location_ = locations[6];
+    yuv_adj_location_ = locations[7];
   }
-  alpha_location_ = locations[10];
-  ya_clamp_rect_location_ = locations[11];
-  uv_clamp_rect_location_ = locations[12];
+  if (highbit_texture_ == HIGHBIT_LUMINANCE_F16) {
+    resource_multiplier_location_ = locations[8];
+    resource_offset_location_ = locations[9];
+  } else if (highbit_texture_ == HIGHBIT_RG88) {
+    max_channel_value_location_ = locations[10];
+    ya_size_location_ = locations[11];
+    uv_size_location_ = locations[12];
+  }
+  alpha_location_ = locations[13];
+  ya_clamp_rect_location_ = locations[14];
+  uv_clamp_rect_location_ = locations[15];
 }
 
 std::string FragmentShaderYUVVideo::GetShaderString(TexCoordPrecision precision,
                                                     SamplerType sampler) const {
   std::string head = SHADER0([]() {
     precision mediump float;
     precision mediump int;
     varying TexCoordPrecision vec2 v_yaTexCoord;
     varying TexCoordPrecision vec2 v_uvTexCoord;
     uniform SamplerType y_texture;
     uniform float alpha;
     uniform vec4 ya_clamp_rect;
     uniform vec4 uv_clamp_rect;
   });
 
   std::string functions = "";
+  if (highbit_texture_ == HIGHBIT_RG88) {

[hubbe, 2016/09/27 20:49:51]

Can we make it so that the if and else branches produce functions with the same
prorotypes? The idea is that the code *after* the if statement can just call the
function without knowing which one it is.

[dshwang, 2016/09/28 10:33:27]

GetY() and GetUV() is kind of public function. Following code just use them
without knowing which one it is.

You might want to go lower layer such as RGTextureLookupBilinear(). However, it
requires texture size which only HIGHBIT_RG88 branch know.

[hubbe, 2016/09/28 17:57:44]

Hmm, you're right. I was hoping to make this a bit cleaner, but it's not easy to
do so unfortunately.

+    head += "  uniform int max_channel_value;\n";

[hubbe, 2016/09/28 17:57:44]

How about reusing resource_multiplier?
The calling code should probably set it to 1/max_channel_value to avoid a
division in this code.

[dshwang, 2016/09/29 18:52:28]

It needs 2048 hardcode or interact code between video_resource_updater to
gl_renderer, while gl_renderer can handle it by itself.

resource_multiplier is defined in video_resource_updater by
external_resources.multiplier = 2048.0 / max_input_value;

Let me change |max_channel_value| to |inverse_max_input_value|

+    head += "  uniform vec2 ya_size;\n";
+    head += "  uniform vec2 uv_size;\n";
+    functions += SHADER0([]() {
+      float GetFloat(vec2 rg) {
+        return ((rg.g * 65536.0) + (rg.r * 256.0)) / float(max_channel_value);

[hubbe, 2016/09/28 17:57:44]

should be (rg.g * 65280.0) + (rg.r * 255.0)

This highlights the need for tests for this code.

[dshwang, 2016/09/29 18:52:27]

oh, you are right. I'm extending renderer_pixeltest test to cover both
LUMINANCE_F16 and RG_88, but it will take time because I need to figure out how
PixelTest create context_provider which support GL_HALF_FLOAT_OES and GL_RG_EXT.

Now I suffer folloing error.
[11470:11471:0929/214423:284857029184:ERROR:texture_manager.cc(2246)]
[GroupMarkerNotSet(crbug.com/242999)!:D0A0867E7B210000]GL ERROR :GL_INVALID_ENUM
: glTexImage2D: type was GL_HALF_FLOAT_OES

+      }
+
+      float RGTextureLookupBilinear(SamplerType sampler, vec2 tex_coord,
+                                    vec4 clamp_rect, vec2 tex_size) {
+        vec2 unit_texel = 1.0 / tex_size;
+        vec2 tex_coord_clamped1 =
+            max(clamp_rect.xy, min(clamp_rect.zw, tex_coord));
+        float s1 = GetFloat(TextureLookup(sampler, tex_coord_clamped1).xy);
+        vec2 tex_coord_clamped2 =
+            max(clamp_rect.xy,
+                min(clamp_rect.zw, tex_coord + vec2(unit_texel.x, 0.)));

[hubbe, 2016/09/28 17:57:44]

This doesn't actually work, because there is no guarantee that the calculation
is accurate enough that adding 1/resolution will give us the next pixel.

We need to snap tex_coord to the middle of a pixel to be sure that it works.

Also, we're clamping text_coord.y the same way as we did before. The compiler
might be able to figure that out, but it's probably better to not rely on it.

[dshwang, 2016/09/29 18:52:28]

Ok, I added snapping code inspired by
http://http.developer.nvidia.com/GPUGems2/gpugems2_chapter27.html

However, the link say
"""
One additional tip is that it is not necessary to snap texture coordinates to
the center of the nearest texel as long as texture lookups do not perform any
type of filtering. This means setting filtering to GL_NEAREST in OpenGL or
D3DTEXF_POINT in Direct3D. This way, coordinate snapping is performed
automatically during texture lookups.
"""

The author thinks if border of tex coord looks up left texel, (border of tex
coord + n/width) always looks up left texel.
WDYT?

[hubbe, 2016/09/29 23:36:50]

If snapping is not needed, I'd like to understand why.

[dshwang, 2016/10/05 15:13:39]

I found snapping is needed when I made pixel test. check new patch

+        float s2 = GetFloat(TextureLookup(sampler, tex_coord_clamped2).xy);
+        vec2 tex_coord_clamped3 =
+            max(clamp_rect.xy,

[hubbe, 2016/09/28 17:57:44]

Do we really need to do the clamping for every sub-sample?
That's not actually what the "normal" GetY() code does...

[dshwang, 2016/09/29 18:52:27]

make sense. it's not really needed.

+                min(clamp_rect.zw, tex_coord + vec2(0., unit_texel.y)));
+        float s3 = GetFloat(TextureLookup(sampler, tex_coord_clamped3).xy);
+        vec2 tex_coord_clamped4 =
+            max(clamp_rect.xy, min(clamp_rect.zw, tex_coord + unit_texel));
+        float s4 = GetFloat(TextureLookup(sampler, tex_coord_clamped4).xy);
+        vec2 f = fract(tex_coord * tex_size);
+        return mix(mix(s1, s2, f.x), mix(s3, s4, f.x), f.y);
+      }
+
+      float GetY() {
+        return RGTextureLookupBilinear(y_texture, v_yaTexCoord, ya_clamp_rect,
+                                       ya_size);
+      }
+    });
+  } else {
+    functions += SHADER0([]() {
+      float GetY() {
+        vec2 ya_clamped =
+            max(ya_clamp_rect.xy, min(ya_clamp_rect.zw, v_yaTexCoord));
+        return TextureLookup(y_texture, ya_clamped).x;
+      }
+    });
+  }
+
   if (use_nv12_) {
     head += "  uniform SamplerType uv_texture;\n";
     functions += SHADER0([]() {
-      vec2 GetUV(vec2 uv_clamped) {
+      vec2 GetUV(vec2 uv_tex_coord) {
+        vec2 uv_clamped =
+            max(uv_clamp_rect.xy, min(uv_clamp_rect.zw, uv_tex_coord));
         return TextureLookup(uv_texture, uv_clamped).xy;
       }
     });
   } else {
     head += "  uniform SamplerType u_texture;\n";
     head += "  uniform SamplerType v_texture;\n";
-    functions += SHADER0([]() {
-      vec2 GetUV(vec2 uv_clamped) {
-        return vec2(TextureLookup(u_texture, uv_clamped).x,
-                    TextureLookup(v_texture, uv_clamped).x);
-      }
-    });
+    if (highbit_texture_ == HIGHBIT_RG88) {
+      functions += SHADER0([]() {
+        vec2 GetUV(vec2 uv_tex_coord) {

[hubbe, 2016/09/27 20:49:51]

I think it would be more efficient if this was a an 8888 texture so that we
could look up U and V at the same time. However, we could do that in a separate
patch. (We'd need to swizzle the values a bit on upload, but that's nearly
free...)

[dshwang, 2016/09/28 10:33:27]

That's good idea. 8888 texture could be fallback option of RG88. What I mean is
8888 replacing both F16 and scaling down when RG88 is not supported.

[hubbe, 2016/09/29 23:36:51]

Almost everything supports RG88 I think.

+          vec2 uv = vec2(RGTextureLookupBilinear(u_texture, uv_tex_coord,
+                                                 uv_clamp_rect, uv_size),
+                         RGTextureLookupBilinear(v_texture, uv_tex_coord,
+                                                 uv_clamp_rect, uv_size));
+          return uv;
+        }
+      });
+    } else {
+      functions += SHADER0([]() {
+        vec2 GetUV(vec2 uv_tex_coord) {
+          vec2 uv_clamped =
+              max(uv_clamp_rect.xy, min(uv_clamp_rect.zw, uv_tex_coord));
+          vec2 uv = vec2(TextureLookup(u_texture, uv_clamped).x,
+                         TextureLookup(v_texture, uv_clamped).x);
+          return uv;
+        }
+      });
+    }
   }
 
   if (use_alpha_texture_) {
     head += "  uniform SamplerType a_texture;\n";
     functions += SHADER0([]() {
-      float GetAlpha(vec2 ya_clamped) {
+      float GetAlpha(vec2 ya_tex_coord) {
+        vec2 ya_clamped =
+            max(ya_clamp_rect.xy, min(ya_clamp_rect.zw, ya_tex_coord));
         return alpha * TextureLookup(a_texture, ya_clamped).x;
       }
     });
   } else {
     functions += SHADER0([]() {
-      float GetAlpha(vec2 ya_clamped) { return alpha; }
+      float GetAlpha(vec2 ya_tex_coord) { return alpha; }
     });
   }
 
   if (use_color_lut_) {
     head += "  uniform sampler2D lut_texture;\n";
-    head += "  uniform float resource_multiplier;\n";
-    head += "  uniform float resource_offset;\n";
     functions += SHADER0([]() {
       vec4 LUT(sampler2D sampler, vec3 pos, float size) {
         pos *= size - 1.0;
         // Select layer
         float layer = min(floor(pos.x), size - 2.0);
         // Compress the yz coordinates so they stay within
         // [0.5 .. 31.5] / 32 (assuming a LUT size of 32^3)
         pos.yz = (pos.yz + vec2(0.5)) / size;
         pos.z = (pos.z + layer) / size;
         return mix(texture2D(sampler, pos.yz),
                    texture2D(sampler, pos.yz + vec2(0, 1.0 / size)),
                    pos.x - layer);
       }
-
-      vec3 yuv2rgb(vec3 yuv) {
-        yuv = (yuv - vec3(resource_offset)) * resource_multiplier;
-        return LUT(lut_texture, yuv, 32.0).xyz;
-      }
     });
+    if (highbit_texture_ == HIGHBIT_LUMINANCE_F16) {
+      head += "  uniform float resource_multiplier;\n";
+      head += "  uniform float resource_offset;\n";
+      functions += SHADER0([]() {
+        vec3 yuv2rgb(vec3 yuv) {
+          yuv = (yuv - vec3(resource_offset)) * resource_multiplier;
+          return LUT(lut_texture, yuv, 32.0).xyz;
+        }
+      });
+    } else {
+      functions += SHADER0([]() {
+        vec3 yuv2rgb(vec3 yuv) { return LUT(lut_texture, yuv, 32.0).xyz; }
+      });
+    }
   } else {
     head += "  uniform mat3 yuv_matrix;\n";
     head += "  uniform vec3 yuv_adj;\n";
     functions += SHADER0([]() {
       vec3 yuv2rgb(vec3 yuv) { return yuv_matrix * (yuv + yuv_adj); }
     });
   }
 
   functions += SHADER0([]() {
     void main() {
-      vec2 ya_clamped =
-          max(ya_clamp_rect.xy, min(ya_clamp_rect.zw, v_yaTexCoord));
-      float y_raw = TextureLookup(y_texture, ya_clamped).x;
-      vec2 uv_clamped =
-          max(uv_clamp_rect.xy, min(uv_clamp_rect.zw, v_uvTexCoord));
-      vec3 yuv = vec3(y_raw, GetUV(uv_clamped));
-      gl_FragColor = vec4(yuv2rgb(yuv), 1.0) * GetAlpha(ya_clamped);
+      float y_raw = GetY();

[hubbe, 2016/09/28 17:57:44]

GetUV and GetAlpha takes a parameter, probably GetY() should too?

[dshwang, 2016/09/29 18:52:28]

to make consistent, let me change it to get |v_yaTexCoord|

+      vec3 yuv = vec3(y_raw, GetUV(v_uvTexCoord));
+      gl_FragColor = vec4(yuv2rgb(yuv), 1.0) * GetAlpha(v_yaTexCoord);
     }
   });
 
   return FRAGMENT_SHADER(head, functions);
 }
 
 FragmentShaderColor::FragmentShaderColor() : color_location_(-1) {
 }
 
 void FragmentShaderColor::Init(GLES2Interface* context,
@@ skipping 69 matching lines @@
     void main() {
       vec4 d4 = min(edge_dist[0], edge_dist[1]);
       vec2 d2 = min(d4.xz, d4.yw);
       float aa = clamp(gl_FragCoord.w * min(d2.x, d2.y), 0.0, 1.0);
       gl_FragColor = color * aa;
     }
   });
 }
 
 }  // namespace cc