WebGL & 16-bit video stream: upload to FLOAT texture.

content/test/data/media/getusermedia-depth-capture.html

Index: content/test/data/media/getusermedia-depth-capture.html
diff --git a/content/test/data/media/getusermedia-depth-capture.html b/content/test/data/media/getusermedia-depth-capture.html
index 521fc0730c17322bf833ccb5588b0ac2c2d4e21f..affd36ac6f8f890ae723ffa51f78f931cdcd8ebd 100644
--- a/content/test/data/media/getusermedia-depth-capture.html
+++ b/content/test/data/media/getusermedia-depth-capture.html
@@ -26,6 +26,25 @@
     },
     failedCallback);
   }
+  
+  // testVideoToWebGLTexture is a layout test that we run here because it
+  // requires --use-fake-device-for-media-capture.
+  function getDepthStreamAndCallGLTexImage2D() {
+    console.log('Calling getDepthStreamAndCallGLTexImage2D');
+    getFake16bitStream().then(function(stream) {
+      detectVideoInLocalView1(stream, function() {
+        testVideoToWebGLTexture('local-view-1', function(skip_info) {
+          if (skip_info) {
+            console.log("SKIP getDepthStreamAndCallGLTexImage2D: " +
+                        skip_info);  
+          }
+          stream.getVideoTracks()[0].stop();
+          waitForVideoToStop('local-view-1');
+        }, failedCallback);
+      });
+    },
+    failedCallback);
+  }
 
   function failedCallback(error) {
     failTest('GetUserMedia call failed with error name ' + error.name);
@@ -40,14 +59,187 @@
     attachMediaStream(stream, 'local-view-1');
     detectVideoPlaying('local-view-1', callback);
   }
+  
+  function testVideoToImageBitmap(videoElementName, success, error)
+  {

[phoglund_chromium, 2016/11/07 09:20:00]

Nit: pull up braces to previous line (apply throughout)

[aleksandar.stojiljkovic, 2016/11/16 20:01:24]

Done.

+    var bitmaps = {};
+    var video = $(videoElementName);
+    var canvas = document.createElement('canvas');
+    canvas.width = 96;
+    canvas.height = 96;
+    document.body.appendChild(canvas);
+    var p1 = createImageBitmap(video).then(function(imageBitmap) {
+      return runImageBitmapTest(imageBitmap, canvas, false); });
+    var p2 = createImageBitmap(video,
+        {imageOrientation: "none", premultiplyAlpha: "premultiply"}).then(
+            function(imageBitmap) {
+              return runImageBitmapTest(imageBitmap, canvas, false); });
+    var p3 = createImageBitmap(video,
+        {imageOrientation: "none", premultiplyAlpha: "default"}).then(
+            function(imageBitmap) {
+              return runImageBitmapTest(imageBitmap, canvas, false); });
+    var p4 = createImageBitmap(video,
+        {imageOrientation: "none", premultiplyAlpha: "none"}).then(
+            function(imageBitmap) {
+              return runImageBitmapTest(imageBitmap, canvas, false); });
+    var p5 = createImageBitmap(video,
+        {imageOrientation: "flipY", premultiplyAlpha: "premultiply"}).then(
+            function(imageBitmap) {
+              return runImageBitmapTest(imageBitmap, canvas, true); });
+    var p6 = createImageBitmap(video,
+        {imageOrientation: "flipY", premultiplyAlpha: "default"}).then(
+            function(imageBitmap) {
+              return runImageBitmapTest(imageBitmap, canvas, true); });
+    var p7 = createImageBitmap(video,
+        {imageOrientation: "flipY", premultiplyAlpha: "none"}).then(
+            function(imageBitmap) {
+              return runImageBitmapTest(imageBitmap, canvas, true); });
+    return Promise.all([p1, p2, p3, p4, p5, p6, p7]).then(success(), reason => {
+      return error({name: reason});
+    });
+  }
+  
+  function runImageBitmapTest(bitmap, canvas, flip_y) {
+    var context = canvas.getContext('2d');
+    context.drawImage(bitmap,0,0);
+    var imageData = context.getImageData(0, 0, canvas.width, canvas.height);
+    // Fake capture device 96x96 depth image is gradient. See also
+    // Draw16BitGradient in fake_video_capture_device.cc.
+    var color_step = 255.0 / (canvas.width + canvas.height);
+    return verifyPixels(imageData.data, canvas.width, canvas.height, flip_y,
+                        color_step, 255, 2);
+  }
+
+  function testVideoToWebGLTexture(videoElementName, success, error)

[phoglund_chromium, 2016/11/07 09:20:00]

Will you get clear test errors when any of the sub-tests in this function fails?
Will test errors obscure other test errors? You're testing a lot of things here
for one browser test, so you run the risk of really confusing test errors. Also,
if one of the sub-tests goes flaky your entire test will probably be disabled
even if the other sub-tests are fine.

You can remedy this by splitting up the test; for instance by splitting up the
below into one 2D and one Cubemap test, or even 2d_unsigned_byte, 2d_float,
cubemap_unsigned_byte, cubemap_float. Each such test gets called by one browser
test on the c++ level. You can solve the resulting duplication problem using a
higher-order function, e.g.

function runOneGlTest(glTestCaseFunction) {
  getFake16bitStream().then(function(stream) {
    detectVideoInLocalView1(stream, function() {
      glTestCaseFunction('local-view-1', function(skip_info) {
      if (skip_info) {
        console.log("SKIP " + glTestCaseFunction + ": " +
            skip_info);  
      }
      stream.getVideoTracks()[0].stop();
      waitForVideoToStop('local-view-1');
    }, failedCallback);
  });
}, failedCallback);

It's your call, but if you keep this as-is, at least test manually introducing
some failures. That way you can check that you get reasonable test failure
messages and that the tests don't interfere with each other.

[aleksandar.stojiljkovic, 2016/11/16 20:01:24]

Done.
The only reason to couple unsigned byte and float was to save time on test
initialization. More tests planned here for different formats and proposed
refactoring made sense. I kept cubemap and texture2d in the same run, but with
clarified error reporting. Thanks.

+  {
+    var video = $(videoElementName);
+    var canvas = document.createElement('canvas');
+    canvas.width = 96;
+    canvas.height = 96;
+    video.width = 96;
+    video.height = 96;
+    var gl = canvas.getContext('webgl');
+    if(!gl)
+      return error({name:"WebGL is not available."});
+    if(!gl.getExtension("OES_texture_float"))
+      return error({name:"OES_texture_float extension is not available."});
+    var p1 = testVideoToTexture2D(gl, gl.UNSIGNED_BYTE, video);
+    var p2 = testVideoToTextureCubemap(gl, gl.UNSIGNED_BYTE, video);
+    var p3 = testVideoToTexture2D(gl, gl.FLOAT, video);
+    var p4 = testVideoToTextureCubemap(gl, gl.FLOAT, video);
+    return Promise.all([p1, p2, p3, p4]).then(success(), reason => {
+      return error({name: reason});
+    });
+  }
+
+  function testVideoToTexture2D(gl, type, video)
+  {
+    // Premultiply alpha is ignored, so we test it in one iteration only.
+    var p1 = runWebGLTextureTest(gl, gl.TEXTURE_2D, gl.TEXTURE_2D, type, video,
+        false/*sub_image*/, false/*flip_y*/, true/*premultiply_alpha*/);
+    var p2 = runWebGLTextureTest(gl, gl.TEXTURE_2D, gl.TEXTURE_2D, type, video,
+        false/*sub_image*/, true/*flip_y*/, false/*premultiply_alpha*/);
+    var p3 = runWebGLTextureTest(gl, gl.TEXTURE_2D, gl.TEXTURE_2D, type, video,
+        true/*sub_image*/, false/*flip_y*/, false/*premultiply_alpha*/);
+    var p4 = runWebGLTextureTest(gl, gl.TEXTURE_2D, gl.TEXTURE_2D, type, video,
+        true/*sub_image*/, true/*flip_y*/, false/*premultiply_alpha*/);
+    return Promise.all([p1, p2, p3, p4]).then(Promise.resolve(), reason => {
+      return Promise.reject(reason);
+    });
+  }
+
+  function testVideoToTextureCubemap(gl, type, video)
+  {
+    var p1 = runWebGLTextureTest(gl, gl.TEXTURE_CUBE_MAP,
+        gl.TEXTURE_CUBE_MAP_POSITIVE_X, type, video, false/*sub_image*/,
+        false/*flip_y*/, true/*premultiply_alpha*/);
+    var p2 = runWebGLTextureTest(gl, gl.TEXTURE_CUBE_MAP,
+        gl.TEXTURE_CUBE_MAP_POSITIVE_Y, type, video, true/*sub_image*/,
+        false/*flip_y*/, true/*premultiply_alpha*/);
+    var p3 = runWebGLTextureTest(gl, gl.TEXTURE_CUBE_MAP,
+        gl.TEXTURE_CUBE_MAP_POSITIVE_Z, type, video, false/*sub_image*/,
+        true/*flip_y*/, true/*premultiply_alpha*/);
+    var p4 = runWebGLTextureTest(gl, gl.TEXTURE_CUBE_MAP,
+        gl.TEXTURE_CUBE_MAP_NEGATIVE_X, type, video, true/*sub_image*/,
+        true/*flip_y*/, true/*premultiply_alpha*/);
+    var p5 = runWebGLTextureTest(gl, gl.TEXTURE_CUBE_MAP,
+        gl.TEXTURE_CUBE_MAP_NEGATIVE_Y, type, video, false/*sub_image*/,
+        false/*flip_y*/, false/*premultiply_alpha*/);
+    var p6 = runWebGLTextureTest(gl, gl.TEXTURE_CUBE_MAP,
+        gl.TEXTURE_CUBE_MAP_NEGATIVE_Z, type, video, true/*sub_image*/,
+        true/*flip_y*/, false/*premultiply_alpha*/);
+    return Promise.all([p1, p2, p3, p4, p5, p6]).then(Promise.resolve(),
+        reason => {
+      return Promise.reject(reason);
+    });
+  }
+
+  //For cubemap, binding_target is gl.TEXTURE_CUBE_MAP and target is a face id.

[phoglund_chromium, 2016/11/07 09:20:00]

Nit: Space between // and For; this and next line

[aleksandar.stojiljkovic, 2016/11/16 20:01:24]

Done.

+  //For gl.TEXTURE_2D, binding_target=target=gl.TEXTURE_2D.
+  function runWebGLTextureTest(gl, binding_target, target, type, video,
+                              use_sub_image_2d, flip_y, premultiply_alpha) {
+   var tex = gl.createTexture();

[phoglund_chromium, 2016/11/07 09:20:00]

Nit: indent 2 here, not 1

[aleksandar.stojiljkovic, 2016/11/16 20:01:24]

Done.

+   gl.bindTexture(binding_target, tex);
+   gl.texParameteri(binding_target, gl.TEXTURE_WRAP_T,     gl.CLAMP_TO_EDGE);
+   gl.texParameteri(binding_target, gl.TEXTURE_WRAP_S,     gl.CLAMP_TO_EDGE);
+   gl.texParameteri(binding_target, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
+   gl.texParameteri(binding_target, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+  
+   // Create framebuffer that we will use for reading back the texture.
+   var fb = gl.createFramebuffer();
+   gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
+   gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0,
+                           target, tex, 0);
+  
+   gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, flip_y);
+   gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, premultiply_alpha);
+  
+   var targets = (binding_target == gl.TEXTURE_CUBE_MAP) ?
+       [gl.TEXTURE_CUBE_MAP_POSITIVE_X,
+        gl.TEXTURE_CUBE_MAP_NEGATIVE_X,
+        gl.TEXTURE_CUBE_MAP_POSITIVE_Y,
+        gl.TEXTURE_CUBE_MAP_NEGATIVE_Y,
+        gl.TEXTURE_CUBE_MAP_POSITIVE_Z,
+        gl.TEXTURE_CUBE_MAP_NEGATIVE_Z] : [target];
+  
+   // Upload the video frame into the texture.
+   for (var i = 0; i < targets.length; ++i) {
+     if (use_sub_image_2d) {

[phoglund_chromium, 2016/11/07 09:20:00]

I don't like this kind of complexity in tests; they should generally have a
cyclomatic complexity of one (e.g. no branches). You're basically at the level
where the test is complex enough to need a test of its own. One suggestion is to
replace

var p3 = runWebGLTextureTest(gl, gl.TEXTURE_CUBE_MAP,
    gl.TEXTURE_CUBE_MAP_POSITIVE_Z, type, video, false/*sub_image*/,
    true/*flip_y*/, true/*premultiply_alpha*/);

with

var texture = createTexture(gl, binding_target, ...);
uploadVideoFrameSubImage2D(texture, binding_target, ...);
var p3 = readAndVerifyPixels(texture, ...);

That way you can compose your tests by calling uploadVideoFrameSubImage2D, which
is now a simple three-line function, or uploadVideoFrame which is a one-liner.

[aleksandar.stojiljkovic, 2016/11/16 20:01:24]

Done. Rewritten, no branching.

+       gl.texImage2D(targets[i], 0, gl.RGBA, video.width, video.height, 0,
+       gl.RGBA, type, null);
+       gl.texSubImage2D(targets[i], 0, 0, 0, gl.RGBA, type, video);
+     } else {
+       gl.texImage2D(targets[i], 0, gl.RGBA, gl.RGBA, type, video);
+     }
+   }
+  
+   var arr = (type == gl.FLOAT) ?
+       new Float32Array(video.width * video.height * 4) :
+       new Uint8Array(video.width * video.height * 4);
+   // Read the texture attached to framebuffer.
+   gl.readPixels(0, 0, video.width, video.height, gl.RGBA, type, arr);
+   gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+   gl.deleteTexture(tex);
+   gl.deleteFramebuffer(fb);
+  
+   if(type == gl.FLOAT) {
+     var color_step = 1.0 / (video.width + video.height);
+     return verifyPixels(arr, video.width, video.height, flip_y, color_step,
+                         1.0 /*wrap_around*/, 1.5/65535 /*tolerance*/);
+   } else {
+     var color_step = 255.0 / (video.width + video.height);
+     return verifyPixels(arr, video.width, video.height, flip_y, color_step,
+                         255 /*wrap_around*/, 2 /*tolerance*/);
+   }
+  }
   </script>
 </head>
 <body>
   <table border="0">
-    <!-- Canvases are named after their corresponding video elements. -->
     <tr>
       <td><video id="local-view-1" width="96" height="96" autoplay
           style="display:none"></video></td>
+      <!-- The canvas is used to detect when video starts and stops. -->
       <td><canvas id="local-view-1-canvas" width="96" height="96"
           style="display:none"></canvas></td>
     </tr>