16 bpp video stream capture, render and createImageBitmap(video) using (CPU) shared memory buffers

content/browser/renderer_host/media/video_capture_controller_unittest.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Unit test for VideoCaptureController.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
 #include <stdint.h>
 #include <string.h>
@@ skipping 34 matching lines @@
   explicit MockVideoCaptureControllerEventHandler(
       VideoCaptureController* controller)
       : controller_(controller),
         resource_utilization_(-1.0) {}
   ~MockVideoCaptureControllerEventHandler() override {}
 
   // These mock methods are delegated to by our fake implementation of
   // VideoCaptureControllerEventHandler, to be used in EXPECT_CALL().
   MOCK_METHOD1(DoBufferCreated, void(VideoCaptureControllerID));
   MOCK_METHOD1(DoBufferDestroyed, void(VideoCaptureControllerID));
-  MOCK_METHOD2(DoI420BufferReady,
-               void(VideoCaptureControllerID, const gfx::Size&));
+  MOCK_METHOD2(DoBufferReady, void(VideoCaptureControllerID, const gfx::Size&));
   MOCK_METHOD1(DoEnded, void(VideoCaptureControllerID));
   MOCK_METHOD1(DoError, void(VideoCaptureControllerID));
 
   void OnError(VideoCaptureControllerID id) override {
     DoError(id);
   }
   void OnBufferCreated(VideoCaptureControllerID id,
                        mojo::ScopedSharedBufferHandle handle,
                        int length, int buffer_id) override {
     DoBufferCreated(id);
   }
   void OnBufferDestroyed(VideoCaptureControllerID id, int buffer_id) override {
     DoBufferDestroyed(id);
   }
   void OnBufferReady(VideoCaptureControllerID id,
                      int buffer_id,
                      const scoped_refptr<media::VideoFrame>& frame) override {
-    EXPECT_EQ(frame->format(), media::PIXEL_FORMAT_I420);
+    EXPECT_EQ(frame->format(), expect_buffer_format_);
     base::TimeTicks reference_time;
     EXPECT_TRUE(frame->metadata()->GetTimeTicks(
         media::VideoFrameMetadata::REFERENCE_TIME, &reference_time));
-    DoI420BufferReady(id, frame->coded_size());
+    DoBufferReady(id, frame->coded_size());
     base::ThreadTaskRunnerHandle::Get()->PostTask(
         FROM_HERE,
         base::Bind(&VideoCaptureController::ReturnBuffer,
                    base::Unretained(controller_), id, this, buffer_id,
                    gpu::SyncToken(), resource_utilization_));
   }
   void OnEnded(VideoCaptureControllerID id) override {
     DoEnded(id);
     // OnEnded() must respond by (eventually) unregistering the client.
     base::ThreadTaskRunnerHandle::Get()->PostTask(
         FROM_HERE,
         base::Bind(base::IgnoreResult(&VideoCaptureController::RemoveClient),
                    base::Unretained(controller_), id, this));
   }
 
   VideoCaptureController* controller_;
+  media::VideoPixelFormat expect_buffer_format_ = media::PIXEL_FORMAT_I420;

[mcasas, 2016/10/25 22:44:43]

nit: s/expect_buffer_format_/expected_pixel_format_/

[aleksandar.stojiljkovic, 2016/10/26 14:02:18]

Done.

   double resource_utilization_;
 };
 
 // Test class.
-class VideoCaptureControllerTest : public testing::Test {
+class VideoCaptureControllerTest
+    : public testing::Test,
+      public testing::WithParamInterface<media::VideoPixelFormat> {
  public:
   VideoCaptureControllerTest() {}
   ~VideoCaptureControllerTest() override {}
 
  protected:
   static const int kPoolSize = 3;
 
   void SetUp() override {
     controller_.reset(new VideoCaptureController(kPoolSize));
     device_ = controller_->NewDeviceClient();
     client_a_.reset(new MockVideoCaptureControllerEventHandler(
         controller_.get()));
     client_b_.reset(new MockVideoCaptureControllerEventHandler(
         controller_.get()));
   }
 
   void TearDown() override { base::RunLoop().RunUntilIdle(); }
 
-  scoped_refptr<media::VideoFrame> WrapI420Buffer(gfx::Size dimensions,
-                                                  uint8_t* data) {
+  scoped_refptr<media::VideoFrame> WrapBuffer(
+      gfx::Size dimensions,
+      uint8_t* data,
+      media::VideoPixelFormat format = media::PIXEL_FORMAT_I420) {
     scoped_refptr<media::VideoFrame> video_frame =
         media::VideoFrame::WrapExternalSharedMemory(
-            media::PIXEL_FORMAT_I420, dimensions, gfx::Rect(dimensions),
-            dimensions, data, media::VideoFrame::AllocationSize(
-                                  media::PIXEL_FORMAT_I420, dimensions),
+            format, dimensions, gfx::Rect(dimensions), dimensions, data,
+            media::VideoFrame::AllocationSize(format, dimensions),
             base::SharedMemory::NULLHandle(), 0u, base::TimeDelta());
     EXPECT_TRUE(video_frame);
     return video_frame;
   }
 
   TestBrowserThreadBundle bundle_;
   std::unique_ptr<MockVideoCaptureControllerEventHandler> client_a_;
   std::unique_ptr<MockVideoCaptureControllerEventHandler> client_b_;
   std::unique_ptr<VideoCaptureController> controller_;
   std::unique_ptr<media::VideoCaptureDevice::Client> device_;
@@ skipping 94 matching lines @@
       controller_->RemoveClient(client_b_route_2, client_b_.get()))
       << "Removing client B/2 should return its session_id.";
   // Clients in controller: []
   ASSERT_EQ(0, controller_->GetClientCount())
       << "Client count should return to zero after all clients are gone.";
 }
 
 // This test will connect and disconnect several clients while simulating an
 // active capture device being started and generating frames. It runs on one
 // thread and is intended to behave deterministically.
-TEST_F(VideoCaptureControllerTest, NormalCaptureMultipleClients) {
+TEST_P(VideoCaptureControllerTest, NormalCaptureMultipleClients) {
   media::VideoCaptureParams session_100;
-  session_100.requested_format = media::VideoCaptureFormat(
-      gfx::Size(320, 240), 30, media::PIXEL_FORMAT_I420);
+  const media::VideoPixelFormat format = GetParam();
+  client_a_->expect_buffer_format_ = format;
+  client_b_->expect_buffer_format_ = format;
+
+  session_100.requested_format =
+      media::VideoCaptureFormat(gfx::Size(320, 240), 30, format);
 
   media::VideoCaptureParams session_200 = session_100;
 
   media::VideoCaptureParams session_300 = session_100;
 
   media::VideoCaptureParams session_1 = session_100;
 
   const gfx::Size capture_resolution(444, 200);
 
   // The device format needn't match the VideoCaptureParams (the camera can do
@@ skipping 19 matching lines @@
                          client_a_.get(),
                          200,
                          session_200);
   ASSERT_EQ(3, controller_->GetClientCount());
 
   // Now, simulate an incoming captured buffer from the capture device. As a
   // side effect this will cause the first buffer to be shared with clients.
   uint8_t buffer_no = 1;
   ASSERT_EQ(0.0, device_->GetBufferPoolUtilization());
   std::unique_ptr<media::VideoCaptureDevice::Client::Buffer> buffer(
-      device_->ReserveOutputBuffer(capture_resolution, media::PIXEL_FORMAT_I420,
+      device_->ReserveOutputBuffer(capture_resolution, format,
                                    media::PIXEL_STORAGE_CPU));
   ASSERT_TRUE(buffer.get());
   ASSERT_EQ(1.0 / kPoolSize, device_->GetBufferPoolUtilization());
   memset(buffer->data(), buffer_no++, buffer->mapped_size());
   {
     InSequence s;
     EXPECT_CALL(*client_a_, DoBufferCreated(client_a_route_1)).Times(1);
-    EXPECT_CALL(*client_a_,
-                DoI420BufferReady(client_a_route_1, capture_resolution))
+    EXPECT_CALL(*client_a_, DoBufferReady(client_a_route_1, capture_resolution))
         .Times(1);

[mcasas, 2016/10/25 22:44:43]

micro-nit: .Times(1) is redundant, you could
safely remove it throughout these few lines.

[aleksandar.stojiljkovic, 2016/10/26 14:02:18]

Checked the code further but could only guess why it is redundant. Propose to
remove it in follow up patch as it is on multiple cases here,... but don't know
when it is redundant.

   }
   {
     InSequence s;
     EXPECT_CALL(*client_b_, DoBufferCreated(client_b_route_1)).Times(1);
-    EXPECT_CALL(*client_b_,
-                DoI420BufferReady(client_b_route_1, capture_resolution))
+    EXPECT_CALL(*client_b_, DoBufferReady(client_b_route_1, capture_resolution))
         .Times(1);
   }
   {
     InSequence s;
     EXPECT_CALL(*client_a_, DoBufferCreated(client_a_route_2)).Times(1);
-    EXPECT_CALL(*client_a_,
-                DoI420BufferReady(client_a_route_2, capture_resolution))
+    EXPECT_CALL(*client_a_, DoBufferReady(client_a_route_2, capture_resolution))
         .Times(1);
   }
-  scoped_refptr<media::VideoFrame> video_frame =
-      WrapI420Buffer(capture_resolution, static_cast<uint8_t*>(buffer->data()));
+  scoped_refptr<media::VideoFrame> video_frame = WrapBuffer(
+      capture_resolution, static_cast<uint8_t*>(buffer->data()), format);
   ASSERT_TRUE(video_frame);
   ASSERT_FALSE(video_frame->metadata()->HasKey(
       media::VideoFrameMetadata::RESOURCE_UTILIZATION));
   client_a_->resource_utilization_ = 0.5;
   client_b_->resource_utilization_ = -1.0;
   video_frame->metadata()->SetTimeTicks(
       media::VideoFrameMetadata::REFERENCE_TIME, base::TimeTicks());
   device_->OnIncomingCapturedVideoFrame(std::move(buffer), video_frame);
 
   base::RunLoop().RunUntilIdle();
   Mock::VerifyAndClearExpectations(client_a_.get());
   Mock::VerifyAndClearExpectations(client_b_.get());
 
   // Expect VideoCaptureController set the metadata in |video_frame| to hold a
   // resource utilization of 0.5 (the largest of all reported values).
   double resource_utilization_in_metadata = -1.0;
   ASSERT_TRUE(video_frame->metadata()->GetDouble(
       media::VideoFrameMetadata::RESOURCE_UTILIZATION,
       &resource_utilization_in_metadata));
   ASSERT_EQ(0.5, resource_utilization_in_metadata);
 
   // Second buffer which ought to use the same shared memory buffer. In this
   // case pretend that the Buffer pointer is held by the device for a long
   // delay. This shouldn't affect anything.
   std::unique_ptr<media::VideoCaptureDevice::Client::Buffer> buffer2 =
-      device_->ReserveOutputBuffer(capture_resolution, media::PIXEL_FORMAT_I420,
+      device_->ReserveOutputBuffer(capture_resolution, format,
                                    media::PIXEL_STORAGE_CPU);
   ASSERT_TRUE(buffer2.get());
   memset(buffer2->data(), buffer_no++, buffer2->mapped_size());
-  video_frame = WrapI420Buffer(capture_resolution,
-                               static_cast<uint8_t*>(buffer2->data()));
+  video_frame = WrapBuffer(capture_resolution,
+                           static_cast<uint8_t*>(buffer2->data()), format);
   ASSERT_TRUE(video_frame);
   ASSERT_FALSE(video_frame->metadata()->HasKey(
       media::VideoFrameMetadata::RESOURCE_UTILIZATION));
   client_a_->resource_utilization_ = 0.5;
   client_b_->resource_utilization_ = 3.14;
   video_frame->metadata()->SetTimeTicks(
       media::VideoFrameMetadata::REFERENCE_TIME, base::TimeTicks());
   device_->OnIncomingCapturedVideoFrame(std::move(buffer2), video_frame);
 
   // The buffer should be delivered to the clients in any order.
   {
     InSequence s;
     EXPECT_CALL(*client_a_, DoBufferCreated(client_a_route_1)).Times(1);
-    EXPECT_CALL(*client_a_,
-                DoI420BufferReady(client_a_route_1, capture_resolution))
+    EXPECT_CALL(*client_a_, DoBufferReady(client_a_route_1, capture_resolution))
         .Times(1);
   }
   {
     InSequence s;
     EXPECT_CALL(*client_b_, DoBufferCreated(client_b_route_1)).Times(1);
-    EXPECT_CALL(*client_b_,
-                DoI420BufferReady(client_b_route_1, capture_resolution))
+    EXPECT_CALL(*client_b_, DoBufferReady(client_b_route_1, capture_resolution))
         .Times(1);
   }
   {
     InSequence s;
     EXPECT_CALL(*client_a_, DoBufferCreated(client_a_route_2)).Times(1);
-    EXPECT_CALL(*client_a_,
-                DoI420BufferReady(client_a_route_2, capture_resolution))
+    EXPECT_CALL(*client_a_, DoBufferReady(client_a_route_2, capture_resolution))
         .Times(1);
   }
   base::RunLoop().RunUntilIdle();
   Mock::VerifyAndClearExpectations(client_a_.get());
   Mock::VerifyAndClearExpectations(client_b_.get());
   // Expect VideoCaptureController set the metadata in |video_frame| to hold a
   // resource utilization of 3.14 (the largest of all reported values).
   resource_utilization_in_metadata = -1.0;
   ASSERT_TRUE(video_frame->metadata()->GetDouble(
       media::VideoFrameMetadata::RESOURCE_UTILIZATION,
       &resource_utilization_in_metadata));
   ASSERT_EQ(3.14, resource_utilization_in_metadata);
 
   // Add a fourth client now that some buffers have come through.
   controller_->AddClient(client_b_route_2,
                          client_b_.get(),
                          1,
                          session_1);
   Mock::VerifyAndClearExpectations(client_b_.get());
 
   // Third, fourth, and fifth buffers. Pretend they all arrive at the same time.
   for (int i = 0; i < kPoolSize; i++) {
     std::unique_ptr<media::VideoCaptureDevice::Client::Buffer> buffer =
-        device_->ReserveOutputBuffer(capture_resolution,
-                                     media::PIXEL_FORMAT_I420,
+        device_->ReserveOutputBuffer(capture_resolution, format,
                                      media::PIXEL_STORAGE_CPU);
     ASSERT_TRUE(buffer.get());
     memset(buffer->data(), buffer_no++, buffer->mapped_size());
-    video_frame = WrapI420Buffer(capture_resolution,
-                                 static_cast<uint8_t*>(buffer->data()));
+    video_frame = WrapBuffer(capture_resolution,
+                             static_cast<uint8_t*>(buffer->data()), format);
     ASSERT_TRUE(video_frame);
     video_frame->metadata()->SetTimeTicks(
         media::VideoFrameMetadata::REFERENCE_TIME, base::TimeTicks());
     device_->OnIncomingCapturedVideoFrame(std::move(buffer), video_frame);
   }
   // ReserveOutputBuffer ought to fail now, because the pool is depleted.
-  ASSERT_FALSE(
-      device_->ReserveOutputBuffer(capture_resolution,
-                                   media::PIXEL_FORMAT_I420,
-                                   media::PIXEL_STORAGE_CPU).get());
+  ASSERT_FALSE(device_
+                   ->ReserveOutputBuffer(capture_resolution, format,
+                                         media::PIXEL_STORAGE_CPU)
+                   .get());
 
   // The new client needs to be notified of the creation of |kPoolSize| buffers;
   // the old clients only |kPoolSize - 2|.
   EXPECT_CALL(*client_b_, DoBufferCreated(client_b_route_2)).Times(kPoolSize);
-  EXPECT_CALL(*client_b_,
-              DoI420BufferReady(client_b_route_2, capture_resolution))
+  EXPECT_CALL(*client_b_, DoBufferReady(client_b_route_2, capture_resolution))
       .Times(kPoolSize);
   EXPECT_CALL(*client_a_, DoBufferCreated(client_a_route_1))
       .Times(kPoolSize - 2);
-  EXPECT_CALL(*client_a_,
-              DoI420BufferReady(client_a_route_1, capture_resolution))
+  EXPECT_CALL(*client_a_, DoBufferReady(client_a_route_1, capture_resolution))
       .Times(kPoolSize);
   EXPECT_CALL(*client_a_, DoBufferCreated(client_a_route_2))
       .Times(kPoolSize - 2);
-  EXPECT_CALL(*client_a_,
-              DoI420BufferReady(client_a_route_2, capture_resolution))
+  EXPECT_CALL(*client_a_, DoBufferReady(client_a_route_2, capture_resolution))
       .Times(kPoolSize);
   EXPECT_CALL(*client_b_, DoBufferCreated(client_b_route_1))
       .Times(kPoolSize - 2);
-  EXPECT_CALL(*client_b_,
-              DoI420BufferReady(client_b_route_1, capture_resolution))
+  EXPECT_CALL(*client_b_, DoBufferReady(client_b_route_1, capture_resolution))
       .Times(kPoolSize);
   base::RunLoop().RunUntilIdle();
   Mock::VerifyAndClearExpectations(client_a_.get());
   Mock::VerifyAndClearExpectations(client_b_.get());
 
   // Now test the interaction of client shutdown and buffer delivery.
   // Kill A1 via renderer disconnect (synchronous).
   controller_->RemoveClient(client_a_route_1, client_a_.get());
   // Kill B1 via session close (posts a task to disconnect).
   EXPECT_CALL(*client_b_, DoEnded(client_b_route_1)).Times(1);
   controller_->StopSession(300);
   // Queue up another buffer.
   std::unique_ptr<media::VideoCaptureDevice::Client::Buffer> buffer3 =
-      device_->ReserveOutputBuffer(capture_resolution, media::PIXEL_FORMAT_I420,
+      device_->ReserveOutputBuffer(capture_resolution, format,
                                    media::PIXEL_STORAGE_CPU);
   ASSERT_TRUE(buffer3.get());
   memset(buffer3->data(), buffer_no++, buffer3->mapped_size());
-  video_frame = WrapI420Buffer(capture_resolution,
-                               static_cast<uint8_t*>(buffer3->data()));
+  video_frame = WrapBuffer(capture_resolution,
+                           static_cast<uint8_t*>(buffer3->data()), format);
   ASSERT_TRUE(video_frame);
   video_frame->metadata()->SetTimeTicks(
       media::VideoFrameMetadata::REFERENCE_TIME, base::TimeTicks());
   device_->OnIncomingCapturedVideoFrame(std::move(buffer3), video_frame);
 
   std::unique_ptr<media::VideoCaptureDevice::Client::Buffer> buffer4 =
-      device_->ReserveOutputBuffer(capture_resolution, media::PIXEL_FORMAT_I420,
+      device_->ReserveOutputBuffer(capture_resolution, format,
                                    media::PIXEL_STORAGE_CPU);
   {
     // Kill A2 via session close (posts a task to disconnect, but A2 must not
     // be sent either of these two buffers).
     EXPECT_CALL(*client_a_, DoEnded(client_a_route_2)).Times(1);
     controller_->StopSession(200);
   }
   ASSERT_TRUE(buffer4.get());
   memset(buffer4->data(), buffer_no++, buffer4->mapped_size());
-  video_frame = WrapI420Buffer(capture_resolution,
-                               static_cast<uint8_t*>(buffer4->data()));
+  video_frame = WrapBuffer(capture_resolution,
+                           static_cast<uint8_t*>(buffer4->data()), format);
   ASSERT_TRUE(video_frame);
   video_frame->metadata()->SetTimeTicks(
       media::VideoFrameMetadata::REFERENCE_TIME, base::TimeTicks());
   device_->OnIncomingCapturedVideoFrame(std::move(buffer4), video_frame);
   // B2 is the only client left, and is the only one that should
   // get the buffer.
-  EXPECT_CALL(*client_b_,
-              DoI420BufferReady(client_b_route_2, capture_resolution))
+  EXPECT_CALL(*client_b_, DoBufferReady(client_b_route_2, capture_resolution))
       .Times(2);
   base::RunLoop().RunUntilIdle();
   Mock::VerifyAndClearExpectations(client_a_.get());
   Mock::VerifyAndClearExpectations(client_b_.get());
 }
 
+INSTANTIATE_TEST_CASE_P(,
+                        VideoCaptureControllerTest,
+                        ::testing::Values(media::PIXEL_FORMAT_I420,
+                                          media::PIXEL_FORMAT_Y16));
+
 // Exercises the OnError() codepath of VideoCaptureController, and tests the
 // behavior of various operations after the error state has been signalled.
 TEST_F(VideoCaptureControllerTest, ErrorBeforeDeviceCreation) {
   media::VideoCaptureParams session_100;
   session_100.requested_format = media::VideoCaptureFormat(
       gfx::Size(320, 240), 30, media::PIXEL_FORMAT_I420);
 
   media::VideoCaptureParams session_200 = session_100;
 
   const gfx::Size capture_resolution(320, 240);
@@ skipping 12 matching lines @@
   EXPECT_CALL(*client_b_, DoError(route_id)).Times(1);
   controller_->AddClient(route_id, client_b_.get(), 200, session_200);
   base::RunLoop().RunUntilIdle();
   Mock::VerifyAndClearExpectations(client_b_.get());
 
   std::unique_ptr<media::VideoCaptureDevice::Client::Buffer> buffer(
       device_->ReserveOutputBuffer(capture_resolution, media::PIXEL_FORMAT_I420,
                                    media::PIXEL_STORAGE_CPU));
   ASSERT_TRUE(buffer.get());
   scoped_refptr<media::VideoFrame> video_frame =
-      WrapI420Buffer(capture_resolution, static_cast<uint8_t*>(buffer->data()));
+      WrapBuffer(capture_resolution, static_cast<uint8_t*>(buffer->data()));
   ASSERT_TRUE(video_frame);
   video_frame->metadata()->SetTimeTicks(
       media::VideoFrameMetadata::REFERENCE_TIME, base::TimeTicks());
   device_->OnIncomingCapturedVideoFrame(std::move(buffer), video_frame);
 
   base::RunLoop().RunUntilIdle();
 }
 
 // Exercises the OnError() codepath of VideoCaptureController, and tests the
 // behavior of various operations after the error state has been signalled.
@@ skipping 17 matching lines @@
   base::RunLoop().RunUntilIdle();
   Mock::VerifyAndClearExpectations(client_a_.get());
 
   const gfx::Size dims(320, 240);
   std::unique_ptr<media::VideoCaptureDevice::Client::Buffer> buffer(
       device_->ReserveOutputBuffer(dims, media::PIXEL_FORMAT_I420,
                                    media::PIXEL_STORAGE_CPU));
   ASSERT_TRUE(buffer.get());
 
   scoped_refptr<media::VideoFrame> video_frame =
-      WrapI420Buffer(dims, static_cast<uint8_t*>(buffer->data()));
+      WrapBuffer(dims, static_cast<uint8_t*>(buffer->data()));
   ASSERT_TRUE(video_frame);
   device_->OnError(FROM_HERE, "Test Error");
   video_frame->metadata()->SetTimeTicks(
       media::VideoFrameMetadata::REFERENCE_TIME, base::TimeTicks());
   device_->OnIncomingCapturedVideoFrame(std::move(buffer), video_frame);
 
   EXPECT_CALL(*client_a_, DoError(route_id)).Times(1);
   base::RunLoop().RunUntilIdle();
   Mock::VerifyAndClearExpectations(client_a_.get());
 
   // Second client connects after the error state. It also should get told of
   // the error.
   EXPECT_CALL(*client_b_, DoError(route_id)).Times(1);
   controller_->AddClient(route_id, client_b_.get(), 200, session_200);
   Mock::VerifyAndClearExpectations(client_b_.get());
 }
 
 }  // namespace content