Android Tango depth camera capture support.

media/capture/video/android/video_capture_device_tango_android_unittest.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "video_capture_device_tango_android.h"
+#include "base/android/jni_android.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/sys_info.h"
+#include "base/test/scoped_task_environment.h"
+#include "media/capture/video/android/tango_client_api_glue.h"
+#include "media/capture/video/android/video_capture_device_factory_android.h"
+#include "media/capture/video/android/video_capture_device_tango_android.h"
+#include "media/capture/video_capture_types.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+using Tango = media::TangoClientApiGlue;
+using Intrinsics = media::TangoClientApiGlue::TangoCameraIntrinsics;
+
+namespace media {
+
+namespace {
+
+const int kWidth = 214;
+const int kHeight = 120;
+// Projected point cloud values to color frame could be are outside color frame
+// bounds. kMargin defines how larger is the depth frame area compared to color.
+const int kMargin = 20;
+
+bool IsTangoDevice() {
+  static const char* tango_models[] = {"Project Tango Tablet Development Kit",
+                                       "Lenovo PB2-690M", "ASUS_A002",
+                                       "ASUS_A002A"};
+  const std::string model = base::SysInfo::HardwareModelName();
+  const char** result =
+      std::find(tango_models, tango_models + arraysize(tango_models), model);
+  return (result < tango_models + arraysize(tango_models));
+}
+
+void OnPointCloudAvailable(void* context,
+                           const Tango::TangoPointCloud* point_cloud) {}
+
+void createTestTangoPointCloud(float* data,
+                               int xstart,
+                               int ystart,
+                               int xend,
+                               int yend,
+                               float cx,
+                               float cy,
+                               float fx,
+                               float fy) {
+  float* out = data;
+  for (int i = ystart; i < yend; i++) {
+    for (int j = xstart; j < xend; j++) {
+      // Skip some elements to mimic Tango behavior.
+      if ((j % 3 && (i + j) % 15 == 0) || j % 20 == 0)
+        continue;
+      // Generate depth value based on row and column. Maximum value is 16m.
+      float depth =
+          16.0 / (yend - ystart + xend - xstart) * (j - xstart + i - ystart);
+      float xvalue = depth * (j - cx) / fx;
+      float yvalue = depth * (i - cy) / fy;
+      out[0] = xvalue;
+      out[1] = yvalue;
+      out[2] = depth;
+      out[3] = 1.0;
+      out += 4;
+    }
+  }
+}
+
+}  // namespace
+
+class MockVideoCaptureClient : public VideoCaptureDevice::Client {
+ public:
+  MOCK_METHOD2(OnError,
+               void(const tracked_objects::Location& from_here,
+                    const std::string& reason));
+  MOCK_CONST_METHOD0(GetBufferPoolUtilization, double(void));
+  MOCK_METHOD0(OnStarted, void(void));
+
+  explicit MockVideoCaptureClient(
+      base::Callback<void(const uint8_t*, int, const VideoCaptureFormat&)>
+          frame_cb)
+      : frame_cb_(frame_cb) {}
+
+  void OnIncomingCapturedData(const uint8_t* data,
+                              int length,
+                              const VideoCaptureFormat& format,
+                              int rotation,
+                              base::TimeTicks reference_time,
+                              base::TimeDelta timestamp,
+                              int frame_feedback_id) override {
+    ASSERT_GT(length, 0);
+    ASSERT_TRUE(data);
+    frame_cb_.Run(data, length, format);
+  }
+
+  // Trampoline methods to workaround GMOCK problems with std::unique_ptr<>.
+  Buffer ReserveOutputBuffer(const gfx::Size& dimensions,
+                             media::VideoPixelFormat format,
+                             media::VideoPixelStorage storage,
+                             int frame_feedback_id) override {
+    NOTREACHED() << "This should never be called";
+    return Buffer();
+  }
+  void OnIncomingCapturedBuffer(Buffer buffer,
+                                const VideoCaptureFormat& format,
+                                base::TimeTicks reference_time,
+                                base::TimeDelta timestamp) override {
+    NOTREACHED() << "This should never be called";
+  }
+  void OnIncomingCapturedBufferExt(
+      Buffer buffer,
+      const VideoCaptureFormat& format,
+      base::TimeTicks reference_time,
+      base::TimeDelta timestamp,
+      gfx::Rect visible_rect,
+      const VideoFrameMetadata& additional_metadata) override {
+    NOTREACHED() << "This should never be called";
+  }
+  Buffer ResurrectLastOutputBuffer(const gfx::Size& dimensions,
+                                   media::VideoPixelFormat format,
+                                   media::VideoPixelStorage storage,
+                                   int frame_feedback_id) {
+    NOTREACHED() << "This should never be called";
+    return Buffer();
+  }
+
+ private:
+  base::Callback<void(const uint8_t*, int, const VideoCaptureFormat&)>
+      frame_cb_;
+};
+
+class VideoCaptureDeviceTangoAndroidTest
+    : public testing::TestWithParam<gfx::Size> {
+ protected:
+  VideoCaptureDeviceTangoAndroidTest()
+      : video_capture_device_factory_(VideoCaptureDeviceFactory::CreateFactory(
+            base::ThreadTaskRunnerHandle::Get())),
+        device_descriptors_(new VideoCaptureDeviceDescriptors()),
+        video_capture_client_(new MockVideoCaptureClient(
+            base::Bind(&VideoCaptureDeviceTangoAndroidTest::OnFrameCaptured,
+                       base::Unretained(this)))) {}
+
+  void SetUp() override {
+    video_capture_device_factory_->GetDeviceDescriptors(
+        device_descriptors_.get());
+    VideoCaptureDeviceTangoAndroid::RegisterVideoCaptureDevice(
+        base::android::AttachCurrentThread());
+
+    static_cast<VideoCaptureDeviceFactoryAndroid*>(
+        video_capture_device_factory_.get())
+        ->ConfigureForTesting();
+  }
+
+  std::unique_ptr<VideoCaptureDeviceDescriptor> GetTangoDepthDescriptor() {
+    for (const auto& descriptor : *device_descriptors_) {
+      if (descriptor.capture_api == VideoCaptureApi::ANDROID_TANGO) {
+        return std::unique_ptr<VideoCaptureDeviceDescriptor>(
+            new VideoCaptureDeviceDescriptor(descriptor));
+      }
+    }
+    return nullptr;
+  }
+
+  void OnFrameCaptured(const uint8_t* data,
+                       int length,
+                       const VideoCaptureFormat& format) {
+    memcpy(last_data_, data, length);
+    last_length_ = length;
+    last_format_ = format;
+  }
+
+  void SetIntrinsics(VideoCaptureDeviceTangoAndroid* tango_device,
+                     double cx,
+                     double cy,
+                     double fx,
+                     double fy) {
+    tango_device->intrinsics_ = std::unique_ptr<Intrinsics>(new Intrinsics());
+    tango_device->intrinsics_->cx = cx;
+    tango_device->intrinsics_->cy = cy;
+    tango_device->intrinsics_->fx = fx;
+    tango_device->intrinsics_->fy = fy;
+  }
+
+  // Non-Tango devices enter error state and we need to force it to configured
+  // in order to deliver the frame to client.
+  void SetConfiguredState(VideoCaptureDeviceTangoAndroid* tango_device) {
+    tango_device->state_ = VideoCaptureDeviceTangoAndroid::kConfigured;
+  }
+
+  base::test::ScopedTaskEnvironment scoped_task_environment_;
+  const std::unique_ptr<VideoCaptureDeviceFactory>
+      video_capture_device_factory_;
+  std::unique_ptr<VideoCaptureDeviceDescriptors> device_descriptors_;
+  std::unique_ptr<MockVideoCaptureClient> video_capture_client_;
+  VideoCaptureFormat last_format_;
+  uint16_t last_data_[kWidth * kHeight] = {0};
+  int last_length_ = 0;
+};
+
+TEST_F(VideoCaptureDeviceTangoAndroidTest, OnPointCloudAvailable) {
+  VideoCaptureDeviceFactoryAndroid* android_factory =
+      static_cast<VideoCaptureDeviceFactoryAndroid*>(
+          video_capture_device_factory_.get());
+
+  std::unique_ptr<VideoCaptureDeviceDescriptor> tango_depth_descriptor(
+      GetTangoDepthDescriptor());
+  ASSERT_TRUE(IsTangoDevice() ^ tango_depth_descriptor == nullptr);
+  if (!tango_depth_descriptor) {
+    // Create mock Tango device. Tango depth id value is equal to number of
+    // system cameras; see VideoCaptureFactory.java.
+    tango_depth_descriptor = std::unique_ptr<VideoCaptureDeviceDescriptor>(
+        new VideoCaptureDeviceDescriptor(
+            "Tango Mock Test Depth",
+            base::IntToString(device_descriptors_->size()),
+            VideoCaptureApi::ANDROID_TANGO));
+  }
+  std::unique_ptr<VideoCaptureDevice> device =
+      android_factory->CreateDevice(*tango_depth_descriptor);
+  ASSERT_TRUE(device);
+
+  VideoCaptureDeviceTangoAndroid* tango_device =
+      static_cast<VideoCaptureDeviceTangoAndroid*>(device.get());
+
+  VideoCaptureParams capture_params;
+  capture_params.requested_format.frame_size.SetSize(kWidth, kHeight);
+  capture_params.requested_format.frame_rate = 5;
+  capture_params.requested_format.pixel_format = PIXEL_FORMAT_Y16;
+  tango_device->AllocateAndStart(capture_params,
+                                 std::move(video_capture_client_));
+  if (!IsTangoDevice())
+    SetConfiguredState(tango_device);
+
+  const int length = (kWidth + 2 * kMargin) * (kHeight + 2 * kMargin);
+  float buffer[length * 4] = {0};
+
+  const double cx = 1920. / 18;
+  const double cy = 1080. / 18;
+  const double fxy = 1920. / 9;
+  SetIntrinsics(tango_device, cx, cy, fxy, fxy);
+
+  createTestTangoPointCloud(buffer, -kMargin, -kMargin, kWidth + kMargin,
+                            kHeight + kMargin, cx, cy, fxy, fxy);
+  tango_device->OnPointCloudAvailable(length, buffer, 0);
+
+  // Verify the conversion of the generated point cloud to depth buffer.
+  for (int i = 0; i < kHeight; i++) {
+    for (int j = 0; j < kWidth; j++) {
+      // Skip the same elements as in createTestTangoPointCloud.
+      if ((j % 3 && (i + j) % 15 == 0) || j % 20 == 0) {
+        continue;
+      }
+      // Generate depth value based on row and column. Maximum value is 16m.
+      float depth =
+          16.0 / (kWidth + kHeight + 4 * kMargin) * (j + kMargin + i + kMargin);
+      ASSERT_TRUE(static_cast<uint16_t>(depth * 4096) ==
+                  last_data_[j + i * kWidth]);
+      EXPECT_EQ(static_cast<uint16_t>(depth * 4096), last_data_[j + i * kWidth])
+          << "x:" << j << " ,y:" << i;
+    }
+  }
+  tango_device->StopAndDeAllocate();
+}
+
+TEST_F(VideoCaptureDeviceTangoAndroidTest, TangoApiNotFound) {
+  if (IsTangoDevice())
+    return;  // Skip if it is a Tango device.
+  JNIEnv* env = base::android::AttachCurrentThread();
+  EXPECT_EQ(Tango::TANGO_METHOD_NOT_FOUND,
+            Tango::TangoService_setBinder(env, nullptr));
+  EXPECT_EQ(Tango::TANGO_METHOD_NOT_FOUND,
+            Tango::TangoService_connect(nullptr, nullptr));
+  EXPECT_EQ(Tango::TANGO_METHOD_NOT_FOUND,
+            Tango::TangoConfig_setBool(nullptr, "config_enable_depth", true));
+  EXPECT_EQ(Tango::TANGO_METHOD_NOT_FOUND,
+            Tango::TangoConfig_setInt32(nullptr, "config_depth_mode", 0));
+  EXPECT_EQ(Tango::TANGO_METHOD_NOT_FOUND,
+            Tango::TangoService_connectOnPointCloudAvailable(
+                OnPointCloudAvailable, this));
+  Intrinsics intrinsics;
+  EXPECT_EQ(Tango::TANGO_METHOD_NOT_FOUND,
+            Tango::TangoService_getCameraIntrinsics(Tango::TANGO_CAMERA_COLOR,
+                                                    &intrinsics));
+  ASSERT_FALSE(Tango::TangoService_getConfig(Tango::TANGO_CONFIG_DEFAULT));
+  Tango::TangoService_disconnect();
+}
+
+TEST_F(VideoCaptureDeviceTangoAndroidTest, TangoApi) {
+  if (!IsTangoDevice())
+    return;  // Skip if not a Tango device.
+
+  // Previous versions of Tango allow calls to some of the methods even without
+  // connecting to service. We want to verify that the methods are resolved and
+  // checking that return error is not TANGO_METHOD_NOT_FOUND is sufficient.
+  JNIEnv* env = base::android::AttachCurrentThread();
+  EXPECT_EQ(Tango::TANGO_ERROR, Tango::TangoService_setBinder(env, nullptr));
+  EXPECT_NE(Tango::TANGO_METHOD_NOT_FOUND,
+            Tango::TangoConfig_setBool(nullptr, "config_enable_depth", true));
+  EXPECT_NE(Tango::TANGO_METHOD_NOT_FOUND,
+            Tango::TangoConfig_setInt32(nullptr, "config_depth_mode", 0));
+  EXPECT_NE(Tango::TANGO_METHOD_NOT_FOUND,
+            Tango::TangoService_connectOnPointCloudAvailable(
+                OnPointCloudAvailable, this));
+  Intrinsics intrinsics;
+  EXPECT_NE(Tango::TANGO_METHOD_NOT_FOUND,
+            Tango::TangoService_getCameraIntrinsics(Tango::TANGO_CAMERA_COLOR,
+                                                    &intrinsics));
+  Tango::TangoService_disconnect();
+}
+
+}  // namespace media