Implement UI hit testing + Reticle scaling

chrome/browser/android/vr_shell/vr_shell.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/android/vr_shell/vr_shell.h"
 
 #include <thread>
 
 #include "chrome/browser/android/vr_shell/vr_compositor.h"
 #include "chrome/browser/android/vr_shell/vr_gl_util.h"
@@ skipping 20 matching lines @@
 
 static constexpr gvr::Vec3f kDesktopPositionDefault = {0.0f, 0.0f, -2.0f};
 static constexpr float kDesktopHeightDefault = 1.6f;
 
 // Screen angle in degrees. 0 = vertical, positive = top closer.
 static constexpr float kDesktopScreenTiltDefault = 0;
 
 static constexpr float kScreenHeightRatio = 1.0f;
 static constexpr float kScreenWidthRatio = 16.0f / 9.0f;
 
-static constexpr float kReticleWidth = 0.05f;
-static constexpr float kReticleHeight = 0.05f;
+static constexpr float kReticleWidth = 0.025f;
+static constexpr float kReticleHeight = 0.025f;
 
 static constexpr float kLaserWidth = 0.01f;
 
 // The neutral direction is fixed in world space, this is the
 // reference angle pointing forward towards the horizon when the
 // controller orientation is reset. This should match the yaw angle
 // where the main screen is placed.
 static constexpr gvr::Vec3f kNeutralPose = {0.0f, 0.0f, -1.0f};
 
+static constexpr gvr::Vec3f kOrigin = {0.0f, 0.0f, 0.0f};
+
 // In lieu of an elbow model, we assume a position for the user's hand.
 // TODO(mthiesse): Handedness options.
 static constexpr gvr::Vec3f kHandPosition = {0.2f, -0.5f, -0.2f};
 
-static constexpr float kReticleZOffset = 0.01f;
+// Fraction of the z-distance to the object the cursor is drawn at to avoid
+// rounding errors drawing the cursor behind the object.
+static constexpr float kReticleZOffset = 0.99f;
 
 }  // namespace
 
 namespace vr_shell {
 
 VrShell::VrShell(JNIEnv* env, jobject obj,
                  content::ContentViewCore* content_core,
                  ui::WindowAndroid* content_window)
     : desktop_screen_tilt_(kDesktopScreenTiltDefault),
       desktop_height_(kDesktopHeightDefault),
       desktop_position_(kDesktopPositionDefault),
+      cursor_distance_(-kDesktopPositionDefault.z),
       content_cvc_(content_core) {
   j_vr_shell_.Reset(env, obj);
   content_compositor_view_.reset(new VrCompositor(content_window));
   ui_rects_.emplace_back(new ContentRectangle());
   desktop_plane_ = ui_rects_.back().get();
   desktop_plane_->id = 0;
   desktop_plane_->copy_rect = {0.0f, 0.0f, 1.0f, 1.0f};
   // TODO(cjgrant): If we use the native path for content clicks, fix this.
   desktop_plane_->window_rect = {0, 0, 0, 0};
   desktop_plane_->translation = {0.0f, 0.0f, 0.0f};
@@ skipping 84 matching lines @@
   gvr::Vec3f translation = getTranslation(mat);
 
   // Use the eye midpoint as the origin for Cardboard mode, but apply an offset
   // for the controller.
   if (controller_active_) {
     translation.x += kHandPosition.x;
     translation.y += kHandPosition.y;
     translation.z += kHandPosition.z;
   }
 
-  // We can only be intersecting the desktop plane at the moment.
-  float distance_to_plane = desktop_plane_->GetRayDistance(translation,
-                                                           forward);
-  look_at_vector_ = GetRayPoint(translation, forward, distance_to_plane);
+  float desktop_dist = ui_rects_[0]->GetRayDistance(translation, forward);

[cjgrant, 2016/09/22 18:59:04]

Should use desktop_plane_ or scene_.GetUiElementById(kBrowserUiElementId) here.

[mthiesse, 2016/09/22 22:01:33]

Done.

+  gvr::Vec3f cursor_position = GetRayPoint(translation, forward, desktop_dist);

[cjgrant, 2016/09/22 18:59:03]

If you want, there's a better way to do this.  Since GetRayDistance() already
finds the intersection, you could expose GetRayPlaneIntersection() in
ui_elements.cc, then find the distance from that.

[mthiesse, 2016/09/22 22:01:33]

I'll get back to this when we have working controllers I can test with (I want
to play with this some more). For now what we have here works so I'd prefer to
leave as is.

+  look_at_vector_ = cursor_position;
+  cursor_distance_ = desktop_dist;
+
+  // Determine which UI element (if any) the cursor is pointing to.
+  float closest_element = std::numeric_limits<float>::infinity();
+  int closest_element_index = -1;
+  int pixel_x, pixel_y;
+
+  for (std::size_t i = 0; i < ui_rects_.size(); ++i) {
+    const ContentRectangle& plane = *ui_rects_[i].get();
+    float distance_to_plane = plane.GetRayDistance(kOrigin, cursor_position);
+    gvr::Vec3f plane_intersection_point =
+        GetRayPoint(kOrigin, cursor_position, distance_to_plane);
+    gvr::Vec3f rect_2d_point =
+        MatrixVectorMul(plane.transform.from_world, plane_intersection_point);
+    float x = rect_2d_point.x + 0.5f;
+    float y = 0.5f - rect_2d_point.y;
+    if (distance_to_plane > 0 && distance_to_plane < closest_element) {
+      bool is_inside = x >= 0.0f && x < 1.0f && y >= 0.0f && y < 1.0f;
+      if (is_inside) {
+        closest_element = distance_to_plane;
+        cursor_distance_ = desktop_dist * distance_to_plane;
+        closest_element_index = i;
+        pixel_x = int((plane.copy_rect.width * x) + plane.copy_rect.x);
+        pixel_y = int((plane.copy_rect.height * y) + plane.copy_rect.y);
+        look_at_vector_ = plane_intersection_point;
+      }
+    }
+  }
+  // TODO(mthiesse): Create input events for CVC using pixel_x/y.
 }
 
 void ApplyNeckModel(gvr::Mat4f& mat_forward) {
   // This assumes that the input matrix is a pure rotation matrix. The
   // input object_from_reference matrix has the inverse rotation of
   // the head rotation. Invert it (this is just a transpose).
   gvr::Mat4f mat = MatrixTranspose(mat_forward);
 
   // Position of the point between the eyes, relative to the neck pivot:
   const float kNeckHorizontalOffset = -0.080f;  // meters in Z
@@ skipping 34 matching lines @@
   frame.Unbind();
   frame.Submit(*buffer_viewport_list_, head_pose_);
 }
 
 void VrShell::DrawVrShell() {
   float screen_width = kScreenWidthRatio * desktop_height_;
   float screen_height = kScreenHeightRatio * desktop_height_;
 
   float screen_tilt = desktop_screen_tilt_ * M_PI / 180.0f;
 
-  forward_vector_ = getForwardVector(head_pose_);
-
   gvr::Vec3f headPos = getTranslation(head_pose_);
   if (headPos.x == 0.0f && headPos.y == 0.0f && headPos.z == 0.0f) {
     // This appears to be a 3DOF pose without a neck model. Add one.
     // The head pose has redundant data. Assume we're only using the
     // object_from_reference_matrix, we're not updating position_external.
     // TODO: Not sure what object_from_reference_matrix is. The new api removed
     // it. For now, removing it seems working fine.
     ApplyNeckModel(head_pose_);
   }
 
+  forward_vector_ = getForwardVector(head_pose_);
+
   desktop_plane_->size = {screen_width, screen_height, 1.0f};
   desktop_plane_->translation.x = desktop_position_.x;
   desktop_plane_->translation.y = desktop_position_.y;
   desktop_plane_->translation.z = desktop_position_.z;
 
   // Update position of all UI elements (including desktop)
   UpdateTransforms(screen_width, screen_height, screen_tilt);
 
   UpdateController();
 
@@ skipping 50 matching lines @@
     combined_matrix = MatrixMul(projection_matrix_, combined_matrix);
     vr_shell_renderer_->GetTexturedQuadRenderer()->Draw(
         ui_rects_[i].get()->content_texture_handle, combined_matrix,
         ui_rects_[i].get()->copy_rect);
   }
 }
 
 void VrShell::DrawCursor() {
   gvr::Mat4f mat;
   SetIdentityM(mat);
+
   // Scale the pointer.

[cjgrant, 2016/09/22 18:59:04]

... to have a fixed FOV size at any distance.  (?)

[mthiesse, 2016/09/22 22:01:33]

Done.

-  ScaleM(mat, mat, kReticleWidth, kReticleHeight, 1.0f);
+  ScaleM(mat, mat, kReticleWidth * cursor_distance_,
+         kReticleHeight * cursor_distance_, 1.0f);
+
   // Place the pointer at the screen plane intersection point.
-  TranslateM(mat, mat, look_at_vector_.x, look_at_vector_.y,
-             look_at_vector_.z + kReticleZOffset);
+  TranslateM(mat, mat, look_at_vector_.x * kReticleZOffset,

[cjgrant, 2016/09/22 18:59:03]

Why are x and y also being offset?

[mthiesse, 2016/09/22 22:01:33]

We're moving it slightly towards the viewer while maintaining its position in
the user's FOV by translating it only 99% of the way along its vector.

+             look_at_vector_.y * kReticleZOffset,
+             look_at_vector_.z * kReticleZOffset);
   gvr::Mat4f mv = MatrixMul(view_matrix_, mat);
   gvr::Mat4f mvp = MatrixMul(projection_matrix_, mv);
   vr_shell_renderer_->GetReticleRenderer()->Draw(mvp);
 
   // Draw the laser only for controllers.
   if (!controller_active_) {
     return;
   }
   // Find the length of the beam (from hand to target).
   float xdiff = (kHandPosition.x - look_at_vector_.x);
@@ skipping 144 matching lines @@
            const JavaParamRef<jobject>& content_web_contents,
            jlong content_window_android) {
   content::ContentViewCore* c_core = content::ContentViewCore::FromWebContents(
       content::WebContents::FromJavaWebContents(content_web_contents));
   return reinterpret_cast<intptr_t>(new VrShell(
       env, obj, c_core,
       reinterpret_cast<ui::WindowAndroid*>(content_window_android)));
 }
 
 }  // namespace vr_shell