Remove Sync GetPose VRService call, implement VRVSyncProvider

chrome/browser/android/vr_shell/vr_shell_gl.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/android/vr_shell/vr_shell_gl.h"
 
 #include "base/android/jni_android.h"
 #include "base/memory/ptr_util.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "chrome/browser/android/vr_shell/ui_elements.h"
 #include "chrome/browser/android/vr_shell/ui_scene.h"
 #include "chrome/browser/android/vr_shell/vr_controller.h"
 #include "chrome/browser/android/vr_shell/vr_gl_util.h"
 #include "chrome/browser/android/vr_shell/vr_input_manager.h"
 #include "chrome/browser/android/vr_shell/vr_math.h"
 #include "chrome/browser/android/vr_shell/vr_shell.h"
 #include "chrome/browser/android/vr_shell/vr_shell_renderer.h"
 #include "third_party/WebKit/public/platform/WebInputEvent.h"
+#include "ui/gfx/transform.h"
+#include "ui/gfx/transform_util.h"
 #include "ui/gfx/vsync_provider.h"
 #include "ui/gl/android/scoped_java_surface.h"
 #include "ui/gl/android/surface_texture.h"
 #include "ui/gl/gl_bindings.h"
 #include "ui/gl/gl_context.h"
 #include "ui/gl/gl_surface.h"
 #include "ui/gl/init/gl_factory.h"
 
 namespace vr_shell {
 
 namespace {
-// Constant taken from treasure_hunt demo.
+// TODO(mthiesse): If gvr::PlatformInfo().GetPosePredictionTime() is ever
+// exposed, use that instead (it defaults to 50ms on most platforms).
 static constexpr long kPredictionTimeWithoutVsyncNanos = 50000000;
 
 static constexpr float kZNear = 0.1f;
 static constexpr float kZFar = 1000.0f;
 
 // Screen angle in degrees. 0 = vertical, positive = top closer.
 static constexpr float kDesktopScreenTiltDefault = 0;
 
 static constexpr float kReticleWidth = 0.025f;
 static constexpr float kReticleHeight = 0.025f;
@@ skipping 93 matching lines @@
   return mouse_event;
 }
 
 enum class ViewerType {
   UNKNOWN_TYPE = 0,
   CARDBOARD = 1,
   DAYDREAM = 2,
   VIEWER_TYPE_MAX,
 };
 
-int GetPixelEncodedPoseIndexByte() {
+bool GetPixelEncodedPoseIndexByte(int* pose_index) {
   TRACE_EVENT0("gpu", "VrShellGl::GetPixelEncodedPoseIndex");
   // Read the pose index encoded in a bottom left pixel as color values.
   // See also third_party/WebKit/Source/modules/vr/VRDisplay.cpp which
   // encodes the pose index, and device/vr/android/gvr/gvr_device.cc
   // which tracks poses. Returns the low byte (0..255) if valid, or -1
   // if not valid due to bad magic number.
   uint8_t pixels[4];
   // Assume we're reading from the framebuffer we just wrote to.
   // That's true currently, we may need to use glReadBuffer(GL_BACK)
   // or equivalent if the rendering setup changes in the future.
   glReadPixels(0, 0, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, pixels);
 
   // Check for the magic number written by VRDevice.cpp on submit.
   // This helps avoid glitches from garbage data in the render
   // buffer that can appear during initialization or resizing. These
   // often appear as flashes of all-black or all-white pixels.
   if (pixels[1] == kWebVrPosePixelMagicNumbers[0] &&
       pixels[2] == kWebVrPosePixelMagicNumbers[1]) {
     // Pose is good.
-    return pixels[0];
+    *pose_index = pixels[0];
+    return true;
   }
-  VLOG(1) << "WebVR: reject decoded pose index " << (int)pixels[0] <<
-      ", bad magic number " << (int)pixels[1] << ", " << (int)pixels[2];
-  return -1;
+  VLOG(1) << "WebVR: reject decoded pose index " << (int)pixels[0]
+          << ", bad magic number " << (int)pixels[1] << ", "
+          << (int)pixels[2];
+  return false;
 }
 
 int64_t TimeInMicroseconds() {
   return std::chrono::duration_cast<std::chrono::microseconds>(
       std::chrono::steady_clock::now().time_since_epoch()).count();
 }
 
 void WaitForSwapAck(const base::Closure& callback, gfx::SwapResult result) {
   callback.Run();
 }
@@ skipping 17 matching lines @@
         main_thread_task_runner_(std::move(main_thread_task_runner)),
         weak_ptr_factory_(this) {
   GvrInit(gvr_api);
 }
 
 VrShellGl::~VrShellGl() {
   draw_task_.Cancel();
 }
 
 void VrShellGl::Initialize() {
-  gvr::Mat4f identity;
-  SetIdentityM(identity);
-  webvr_head_pose_.resize(kPoseRingBufferSize, identity);
-  webvr_head_pose_valid_.resize(kPoseRingBufferSize, false);
-
   scene_.reset(new UiScene);
 
   if (surfaceless_rendering_) {
     // If we're rendering surfaceless, we'll never get a java surface to render
     // into, so we can initialize GL right away.
     InitializeGl(nullptr);
   }
 }
 
 void VrShellGl::InitializeGl(gfx::AcceleratedWidget window) {
@@ skipping 22 matching lines @@
     LOG(ERROR) << "gl::init::CreateGLContext failed";
     ForceExitVr();
     return;
   }
   if (!context_->MakeCurrent(surface_.get())) {
     LOG(ERROR) << "gl::GLContext::MakeCurrent() failed";
     ForceExitVr();
     return;
   }
 
-  // TODO(mthiesse): We don't appear to have a VSync provider ever here. This is
-  // sort of okay, because the GVR swap chain will block if we render too fast,
-  // but we should address this properly.
-  if (surface_->GetVSyncProvider()) {
-    surface_->GetVSyncProvider()->GetVSyncParameters(base::Bind(
-        &VrShellGl::UpdateVSyncParameters, weak_ptr_factory_.GetWeakPtr()));
-  } else {
-    LOG(ERROR) << "No VSync Provider";
-  }
-
   unsigned int textures[2];
   glGenTextures(2, textures);
   ui_texture_id_ = textures[0];
   content_texture_id_ = textures[1];
   ui_surface_texture_ = gl::SurfaceTexture::Create(ui_texture_id_);
   content_surface_texture_ = gl::SurfaceTexture::Create(content_texture_id_);
   ui_surface_.reset(new gl::ScopedJavaSurface(ui_surface_texture_.get()));
   content_surface_.reset(new gl::ScopedJavaSurface(
       content_surface_texture_.get()));
   ui_surface_texture_->SetFrameAvailableCallback(base::Bind(
         &VrShellGl::OnUIFrameAvailable, weak_ptr_factory_.GetWeakPtr()));
   content_surface_texture_->SetFrameAvailableCallback(base::Bind(
         &VrShellGl::OnContentFrameAvailable, weak_ptr_factory_.GetWeakPtr()));
 
   content_surface_texture_->SetDefaultBufferSize(
       content_tex_physical_size_.width, content_tex_physical_size_.height);
   ui_surface_texture_->SetDefaultBufferSize(ui_tex_physical_size_.width,
                                             ui_tex_physical_size_.height);
 
   main_thread_task_runner_->PostTask(FROM_HERE, base::Bind(
       &VrShell::SurfacesChanged, weak_vr_shell_,
       content_surface_->j_surface().obj(),
       ui_surface_->j_surface().obj()));
 
   InitializeRenderer();
 
   draw_task_.Reset(base::Bind(&VrShellGl::DrawFrame, base::Unretained(this)));
   ScheduleNextDrawFrame();
 
+  main_thread_task_runner_->PostTask(FROM_HERE, base::Bind(
+      &VrShell::OnVRVsyncProviderReady, weak_vr_shell_));
+
   ready_to_draw_ = true;
 }
 
 void VrShellGl::OnUIFrameAvailable() {
   ui_surface_texture_->UpdateTexImage();
 }
 
 void VrShellGl::OnContentFrameAvailable() {
   content_surface_texture_->UpdateTexImage();
 }
@@ skipping 18 matching lines @@
   UMA_HISTOGRAM_ENUMERATION("VRViewerType", static_cast<int>(viewerType),
       static_cast<int>(ViewerType::VIEWER_TYPE_MAX));
 }
 
 void VrShellGl::InitializeRenderer() {
   // While WebVR is going through the compositor path, it shares
   // the same texture ID. This will change once it gets its own
   // surface, but store it separately to avoid future confusion.
   // TODO(klausw,crbug.com/655722): remove this.
   webvr_texture_id_ = content_texture_id_;
-  // Out of paranoia, explicitly reset the "pose valid" flags to false
-  // from the GL thread. The constructor ran in the UI thread.
-  // TODO(klausw,crbug.com/655722): remove this.
-  webvr_head_pose_valid_.assign(kPoseRingBufferSize, false);
 
   gvr_api_->InitializeGl();
+  webvr_head_pose_.assign(kPoseRingBufferSize,
+                          gvr_api_->GetHeadSpaceFromStartSpaceRotation(
+                              gvr::GvrApi::GetTimePointNow()));
+
   std::vector<gvr::BufferSpec> specs;
   // For kFramePrimaryBuffer (primary VrShell and WebVR content)
   specs.push_back(gvr_api_->CreateBufferSpec());
   render_size_primary_ = specs[kFramePrimaryBuffer].GetSize();
 
   // For kFrameHeadlockedBuffer (for WebVR insecure content warning).
   // Set this up at fixed resolution, the (smaller) FOV gets set below.
   specs.push_back(gvr_api_->CreateBufferSpec());
   specs.back().SetSize(kHeadlockedBufferDimensions);
   render_size_headlocked_ = specs[kFrameHeadlockedBuffer].GetSize();
@@ skipping 239 matching lines @@
   DCHECK(input_target != InputTarget::NONE);
   const base::WeakPtr<VrInputManager>& weak_ptr =
       input_target == InputTarget::CONTENT ? content_input_manager_
                                            : ui_input_manager_;
   main_thread_task_runner_->PostTask(
       FROM_HERE,
       base::Bind(&VrInputManager::ProcessUpdatedGesture, weak_ptr,
                  base::Passed(std::move(event))));
 }
 
-void VrShellGl::SetGvrPoseForWebVr(const gvr::Mat4f& pose, uint32_t pose_num) {
-  webvr_head_pose_[pose_num % kPoseRingBufferSize] = pose;
-  webvr_head_pose_valid_[pose_num % kPoseRingBufferSize] = true;
-}
-
-bool VrShellGl::WebVrPoseByteIsValid(int pose_index_byte) {
-  if (pose_index_byte < 0) {
-    return false;
-  }
-  if (!webvr_head_pose_valid_[pose_index_byte % kPoseRingBufferSize]) {
-    VLOG(1) << "WebVR: reject decoded pose index " << pose_index_byte <<
-        ", not a valid pose";
-    return false;
-  }
-  return true;
-}
-
 void VrShellGl::DrawFrame() {
   TRACE_EVENT0("gpu", "VrShellGl::DrawFrame");
+
   // Reset the viewport list to just the pair of viewports for the
   // primary buffer each frame. Head-locked viewports get added by
   // DrawVrShell if needed.
   buffer_viewport_list_->SetToRecommendedBufferViewports();
 
   gvr::Frame frame = swap_chain_->AcquireFrame();
-  gvr::ClockTimePoint target_time = gvr::GvrApi::GetTimePointNow();
-  target_time.monotonic_system_time_nanos += kPredictionTimeWithoutVsyncNanos;
 
-  gvr::Mat4f head_pose =
-      gvr_api_->GetHeadSpaceFromStartSpaceRotation(target_time);
+  frame.BindBuffer(kFramePrimaryBuffer);
+  if (web_vr_mode_) {
+    DrawWebVr();
+  }
+
+  int pose_index;
+  gvr::Mat4f head_pose;
+
+  // When using async reprojection, we need to know which pose was used in
+  // the WebVR app for drawing this frame. Due to unknown amounts of
+  // buffering in the compositor and SurfaceTexture, we read the pose number
+  // from a corner pixel. There's no point in doing this for legacy
+  // distortion rendering since that doesn't need a pose, and reading back
+  // pixels is an expensive operation. TODO(klausw,crbug.com/655722): stop
+  // doing this once we have working no-compositor rendering for WebVR.
+  if (web_vr_mode_ && gvr_api_->GetAsyncReprojectionEnabled() &&
+      GetPixelEncodedPoseIndexByte(&pose_index)) {
+    head_pose = webvr_head_pose_[pose_index % kPoseRingBufferSize];
+  } else {
+    gvr::ClockTimePoint target_time = gvr::GvrApi::GetTimePointNow();
+    target_time.monotonic_system_time_nanos += kPredictionTimeWithoutVsyncNanos;
+    head_pose = gvr_api_->GetHeadSpaceFromStartSpaceRotation(target_time);
+  }
 
   gvr::Vec3f position = GetTranslation(head_pose);
   if (position.x == 0.0f && position.y == 0.0f && position.z == 0.0f) {
     // This appears to be a 3DOF pose without a neck model. Add one.
     // The head pose has redundant data. Assume we're only using the
     // object_from_reference_matrix, we're not updating position_external.
     // TODO: Not sure what object_from_reference_matrix is. The new api removed
     // it. For now, removing it seems working fine.
     gvr_api_->ApplyNeckModel(head_pose, 1.0f);
   }
 
-  frame.BindBuffer(kFramePrimaryBuffer);
-
   // Update the render position of all UI elements (including desktop).
   const float screen_tilt = kDesktopScreenTiltDefault * M_PI / 180.0f;
   scene_->UpdateTransforms(screen_tilt, TimeInMicroseconds());
 
   UpdateController(GetForwardVector(head_pose));
 
-  if (web_vr_mode_) {
-    DrawWebVr();
-
-    // When using async reprojection, we need to know which pose was used in
-    // the WebVR app for drawing this frame. Due to unknown amounts of
-    // buffering in the compositor and SurfaceTexture, we read the pose number
-    // from a corner pixel. There's no point in doing this for legacy
-    // distortion rendering since that doesn't need a pose, and reading back
-    // pixels is an expensive operation. TODO(klausw,crbug.com/655722): stop
-    // doing this once we have working no-compositor rendering for WebVR.
-    if (gvr_api_->GetAsyncReprojectionEnabled()) {
-      int pose_index_byte = GetPixelEncodedPoseIndexByte();
-      if (WebVrPoseByteIsValid(pose_index_byte)) {
-        // We have a valid pose, use it for reprojection.
-        webvr_left_viewport_->SetReprojection(GVR_REPROJECTION_FULL);
-        webvr_right_viewport_->SetReprojection(GVR_REPROJECTION_FULL);
-        head_pose = webvr_head_pose_[pose_index_byte % kPoseRingBufferSize];
-        // We can't mark the used pose as invalid since unfortunately
-        // we have to reuse them. The compositor will re-submit stale
-        // frames on vsync, and we can't tell that this has happened
-        // until we've read the pose index from it, and at that point
-        // it's too late to skip rendering.
-      } else {
-        // If we don't get a valid frame ID back we shouldn't attempt
-        // to reproject by an invalid matrix, so turn off reprojection
-        // instead. Invalid poses can permanently break reprojection
-        // for this GVR instance: http://crbug.com/667327
-        webvr_left_viewport_->SetReprojection(GVR_REPROJECTION_NONE);
-        webvr_right_viewport_->SetReprojection(GVR_REPROJECTION_NONE);
-      }
-    }
-  }
-
   DrawVrShell(head_pose, frame);
 
   frame.Unbind();
   frame.Submit(*buffer_viewport_list_, head_pose);
 
   // No need to swap buffers for surfaceless rendering.
   if (surfaceless_rendering_) {
     ScheduleNextDrawFrame();
     return;
   }
@@ skipping 278 matching lines @@
   if (ui_surface_texture_.get())
     ui_surface_texture_->SetDefaultBufferSize(width, height);
   ui_tex_physical_size_.width = width;
   ui_tex_physical_size_.height = height;
 }
 
 base::WeakPtr<VrShellGl> VrShellGl::GetWeakPtr() {
   return weak_ptr_factory_.GetWeakPtr();
 }
 
-void VrShellGl::UpdateVSyncParameters(const base::TimeTicks timebase,
-                                      const base::TimeDelta interval) {
-  vsync_timebase_ = timebase;
-  vsync_interval_ = interval;
-}
-
 void VrShellGl::ScheduleNextDrawFrame() {
   base::TimeTicks now = base::TimeTicks::Now();
   base::TimeTicks target;
 
   if (vsync_interval_.is_zero()) {
     target = now;
   } else {
     target = now + vsync_interval_;
     int64_t intervals = (target - vsync_timebase_) / vsync_interval_;
     target = vsync_timebase_ + intervals * vsync_interval_;
   }
+  task_runner_->PostDelayedTask(FROM_HERE, draw_task_.callback(), target - now);
 
-  task_runner_->PostDelayedTask(FROM_HERE, draw_task_.callback(), target - now);
+  if (!client_.is_bound())
+    return;
+  int64_t intervals = (target - vsync_timebase_) / vsync_interval_;
+  double time = (intervals * vsync_interval_).InSecondsF();
+  if (!pending_ack_) {
+    client_->OnVSync(
+        GetPose(), time,
+        base::Bind(&VrShellGl::OnVSyncAck, weak_ptr_factory_.GetWeakPtr()));
+    pending_ack_ = true;
+  } else {
+    pending_vsync_ = true;
+    pending_time_ = time;
+  }
 }
 
 void VrShellGl::ForceExitVr() {
   main_thread_task_runner_->PostTask(
       FROM_HERE, base::Bind(&VrShell::ForceExitVr, weak_vr_shell_));
 }
 
+void VrShellGl::OnVRVSyncProviderClientConnected(
+    device::mojom::VRVSyncProviderClientPtr client) {
+  client_.Bind(client.PassInterface());
+}
+
+void VrShellGl::OnVSyncAck() {
+  pending_ack_ = false;
+  if (!pending_vsync_) {
+    triggered_vsync_from_ack_ = false;
+    return;
+  }
+  pending_vsync_ = false;
+  if (triggered_vsync_from_ack_)
+    return;
+  if (!client_.is_bound())
+    return;
+
+  // Allow VSyncAck to trigger a VSync in the case that we missed the previous
+  // frame to make it less likely multiple frames are missed in a row. However,
+  // we don't want to get into a state where VSync is only ever triggered from
+  // VSyncAck so we make sure the next VSync is really from a VSync and not an
+  // Ack.
+  client_->OnVSync(
+      GetPose(), pending_time_,
+      base::Bind(&VrShellGl::OnVSyncAck, weak_ptr_factory_.GetWeakPtr()));
+  pending_ack_ = true;
+  triggered_vsync_from_ack_ = true;
+}
+
+void VrShellGl::UpdateVSyncInterval(long timebase_nanos,
+                                    double interval_seconds) {
+  vsync_timebase_ = base::TimeTicks();
+  vsync_timebase_ += base::TimeDelta::FromMicroseconds(timebase_nanos / 1000);
+  vsync_interval_ = base::TimeDelta::FromSecondsD(interval_seconds);
+}
+
 void VrShellGl::UpdateScene(std::unique_ptr<base::ListValue> commands) {
   scene_->HandleCommands(std::move(commands), TimeInMicroseconds());
 }
 
+device::mojom::VRPosePtr VrShellGl::GetPose() {
+  TRACE_EVENT0("input", "GvrDevice::GetSensorState");
+
+  device::mojom::VRPosePtr pose = device::mojom::VRPose::New();
+
+  pose->timestamp = base::Time::Now().ToJsTime();
+
+  // Increment pose frame counter always, even if it's a faked pose.
+  pose->poseIndex = ++pose_index_;
+
+  pose->orientation.emplace(4);
+
+  gvr::ClockTimePoint target_time = gvr::GvrApi::GetTimePointNow();
+  target_time.monotonic_system_time_nanos += kPredictionTimeWithoutVsyncNanos;
+
+  gvr::Mat4f head_mat =
+      gvr_api_->GetHeadSpaceFromStartSpaceRotation(target_time);
+  head_mat = gvr_api_->ApplyNeckModel(head_mat, 1.0f);
+
+  // Save the underlying GVR pose for use by rendering. It can't use a
+  // VRPosePtr since that's a different data type.
+  webvr_head_pose_[pose_index_ % kPoseRingBufferSize] = head_mat;
+
+  gfx::Transform inv_transform(
+      head_mat.m[0][0], head_mat.m[0][1], head_mat.m[0][2], head_mat.m[0][3],
+      head_mat.m[1][0], head_mat.m[1][1], head_mat.m[1][2], head_mat.m[1][3],
+      head_mat.m[2][0], head_mat.m[2][1], head_mat.m[2][2], head_mat.m[2][3],
+      head_mat.m[3][0], head_mat.m[3][1], head_mat.m[3][2], head_mat.m[3][3]);
+
+  gfx::Transform transform;
+  if (inv_transform.GetInverse(&transform)) {
+    gfx::DecomposedTransform decomposed_transform;
+    gfx::DecomposeTransform(&decomposed_transform, transform);
+
+    pose->orientation.value()[0] = decomposed_transform.quaternion[0];
+    pose->orientation.value()[1] = decomposed_transform.quaternion[1];
+    pose->orientation.value()[2] = decomposed_transform.quaternion[2];
+    pose->orientation.value()[3] = decomposed_transform.quaternion[3];
+
+    pose->position.emplace(3);
+    pose->position.value()[0] = decomposed_transform.translate[0];
+    pose->position.value()[1] = decomposed_transform.translate[1];
+    pose->position.value()[2] = decomposed_transform.translate[2];
+  }
+
+  return pose;
+}
+
 }  // namespace vr_shell