WIP: working copy-no-compositor path

device/vr/android/gvr/gvr_device.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "device/vr/android/gvr/gvr_device.h"
 
 #include <math.h>
 #include <algorithm>
 
 #include "base/time/time.h"
 #include "base/trace_event/trace_event.h"
 #include "device/vr/android/gvr/gvr_delegate.h"
 #include "device/vr/android/gvr/gvr_device_provider.h"
 #include "device/vr/vr_device_manager.h"
 #include "third_party/gvr-android-sdk/src/libraries/headers/vr/gvr/capi/include/gvr.h"
 #include "third_party/gvr-android-sdk/src/libraries/headers/vr/gvr/capi/include/gvr_types.h"
 #include "ui/gfx/transform.h"
 #include "ui/gfx/transform_util.h"
 
 namespace device {
 
 namespace {
 
-static const uint64_t kPredictionTimeWithoutVsyncNanos = 50000000;
+// TODO(klausw): adjust this based on historic render times
+static const double kDefaultRenderTimeMillis = 15.0;
+static const double kPredictionTimeWithoutVsyncMillis = 24.0;
 
 }  // namespace
 
+// THIS IS A COPY, keep in sync with VRDisplay.cpp ?
+StatTracker::StatTracker(unsigned int capacity) : m_capacity(capacity) {}
+
+StatTracker::~StatTracker() = default;
+
+void StatTracker::add(double item) {
+  if (m_items.size() >= m_capacity) {
+    m_items.pop_front();
+  }
+  m_items.push_back(item);
+}
+
+void StatTracker::clear() {
+  m_items.clear();
+}
+
+bool StatTracker::hasPrediction() {
+  return m_items.size() > 0;
+}
+
+double StatTracker::getPrediction() {
+  assert(hasPrediction());
+
+  // If we have 3 or more items, ignore min and max outliers and
+  // average the rest. For 2 or less, minmax.first and minmax.second
+  // will both be m_items.end(), so it's just a plain average.
+  auto minmax = m_items.size() > 2 ?
+      std::minmax_element(m_items.begin(), m_items.end()) :
+      std::minmax_element(m_items.end(), m_items.end());
+
+  double sum = 0.0;
+  int count = 0;
+  //VLOG(2) << __FUNCTION__ << ": stat start";
+  for (auto it = m_items.begin(); it != m_items.end(); ++it) {
+    //VLOG(2) << __FUNCTION__ << ":    val=" << *it;
+    if (it == minmax.first || it == minmax.second) continue;
+    sum += *it;
+    ++count;
+  }
+  //VLOG(2) << __FUNCTION__ << ": stat return " << sum / count;
+  return sum / count;
+}
+// COPY end
+
 GvrDevice::GvrDevice(GvrDeviceProvider* provider, GvrDelegate* delegate)
-    : VRDevice(), delegate_(delegate), gvr_provider_(provider) {}
+    : VRDevice(), delegate_(delegate), gvr_provider_(provider) {
+  VLOG(1) << __FUNCTION__ << ": CONSTRUCTOR this=" << (void*)this << " ***********************************************************************************************************";
+}
 
-GvrDevice::~GvrDevice() {}
+GvrDevice::~GvrDevice() {
+  VLOG(1) << __FUNCTION__ << ": DESTRUCTOR this=" << (void*)this << " ***********************************************************************************************************";

[artem.bolgar, 2017/02/14 05:04:24]

It is possible that callback_map_ is not empty here. In this case, you'll get an
assertion, if the callback is not executed. Must go through them and call, to
avoid the assertion:
====
  // Run pending "frame complete" callbacks
  for (auto it : callback_map_) {
    VLOG(2) << __FUNCTION__ << ": running pending callback for frame: " <<
it.first;
    it.second.Run(0, it.first, -1.0);
  }
  callback_map_.clear();
====

+}
 
 mojom::VRDisplayInfoPtr GvrDevice::GetVRDevice() {
   TRACE_EVENT0("input", "GvrDevice::GetVRDevice");
 
   mojom::VRDisplayInfoPtr device = mojom::VRDisplayInfo::New();
 
   device->index = id();
 
   device->capabilities = mojom::VRDisplayCapabilities::New();
   device->capabilities->hasOrientation = true;
@@ skipping 16 matching lines @@
   // we can't render into it yet. Other code uses this to check
   // for valid state.
   gvr::Sizei render_target_size = kInvalidRenderTargetSize;
   left_eye->renderWidth = render_target_size.width / 2;
   left_eye->renderHeight = render_target_size.height;
 
   right_eye->renderWidth = left_eye->renderWidth;
   right_eye->renderHeight = left_eye->renderHeight;
 
   gvr::GvrApi* gvr_api = GetGvrApi();
+  VLOG(1) << __FUNCTION__ << ": gvr_api=" << gvr_api;
   if (!gvr_api) {
     // We may not be able to get an instance of GvrApi right away, so
     // stub in some data till we have one.
+
+    // TODO(klausw): I think we never get here ?!
     device->displayName = "Unknown";
 
     left_eye->fieldOfView->upDegrees = 45;
     left_eye->fieldOfView->downDegrees = 45;
     left_eye->fieldOfView->leftDegrees = 45;
     left_eye->fieldOfView->rightDegrees = 45;
 
     right_eye->fieldOfView->upDegrees = 45;
     right_eye->fieldOfView->downDegrees = 45;
     right_eye->fieldOfView->leftDegrees = 45;
     right_eye->fieldOfView->rightDegrees = 45;
 
     left_eye->offset[0] = -0.0;
     left_eye->offset[1] = -0.0;
     left_eye->offset[2] = -0.03;
 
     right_eye->offset[0] = 0.0;
     right_eye->offset[1] = 0.0;
     right_eye->offset[2] = 0.03;
 
+#if 0
     // Tell the delegate not to draw yet, to avoid a race condition
     // (and visible wobble) on entering VR.
     if (delegate_) {
       delegate_->SetWebVRRenderSurfaceSize(kInvalidRenderTargetSize.width,
                                            kInvalidRenderTargetSize.height);
     }
-
+#endif
     return device;
   }
 
-  // In compositor mode, we have to use the current compositor window's
-  // surface size. Would be nice to change it, but that needs more browser
-  // internals to be modified. TODO(klausw,crbug.com/655722): remove this once
-  // we can pick our own surface size.
-  gvr::Sizei compositor_size = delegate_->GetWebVRCompositorSurfaceSize();
-  left_eye->renderWidth = compositor_size.width / 2;
-  left_eye->renderHeight = compositor_size.height;
+  render_target_size = gvr_api->GetMaximumEffectiveRenderTargetSize();
+  // Render at less than the maximum effective render target size as a
+  // compromise between image quality and performance, similar to
+  // current polyfill. TODO(klausw): allow clients to override this to
+  // get full resolution.
+  left_eye->renderWidth = render_target_size.width * 55 / 100 / 2;
+  left_eye->renderHeight = render_target_size.height * 55 / 100;
   right_eye->renderWidth = left_eye->renderWidth;
   right_eye->renderHeight = left_eye->renderHeight;
+  LOG(INFO) << "klausw: render_target_size=" << render_target_size.width << "x"
+            << render_target_size.height;
 
   std::string vendor = gvr_api->GetViewerVendor();
   std::string model = gvr_api->GetViewerModel();
   device->displayName = vendor + " " + model;
 
+  VLOG(1) << __FUNCTION__ << ": gvr_api creating buffer viewports";
   gvr::BufferViewportList gvr_buffer_viewports =
       gvr_api->CreateEmptyBufferViewportList();
   gvr_buffer_viewports.SetToRecommendedBufferViewports();
 
   gvr::BufferViewport eye_viewport = gvr_api->CreateBufferViewport();
   gvr_buffer_viewports.GetBufferViewport(GVR_LEFT_EYE, &eye_viewport);
   gvr::Rectf eye_fov = eye_viewport.GetSourceFov();
   left_eye->fieldOfView->upDegrees = eye_fov.top;
   left_eye->fieldOfView->downDegrees = eye_fov.bottom;
   left_eye->fieldOfView->leftDegrees = eye_fov.left;
@@ skipping 10 matching lines @@
   gvr::Mat4f left_eye_mat = gvr_api->GetEyeFromHeadMatrix(GVR_LEFT_EYE);
   left_eye->offset[0] = -left_eye_mat.m[0][3];
   left_eye->offset[1] = -left_eye_mat.m[1][3];
   left_eye->offset[2] = -left_eye_mat.m[2][3];
 
   gvr::Mat4f right_eye_mat = gvr_api->GetEyeFromHeadMatrix(GVR_RIGHT_EYE);
   right_eye->offset[0] = -right_eye_mat.m[0][3];
   right_eye->offset[1] = -right_eye_mat.m[1][3];
   right_eye->offset[2] = -right_eye_mat.m[2][3];
 
-  if (delegate_) {
-    delegate_->SetWebVRRenderSurfaceSize(2 * left_eye->renderWidth,
-                                         left_eye->renderHeight);
-  }
-
   return device;
 }
 
 mojom::VRPosePtr GvrDevice::GetPose() {
   TRACE_EVENT0("input", "GvrDevice::GetSensorState");
 
+  // Increment pose frame counter always, even if it's a faked pose.
+  ++pose_index_;
+  // Don't allow an actual pose index to be negative or zero, those
+  // are reserved for error or "no pose" states.
+  if (pose_index_ <= 0) pose_index_ = 1;
+
+  VLOG(2) << __FUNCTION__ << ": frame " << pose_index_;
+  TRACE_EVENT1("media", "klausw:GetPose", "frame", pose_index_);
+
   mojom::VRPosePtr pose = mojom::VRPose::New();
-
   pose->timestamp = base::Time::Now().ToJsTime();
-
-  // Increment pose frame counter always, even if it's a faked pose.
-  pose->poseIndex = ++pose_index_;
-
+  pose->poseIndex = pose_index_;
   pose->orientation.emplace(4);
 
   gvr::GvrApi* gvr_api = GetGvrApi();
   if (!gvr_api) {
     // If we don't have a GvrApi instance return a static forward orientation.
     pose->orientation.value()[0] = 0.0;
     pose->orientation.value()[1] = 0.0;
     pose->orientation.value()[2] = 0.0;
     pose->orientation.value()[3] = 1.0;
 
     return pose;
   }
 
-  if (!delegate_)
+  if (!delegate_) {
+    VLOG(2) << __FUNCTION__ << ": no delegate, no pose!";
     return nullptr;
+  }
 
   gvr::ClockTimePoint target_time = gvr::GvrApi::GetTimePointNow();
-  target_time.monotonic_system_time_nanos += kPredictionTimeWithoutVsyncNanos;
+  int64_t pose_time = target_time.monotonic_system_time_nanos;
+
+#if 0
+  int64_t nstime = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
+  LOG(INFO) << "klausw: gvr ns=" << pose_time << " steady ns=" << steady_nanos << " delta=" << pose_time - nstime;
+#endif
+
+  // TODO(klausw): sanity check
+  double render_time_ms = last_processing_ms_.hasPrediction() && last_render_ms_.hasPrediction() ?
+      last_processing_ms_.getPrediction() + last_render_ms_.getPrediction() :
+      kDefaultRenderTimeMillis;
+  double predict_time_ms = render_time_ms + kPredictionTimeWithoutVsyncMillis;
+  VLOG(2) << __FUNCTION__ << ": predict_time_ms=" << predict_time_ms;
+  target_time.monotonic_system_time_nanos += static_cast<uint64_t>(predict_time_ms * 1e6);
 
   gvr::Mat4f head_mat =
       gvr_api->GetHeadSpaceFromStartSpaceRotation(target_time);
   head_mat = gvr_api->ApplyNeckModel(head_mat, 1.0f);
 
   gfx::Transform inv_transform(
       head_mat.m[0][0], head_mat.m[0][1], head_mat.m[0][2], head_mat.m[0][3],
       head_mat.m[1][0], head_mat.m[1][1], head_mat.m[1][2], head_mat.m[1][3],
       head_mat.m[2][0], head_mat.m[2][1], head_mat.m[2][2], head_mat.m[2][3],
       head_mat.m[3][0], head_mat.m[3][1], head_mat.m[3][2], head_mat.m[3][3]);
 
   gfx::Transform transform;
   if (inv_transform.GetInverse(&transform)) {
     gfx::DecomposedTransform decomposed_transform;
     gfx::DecomposeTransform(&decomposed_transform, transform);
 
     pose->orientation.value()[0] = decomposed_transform.quaternion[0];
     pose->orientation.value()[1] = decomposed_transform.quaternion[1];
     pose->orientation.value()[2] = decomposed_transform.quaternion[2];
     pose->orientation.value()[3] = decomposed_transform.quaternion[3];
 
     pose->position.emplace(3);
     pose->position.value()[0] = decomposed_transform.translate[0];
     pose->position.value()[1] = decomposed_transform.translate[1];
     pose->position.value()[2] = decomposed_transform.translate[2];
   }
 
   // Save the underlying GVR pose for use by rendering. It can't use a
   // VRPosePtr since that's a different data type.
-  delegate_->SetGvrPoseForWebVr(head_mat, pose_index_);
+  delegate_->SetWebVRGvrPose(head_mat, pose_index_, pose_time);
 
   return pose;
 }
 
 void GvrDevice::ResetPose() {
   gvr::GvrApi* gvr_api = GetGvrApi();
 
   // Should never call RecenterTracking when using with Daydream viewers. On
   // those devices recentering should only be done via the controller.
   if (gvr_api && gvr_api->GetViewerType() == GVR_VIEWER_TYPE_CARDBOARD)
     gvr_api->RecenterTracking();
 }
 
 void GvrDevice::RequestPresent(const base::Callback<void(bool)>& callback) {
   gvr_provider_->RequestPresent(callback);
 }
 
 void GvrDevice::SetSecureOrigin(bool secure_origin) {
   secure_origin_ = secure_origin;
   if (delegate_)
     delegate_->SetWebVRSecureOrigin(secure_origin_);
 }
 
 void GvrDevice::ExitPresent() {
+  // Run pending "frame complete" callbacks
+  for (auto it : callback_map_) {

[artem.bolgar, 2017/02/14 05:04:24]

This cleanup code block should be moved to GvrDevice::OnExitPresent (of course,
such override method should be created first). The reason: you may exit
presentation w/o calling exitPresent (by pressing back button, for example, that
calls shutdownVR, that calls GvrDeviceProvider::OnDelegateRemoved(), that calls
Device's OnExitPresent (currently defined only in VRDevice).

+    VLOG(2) << __FUNCTION__ << ": ExitPresent running pending callback for frame: " << it.first;
+    it.second.Run(0, it.first, -1.0);
+  }
+  callback_map_.clear();
   gvr_provider_->ExitPresent();
   OnExitPresent();
 }
 
-void GvrDevice::SubmitFrame(mojom::VRPosePtr pose) {
-  if (delegate_)
-    delegate_->SubmitWebVRFrame();
+void GvrDevice::SubmitFrame(int32_t surfaceHandle,
+                            mojom::VRPosePtr pose,
+                            const mojom::VRDisplay::SubmitFrameCallback& callback) {
+  TRACE_EVENT1("media", "klausw:SubmitFrame", "frame", pose->poseIndex);
+  VLOG(2) << __FUNCTION__ << ": frame " << pose->poseIndex;
+  if (delegate_) {
+    VLOG(2) << __FUNCTION__ << ": save callback for frame index " << pose->poseIndex;
+    callback_map_.insert(std::make_pair(pose->poseIndex, std::move(callback)));
+    last_processing_ms_.add(pose->ts_submit - pose->ts_getPose);
+    delegate_->SubmitWebVRFrame(surfaceHandle, std::move(pose));
+  } else {
+    VLOG(2) << __FUNCTION__ << ": No delegate, calling callback now for frame index " << pose->poseIndex;
+    callback.Run(surfaceHandle, pose->poseIndex, -1.0);
+    VLOG(2) << __FUNCTION__ << ": run callback done";
+  }
+}
+
+void GvrDevice::OnFrameSubmitted(int32_t surfaceHandle, uint32_t frame_index, double elapsed) {
+  auto it = callback_map_.find(frame_index);
+  if (it == callback_map_.end()) {
+    VLOG(2) << __FUNCTION__ << ": no callback found for frame index " << frame_index;
+  } else {
+    VLOG(2) << __FUNCTION__ << ": run callback for frame index " << frame_index;
+    auto callback = std::move(it->second);
+    callback.Run(surfaceHandle, frame_index, elapsed);
+    last_render_ms_.add(elapsed);
+    VLOG(2) << __FUNCTION__ << ": run callback done, erasing";
+    callback_map_.erase(it);
+    VLOG(2) << __FUNCTION__ << ": erasing done";
+  }
+
+  for (auto it : callback_map_) {
+    VLOG(2) << __FUNCTION__ << ":     pending frame: " << it.first;
+  }
 }
 
 void GvrDevice::UpdateLayerBounds(mojom::VRLayerBoundsPtr left_bounds,
                                   mojom::VRLayerBoundsPtr right_bounds) {
   if (!delegate_)
     return;
 
   gvr::Rectf left_gvr_bounds;
   left_gvr_bounds.left = left_bounds->left;
   left_gvr_bounds.top = 1.0f - left_bounds->top;
   left_gvr_bounds.right = left_bounds->left + left_bounds->width;
   left_gvr_bounds.bottom = 1.0f - (left_bounds->top + left_bounds->height);
 
   gvr::Rectf right_gvr_bounds;
   right_gvr_bounds.left = right_bounds->left;
   right_gvr_bounds.top = 1.0f - right_bounds->top;
   right_gvr_bounds.right = right_bounds->left + right_bounds->width;
   right_gvr_bounds.bottom = 1.0f - (right_bounds->top + right_bounds->height);
 
-  delegate_->UpdateWebVRTextureBounds(left_gvr_bounds, right_gvr_bounds);
+  // TODO(klausw): tie this to pose_index_ for future execution
+  delegate_->UpdateWebVRTextureBounds(left_bounds->forPoseIndex, left_gvr_bounds, right_gvr_bounds);
+}
+
+void GvrDevice::GetSurfaceHandle(int32_t width, int32_t height, const mojom::VRDisplay::GetSurfaceHandleCallback& callback) {
+  // LOG(ERROR) << "klausw:GvrDevice.GetSurfaceHandle, delegate=" << delegate_;
+  if (!delegate_)
+    callback.Run(0);
+
+  delegate_->GetWebVRSurfaceHandle(width, height, callback);
 }
 
 void GvrDevice::SetDelegate(GvrDelegate* delegate) {
   delegate_ = delegate;
 
   // Notify the clients that this device has changed
   if (delegate_) {
     delegate_->SetWebVRSecureOrigin(secure_origin_);
     OnChanged();
   }
 }
 
 gvr::GvrApi* GvrDevice::GetGvrApi() {
   if (!delegate_)
     return nullptr;
 
   return delegate_->gvr_api();
 }
 
 }  // namespace device