Re-land^2 WebVR compositor bypass via BrowserMain context + mailbox

chrome/browser/android/vr_shell/vr_shell_gl.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/android/vr_shell/vr_shell_gl.h"
 
 #include <limits>
 #include <utility>
 
 #include "base/android/jni_android.h"
@@ skipping 12 matching lines @@
 #include "device/vr/android/gvr/gvr_device.h"
 #include "third_party/WebKit/public/platform/WebInputEvent.h"
 #include "third_party/WebKit/public/platform/WebMouseEvent.h"
 #include "ui/gl/android/scoped_java_surface.h"
 #include "ui/gl/android/surface_texture.h"
 #include "ui/gl/gl_bindings.h"
 #include "ui/gl/gl_context.h"
 #include "ui/gl/gl_surface.h"
 #include "ui/gl/init/gl_factory.h"
 
+#include "gpu/ipc/common/gpu_surface_tracker.h"
+
 namespace vr_shell {
 
 namespace {
 // TODO(mthiesse): If gvr::PlatformInfo().GetPosePredictionTime() is ever
 // exposed, use that instead (it defaults to 50ms on most platforms).
 static constexpr int64_t kPredictionTimeWithoutVsyncNanos = 50000000;
 
 static constexpr float kZNear = 0.1f;
 static constexpr float kZFar = 1000.0f;
 
@@ skipping 33 matching lines @@
 
 // The GVR viewport list has two entries (left eye and right eye) for each
 // GVR buffer.
 static constexpr int kViewportListPrimaryOffset = 0;
 static constexpr int kViewportListHeadlockedOffset = 2;
 
 // Buffer size large enough to handle the current backlog of poses which is
 // 2-3 frames.
 static constexpr unsigned kPoseRingBufferSize = 8;
 
+#if 0
 // Magic numbers used to mark valid pose index values encoded in frame
 // data. Must match the magic numbers used in blink's VRDisplay.cpp.
 static constexpr std::array<uint8_t, 2> kWebVrPosePixelMagicNumbers{{42, 142}};
+#endif
 
 float Distance(const gvr::Vec3f& vec1, const gvr::Vec3f& vec2) {
   float xdiff = (vec1.x - vec2.x);
   float ydiff = (vec1.y - vec2.y);
   float zdiff = (vec1.z - vec2.z);
   float scale = xdiff * xdiff + ydiff * ydiff + zdiff * zdiff;
   return std::sqrt(scale);
 }
 
 // Generate a quaternion representing the rotation from the negative Z axis
@@ skipping 129 matching lines @@
     LOG(ERROR) << "gl::init::CreateGLContext failed";
     ForceExitVr();
     return;
   }
   if (!context_->MakeCurrent(surface_.get())) {
     LOG(ERROR) << "gl::GLContext::MakeCurrent() failed";
     ForceExitVr();
     return;
   }
 
-  unsigned int textures[2];
-  glGenTextures(2, textures);
+  unsigned int textures[3];
+  glGenTextures(3, textures);
   ui_texture_id_ = textures[0];
   content_texture_id_ = textures[1];
+  webvr_texture_id_ = textures[2];
   ui_surface_texture_ = gl::SurfaceTexture::Create(ui_texture_id_);
   content_surface_texture_ = gl::SurfaceTexture::Create(content_texture_id_);
+  webvr_surface_texture_ = gl::SurfaceTexture::Create(webvr_texture_id_);
   CreateUiSurface();
   CreateContentSurface();
+  CreateWebVRSurface();
   ui_surface_texture_->SetFrameAvailableCallback(base::Bind(
       &VrShellGl::OnUIFrameAvailable, weak_ptr_factory_.GetWeakPtr()));
   content_surface_texture_->SetFrameAvailableCallback(base::Bind(
       &VrShellGl::OnContentFrameAvailable, weak_ptr_factory_.GetWeakPtr()));
+  webvr_surface_texture_->SetFrameAvailableCallback(base::Bind(
+      &VrShellGl::OnWebVRFrameAvailable, weak_ptr_factory_.GetWeakPtr()));
+  ui_surface_texture_->SetDefaultBufferSize(ui_tex_physical_size_.width,
+                                            ui_tex_physical_size_.height);
   content_surface_texture_->SetDefaultBufferSize(
       content_tex_physical_size_.width, content_tex_physical_size_.height);
-  ui_surface_texture_->SetDefaultBufferSize(ui_tex_physical_size_.width,
-                                            ui_tex_physical_size_.height);
+  VLOG(1) << __FUNCTION__ << ";;; content_tex_physical_size=" <<
+      content_tex_physical_size_.width << "x" <<
+      content_tex_physical_size_.height;
+  VLOG(1) << __FUNCTION__ << ";;; render_size_primary=" <<
+      render_size_primary_.width << "x" << render_size_primary_.height;
+  webvr_surface_texture_->SetDefaultBufferSize(
+      //content_tex_physical_size_.width, content_tex_physical_size_.height);
+      2560, 1440); // FIXME!
   InitializeRenderer();
 
   vsync_task_.Reset(base::Bind(&VrShellGl::OnVSync, base::Unretained(this)));
   OnVSync();
 
   ready_to_draw_ = true;
 }
 
 void VrShellGl::CreateContentSurface() {
   content_surface_ =
       base::MakeUnique<gl::ScopedJavaSurface>(content_surface_texture_.get());
   main_thread_task_runner_->PostTask(
       FROM_HERE, base::Bind(&VrShell::ContentSurfaceChanged, weak_vr_shell_,
                             content_surface_->j_surface().obj()));
 }
 
 void VrShellGl::CreateUiSurface() {
   ui_surface_ =
       base::MakeUnique<gl::ScopedJavaSurface>(ui_surface_texture_.get());
   main_thread_task_runner_->PostTask(
       FROM_HERE, base::Bind(&VrShell::UiSurfaceChanged, weak_vr_shell_,
                             ui_surface_->j_surface().obj()));
 }
 
+void VrShellGl::CreateWebVRSurface() {
+  ANativeWindow* window = webvr_surface_texture_->CreateSurface();
+  gpu::GpuSurfaceTracker* tracker = gpu::GpuSurfaceTracker::Get();
+  ANativeWindow_acquire(window);
+  // setBuffersGeometry seems optional, it uses the SurfaceTexture's
+  // default size by default?
+  // ANativeWindow_setBuffersGeometry(
+  //   window, 2048, 1024, WINDOW_FORMAT_RGBA_8888); // FIXME!
+  webvr_surface_handle_ = tracker->AddSurfaceForNativeWidget(window);
+  VLOG(1) << __FUNCTION__ << ";;;: webvr_surface_handle_=" <<
+      webvr_surface_handle_;
+
+  webvr_surface_ =
+      base::MakeUnique<gl::ScopedJavaSurface>(webvr_surface_texture_.get());
+  tracker->RegisterViewSurface(
+      webvr_surface_handle_, webvr_surface_->j_surface().obj());
+  // When should this be released? Does registering it keep it alive?
+  ANativeWindow_release(window);
+
+  main_thread_task_runner_->PostTask(
+      FROM_HERE, base::Bind(&VrShell::WebVRSurfaceChanged, weak_vr_shell_,
+                            webvr_surface_handle_));
+}
+
 void VrShellGl::OnUIFrameAvailable() {
+  VLOG(1) << __FUNCTION__ << ";;; UI UpdateTexImage start";
   ui_surface_texture_->UpdateTexImage();
+  VLOG(1) << __FUNCTION__ << ";;; UI UpdateTexImage end";
 }
 
 void VrShellGl::OnContentFrameAvailable() {
+  VLOG(1) << __FUNCTION__ << ";;; Content UpdateTexImage start";
   content_surface_texture_->UpdateTexImage();
   received_frame_ = true;
+  VLOG(1) << __FUNCTION__ << ";;; Content UpdateTexImage end";
 }
 
+void VrShellGl::OnWebVRFrameAvailable() {
+  VLOG(1) << __FUNCTION__ << ";;; pending count=" << pending_frames_.size();
+  // A "while" loop here is a bad idea. It's legal to call
+  // UpdateTexImage repeatedly even if no frames are available, but
+  // that does *not* wait for a new frame, it just reuses the most
+  // recent one. That would mess up the count.
+  if (pending_frames_.empty()) {
+    VLOG(1) << __FUNCTION__ << ";;; no pending frames? Please retry! premature_received_frames " << premature_received_frames_ << " => " << (premature_received_frames_ + 1);
+    ++premature_received_frames_;
+    return;
+  }
+
+  VLOG(1) << __FUNCTION__ << ";;; requested_frames " << requested_frames_ << " => " << (requested_frames_ - 1);
+  --requested_frames_;
+  VLOG(1) << __FUNCTION__ << ";;; WebVR UpdateTexImage start";
+  webvr_surface_texture_->UpdateTexImage();
+  int frame_index = pending_frames_.front();
+  pending_frames_.pop();
+  VLOG(1) << __FUNCTION__ << ";;; WebVR UpdateTexImage end, got frame=" << frame_index;
+  DrawFrame(frame_index);
+  main_thread_task_runner_->PostTask(
+      FROM_HERE, base::Bind(&VrShell::WebVRFrameCompleted, weak_vr_shell_,
+                            frame_index));
+
+  if (pending_frames_.empty() && requested_frames_ > 0) {
+    // Heuristic got confused? TODO(klausw): do proper SubmitFrame callback?
+    VLOG(1) << __FUNCTION__ << ";;; Resetting requested_frames " << requested_frames_ << " => " << 0;
+    requested_frames_ = 0;
+  }
+}
+
+#if 0
 bool VrShellGl::GetPixelEncodedFrameIndex(uint16_t* frame_index) {
+  //*frame_index = 1;
+  //return true;
   TRACE_EVENT0("gpu", "VrShellGl::GetPixelEncodedFrameIndex");
   if (!received_frame_) {
     if (last_frame_index_ == (uint16_t)-1)
       return false;
     *frame_index = last_frame_index_;
     return true;
   }
   received_frame_ = false;
 
   // Read the pose index encoded in a bottom left pixel as color values.
@@ skipping 16 matching lines @@
     // Pose is good.
     *frame_index = pixels[0];
     last_frame_index_ = pixels[0];
     return true;
   }
   VLOG(1) << "WebVR: reject decoded pose index " << static_cast<int>(pixels[0])
           << ", bad magic number " << static_cast<int>(pixels[1]) << ", "
           << static_cast<int>(pixels[2]);
   return false;
 }
+#endif
 
 void VrShellGl::GvrInit(gvr_context* gvr_api) {
   gvr_api_ = gvr::GvrApi::WrapNonOwned(gvr_api);
   controller_.reset(new VrController(gvr_api));
 
   ViewerType viewerType;
   switch (gvr_api_->GetViewerType()) {
     case gvr::ViewerType::GVR_VIEWER_TYPE_DAYDREAM:
       viewerType = ViewerType::DAYDREAM;
       break;
     case gvr::ViewerType::GVR_VIEWER_TYPE_CARDBOARD:
       viewerType = ViewerType::CARDBOARD;
       break;
     default:
       NOTREACHED();
       viewerType = ViewerType::UNKNOWN_TYPE;
       break;
   }
   UMA_HISTOGRAM_ENUMERATION("VRViewerType", static_cast<int>(viewerType),
                             static_cast<int>(ViewerType::VIEWER_TYPE_MAX));
 }
 
 void VrShellGl::InitializeRenderer() {
-  // While WebVR is going through the compositor path, it shares
-  // the same texture ID. This will change once it gets its own
-  // surface, but store it separately to avoid future confusion.
-  // TODO(klausw,crbug.com/655722): remove this.
-  webvr_texture_id_ = content_texture_id_;
-
   gvr_api_->InitializeGl();
   webvr_head_pose_.assign(kPoseRingBufferSize,
                           gvr_api_->GetHeadSpaceFromStartSpaceRotation(
                               gvr::GvrApi::GetTimePointNow()));
 
   std::vector<gvr::BufferSpec> specs;
   // For kFramePrimaryBuffer (primary VrShell and WebVR content)
   specs.push_back(gvr_api_->CreateBufferSpec());
   render_size_primary_ = specs[kFramePrimaryBuffer].GetSize();
 
@@ skipping 260 matching lines @@
                             std::unique_ptr<blink::WebInputEvent> event) {
   DCHECK(input_target != InputTarget::NONE);
   auto&& target = input_target == InputTarget::CONTENT
                       ? &VrShell::ProcessContentGesture
                       : &VrShell::ProcessUIGesture;
   main_thread_task_runner_->PostTask(
       FROM_HERE,
       base::Bind(target, weak_vr_shell_, base::Passed(std::move(event))));
 }
 
-void VrShellGl::DrawFrame() {
+void VrShellGl::DrawFrame(int frame_index) {
   TRACE_EVENT0("gpu", "VrShellGl::DrawFrame");
 
   // Reset the viewport list to just the pair of viewports for the
   // primary buffer each frame. Head-locked viewports get added by
   // DrawVrShell if needed.
   buffer_viewport_list_->SetToRecommendedBufferViewports();
 
+  TRACE_EVENT_BEGIN0("gpu", "VrShellGl::AcquireFrame");
   gvr::Frame frame = swap_chain_->AcquireFrame();
+  TRACE_EVENT_END0("gpu", "VrShellGl::AcquireFrame");
   if (!frame.is_valid()) {
     return;
   }
   frame.BindBuffer(kFramePrimaryBuffer);
   if (web_vr_mode_) {
     DrawWebVr();
   }
 
-  uint16_t frame_index;
   gvr::Mat4f head_pose;
 
   // When using async reprojection, we need to know which pose was used in
   // the WebVR app for drawing this frame. Due to unknown amounts of
   // buffering in the compositor and SurfaceTexture, we read the pose number
   // from a corner pixel. There's no point in doing this for legacy
   // distortion rendering since that doesn't need a pose, and reading back
   // pixels is an expensive operation. TODO(klausw,crbug.com/655722): stop
   // doing this once we have working no-compositor rendering for WebVR.
-  if (web_vr_mode_ && gvr_api_->GetAsyncReprojectionEnabled() &&
-      GetPixelEncodedFrameIndex(&frame_index)) {
+  if (web_vr_mode_ && gvr_api_->GetAsyncReprojectionEnabled()) {
     static_assert(!((kPoseRingBufferSize - 1) & kPoseRingBufferSize),
                   "kPoseRingBufferSize must be a power of 2");
     head_pose = webvr_head_pose_[frame_index % kPoseRingBufferSize];
     // Process all pending_bounds_ changes targeted for before this frame, being
     // careful of wrapping frame indices.
     static constexpr unsigned max =
         std::numeric_limits<decltype(frame_index_)>::max();
     static_assert(max > kPoseRingBufferSize * 2,
                   "To detect wrapping, kPoseRingBufferSize must be smaller "
                   "than half of frame_index_ range.");
     while (!pending_bounds_.empty()) {
       uint16_t index = pending_bounds_.front().first;
+      VLOG(1) << __FUNCTION__ << ";;; new bounds, index=" << index;
       // If index is less than the frame_index it's possible we've wrapped, so
       // we extend the range and 'un-wrap' to account for this.
       if (index < frame_index)
         index += max;
       // If the pending bounds change is for an upcoming frame within our buffer
       // size, wait to apply it. Otherwise, apply it immediately. This
       // guarantees that even if we miss many frames, the queue can't fill up
       // with stale bounds.
       if (index > frame_index && index <= frame_index + kPoseRingBufferSize)
         break;
@@ skipping 22 matching lines @@
     // it. For now, removing it seems working fine.
     gvr_api_->ApplyNeckModel(head_pose, 1.0f);
   }
 
   // Update the render position of all UI elements (including desktop).
   const float screen_tilt = kDesktopScreenTiltDefault * M_PI / 180.0f;
   scene_->UpdateTransforms(screen_tilt, TimeInMicroseconds());
 
   UpdateController(GetForwardVector(head_pose));
 
-  DrawVrShell(head_pose, frame);
+  // Drawing VrShell causes GL error 0x501 GL_INVALID_VALUE while in
+  // WebVR mode. FIXME.
+  if (!web_vr_mode_) DrawVrShell(head_pose, frame);
 
   frame.Unbind();
   frame.Submit(*buffer_viewport_list_, head_pose);
 
   // No need to swap buffers for surfaceless rendering.
   if (!surfaceless_rendering_) {
     // TODO(mthiesse): Support asynchronous SwapBuffers.
     surface_->SwapBuffers();
   }
 }
 
+void VrShellGl::ScheduleWebVRFrame(int frame_index) {
+  VLOG(1) << __FUNCTION__ << ";;; schedule frame=" << frame_index;
+  pending_frames_.emplace(frame_index);
+}
+
+void VrShellGl::DropWebVRFrame(int frame_index) {
+  VLOG(1) << __FUNCTION__ << ";;; drop frame=" << frame_index;
+  if (requested_frames_ > 0) {
+    VLOG(1) << __FUNCTION__ << ";;; requested_frames " << requested_frames_ <<
+        " => " << (requested_frames_ - 1);
+    --requested_frames_;
+  }
+}
+
 void VrShellGl::DrawVrShell(const gvr::Mat4f& head_pose, gvr::Frame& frame) {
   TRACE_EVENT0("gpu", "VrShellGl::DrawVrShell");
   std::vector<const ContentRectangle*> head_locked_elements;
   std::vector<const ContentRectangle*> world_elements;
   for (const auto& rect : scene_->GetUiElements()) {
     if (!rect->IsVisible())
       continue;
     if (rect->lock_to_fov) {
       head_locked_elements.push_back(rect.get());
     } else {
@@ skipping 284 matching lines @@
   }
 }
 
 void VrShellGl::SetWebVrMode(bool enabled) {
   web_vr_mode_ = enabled;
 }
 
 void VrShellGl::UpdateWebVRTextureBounds(int16_t frame_index,
                                          const gvr::Rectf& left_bounds,
                                          const gvr::Rectf& right_bounds) {
+  VLOG(1) << __FUNCTION__ << ";;; frame_index=" << frame_index <<
+      " left=" << left_bounds.left << "," << left_bounds.bottom <<
+      "," << left_bounds.right << "," << left_bounds.top <<
+      " right=" << right_bounds.left << "," << right_bounds.bottom <<
+      "," << right_bounds.right << "," << right_bounds.top;
   if (frame_index < 0) {
     webvr_left_viewport_->SetSourceUv(left_bounds);
     webvr_right_viewport_->SetSourceUv(right_bounds);
   } else {
     pending_bounds_.emplace(
         std::make_pair(frame_index, std::make_pair(left_bounds, right_bounds)));
   }
 }
 
 void VrShellGl::ContentBoundsChanged(int width, int height) {
@@ skipping 19 matching lines @@
     ui_surface_texture_->SetDefaultBufferSize(width, height);
   ui_tex_physical_size_.width = width;
   ui_tex_physical_size_.height = height;
 }
 
 base::WeakPtr<VrShellGl> VrShellGl::GetWeakPtr() {
   return weak_ptr_factory_.GetWeakPtr();
 }
 
 void VrShellGl::OnVSync() {
+  while (premature_received_frames_ > 0) {
+    VLOG(1) << __FUNCTION__ << ";;; Retrying premature received frame " <<
+        premature_received_frames_ << " => " <<
+        (premature_received_frames_ - 1);
+    --premature_received_frames_;
+    OnWebVRFrameAvailable();
+  }
+
   base::TimeTicks now = base::TimeTicks::Now();
   base::TimeTicks target;
 
   // Don't send VSyncs until we have a timebase/interval.
   if (vsync_interval_.is_zero())
     return;
   target = now + vsync_interval_;
   int64_t intervals = (target - vsync_timebase_) / vsync_interval_;
   target = vsync_timebase_ + intervals * vsync_interval_;
   task_runner_->PostDelayedTask(FROM_HERE, vsync_task_.callback(),
                                 target - now);
 
   base::TimeDelta time = intervals * vsync_interval_;
+  // Send a small rate of vsyncs even while backlogged to avoid
+  // locking up when the rAF loop doesn't submit a frame.
+  static int skip_counter = 0;
   if (!callback_.is_null()) {
-    SendVSync(time, base::ResetAndReturn(&callback_));
+    if (requested_frames_ < 1 || (++skip_counter % 30) == 0) {
+      VLOG(1) << __FUNCTION__ << ";;; vsync B, interval=" << vsync_interval_;
+      SendVSync(time, base::ResetAndReturn(&callback_));
+    } else {
+      VLOG(1) << __FUNCTION__ << ";;; Skip vsync B, already have frame requested";
+      return;
+    }
   } else {
     pending_vsync_ = true;
     pending_time_ = time;
   }
-  DrawFrame();
+  if (!web_vr_mode_) {
+    DrawFrame(-1);
+  }
 }
 
 void VrShellGl::OnRequest(device::mojom::VRVSyncProviderRequest request) {
   binding_.Close();
   binding_.Bind(std::move(request));
 }
 
 void VrShellGl::GetVSync(const GetVSyncCallback& callback) {
   if (!pending_vsync_) {
     if (!callback_.is_null()) {
       mojo::ReportBadMessage(
           "Requested VSync before waiting for response to previous request.");
       binding_.Close();
       return;
     }
     callback_ = callback;
     return;
   }
   pending_vsync_ = false;
+  VLOG(1) << __FUNCTION__ << ";;; vsync A, pending time=" << pending_time_;
   SendVSync(pending_time_, callback);
 }
 
 void VrShellGl::UpdateVSyncInterval(int64_t timebase_nanos,
                                     double interval_seconds) {
   vsync_timebase_ = base::TimeTicks();
   vsync_timebase_ += base::TimeDelta::FromMicroseconds(timebase_nanos / 1000);
   vsync_interval_ = base::TimeDelta::FromSecondsD(interval_seconds);
   vsync_task_.Reset(base::Bind(&VrShellGl::OnVSync, base::Unretained(this)));
+  VLOG(1) << __FUNCTION__ << ";;; vsync_interval=" << vsync_interval_;
   OnVSync();
 }
 
 void VrShellGl::ForceExitVr() {
   main_thread_task_runner_->PostTask(
       FROM_HERE, base::Bind(&VrShell::ForceExitVr, weak_vr_shell_));
 }
 
 void VrShellGl::UpdateScene(std::unique_ptr<base::ListValue> commands) {
   scene_->HandleCommands(std::move(commands), TimeInMicroseconds());
 }
 
 void VrShellGl::SendVSync(base::TimeDelta time,
                           const GetVSyncCallback& callback) {
   TRACE_EVENT0("input", "VrShellGl::SendVSync");
 
+  VLOG(1) << __FUNCTION__ << ";;; requested_frames " << requested_frames_ <<
+      " => " << (requested_frames_ + 1);
+  ++requested_frames_;
+
   uint8_t frame_index = frame_index_++;
+  VLOG(1) << __FUNCTION__ << ";;; vsync for frame=" << (int)frame_index;
 
   gvr::ClockTimePoint target_time = gvr::GvrApi::GetTimePointNow();
   target_time.monotonic_system_time_nanos += kPredictionTimeWithoutVsyncNanos;
 
   gvr::Mat4f head_mat =
       gvr_api_->GetHeadSpaceFromStartSpaceRotation(target_time);
   head_mat = gvr_api_->ApplyNeckModel(head_mat, 1.0f);
 
   webvr_head_pose_[frame_index % kPoseRingBufferSize] = head_mat;
 
@@ skipping 12 matching lines @@
     const base::Callback<void(device::mojom::VRDisplayInfoPtr)>& callback,
     uint32_t device_id) {
   device::mojom::VRDisplayInfoPtr info = VrShell::CreateVRDisplayInfo(
       gvr_api_.get(), content_tex_physical_size_, device_id);
   main_thread_task_runner_->PostTask(
       FROM_HERE,
       base::Bind(&RunVRDisplayInfoCallback, callback, base::Passed(&info)));
 }
 
 }  // namespace vr_shell