| OLD | NEW |
| (Empty) |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 package org.chromium.chromoting.cardboard; | |
| 6 | |
| 7 import android.app.Activity; | |
| 8 import android.graphics.Point; | |
| 9 import android.graphics.PointF; | |
| 10 import android.opengl.GLES20; | |
| 11 import android.opengl.Matrix; | |
| 12 | |
| 13 import com.google.vrtoolkit.cardboard.CardboardView; | |
| 14 import com.google.vrtoolkit.cardboard.Eye; | |
| 15 import com.google.vrtoolkit.cardboard.HeadTransform; | |
| 16 import com.google.vrtoolkit.cardboard.Viewport; | |
| 17 | |
| 18 import org.chromium.chromoting.jni.Client; | |
| 19 import org.chromium.chromoting.jni.Display; | |
| 20 | |
| 21 import javax.microedition.khronos.egl.EGLConfig; | |
| 22 | |
| 23 /** | |
| 24 * Renderer for Cardboard view. | |
| 25 */ | |
| 26 public class CardboardRenderer implements CardboardView.StereoRenderer { | |
| 27 private static final String TAG = "cr.CardboardRenderer"; | |
| 28 | |
| 29 private static final int BYTE_PER_FLOAT = 4; | |
| 30 private static final int POSITION_DATA_SIZE = 3; | |
| 31 private static final int TEXTURE_COORDINATE_DATA_SIZE = 2; | |
| 32 private static final float Z_NEAR = 0.1f; | |
| 33 private static final float Z_FAR = 1000.0f; | |
| 34 | |
| 35 // Desktop position is fixed in world coordinates. | |
| 36 private static final float DESKTOP_POSITION_X = 0.0f; | |
| 37 private static final float DESKTOP_POSITION_Y = 0.0f; | |
| 38 private static final float DESKTOP_POSITION_Z = -2.0f; | |
| 39 | |
| 40 // Menu bar position is relative to the view point. | |
| 41 private static final float MENU_BAR_POSITION_X = 0.0f; | |
| 42 private static final float MENU_BAR_POSITION_Y = 0.0f; | |
| 43 private static final float MENU_BAR_POSITION_Z = -0.9f; | |
| 44 | |
| 45 private static final float HALF_SKYBOX_SIZE = 100.0f; | |
| 46 private static final float VIEW_POSITION_MIN = -1.0f; | |
| 47 private static final float VIEW_POSITION_MAX = 3.0f; | |
| 48 | |
| 49 // Allows user to click even when looking outside the desktop | |
| 50 // but within edge margin. | |
| 51 private static final float EDGE_MARGIN = 0.1f; | |
| 52 | |
| 53 // Distance to move camera each time. | |
| 54 private static final float CAMERA_MOTION_STEP = 0.5f; | |
| 55 | |
| 56 // This ratio is used by {@link isLookingFarawayFromDesktop()} to determine
the | |
| 57 // angle beyond which the user is looking faraway from the desktop. | |
| 58 // The ratio is based on half of the desktop's angular width, as seen from | |
| 59 // the camera position. | |
| 60 // If the user triggers the button while looking faraway, this will cause th
e | |
| 61 // desktop to be re-positioned in the center of the view. | |
| 62 private static final float FARAWAY_ANGLE_RATIO = 1.6777f; | |
| 63 | |
| 64 // Small number used to avoid division-overflow or other problems with | |
| 65 // floating-point imprecision. | |
| 66 private static final float EPSILON = 1e-5f; | |
| 67 | |
| 68 private final Activity mActivity; | |
| 69 private final Client mClient; | |
| 70 private final Display mDisplay; | |
| 71 | |
| 72 private float mCameraPosition; | |
| 73 | |
| 74 // Lock to allow multithreaded access to mCameraPosition. | |
| 75 private final Object mCameraPositionLock = new Object(); | |
| 76 | |
| 77 private float[] mCameraMatrix; | |
| 78 private float[] mViewMatrix; | |
| 79 private float[] mProjectionMatrix; | |
| 80 | |
| 81 // Make matrix member variable to avoid unnecessary initialization. | |
| 82 private float[] mDesktopModelMatrix; | |
| 83 private float[] mDesktopCombinedMatrix; | |
| 84 private float[] mEyePointModelMatrix; | |
| 85 private float[] mEyePointCombinedMatrix; | |
| 86 private float[] mPhotosphereCombinedMatrix; | |
| 87 | |
| 88 // Direction that user is looking towards. | |
| 89 private float[] mForwardVector; | |
| 90 | |
| 91 // Eye position at the desktop distance. | |
| 92 private PointF mEyeDesktopPosition; | |
| 93 | |
| 94 // Eye position at the menu bar distance; | |
| 95 private PointF mEyeMenuBarPosition; | |
| 96 | |
| 97 private Desktop mDesktop; | |
| 98 private MenuBar mMenuBar; | |
| 99 private Photosphere mPhotosphere; | |
| 100 private Cursor mCursor; | |
| 101 | |
| 102 // Lock for eye position related operations. | |
| 103 // This protects access to mEyeDesktopPosition. | |
| 104 private final Object mEyeDesktopPositionLock = new Object(); | |
| 105 | |
| 106 // Flag to indicate whether to show menu bar. | |
| 107 private boolean mMenuBarVisible; | |
| 108 | |
| 109 public CardboardRenderer(Activity activity, Client client, Display display)
{ | |
| 110 mActivity = activity; | |
| 111 mClient = client; | |
| 112 mDisplay = display; | |
| 113 mCameraPosition = 0.0f; | |
| 114 | |
| 115 mCameraMatrix = new float[16]; | |
| 116 mViewMatrix = new float[16]; | |
| 117 mProjectionMatrix = new float[16]; | |
| 118 mDesktopModelMatrix = new float[16]; | |
| 119 mDesktopCombinedMatrix = new float[16]; | |
| 120 mEyePointModelMatrix = new float[16]; | |
| 121 mEyePointCombinedMatrix = new float[16]; | |
| 122 mPhotosphereCombinedMatrix = new float[16]; | |
| 123 | |
| 124 mForwardVector = new float[3]; | |
| 125 } | |
| 126 | |
| 127 private void initializeRedrawCallback() { | |
| 128 mActivity.runOnUiThread(new Runnable() { | |
| 129 public void run() { | |
| 130 mDisplay.provideRedrawCallback(new Runnable() { | |
| 131 @Override | |
| 132 public void run() { | |
| 133 mDesktop.reloadTexture(); | |
| 134 mCursor.reloadTexture(); | |
| 135 } | |
| 136 }); | |
| 137 | |
| 138 mDisplay.redrawGraphics(); | |
| 139 } | |
| 140 }); | |
| 141 } | |
| 142 | |
| 143 @Override | |
| 144 public void onSurfaceCreated(EGLConfig config) { | |
| 145 // Set the background clear color to black. | |
| 146 GLES20.glClearColor(0.0f, 0.0f, 0.0f, 0.0f); | |
| 147 | |
| 148 // Use culling to remove back faces. | |
| 149 GLES20.glEnable(GLES20.GL_CULL_FACE); | |
| 150 | |
| 151 // Enable depth testing. | |
| 152 GLES20.glEnable(GLES20.GL_DEPTH_TEST); | |
| 153 | |
| 154 mDesktop = new Desktop(mDisplay); | |
| 155 mMenuBar = new MenuBar(mActivity); | |
| 156 mPhotosphere = new Photosphere(mActivity); | |
| 157 mCursor = new Cursor(mClient, mDisplay); | |
| 158 | |
| 159 initializeRedrawCallback(); | |
| 160 } | |
| 161 | |
| 162 @Override | |
| 163 public void onSurfaceChanged(int width, int height) { | |
| 164 } | |
| 165 | |
| 166 @Override | |
| 167 public void onNewFrame(HeadTransform headTransform) { | |
| 168 // Position the eye at the origin. | |
| 169 float eyeX = 0.0f; | |
| 170 float eyeY = 0.0f; | |
| 171 float eyeZ; | |
| 172 synchronized (mCameraPositionLock) { | |
| 173 eyeZ = mCameraPosition; | |
| 174 } | |
| 175 | |
| 176 // We are looking toward the negative Z direction. | |
| 177 float lookX = DESKTOP_POSITION_X; | |
| 178 float lookY = DESKTOP_POSITION_Y; | |
| 179 float lookZ = DESKTOP_POSITION_Z; | |
| 180 | |
| 181 // Set our up vector. This is where our head would be pointing were we h
olding the camera. | |
| 182 float upX = 0.0f; | |
| 183 float upY = 1.0f; | |
| 184 float upZ = 0.0f; | |
| 185 | |
| 186 Matrix.setLookAtM(mCameraMatrix, 0, eyeX, eyeY, eyeZ, lookX, lookY, look
Z, upX, upY, upZ); | |
| 187 | |
| 188 headTransform.getForwardVector(mForwardVector, 0); | |
| 189 mEyeDesktopPosition = getLookingPosition(Math.abs(DESKTOP_POSITION_Z - e
yeZ)); | |
| 190 mEyeMenuBarPosition = getLookingPosition(Math.abs(MENU_BAR_POSITION_Z)); | |
| 191 mDesktop.maybeLoadDesktopTexture(); | |
| 192 mPhotosphere.maybeLoadTextureAndCleanImage(); | |
| 193 mCursor.maybeLoadTexture(mDesktop); | |
| 194 mCursor.moveTo(getMouseCoordinates()); | |
| 195 } | |
| 196 | |
| 197 @Override | |
| 198 public void onDrawEye(Eye eye) { | |
| 199 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT); | |
| 200 | |
| 201 // Apply the eye transformation to the camera. | |
| 202 Matrix.multiplyMM(mViewMatrix, 0, eye.getEyeView(), 0, mCameraMatrix, 0)
; | |
| 203 | |
| 204 mProjectionMatrix = eye.getPerspective(Z_NEAR, Z_FAR); | |
| 205 | |
| 206 drawDesktop(); | |
| 207 drawPhotosphere(eye.getType()); | |
| 208 drawMenuBar(); | |
| 209 drawCursor(); | |
| 210 } | |
| 211 | |
| 212 @Override | |
| 213 public void onRendererShutdown() { | |
| 214 mDesktop.cleanup(); | |
| 215 mMenuBar.cleanup(); | |
| 216 mPhotosphere.cleanup(); | |
| 217 mCursor.cleanup(); | |
| 218 } | |
| 219 | |
| 220 @Override | |
| 221 public void onFinishFrame(Viewport viewport) { | |
| 222 } | |
| 223 | |
| 224 private void drawCursor() { | |
| 225 if (!isLookingAtDesktop() || (isMenuBarVisible() && isLookingAtMenuBar()
) | |
| 226 || !mCursor.hasImageFrame()) { | |
| 227 return; | |
| 228 } | |
| 229 | |
| 230 float eyePointX = clamp(mEyeDesktopPosition.x, -mDesktop.getHalfWidth(), | |
| 231 mDesktop.getHalfWidth()); | |
| 232 float eyePointY = clamp(mEyeDesktopPosition.y, -mDesktop.getHalfHeight()
, | |
| 233 mDesktop.getHalfHeight()); | |
| 234 | |
| 235 Matrix.setIdentityM(mEyePointModelMatrix, 0); | |
| 236 Matrix.translateM(mEyePointModelMatrix, 0, eyePointX , eyePointY, DESKTO
P_POSITION_Z); | |
| 237 | |
| 238 Matrix.multiplyMM(mEyePointCombinedMatrix, 0, mViewMatrix, 0, mEyePointM
odelMatrix, 0); | |
| 239 Matrix.multiplyMM(mEyePointCombinedMatrix, 0, mProjectionMatrix, | |
| 240 0, mEyePointCombinedMatrix, 0); | |
| 241 | |
| 242 mCursor.draw(mEyePointCombinedMatrix); | |
| 243 } | |
| 244 | |
| 245 private void drawPhotosphere(int eyeType) { | |
| 246 // Since we will always put the photosphere center in the origin, the | |
| 247 // model matrix will always be identity matrix which we can ignore. | |
| 248 Matrix.multiplyMM(mPhotosphereCombinedMatrix, 0, mProjectionMatrix, | |
| 249 0, mViewMatrix, 0); | |
| 250 | |
| 251 mPhotosphere.draw(mPhotosphereCombinedMatrix, eyeType); | |
| 252 } | |
| 253 | |
| 254 private void drawDesktop() { | |
| 255 if (!mDesktop.hasVideoFrame()) { | |
| 256 // This can happen if the client is connected, but a complete | |
| 257 // video frame has not yet been decoded. | |
| 258 return; | |
| 259 } | |
| 260 | |
| 261 Matrix.setIdentityM(mDesktopModelMatrix, 0); | |
| 262 Matrix.translateM(mDesktopModelMatrix, 0, DESKTOP_POSITION_X, | |
| 263 DESKTOP_POSITION_Y, DESKTOP_POSITION_Z); | |
| 264 | |
| 265 // Pass in Model View Matrix and Model View Project Matrix. | |
| 266 Matrix.multiplyMM(mDesktopCombinedMatrix, 0, mViewMatrix, 0, mDesktopMod
elMatrix, 0); | |
| 267 Matrix.multiplyMM(mDesktopCombinedMatrix, 0, mProjectionMatrix, | |
| 268 0, mDesktopCombinedMatrix, 0); | |
| 269 | |
| 270 mDesktop.draw(mDesktopCombinedMatrix, mMenuBarVisible); | |
| 271 } | |
| 272 | |
| 273 private void drawMenuBar() { | |
| 274 if (!mMenuBarVisible) { | |
| 275 return; | |
| 276 } | |
| 277 | |
| 278 float menuBarZ; | |
| 279 synchronized (mCameraPositionLock) { | |
| 280 menuBarZ = mCameraPosition + MENU_BAR_POSITION_Z; | |
| 281 } | |
| 282 | |
| 283 | |
| 284 mMenuBar.draw(mViewMatrix, mProjectionMatrix, mEyeMenuBarPosition, MENU_
BAR_POSITION_X, | |
| 285 MENU_BAR_POSITION_Y, menuBarZ); | |
| 286 } | |
| 287 | |
| 288 /** | |
| 289 * Return menu item that is currently looking at or null if not looking at m
enu bar. | |
| 290 */ | |
| 291 public MenuItem getMenuItem() { | |
| 292 // Transform world view to model view. | |
| 293 return mMenuBar.getLookingItem(new PointF(mEyeMenuBarPosition.x - MENU_B
AR_POSITION_X, | |
| 294 mEyeMenuBarPosition.y - MENU_BAR_POSITION_Y)); | |
| 295 } | |
| 296 | |
| 297 /** | |
| 298 * Returns coordinates in units of pixels in the desktop bitmap. | |
| 299 * This can be called on any thread. | |
| 300 */ | |
| 301 public PointF getMouseCoordinates() { | |
| 302 PointF result = new PointF(); | |
| 303 Point shapePixels = mDesktop.getFrameSizePixels(); | |
| 304 int widthPixels = shapePixels.x; | |
| 305 int heightPixels = shapePixels.y; | |
| 306 | |
| 307 synchronized (mEyeDesktopPositionLock) { | |
| 308 // Due to the coordinate direction, we only have to inverse x. | |
| 309 result.x = (mEyeDesktopPosition.x + mDesktop.getHalfWidth()) | |
| 310 / (2 * mDesktop.getHalfWidth()) * widthPixels; | |
| 311 result.y = (-mEyeDesktopPosition.y + mDesktop.getHalfHeight()) | |
| 312 / (2 * mDesktop.getHalfHeight()) * heightPixels; | |
| 313 result.x = clamp(result.x, 0, widthPixels); | |
| 314 result.y = clamp(result.y, 0, heightPixels); | |
| 315 } | |
| 316 return result; | |
| 317 } | |
| 318 | |
| 319 /** | |
| 320 * Returns the passed in value if it resides within the specified range (inc
lusive). If not, | |
| 321 * it will return the closest boundary from the range. The ordering of the
boundary values | |
| 322 * does not matter. | |
| 323 * | |
| 324 * @param value The value to be compared against the range. | |
| 325 * @param a First boundary range value. | |
| 326 * @param b Second boundary range value. | |
| 327 * @return The passed in value if it is within the range, otherwise the clos
est boundary value. | |
| 328 */ | |
| 329 private static float clamp(float value, float a, float b) { | |
| 330 float min = (a > b) ? b : a; | |
| 331 float max = (a > b) ? a : b; | |
| 332 if (value < min) { | |
| 333 value = min; | |
| 334 } else if (value > max) { | |
| 335 value = max; | |
| 336 } | |
| 337 return value; | |
| 338 } | |
| 339 | |
| 340 /** | |
| 341 * Move the camera towards desktop. | |
| 342 * This method can be called on any thread. | |
| 343 */ | |
| 344 public void moveTowardsDesktop() { | |
| 345 synchronized (mCameraPositionLock) { | |
| 346 float newPosition = mCameraPosition - CAMERA_MOTION_STEP; | |
| 347 if (newPosition >= VIEW_POSITION_MIN) { | |
| 348 mCameraPosition = newPosition; | |
| 349 } | |
| 350 } | |
| 351 } | |
| 352 | |
| 353 /** | |
| 354 * Move the camera away from desktop. | |
| 355 * This method can be called on any thread. | |
| 356 */ | |
| 357 public void moveAwayFromDesktop() { | |
| 358 synchronized (mCameraPositionLock) { | |
| 359 float newPosition = mCameraPosition + CAMERA_MOTION_STEP; | |
| 360 if (newPosition <= VIEW_POSITION_MAX) { | |
| 361 mCameraPosition = newPosition; | |
| 362 } | |
| 363 } | |
| 364 } | |
| 365 | |
| 366 /** | |
| 367 * Return true if user is looking at the desktop. | |
| 368 * This method can be called on any thread. | |
| 369 */ | |
| 370 public boolean isLookingAtDesktop() { | |
| 371 synchronized (mEyeDesktopPositionLock) { | |
| 372 // TODO(shichengfeng): Move logic to CardboardActivityDesktop. | |
| 373 return mForwardVector[2] < 0 | |
| 374 && Math.abs(mEyeDesktopPosition.x) <= (mDesktop.getHalfWidth
() + EDGE_MARGIN) | |
| 375 && Math.abs(mEyeDesktopPosition.y) <= (mDesktop.getHalfHeigh
t() + EDGE_MARGIN); | |
| 376 } | |
| 377 } | |
| 378 | |
| 379 /** | |
| 380 * Return true if user is looking at the menu bar. | |
| 381 */ | |
| 382 public boolean isLookingAtMenuBar() { | |
| 383 return mForwardVector[2] < 0 | |
| 384 && mMenuBar.contains(new PointF(mEyeMenuBarPosition.x - MENU_BAR
_POSITION_X, | |
| 385 mEyeMenuBarPosition.y - MENU_BAR_POSITION_Y)); | |
| 386 } | |
| 387 | |
| 388 /** | |
| 389 * Get eye position at the given distance. | |
| 390 */ | |
| 391 private PointF getLookingPosition(float distance) { | |
| 392 if (Math.abs(mForwardVector[2]) < EPSILON) { | |
| 393 return new PointF(Math.copySign(Float.MAX_VALUE, mForwardVector[0]), | |
| 394 Math.copySign(Float.MAX_VALUE, mForwardVector[1])); | |
| 395 } else { | |
| 396 return new PointF(mForwardVector[0] * distance / mForwardVector[2], | |
| 397 mForwardVector[1] * distance / mForwardVector[2]); | |
| 398 } | |
| 399 } | |
| 400 | |
| 401 /** | |
| 402 * Set the visibility of the menu bar. | |
| 403 */ | |
| 404 public void setMenuBarVisible(boolean visible) { | |
| 405 mMenuBarVisible = visible; | |
| 406 } | |
| 407 | |
| 408 /** | |
| 409 * Return true if menu bar is visible. | |
| 410 */ | |
| 411 public boolean isMenuBarVisible() { | |
| 412 return mMenuBarVisible; | |
| 413 } | |
| 414 | |
| 415 | |
| 416 /** | |
| 417 * Return true if user is looking faraway from desktop. | |
| 418 */ | |
| 419 public boolean isLookingFarawayFromDesktop() { | |
| 420 if (mForwardVector[2] > -EPSILON) { | |
| 421 // If user is looking towards the back. | |
| 422 return true; | |
| 423 } | |
| 424 | |
| 425 // Calculate half desktop looking angle. | |
| 426 double theta; | |
| 427 synchronized (mCameraPositionLock) { | |
| 428 theta = Math.atan(mDesktop.getHalfWidth() | |
| 429 / (DESKTOP_POSITION_Z - mCameraPosition)); | |
| 430 } | |
| 431 | |
| 432 // Calculate current looking angle. | |
| 433 double phi = Math.atan(mForwardVector[0] / mForwardVector[2]); | |
| 434 | |
| 435 return Math.abs(phi) > FARAWAY_ANGLE_RATIO * Math.abs(theta); | |
| 436 } | |
| 437 } | |
| OLD | NEW |