OLD | NEW |
| (Empty) |
1 // Copyright 2015 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 package org.chromium.chromoting.cardboard; | |
6 | |
7 import android.app.Activity; | |
8 import android.graphics.Point; | |
9 import android.graphics.PointF; | |
10 import android.opengl.GLES20; | |
11 import android.opengl.Matrix; | |
12 | |
13 import com.google.vrtoolkit.cardboard.CardboardView; | |
14 import com.google.vrtoolkit.cardboard.Eye; | |
15 import com.google.vrtoolkit.cardboard.HeadTransform; | |
16 import com.google.vrtoolkit.cardboard.Viewport; | |
17 | |
18 import org.chromium.chromoting.jni.Client; | |
19 import org.chromium.chromoting.jni.Display; | |
20 | |
21 import javax.microedition.khronos.egl.EGLConfig; | |
22 | |
23 /** | |
24 * Renderer for Cardboard view. | |
25 */ | |
26 public class CardboardRenderer implements CardboardView.StereoRenderer { | |
27 private static final String TAG = "cr.CardboardRenderer"; | |
28 | |
29 private static final int BYTE_PER_FLOAT = 4; | |
30 private static final int POSITION_DATA_SIZE = 3; | |
31 private static final int TEXTURE_COORDINATE_DATA_SIZE = 2; | |
32 private static final float Z_NEAR = 0.1f; | |
33 private static final float Z_FAR = 1000.0f; | |
34 | |
35 // Desktop position is fixed in world coordinates. | |
36 private static final float DESKTOP_POSITION_X = 0.0f; | |
37 private static final float DESKTOP_POSITION_Y = 0.0f; | |
38 private static final float DESKTOP_POSITION_Z = -2.0f; | |
39 | |
40 // Menu bar position is relative to the view point. | |
41 private static final float MENU_BAR_POSITION_X = 0.0f; | |
42 private static final float MENU_BAR_POSITION_Y = 0.0f; | |
43 private static final float MENU_BAR_POSITION_Z = -0.9f; | |
44 | |
45 private static final float HALF_SKYBOX_SIZE = 100.0f; | |
46 private static final float VIEW_POSITION_MIN = -1.0f; | |
47 private static final float VIEW_POSITION_MAX = 3.0f; | |
48 | |
49 // Allows user to click even when looking outside the desktop | |
50 // but within edge margin. | |
51 private static final float EDGE_MARGIN = 0.1f; | |
52 | |
53 // Distance to move camera each time. | |
54 private static final float CAMERA_MOTION_STEP = 0.5f; | |
55 | |
56 // This ratio is used by {@link isLookingFarawayFromDesktop()} to determine
the | |
57 // angle beyond which the user is looking faraway from the desktop. | |
58 // The ratio is based on half of the desktop's angular width, as seen from | |
59 // the camera position. | |
60 // If the user triggers the button while looking faraway, this will cause th
e | |
61 // desktop to be re-positioned in the center of the view. | |
62 private static final float FARAWAY_ANGLE_RATIO = 1.6777f; | |
63 | |
64 // Small number used to avoid division-overflow or other problems with | |
65 // floating-point imprecision. | |
66 private static final float EPSILON = 1e-5f; | |
67 | |
68 private final Activity mActivity; | |
69 private final Client mClient; | |
70 private final Display mDisplay; | |
71 | |
72 private float mCameraPosition; | |
73 | |
74 // Lock to allow multithreaded access to mCameraPosition. | |
75 private final Object mCameraPositionLock = new Object(); | |
76 | |
77 private float[] mCameraMatrix; | |
78 private float[] mViewMatrix; | |
79 private float[] mProjectionMatrix; | |
80 | |
81 // Make matrix member variable to avoid unnecessary initialization. | |
82 private float[] mDesktopModelMatrix; | |
83 private float[] mDesktopCombinedMatrix; | |
84 private float[] mEyePointModelMatrix; | |
85 private float[] mEyePointCombinedMatrix; | |
86 private float[] mPhotosphereCombinedMatrix; | |
87 | |
88 // Direction that user is looking towards. | |
89 private float[] mForwardVector; | |
90 | |
91 // Eye position at the desktop distance. | |
92 private PointF mEyeDesktopPosition; | |
93 | |
94 // Eye position at the menu bar distance; | |
95 private PointF mEyeMenuBarPosition; | |
96 | |
97 private Desktop mDesktop; | |
98 private MenuBar mMenuBar; | |
99 private Photosphere mPhotosphere; | |
100 private Cursor mCursor; | |
101 | |
102 // Lock for eye position related operations. | |
103 // This protects access to mEyeDesktopPosition. | |
104 private final Object mEyeDesktopPositionLock = new Object(); | |
105 | |
106 // Flag to indicate whether to show menu bar. | |
107 private boolean mMenuBarVisible; | |
108 | |
109 public CardboardRenderer(Activity activity, Client client, Display display)
{ | |
110 mActivity = activity; | |
111 mClient = client; | |
112 mDisplay = display; | |
113 mCameraPosition = 0.0f; | |
114 | |
115 mCameraMatrix = new float[16]; | |
116 mViewMatrix = new float[16]; | |
117 mProjectionMatrix = new float[16]; | |
118 mDesktopModelMatrix = new float[16]; | |
119 mDesktopCombinedMatrix = new float[16]; | |
120 mEyePointModelMatrix = new float[16]; | |
121 mEyePointCombinedMatrix = new float[16]; | |
122 mPhotosphereCombinedMatrix = new float[16]; | |
123 | |
124 mForwardVector = new float[3]; | |
125 } | |
126 | |
127 private void initializeRedrawCallback() { | |
128 mActivity.runOnUiThread(new Runnable() { | |
129 public void run() { | |
130 mDisplay.provideRedrawCallback(new Runnable() { | |
131 @Override | |
132 public void run() { | |
133 mDesktop.reloadTexture(); | |
134 mCursor.reloadTexture(); | |
135 } | |
136 }); | |
137 | |
138 mDisplay.redrawGraphics(); | |
139 } | |
140 }); | |
141 } | |
142 | |
143 @Override | |
144 public void onSurfaceCreated(EGLConfig config) { | |
145 // Set the background clear color to black. | |
146 GLES20.glClearColor(0.0f, 0.0f, 0.0f, 0.0f); | |
147 | |
148 // Use culling to remove back faces. | |
149 GLES20.glEnable(GLES20.GL_CULL_FACE); | |
150 | |
151 // Enable depth testing. | |
152 GLES20.glEnable(GLES20.GL_DEPTH_TEST); | |
153 | |
154 mDesktop = new Desktop(mDisplay); | |
155 mMenuBar = new MenuBar(mActivity); | |
156 mPhotosphere = new Photosphere(mActivity); | |
157 mCursor = new Cursor(mClient, mDisplay); | |
158 | |
159 initializeRedrawCallback(); | |
160 } | |
161 | |
162 @Override | |
163 public void onSurfaceChanged(int width, int height) { | |
164 } | |
165 | |
166 @Override | |
167 public void onNewFrame(HeadTransform headTransform) { | |
168 // Position the eye at the origin. | |
169 float eyeX = 0.0f; | |
170 float eyeY = 0.0f; | |
171 float eyeZ; | |
172 synchronized (mCameraPositionLock) { | |
173 eyeZ = mCameraPosition; | |
174 } | |
175 | |
176 // We are looking toward the negative Z direction. | |
177 float lookX = DESKTOP_POSITION_X; | |
178 float lookY = DESKTOP_POSITION_Y; | |
179 float lookZ = DESKTOP_POSITION_Z; | |
180 | |
181 // Set our up vector. This is where our head would be pointing were we h
olding the camera. | |
182 float upX = 0.0f; | |
183 float upY = 1.0f; | |
184 float upZ = 0.0f; | |
185 | |
186 Matrix.setLookAtM(mCameraMatrix, 0, eyeX, eyeY, eyeZ, lookX, lookY, look
Z, upX, upY, upZ); | |
187 | |
188 headTransform.getForwardVector(mForwardVector, 0); | |
189 mEyeDesktopPosition = getLookingPosition(Math.abs(DESKTOP_POSITION_Z - e
yeZ)); | |
190 mEyeMenuBarPosition = getLookingPosition(Math.abs(MENU_BAR_POSITION_Z)); | |
191 mDesktop.maybeLoadDesktopTexture(); | |
192 mPhotosphere.maybeLoadTextureAndCleanImage(); | |
193 mCursor.maybeLoadTexture(mDesktop); | |
194 mCursor.moveTo(getMouseCoordinates()); | |
195 } | |
196 | |
197 @Override | |
198 public void onDrawEye(Eye eye) { | |
199 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT); | |
200 | |
201 // Apply the eye transformation to the camera. | |
202 Matrix.multiplyMM(mViewMatrix, 0, eye.getEyeView(), 0, mCameraMatrix, 0)
; | |
203 | |
204 mProjectionMatrix = eye.getPerspective(Z_NEAR, Z_FAR); | |
205 | |
206 drawDesktop(); | |
207 drawPhotosphere(eye.getType()); | |
208 drawMenuBar(); | |
209 drawCursor(); | |
210 } | |
211 | |
212 @Override | |
213 public void onRendererShutdown() { | |
214 mDesktop.cleanup(); | |
215 mMenuBar.cleanup(); | |
216 mPhotosphere.cleanup(); | |
217 mCursor.cleanup(); | |
218 } | |
219 | |
220 @Override | |
221 public void onFinishFrame(Viewport viewport) { | |
222 } | |
223 | |
224 private void drawCursor() { | |
225 if (!isLookingAtDesktop() || (isMenuBarVisible() && isLookingAtMenuBar()
) | |
226 || !mCursor.hasImageFrame()) { | |
227 return; | |
228 } | |
229 | |
230 float eyePointX = clamp(mEyeDesktopPosition.x, -mDesktop.getHalfWidth(), | |
231 mDesktop.getHalfWidth()); | |
232 float eyePointY = clamp(mEyeDesktopPosition.y, -mDesktop.getHalfHeight()
, | |
233 mDesktop.getHalfHeight()); | |
234 | |
235 Matrix.setIdentityM(mEyePointModelMatrix, 0); | |
236 Matrix.translateM(mEyePointModelMatrix, 0, eyePointX , eyePointY, DESKTO
P_POSITION_Z); | |
237 | |
238 Matrix.multiplyMM(mEyePointCombinedMatrix, 0, mViewMatrix, 0, mEyePointM
odelMatrix, 0); | |
239 Matrix.multiplyMM(mEyePointCombinedMatrix, 0, mProjectionMatrix, | |
240 0, mEyePointCombinedMatrix, 0); | |
241 | |
242 mCursor.draw(mEyePointCombinedMatrix); | |
243 } | |
244 | |
245 private void drawPhotosphere(int eyeType) { | |
246 // Since we will always put the photosphere center in the origin, the | |
247 // model matrix will always be identity matrix which we can ignore. | |
248 Matrix.multiplyMM(mPhotosphereCombinedMatrix, 0, mProjectionMatrix, | |
249 0, mViewMatrix, 0); | |
250 | |
251 mPhotosphere.draw(mPhotosphereCombinedMatrix, eyeType); | |
252 } | |
253 | |
254 private void drawDesktop() { | |
255 if (!mDesktop.hasVideoFrame()) { | |
256 // This can happen if the client is connected, but a complete | |
257 // video frame has not yet been decoded. | |
258 return; | |
259 } | |
260 | |
261 Matrix.setIdentityM(mDesktopModelMatrix, 0); | |
262 Matrix.translateM(mDesktopModelMatrix, 0, DESKTOP_POSITION_X, | |
263 DESKTOP_POSITION_Y, DESKTOP_POSITION_Z); | |
264 | |
265 // Pass in Model View Matrix and Model View Project Matrix. | |
266 Matrix.multiplyMM(mDesktopCombinedMatrix, 0, mViewMatrix, 0, mDesktopMod
elMatrix, 0); | |
267 Matrix.multiplyMM(mDesktopCombinedMatrix, 0, mProjectionMatrix, | |
268 0, mDesktopCombinedMatrix, 0); | |
269 | |
270 mDesktop.draw(mDesktopCombinedMatrix, mMenuBarVisible); | |
271 } | |
272 | |
273 private void drawMenuBar() { | |
274 if (!mMenuBarVisible) { | |
275 return; | |
276 } | |
277 | |
278 float menuBarZ; | |
279 synchronized (mCameraPositionLock) { | |
280 menuBarZ = mCameraPosition + MENU_BAR_POSITION_Z; | |
281 } | |
282 | |
283 | |
284 mMenuBar.draw(mViewMatrix, mProjectionMatrix, mEyeMenuBarPosition, MENU_
BAR_POSITION_X, | |
285 MENU_BAR_POSITION_Y, menuBarZ); | |
286 } | |
287 | |
288 /** | |
289 * Return menu item that is currently looking at or null if not looking at m
enu bar. | |
290 */ | |
291 public MenuItem getMenuItem() { | |
292 // Transform world view to model view. | |
293 return mMenuBar.getLookingItem(new PointF(mEyeMenuBarPosition.x - MENU_B
AR_POSITION_X, | |
294 mEyeMenuBarPosition.y - MENU_BAR_POSITION_Y)); | |
295 } | |
296 | |
297 /** | |
298 * Returns coordinates in units of pixels in the desktop bitmap. | |
299 * This can be called on any thread. | |
300 */ | |
301 public PointF getMouseCoordinates() { | |
302 PointF result = new PointF(); | |
303 Point shapePixels = mDesktop.getFrameSizePixels(); | |
304 int widthPixels = shapePixels.x; | |
305 int heightPixels = shapePixels.y; | |
306 | |
307 synchronized (mEyeDesktopPositionLock) { | |
308 // Due to the coordinate direction, we only have to inverse x. | |
309 result.x = (mEyeDesktopPosition.x + mDesktop.getHalfWidth()) | |
310 / (2 * mDesktop.getHalfWidth()) * widthPixels; | |
311 result.y = (-mEyeDesktopPosition.y + mDesktop.getHalfHeight()) | |
312 / (2 * mDesktop.getHalfHeight()) * heightPixels; | |
313 result.x = clamp(result.x, 0, widthPixels); | |
314 result.y = clamp(result.y, 0, heightPixels); | |
315 } | |
316 return result; | |
317 } | |
318 | |
319 /** | |
320 * Returns the passed in value if it resides within the specified range (inc
lusive). If not, | |
321 * it will return the closest boundary from the range. The ordering of the
boundary values | |
322 * does not matter. | |
323 * | |
324 * @param value The value to be compared against the range. | |
325 * @param a First boundary range value. | |
326 * @param b Second boundary range value. | |
327 * @return The passed in value if it is within the range, otherwise the clos
est boundary value. | |
328 */ | |
329 private static float clamp(float value, float a, float b) { | |
330 float min = (a > b) ? b : a; | |
331 float max = (a > b) ? a : b; | |
332 if (value < min) { | |
333 value = min; | |
334 } else if (value > max) { | |
335 value = max; | |
336 } | |
337 return value; | |
338 } | |
339 | |
340 /** | |
341 * Move the camera towards desktop. | |
342 * This method can be called on any thread. | |
343 */ | |
344 public void moveTowardsDesktop() { | |
345 synchronized (mCameraPositionLock) { | |
346 float newPosition = mCameraPosition - CAMERA_MOTION_STEP; | |
347 if (newPosition >= VIEW_POSITION_MIN) { | |
348 mCameraPosition = newPosition; | |
349 } | |
350 } | |
351 } | |
352 | |
353 /** | |
354 * Move the camera away from desktop. | |
355 * This method can be called on any thread. | |
356 */ | |
357 public void moveAwayFromDesktop() { | |
358 synchronized (mCameraPositionLock) { | |
359 float newPosition = mCameraPosition + CAMERA_MOTION_STEP; | |
360 if (newPosition <= VIEW_POSITION_MAX) { | |
361 mCameraPosition = newPosition; | |
362 } | |
363 } | |
364 } | |
365 | |
366 /** | |
367 * Return true if user is looking at the desktop. | |
368 * This method can be called on any thread. | |
369 */ | |
370 public boolean isLookingAtDesktop() { | |
371 synchronized (mEyeDesktopPositionLock) { | |
372 // TODO(shichengfeng): Move logic to CardboardActivityDesktop. | |
373 return mForwardVector[2] < 0 | |
374 && Math.abs(mEyeDesktopPosition.x) <= (mDesktop.getHalfWidth
() + EDGE_MARGIN) | |
375 && Math.abs(mEyeDesktopPosition.y) <= (mDesktop.getHalfHeigh
t() + EDGE_MARGIN); | |
376 } | |
377 } | |
378 | |
379 /** | |
380 * Return true if user is looking at the menu bar. | |
381 */ | |
382 public boolean isLookingAtMenuBar() { | |
383 return mForwardVector[2] < 0 | |
384 && mMenuBar.contains(new PointF(mEyeMenuBarPosition.x - MENU_BAR
_POSITION_X, | |
385 mEyeMenuBarPosition.y - MENU_BAR_POSITION_Y)); | |
386 } | |
387 | |
388 /** | |
389 * Get eye position at the given distance. | |
390 */ | |
391 private PointF getLookingPosition(float distance) { | |
392 if (Math.abs(mForwardVector[2]) < EPSILON) { | |
393 return new PointF(Math.copySign(Float.MAX_VALUE, mForwardVector[0]), | |
394 Math.copySign(Float.MAX_VALUE, mForwardVector[1])); | |
395 } else { | |
396 return new PointF(mForwardVector[0] * distance / mForwardVector[2], | |
397 mForwardVector[1] * distance / mForwardVector[2]); | |
398 } | |
399 } | |
400 | |
401 /** | |
402 * Set the visibility of the menu bar. | |
403 */ | |
404 public void setMenuBarVisible(boolean visible) { | |
405 mMenuBarVisible = visible; | |
406 } | |
407 | |
408 /** | |
409 * Return true if menu bar is visible. | |
410 */ | |
411 public boolean isMenuBarVisible() { | |
412 return mMenuBarVisible; | |
413 } | |
414 | |
415 | |
416 /** | |
417 * Return true if user is looking faraway from desktop. | |
418 */ | |
419 public boolean isLookingFarawayFromDesktop() { | |
420 if (mForwardVector[2] > -EPSILON) { | |
421 // If user is looking towards the back. | |
422 return true; | |
423 } | |
424 | |
425 // Calculate half desktop looking angle. | |
426 double theta; | |
427 synchronized (mCameraPositionLock) { | |
428 theta = Math.atan(mDesktop.getHalfWidth() | |
429 / (DESKTOP_POSITION_Z - mCameraPosition)); | |
430 } | |
431 | |
432 // Calculate current looking angle. | |
433 double phi = Math.atan(mForwardVector[0] / mForwardVector[2]); | |
434 | |
435 return Math.abs(phi) > FARAWAY_ANGLE_RATIO * Math.abs(theta); | |
436 } | |
437 } | |
OLD | NEW |