Allow Desktop Canvas to be scrolled out from under System UI.

remoting/android/java/src/org/chromium/chromoting/DesktopCanvas.java

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package org.chromium.chromoting;
 
 import android.graphics.Matrix;
 import android.graphics.PointF;
 import android.graphics.Rect;
 import android.graphics.RectF;
 
 /**
  * This class is responsible for transforming the desktop image matrix.
  */
 public class DesktopCanvas {
     /**
-     * Maximum allowed zoom level - see {@link #repositionImageWithZoom()}.
+     * Maximum allowed zoom level - see {@link #scaleAndRepositionImage()}.
      */
     private static final float MAX_ZOOM_FACTOR = 100.0f;
 
+    /**
+     * Used to smoothly reduce the amount of padding while the user is zooming.
+     */
+    private static final float PADDING_REDUCTION_FACTOR = 0.85f;
+
     private final RenderStub mRenderStub;
     private final RenderData mRenderData;
 
     /**
-     * Represents the actual center of the viewport.  This value needs to be a pair of floats so the
-     * desktop image can be positioned with sub-pixel accuracy for smoother panning animations at
-     * high zoom levels.
+     * Represents the actual center of the viewport in image space.  This value needs to be a pair
+     * of floats so the desktop image can be positioned with sub-pixel accuracy for smoother panning
+     * animations at high zoom levels.
      */
     private PointF mViewportPosition = new PointF();

[Lambros, 2016/10/04 22:06:35]

Optional, not really part of this CL:
Is there any reason we need to store this? Can't it always be calculated from
mCursorPosition?

[joedow, 2016/10/04 23:32:11]

This was more important before I rewrote the positioning code.  I will follow up
and see if I can remove it in a later CL.

 
     /**
-     * Represents the desired center of the viewport.  This value may not represent the actual
-     * center of the viewport as adjustments are made to ensure as much of the desktop is visible as
-     * possible.  This value needs to be a pair of floats so the desktop image can be positioned
-     * with sub-pixel accuracy for smoother panning animations at high zoom levels.
+     * Represents the desired center of the viewport in image space.  This value may not represent

[Lambros, 2016/10/04 22:06:35]

Clarify that this position may extend beyond the boundary of the image, even
though the injected cursor position is restricted to the image rectangle.

[joedow, 2016/10/04 23:32:11]

Done.

+     * the actual center of the viewport as adjustments are made to ensure as much of the desktop is
+     * visible as possible.  This value needs to be a pair of floats so the desktop image can be
+     * positioned with sub-pixel accuracy for smoother panning animations at high zoom levels.
      */
     private PointF mCursorPosition = new PointF();
 
     /**
-     * Represents the amount of space, in pixels, used by System UI.
+     * Represents the amount of space, in pixels, used by System UI on each edge of the screen.
      */
-    private Rect mSystemUiOffsetPixels = new Rect();
+    private Rect mSystemUiScreenSize = new Rect();

[Lambros, 2016/10/04 22:06:35]

Now you've renamed this to remove 'offset', maybe clarify that this is not a
true rectangle?

[joedow, 2016/10/04 23:32:11]

Acknowledged.  Added TODO.

+
+    /**
+     * Represents the amount of padding, in screen pixels, added to each edge of the desktop image.
+     * This extra space allows the user to reveal portions of the desktop image which are obscured
+     * by System UI.
+     */
+    private RectF mVisibleImagePadding = new RectF();

[Lambros, 2016/10/04 22:06:35]

optional:
Would be nice if you could remove this member.
It's currently circular: it is determined by reality, and also updates reality.
You could just calculate it from reality everywhere, and then have a method,
reduceVisibleImagePadding() that figures out how to change reality to reduce the
padding by PADDING_REDUCTION_FACTOR.

[joedow, 2016/10/04 23:32:11]

Acknowledged.

 
     public DesktopCanvas(RenderStub renderStub, RenderData renderData) {
         mRenderStub = renderStub;
         mRenderData = renderData;
     }
 
     /**
      * Sets the desired center position of the viewport (a.k.a. the cursor position) and ensures
      * the viewport is updated to include the cursor within it.
      *
@@ skipping 44 matching lines @@
 
     /**
      * Sets the offset values used to calculate the space used by System UI.
      *
      * @param left The space used by System UI on the left edge of the screen.
      * @param top The space used by System UI on the top edge of the screen.
      * @param right The space used by System UI on the right edge of the screen.
      * @param bottom The space used by System UI on the bottom edge of the screen.
      */
     public void setSystemUiOffsetValues(int left, int top, int right, int bottom) {
-        mSystemUiOffsetPixels.set(left, top, right, bottom);
+        mSystemUiScreenSize.set(left, top, right, bottom);
     }
 
     /** Called to indicate that no System UI is visible. */
     public void clearSystemUiOffsets() {
-        mSystemUiOffsetPixels.setEmpty();
+        mSystemUiScreenSize.setEmpty();
     }
 
     /** Resizes the image by zooming it such that the image is displayed without borders. */
     public void resizeImageToFitScreen() {
         // Protect against being called before the image has been initialized.
         if (mRenderData.imageWidth == 0 || mRenderData.imageHeight == 0) {
             return;
         }
 
         float widthRatio = (float) mRenderData.screenWidth / mRenderData.imageWidth;
@@ skipping 39 matching lines @@
         mRenderData.transform.mapVectors(imageSize);
 
         if (imageSize[0] < mRenderData.screenWidth && imageSize[1] < mRenderData.screenHeight) {
             // Displayed image is too small in both directions, so apply the minimum zoom
             // level needed to fit either the width or height.
             float scale = Math.min((float) mRenderData.screenWidth / mRenderData.imageWidth,
                                    (float) mRenderData.screenHeight / mRenderData.imageHeight);
             mRenderData.transform.setScale(scale, scale);
         }
 
+        // Trim the image padding if the user is zooming out.  This prevents cases where the image
+        // pops to the center when it reaches its minimum size.  Note that we do not need to do
+        // anything when the user is zooming in as the canvas will expand and absorb the padding.
+        if (scaleFactor < 1.0f) {
+            mVisibleImagePadding.set(mVisibleImagePadding.left * PADDING_REDUCTION_FACTOR,
+                    mVisibleImagePadding.top * PADDING_REDUCTION_FACTOR,
+                    mVisibleImagePadding.right * PADDING_REDUCTION_FACTOR,
+                    mVisibleImagePadding.bottom * PADDING_REDUCTION_FACTOR);
+        }
+
         if (centerOnCursor) {
             setCursorPosition(mCursorPosition.x, mCursorPosition.y);
         } else {
             // Find the new screen center (it was probably changed during the zoom operation) and
             // update the viewport and cursor.
             float[] mappedPoints = {
                     ((float) mRenderData.screenWidth / 2), ((float) mRenderData.screenHeight / 2)};
             Matrix screenToImage = new Matrix();
             mRenderData.transform.invert(screenToImage);
             screenToImage.mapPoints(mappedPoints);
             // The cursor is mapped to the center of the viewport in this case.
             setCursorPosition(mappedPoints[0], mappedPoints[1]);
         }
     }
 
     /**
      * Repositions the image by translating it (without affecting the zoom level).
      */
     private void repositionImage() {
         // Map the current viewport position to screen coordinates and adjust the image position.
         float[] viewportPosition = {mViewportPosition.x, mViewportPosition.y};
         mRenderData.transform.mapPoints(viewportPosition);
 
         float viewportTransX = ((float) mRenderData.screenWidth / 2) - viewportPosition[0];
         float viewportTransY = ((float) mRenderData.screenHeight / 2) - viewportPosition[1];
 
         // Translate the image so the viewport center is displayed in the middle of the screen.
         mRenderData.transform.postTranslate(viewportTransX, viewportTransY);
 
+        updateVisibleImagePadding();
+
         mRenderStub.setTransformation(mRenderData.transform);
     }
 
     /**
      * Updates the given point such that it refers to a coordinate within the bounds provided.
      *
      * @param point The point to adjust, the original object is modified.
      * @param bounds The bounds used to constrain the point.
      */
     private void constrainPointToBounds(PointF point, RectF bounds) {
         if (point.x < bounds.left) {
             point.x = bounds.left;
         } else if (point.x > bounds.right) {
             point.x = bounds.right;
         }
 
         if (point.y < bounds.top) {
             point.y = bounds.top;
         } else if (point.y > bounds.bottom) {
             point.y = bounds.bottom;
         }
     }
 
-    /** Returns a region which defines the set of valid cursor values in image space. */
+    /** Returns a region which defines the set of valid cursor positions in image space. */
     private RectF getImageBounds() {
-        return new RectF(0, 0, mRenderData.imageWidth, mRenderData.imageHeight);
+        // The set of valid cursor positions includes any point on the image as well as the padding.
+        // Padding is additional space added to the image which is the larger value of:
+        //   - Potential overlap of the System UI and image content
+        //   - Actual amount of padding already being used
+        //
+        // By expanding the area, we allow the user to move the cursor 'under' the System UI which
+        // pulls the content out from under it and allows it to be visible.  Once the System UI has
+        // been dismissed or changes size, we use the actual padding value instead which prevents
+        // the desktop image from 'snapping' back to pre-System UI state.
+        RectF systemUiOverlap = getSystemUiOverlap();
+        float[] padding = {Math.max(mVisibleImagePadding.left, systemUiOverlap.left),
+                Math.max(mVisibleImagePadding.top, systemUiOverlap.top),
+                Math.max(mVisibleImagePadding.right, systemUiOverlap.right),
+                Math.max(mVisibleImagePadding.bottom, systemUiOverlap.bottom)};
+        Matrix screenToImage = new Matrix();
+        mRenderData.transform.invert(screenToImage);
+        screenToImage.mapVectors(padding);
+
+        return new RectF(-padding[0], -padding[1], mRenderData.imageWidth + padding[2],
+                mRenderData.imageHeight + padding[3]);
     }
 
     /** Returns a region which defines the set of valid viewport center values in image space. */
     private RectF getViewportBounds() {
-        float[] screenVectors = {(float) mRenderData.screenWidth, (float) mRenderData.screenHeight};
+        // The region of allowable viewport values is the imageBound rect, inset by the size of the
+        // viewport itself.  This prevent over and under panning of the viewport while still

[Lambros, 2016/10/04 22:06:35]

prevents

[joedow, 2016/10/04 23:32:11]

Done.

+        // allowing the user to see and interact with all pixels one the desktop image.
         Matrix screenToImage = new Matrix();
         mRenderData.transform.invert(screenToImage);
+
+        float[] screenVectors = {(float) mRenderData.screenWidth, (float) mRenderData.screenHeight};
         screenToImage.mapVectors(screenVectors);
 
-        float xAdjust = 0.0f;
-        if (mRenderData.imageWidth < screenVectors[0]) {
-            // Image is narrower than the screen, so adjust the bounds to center it.
-            xAdjust = (screenVectors[0] - mRenderData.imageWidth) / 2.0f;
-        }
-
-        float yAdjust = 0.0f;
-        if (mRenderData.imageHeight < screenVectors[1]) {
-            // Image is shorter than the screen, so adjust the bounds to center it.
-            yAdjust = (screenVectors[1] - mRenderData.imageHeight) / 2.0f;
-        }
+        PointF letterboxPadding = getLetterboxPadding();
+        float[] letterboxPaddingVectors = {letterboxPadding.x, letterboxPadding.y};
+        screenToImage.mapVectors(letterboxPaddingVectors);
 
         // screenCenter values are 1/2 of a particular screen dimension mapped to image space.
-        float screenCenterX = screenVectors[0] / 2.0f;
-        float screenCenterY = screenVectors[1] / 2.0f;
-        return new RectF(screenCenterX - xAdjust, screenCenterY - yAdjust,
-                mRenderData.imageWidth - screenCenterX + xAdjust,
-                mRenderData.imageHeight - screenCenterY + yAdjust);
+        float screenCenterX = (screenVectors[0] / 2.0f) - letterboxPaddingVectors[0];
+        float screenCenterY = (screenVectors[1] / 2.0f) - letterboxPaddingVectors[1];
+        RectF imageBounds = getImageBounds();
+        imageBounds.inset(screenCenterX, screenCenterY);
+        return imageBounds;
+    }
+
+    /**
+     * Provides the amount of padding needed to center the image content on the screen.
+     */
+    private PointF getLetterboxPadding() {
+        float[] imageVectors = {mRenderData.imageWidth, mRenderData.imageHeight};
+        mRenderData.transform.mapVectors(imageVectors);
+
+        // We want to letterbox when the image is smaller than the screen in a specific dimension.
+        // Since we center the image, split the difference so it is equally distributed.
+        float widthAdjust =
+                Math.max(((float) mRenderData.screenWidth - imageVectors[0]) / 2.0f, 0.0f);
+        float heightAdjust =
+                Math.max(((float) mRenderData.screenHeight - imageVectors[1]) / 2.0f, 0.0f);
+
+        return new PointF(widthAdjust, heightAdjust);
+    }
+
+    /**
+     * Returns the amount of System UI along each edge of the screen which could overlap the remote
+     * desktop image below it.  This is the maximum amount that could overlap, not the actual value.
+     */
+    private RectF getSystemUiOverlap() {
+        // letterBox padding represents the space added to the image to center it on the screen.
+        // Since it does not contain any interactable UI, we ignore it when calculating the overlap
+        // between the System UI and the remote desktop image.
+        // Note: Ignore negative padding (clamp to 0) since that means no overlap exists.
+        PointF letterboxPadding = getLetterboxPadding();
+        return new RectF(Math.max(mSystemUiScreenSize.left - letterboxPadding.x, 0.0f),
+                Math.max(mSystemUiScreenSize.top - letterboxPadding.y, 0.0f),
+                Math.max(mSystemUiScreenSize.right - letterboxPadding.x, 0.0f),
+                Math.max(mSystemUiScreenSize.bottom - letterboxPadding.y, 0.0f));
+    }
+
+    /**
+     * Calculates the amount of padding visible on each edge of the desktop image.
+     */
+    private void updateVisibleImagePadding() {
+        PointF letterboxPadding = getLetterboxPadding();
+        float[] imagePoints = {0.0f, 0.0f, mRenderData.imageWidth, mRenderData.imageHeight};
+        mRenderData.transform.mapPoints(imagePoints);
+
+        mVisibleImagePadding.set(Math.max(imagePoints[0] - letterboxPadding.x, 0.0f),
+                Math.max(imagePoints[1] - letterboxPadding.y, 0.0f),
+                Math.max(mRenderData.screenWidth - imagePoints[2] - letterboxPadding.x, 0.0f),
+                Math.max(mRenderData.screenHeight - imagePoints[3] - letterboxPadding.y, 0.0f));
     }
 }