Emulate Android joystick scroll with synthetic mouse wheel event

content/public/android/java/src/org/chromium/content/browser/input/JoystickScrollProvider.java

Index: content/public/android/java/src/org/chromium/content/browser/input/JoystickScrollProvider.java
diff --git a/content/public/android/java/src/org/chromium/content/browser/input/JoystickScrollProvider.java b/content/public/android/java/src/org/chromium/content/browser/input/JoystickScrollProvider.java
index 1336b78b3982b6630d0ad30e90bbc4cf88c6ee73..6bf60b23f7154447983efd3793e6053ac6257396 100644
--- a/content/public/android/java/src/org/chromium/content/browser/input/JoystickScrollProvider.java
+++ b/content/public/android/java/src/org/chromium/content/browser/input/JoystickScrollProvider.java
@@ -4,17 +4,24 @@
 
 package org.chromium.content.browser.input;
 
+import android.content.Context;
+import android.os.SystemClock;
 import android.util.TypedValue;
 import android.view.InputDevice;
 import android.view.MotionEvent;
-import android.view.animation.AnimationUtils;
+import android.view.View;
 
 import org.chromium.base.Log;
-import org.chromium.content.browser.ContentViewCore;
+import org.chromium.base.annotations.CalledByNative;
+import org.chromium.base.annotations.JNINamespace;
+import org.chromium.content_public.browser.WebContents;
+
+import java.lang.ref.WeakReference;
 
 /**
  * This class implements auto scrolling and panning for gamepad left joystick motion event.
  */
+@JNINamespace("content")
 public class JoystickScrollProvider {
     private static final String TAG = "JoystickScroll";
 
@@ -24,11 +31,13 @@ public class JoystickScrollProvider {
     private static final float JOYSTICK_SCROLL_DEADZONE = 0.2f;
     private static final float SCROLL_FACTOR_FALLBACK = 128f;
 
-    private final ContentViewCore mView;
+    private View mContainerView;
+    private long mNativeJoystickScrollProvider;
 
     private float mScrollVelocityX;
     private float mScrollVelocityY;
     private float mScrollFactor;
+    private float mDipScale = 1.0f;
 
     private long mLastAnimateTimeMillis;
 
@@ -38,14 +47,23 @@ public class JoystickScrollProvider {
 
     /**
      * Constructs a new JoystickScrollProvider.
-     *
-     * @param contentview The ContentViewCore used to create this.
      */
-    public JoystickScrollProvider(ContentViewCore contentView) {
-        mView = contentView;
+    public JoystickScrollProvider() {
         mEnabled = true;
     }
 
+    public void initialize(WebContents webContents, View containerView) {

[boliu, 2016/12/07 06:06:44]

just pass in everything in the constructor? having a separate init method
doesn't feel necessary here

yes the ref won't be final anymore in CVC, but doesn't matter much I think

[Tima Vaisburd, 2016/12/08 00:35:48]

Done, I had to add the check in CVC that |mJoystickScrollProvider| is not null
everywhere though.

[boliu, 2016/12/08 06:03:09]

you don't have to, CVC isn't considered initialized until initialize is called,
CVC.initialize should be the first thing after constructor

[Tima Vaisburd, 2016/12/08 20:32:19]

Ok, removed these checks!

+        Log.v(TAG, "initialize");

[boliu, 2016/12/07 06:06:44]

remove

[Tima Vaisburd, 2016/12/08 00:35:48]

Done.

+        mNativeJoystickScrollProvider = nativeInit(webContents);
+        mContainerView = containerView;
+    }
+
+    @CalledByNative
+    private void onNativeObjectDestroyed(long nativePointer) {
+        assert nativePointer == mNativeJoystickScrollProvider;
+        mNativeJoystickScrollProvider = 0;
+    }
+
     /**
      * This function enables or disables scrolling through joystick.
      * @param enabled Decides whether joystick scrolling should be
@@ -56,6 +74,21 @@ public class JoystickScrollProvider {
         if (!enabled) stop();
     }
 
+    public void setDeviceScaleFactor(float dipScale, WeakReference<Context> displayContext) {

[boliu, 2016/12/07 06:06:44]

package

you could remove further dependency on CVC by giving WindowAndroid here, and
then directly observe the display that way

[boliu, 2016/12/08 06:03:09]

not done? at least make it package visible

[Tima Vaisburd, 2016/12/08 18:23:20]

Not yet. I want to try before giving up though. Package scope: this file is in
browser.input package and CVC is just in browser, so it has to be public,
otherwise it did not compile (I checked).

[Tima Vaisburd, 2016/12/08 20:32:19]

JoystickScrollProvider subclasses DisplayObserver in PS4. The essential code is
copied from ContentViewCore. Please take another look.

+        mDipScale = dipScale;
+
+        Context context = displayContext.get();
+        TypedValue outValue = new TypedValue();
+        if (context != null && context.getTheme().resolveAttribute(
+                android.R.attr.listPreferredItemHeight, outValue, true)) {
+            mScrollFactor = outValue.getDimension(context.getResources().getDisplayMetrics());
+        } else {
+            Log.d(TAG, "Theme attribute listPreferredItemHeight not defined"
+                            + " switching to fallback scroll factor ");
+            mScrollFactor = SCROLL_FACTOR_FALLBACK * mDipScale;
+        }
+    }
+
     /**
      * This function processes motion event and computes new
      * scroll offest in pixels which is propertional to left joystick
@@ -72,7 +105,13 @@ public class JoystickScrollProvider {
         Log.d(TAG, "Joystick left stick axis: " + event.getAxisValue(MotionEvent.AXIS_X) + ","
                 + event.getAxisValue(MotionEvent.AXIS_Y));
 
-        computeNewScrollVelocity(event);
+        assert mScrollFactor != 0;
+
+        mScrollVelocityX = getFilteredAxisValue(event, MotionEvent.AXIS_X) * mScrollFactor
+                * JOYSTICK_SCROLL_FACTOR_MULTIPLIER;
+        mScrollVelocityY = getFilteredAxisValue(event, MotionEvent.AXIS_Y) * mScrollFactor
+                * JOYSTICK_SCROLL_FACTOR_MULTIPLIER;
+
         if (mScrollVelocityX == 0 && mScrollVelocityY == 0) {
             stop();
             return false;
@@ -86,8 +125,8 @@ public class JoystickScrollProvider {
             };
         }
         if (mLastAnimateTimeMillis == 0) {
-            mView.getContainerView().postOnAnimation(mScrollRunnable);
-            mLastAnimateTimeMillis = AnimationUtils.currentAnimationTimeMillis();
+            mContainerView.postOnAnimation(mScrollRunnable);
+            mLastAnimateTimeMillis = SystemClock.uptimeMillis();

[boliu, 2016/12/07 06:06:44]

why switch this? should never tick animations using the system clock

[Tima Vaisburd, 2016/12/07 18:14:40]

I made it do be consistent with getEventTime() which returns uptimeMillis():
https://developer.android.com/reference/android/view/MotionEvent.html#getEven...

As far as I got we are not doing animations, we just post an event to UI thread
and take the time it arrives?

[boliu, 2016/12/07 18:23:21]

You are implementing a smooth scroll animation here, no? Using system clock for
animation is generally not ok, since the time may drift within the frame,
causing "non-smooth" appearance for animations.

If getEventTime is using the system clock, then that should not be used for
animations.

Fwiw, android makes the same mistake all the time. The scroller thing most views
use to implement the fling animation was using the system clock, and was only
fixed in N.

[Tima Vaisburd, 2016/12/08 00:35:48]

I put the animation clock back.

         }
         return true;
     }
@@ -96,13 +135,18 @@ public class JoystickScrollProvider {
         if (mLastAnimateTimeMillis == 0) {
             return;
         }
-        final long timeMillis = AnimationUtils.currentAnimationTimeMillis();
+        final long timeMillis = SystemClock.uptimeMillis();

[boliu, 2016/12/07 06:06:44]

ditto

         final long dt = timeMillis - mLastAnimateTimeMillis;
         final float dx = (mScrollVelocityX * dt / 1000.f);
         final float dy = (mScrollVelocityY * dt / 1000.f);
-        mView.scrollBy(dx, dy, true);
+
+        if (mNativeJoystickScrollProvider != 0) {
+            nativeScrollBy(

[boliu, 2016/12/07 06:06:44]

new code isn't exactly the same as calling scrollBy on the view, where in
webview the app could actually override the method and do something else. why is
that ok?

[Tima Vaisburd, 2016/12/07 18:14:40]

Right away I can give you a shallow answer that in the new code the scrolling
with joystick seems no different than the scrolling with mouse wheel. In
CVC.onGenericMotionEvent() we send the same mouse event in react to
ACTION_SCROLL.

[boliu, 2016/12/07 18:23:21]

That did not answer my question at all. You just restated what I said.

I mean if the belief is no app is using this, and since we have no way to
measure usage in webview, then sure, let's break and see if anyone complains.
But is that the thought here?

[Tima Vaisburd, 2016/12/07 18:54:44]

I said it was shallow.
No, honestly I did not see the problem. Still, I do not see how the WebView
subclassing matters in the original code.
When a third party app writer subclasses WebView.scrollBy() (e.g. blocks it),
does the original code still calls CVC.scrollBy(x, y, true) upon Joystick
operation? I think yes. I also think we do not call the subclassed scrollBy()
method from CVC anyhow?

[boliu, 2016/12/07 19:05:01]

Not necessarily. Depends on what the subclass implementation of scrollBy does.
If app made it no-op, then no, CVC.scrollBy won't be called.

I'm not saying there are any apps that actually do this, or that apps should be
allowed to do this. But it is a change in behavior, and I'm asking what is your
reasoning why that's ok.

[Tima Vaisburd, 2016/12/07 19:29:57]

Exactly my point, how the subclassing (blocking) of WebView.scrollBy() will
prevent the original call to mView.scrollBy(dx, dy, true) on l.103? It seems
onMotion() will still be called, it comes from CVC.onGenericMotionEvent()? Will
they need to also subclass and block WebView.onGenenericMotionEvent()?
After I understand how it is possible in the old code I would be able to say
that I missed that important case.

[boliu, 2016/12/07 19:58:30]

It won't. But since scrollBy is no-op, it will not propagate the scroll any
further, ie app intentionally do *not* want to scroll.

After this change, those apps may start seeing unwanted scrolls.

[Tima Vaisburd, 2016/12/07 20:56:03]

But I thought *that* scrollBy() (i.e. CVC.scrollBy()) is past the user's block?
In other words, the WebView.scrollBy subclassing does not stand in the chain
onMotion() -> CVC.scrollBy() -> CVC.nativeScrollBegin()?

Am I right that CVC.scrollBy(float, float, boolean) is never subclassed, only a
glue method which calls CVC.scrollBy can be subclassed?

[boliu, 2016/12/07 21:02:26]

My bad. mView on the left is ContentViewCore, not container view. Burrrrr bad
variable names.

+                    mNativeJoystickScrollProvider, timeMillis, dx / mDipScale, dy / mDipScale);
+        }
+
         mLastAnimateTimeMillis = timeMillis;
-        mView.getContainerView().postOnAnimation(mScrollRunnable);
+        mContainerView.postOnAnimation(mScrollRunnable);
     }
 
     private void stop() {
@@ -110,29 +154,6 @@ public class JoystickScrollProvider {
     }
 
     /**
-     * Translates joystick axes movement to a scroll velocity.
-     */
-    private void computeNewScrollVelocity(MotionEvent event) {
-        if (mScrollFactor == 0) {
-            TypedValue outValue = new TypedValue();
-            if (!mView.getContext().getTheme().resolveAttribute(
-                        android.R.attr.listPreferredItemHeight, outValue, true)) {
-                mScrollFactor = outValue.getDimension(
-                        mView.getContext().getResources().getDisplayMetrics());
-            } else {
-                Log.d(TAG, "Theme attribute listPreferredItemHeight not defined"
-                                + "switching to fallback scroll factor ");
-                mScrollFactor = SCROLL_FACTOR_FALLBACK
-                        * mView.getRenderCoordinates().getDeviceScaleFactor();
-            }
-        }
-        mScrollVelocityX = getFilteredAxisValue(event, MotionEvent.AXIS_X) * mScrollFactor
-                * JOYSTICK_SCROLL_FACTOR_MULTIPLIER;
-        mScrollVelocityY = getFilteredAxisValue(event, MotionEvent.AXIS_Y) * mScrollFactor
-                * JOYSTICK_SCROLL_FACTOR_MULTIPLIER;
-    }
-
-    /**
      * Removes noise from joystick motion events.
      */
     private float getFilteredAxisValue(MotionEvent event, int axis) {
@@ -143,4 +164,8 @@ public class JoystickScrollProvider {
         }
         return 0f;
     }
+
+    private native long nativeInit(WebContents webContents);
+    private native void nativeScrollBy(
+            long nativeJoystickScrollProvider, long timeMs, float dxDip, float dyDip);
 }