Native Android accessibility.

content/public/android/java/src/org/chromium/content/browser/accessibility/BrowserAccessibilityManager.java

Index: content/public/android/java/src/org/chromium/content/browser/accessibility/BrowserAccessibilityManager.java
diff --git a/content/public/android/java/src/org/chromium/content/browser/accessibility/BrowserAccessibilityManager.java b/content/public/android/java/src/org/chromium/content/browser/accessibility/BrowserAccessibilityManager.java
new file mode 100644
index 0000000000000000000000000000000000000000..edc8246d70c7c3b8825b8b1aff2ffeeb15602d50
--- /dev/null
+++ b/content/public/android/java/src/org/chromium/content/browser/accessibility/BrowserAccessibilityManager.java
@@ -0,0 +1,394 @@
+// Copyright (c) 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.content.browser.accessibility;
+
+import android.accessibilityservice.AccessibilityServiceInfo;
+import android.content.Context;
+import android.graphics.Rect;
+import android.os.Build;
+import android.os.Bundle;
+import android.os.Vibrator;
+import android.provider.Settings;
+import android.speech.tts.TextToSpeech;
+import android.util.Log;
+import android.view.MotionEvent;
+import android.view.View;
+import android.view.accessibility.AccessibilityEvent;
+import android.view.accessibility.AccessibilityManager;
+import android.view.accessibility.AccessibilityNodeInfo;
+import android.view.accessibility.AccessibilityNodeProvider;
+import android.view.inputmethod.InputMethodManager;
+
+import org.chromium.base.CalledByNative;
+import org.chromium.base.JNINamespace;
+import org.chromium.content.browser.ContentViewCore;
+import org.chromium.content.browser.RenderCoordinates;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Native accessibility for a {@link ContentViewCore}.
+ */
+@JNINamespace("content")
+public class BrowserAccessibilityManager extends AccessibilityNodeProvider {
+    private static final String TAG = BrowserAccessibilityManager.class.getSimpleName();
+
+    private ContentViewCore mContentViewCore;
+    private RenderCoordinates mRenderCoordinates;
+    private int mNativeObj;
+    private int mA11yFocusId;
+    private final int[] mTempLocation = new int[2];
+    private View mView;
+    private boolean mUserHasTouchExplored = false;
+    private boolean mFrameInfoInitialized = false;
+    private boolean mWebFocusFollowsA11yFocus = false;
+
+    @CalledByNative
+    public static BrowserAccessibilityManager create(int nativeBrowserAccessibilityManagerAndroid,
+                                                     ContentViewCore contentViewCore) {
+        Log.i("AX", "BrowserAccessibilityManager java create");
+        return new BrowserAccessibilityManager(
+            nativeBrowserAccessibilityManagerAndroid, contentViewCore);
+    }
+
+    BrowserAccessibilityManager(int nativeBrowserAccessibilityManagerAndroid,
+                                ContentViewCore contentViewCore) {
+        mNativeObj = nativeBrowserAccessibilityManagerAndroid;
+        mContentViewCore = contentViewCore;
+        mContentViewCore.setBrowserAccessibilityManager(this);
+        mA11yFocusId = View.NO_ID;
+        mView = mContentViewCore.getContainerView();
+        mRenderCoordinates = mContentViewCore.getRenderCoordinates();
+    }
+
+    @CalledByNative
+    private void onNativeObjectDestroyed() {
+        Log.i("AX", "BrowserAccessibilityManager onNativeObjectDestroyed");
+        if (mContentViewCore.getBrowserAccessibilityManager() == this)
+            mContentViewCore.setBrowserAccessibilityManager(null);
+        mNativeObj = 0;
+        mContentViewCore = null;
+    }
+
+    public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) {
+        int rootId = nativeGetRootId(mNativeObj);
+        if (virtualViewId == View.NO_ID)
+            virtualViewId = rootId;
+        if (mA11yFocusId == View.NO_ID)
+            mA11yFocusId = rootId;
+
+        int nativeNode = nativeGetNativeNodeById(mNativeObj, virtualViewId);
+        if (nativeNode == 0)
+            return null;
+        if (!mFrameInfoInitialized)
+            return null;
+
+        final AccessibilityNodeInfo info = AccessibilityNodeInfo.obtain(mView);
+        info.setPackageName(mContentViewCore.getContext().getPackageName());
+        info.setClassName(BrowserAccessibility.nativeGetClassNameJNI(nativeNode));
+
+        info.setSource(mView, virtualViewId);
+        Rect boundsInParent = (Rect)BrowserAccessibility.nativeGetRectInParentJNI(nativeNode);
+        if (virtualViewId == rootId) {
+            // Offset of the web content relative to the View.
+            boundsInParent.offset(0, (int)mRenderCoordinates.getContentOffsetYPix());
+        }
+        info.setBoundsInParent(boundsInParent);
+        Rect rect = (Rect)BrowserAccessibility.nativeGetAbsoluteRectJNI(nativeNode);
+
+        // Offset by the scroll position.
+        rect.offset(-(int)mRenderCoordinates.getScrollX(),
+                    -(int)mRenderCoordinates.getScrollY());
+
+        // Convert CSS (web) pixels to Android View pixels
+        rect.left = (int)mRenderCoordinates.fromLocalCssToPix(rect.left);  // TODO: from float
+        rect.top = (int)mRenderCoordinates.fromLocalCssToPix(rect.top);
+        rect.bottom = (int)mRenderCoordinates.fromLocalCssToPix(rect.bottom);
+        rect.right = (int)mRenderCoordinates.fromLocalCssToPix(rect.right);
+
+        // Offset by the location of the web content within the view.
+        rect.offset(0,
+                    (int)mRenderCoordinates.getContentOffsetYPix());
+        // Additionally offset by the location of the view within the screen.
+        final int[] viewLocation = new int[2];
+        mView.getLocationOnScreen(viewLocation);
+        rect.offset(viewLocation[0], viewLocation[1]);
+        info.setBoundsInScreen(rect);
+        info.setContentDescription(BrowserAccessibility.nativeGetNameJNI(nativeNode));
+        info.setEnabled(true);

[alanv, 2013/06/04 17:50:22]

Gets set later, can drop this line.

[dmazzoni, 2013/06/07 20:23:15]

Done.

+        info.setVisibleToUser(true);

[alanv, 2013/06/04 17:50:22]

Gets set later, can drop this line.

[dmazzoni, 2013/06/07 20:23:15]

Done.

+        if (virtualViewId != rootId)
+            info.setParent(mView, BrowserAccessibility.nativeGetParentJNI(nativeNode));
+
+        boolean focusable = BrowserAccessibility.nativeIsFocusableJNI(nativeNode);
+        info.setFocusable(focusable);
+        if (focusable) {
+            if (BrowserAccessibility.nativeIsFocusedJNI(nativeNode))
+                info.addAction(AccessibilityNodeInfo.ACTION_CLEAR_FOCUS);
+            else
+                info.addAction(AccessibilityNodeInfo.ACTION_FOCUS);
+        }
+
+        int childCount = BrowserAccessibility.nativeGetChildCountJNI(nativeNode);

[alanv, 2013/06/04 17:50:22]

Move closer to loop that uses this var.

[dmazzoni, 2013/06/07 20:23:15]

Done.

+
+        if (mA11yFocusId == virtualViewId) {
+            info.setAccessibilityFocused(true);
+            info.addAction(AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS);
+        } else {
+            info.setAccessibilityFocused(false);
+            info.addAction(AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS);
+        }
+
+        boolean clickable = BrowserAccessibility.nativeGetClickableJNI(nativeNode);
+        if (clickable)
+            info.addAction(AccessibilityNodeInfo.ACTION_CLICK);
+
+        for (int i = 0; i < childCount; i++)
+            info.addChild(mView, BrowserAccessibility.nativeGetChildIdAtJNI(nativeNode, i));
+
+        info.setCheckable(BrowserAccessibility.nativeIsCheckableJNI(nativeNode));
+        info.setChecked(BrowserAccessibility.nativeIsCheckedJNI(nativeNode));
+        info.setEnabled(BrowserAccessibility.nativeIsEnabledJNI(nativeNode));
+        info.setFocused(BrowserAccessibility.nativeIsFocusedJNI(nativeNode));
+        info.setPassword(BrowserAccessibility.nativeIsPasswordJNI(nativeNode));
+        info.setScrollable(BrowserAccessibility.nativeIsScrollableJNI(nativeNode));
+        info.setSelected(BrowserAccessibility.nativeIsSelectedJNI(nativeNode));
+        info.setVisibleToUser(BrowserAccessibility.nativeIsVisibleJNI(nativeNode));
+
+        return info;
+    }
+
+    public List<AccessibilityNodeInfo> findAccessibilityNodeInfosByText(String text,
+                                                                        int virtualViewId) {
+        return new ArrayList<AccessibilityNodeInfo>();
+    }
+
+    public boolean dispatchHoverEvent(MotionEvent event) {
+        mUserHasTouchExplored = true;
+        float x = event.getX();
+        float y = event.getY();
+        // Note: the MotionEvent seems to have view-relative coordinates already.
+        // Offset by the location of the web content within the view.
+        y -= mRenderCoordinates.getContentOffsetYPix();
+        // Convert to CSS coordinates.
+        int cssX = (int)(mRenderCoordinates.fromPixToLocalCss(x) + mRenderCoordinates.getScrollX());
+        int cssY = (int)(mRenderCoordinates.fromPixToLocalCss(y) + mRenderCoordinates.getScrollY());
+        int id = nativeHitTest(mNativeObj, cssX, cssY);
+        int nativeNode = nativeGetNativeNodeById(mNativeObj, id);
+        if (nativeNode == 0)
+            return false;
+
+        if (mA11yFocusId != id) {
+            mA11yFocusId = id;
+            sendAccessibilityEvent(id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED);

[alanv, 2013/06/04 17:50:22]

Views should never send this event or manage accessibility focus state except in
response to ACTION_ACCESSIBILITY_FOCUS. Instead, keep track of the currently
hovered id and send TYPE_VIEW_HOVER_ENTER and _EXIT pairs when the id changes
(or the user leaves the view completely). The accessibility service will handle
focus placement.

+        }
+
+        // If this is a hover end event, web-focus the element and pop up the keyboard
+        // if necessary.  TODO: figure out what happens when there's an ACTION_HOVER_EXIT
+        // because touch exploration exited the ContentViewCore.
+        if (event.getActionMasked() == MotionEvent.ACTION_HOVER_EXIT)
+            focusAndHideOrShowKeyboardIfNecessary(id);
+        return true;
+    }
+
+    public void focusAndHideOrShowKeyboardIfNecessary(int virtualViewId) {

[alanv, 2013/06/04 17:50:22]

The keyboard should only be shown when an item gets system focus, and should not
need special handling for accessibility.

[dmazzoni, 2013/06/07 20:23:15]

We added support for an "auto focus mode", which needs this code - but we're
leaving it disabled for now, to keep things simple.

+        if (!mWebFocusFollowsA11yFocus)
+            return;
+
+        int nativeNode = nativeGetNativeNodeById(mNativeObj, virtualViewId);
+        if (nativeNode == 0)
+            return;
+
+        InputMethodManager imm =
+            (InputMethodManager)mContentViewCore.getContext().getSystemService(
+            Context.INPUT_METHOD_SERVICE);
+        if (!BrowserAccessibility.nativeIsFocusableJNI(nativeNode)) {
+            imm.hideSoftInputFromWindow(mView.getWindowToken(),
+                                        InputMethodManager.HIDE_IMPLICIT_ONLY);
+            return;
+        }
+
+        if (BrowserAccessibility.nativeIsFocusedJNI(nativeNode))
+            return;
+
+        BrowserAccessibility.nativeFocusJNI(nativeNode);
+        sendAccessibilityEvent(virtualViewId,
+                               AccessibilityEvent.TYPE_VIEW_FOCUSED);
+        // Show/hide keyboard
+        if (BrowserAccessibility.nativeIsEditableTextJNI(nativeNode)) {
+            imm.showSoftInput(mView, InputMethodManager.SHOW_IMPLICIT);
+        } else {
+            imm.hideSoftInputFromWindow(mView.getWindowToken(),
+                                        InputMethodManager.HIDE_IMPLICIT_ONLY);
+        }
+    }
+
+    public boolean performAction(int virtualViewId, int action, Bundle arguments) {
+        int rootId = nativeGetRootId(mNativeObj);
+        int nativeRootNode = nativeGetNativeNodeById(mNativeObj, rootId);
+        int nativeNode = nativeGetNativeNodeById(mNativeObj, virtualViewId);
+        if (nativeNode == 0 || nativeRootNode == 0)
+            return false;
+
+        String nodeName = BrowserAccessibility.nativeGetNameJNI(nativeNode);
+
+        switch (action) {
+        case AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS:
+            if (mA11yFocusId == virtualViewId)
+                return true;
+            mA11yFocusId = virtualViewId;
+            focusAndHideOrShowKeyboardIfNecessary(virtualViewId);
+            sendAccessibilityEvent(virtualViewId,

[alanv, 2013/06/04 17:50:22]

What happens when a user is moving focus with a keyboard? Does the web content
still send the correct events?

[dmazzoni, 2013/06/07 20:23:15]

Yes, those are the handleXXX methods below.

+                                   AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED);
+            return true;
+        case AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS:
+            if (mA11yFocusId == virtualViewId)
+                mA11yFocusId = View.NO_ID;
+            /*            sendAccessibilityEvent(virtualViewId,
+                          AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED); */
+            return true;
+        case AccessibilityNodeInfo.ACTION_CLICK:
+            BrowserAccessibility.nativeClickJNI(nativeNode);
+            break;
+        case AccessibilityNodeInfo.ACTION_FOCUS:
+            BrowserAccessibility.nativeFocusJNI(nativeNode);
+            break;
+        case AccessibilityNodeInfo.ACTION_CLEAR_FOCUS:
+            BrowserAccessibility.nativeFocusJNI(nativeRootNode);
+            break;
+        }
+        return false;
+    }
+
+    private void sendAccessibilityEvent(int virtualViewId, int eventType) {
+        int nativeNode = nativeGetNativeNodeById(mNativeObj, virtualViewId);
+        if (nativeNode == 0)
+            return;
+
+        final AccessibilityEvent event = AccessibilityEvent.obtain(eventType);
+        event.setPackageName(mContentViewCore.getContext().getPackageName());
+        event.setClassName(BrowserAccessibility.nativeGetClassNameJNI(nativeNode));
+        event.setChecked(BrowserAccessibility.nativeIsCheckedJNI(nativeNode));
+        event.setEnabled(BrowserAccessibility.nativeIsEnabledJNI(nativeNode));
+        event.setPassword(BrowserAccessibility.nativeIsPasswordJNI(nativeNode));
+        event.setScrollable(BrowserAccessibility.nativeIsScrollableJNI(nativeNode));
+        event.setCurrentItemIndex(BrowserAccessibility.nativeGetItemIndexJNI(nativeNode));
+        event.setItemCount(BrowserAccessibility.nativeGetItemCountJNI(nativeNode));
+        event.setScrollX(BrowserAccessibility.nativeGetScrollXJNI(nativeNode));
+        event.setScrollY(BrowserAccessibility.nativeGetScrollYJNI(nativeNode));
+        event.setMaxScrollX(BrowserAccessibility.nativeGetScrollXJNI(nativeNode));
+        event.setMaxScrollY(BrowserAccessibility.nativeGetScrollYJNI(nativeNode));
+
+        switch (eventType) {
+        case AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED:
+            event.setFromIndex(BrowserAccessibility.nativeGetTextChangeFromIndexJNI(nativeNode));
+            event.setAddedCount(BrowserAccessibility.nativeGetTextChangeAddedCountJNI(nativeNode));
+            event.setRemovedCount(BrowserAccessibility.nativeGetTextChangeRemovedCountJNI(
+                nativeNode));
+            event.setBeforeText(BrowserAccessibility.nativeGetTextChangeBeforeTextJNI(nativeNode));
+            event.getText().add(BrowserAccessibility.nativeGetNameJNI(nativeNode));
+            break;
+        case AccessibilityEvent.TYPE_VIEW_TEXT_SELECTION_CHANGED:
+            event.setFromIndex(BrowserAccessibility.nativeGetSelectionStartJNI(nativeNode));
+            event.setToIndex(BrowserAccessibility.nativeGetSelectionEndJNI(nativeNode));
+            event.setItemCount(BrowserAccessibility.nativeGetEditableTextLengthJNI(nativeNode));
+            event.getText().add(BrowserAccessibility.nativeGetNameJNI(nativeNode));
+            break;
+        }
+
+        int rootId = nativeGetRootId(mNativeObj);
+        if (virtualViewId == rootId)
+            virtualViewId = View.NO_ID;
+        event.setSource(mView, virtualViewId);
+
+        mContentViewCore.getContainerView().invalidate();
+        mContentViewCore.getContainerView().requestSendAccessibilityEvent(
+            mView, event);
+    }
+
+    public void notifyFrameInfoInitialized() {
+        if (mFrameInfoInitialized)
+            return;
+
+        mFrameInfoInitialized = true;
+        // (re-) focus focused element, since it didn't have correct coordinates before
+        if (mA11yFocusId != View.NO_ID)
+            sendAccessibilityEvent(mA11yFocusId, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED);
+    }
+
+    @CalledByNative
+    private void handlePageLoaded(int id) {
+        if (mUserHasTouchExplored)
+            return;
+
+        // Otherwise, focus the natively focused node (usually document).
+        mA11yFocusId = id;
+        sendAccessibilityEvent(id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED);

[alanv, 2013/06/04 17:50:22]

Views should not manually move accessibility focus.

[dmazzoni, 2013/06/07 20:23:15]

I think this can be a good experience - we specifically don't do anything
if the user has already explored since initiating loading of the page.

What if we send TYPE_VIEW_FOCUSED instead?

[alanv, 2013/06/17 22:53:28]

You can talk with Casey about allowing apps to automatically place focus, but
for not it is not a supported use case and may result in breakage / unexpected
behavior. Sending VIEW_FOCUSED would work, but the corresponding node info
should have its focused bit set to true. There are some apps that use
VIEW_FOCUSED in a way that breaks TalkBack, so it does some simple checks for
event validity.

[dmazzoni, 2013/06/18 20:22:35]

OK, I've disabled this for now. Maybe we can make it a user option later
if we can get it to work well.

+    }
+
+    @CalledByNative
+    private void handleFocusChanged(int id) {
+        if (mA11yFocusId == id)
+            return;
+        mA11yFocusId = id;
+        sendAccessibilityEvent(id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED);

[alanv, 2013/06/04 17:50:22]

If focus here means system focus, please only send a VIEW_FOCUSED event and let
the service handle accessibility focus placement.

[dmazzoni, 2013/06/07 20:23:15]

Done.

+    }
+
+    @CalledByNative
+    private void handleCheckStateChanged(int id) {
+        sendAccessibilityEvent(id, AccessibilityEvent.TYPE_VIEW_CLICKED);
+    }
+
+    @CalledByNative
+    private void handleTextSelectionChanged(int id) {
+        sendAccessibilityEvent(id, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);
+        sendAccessibilityEvent(id, AccessibilityEvent.TYPE_VIEW_TEXT_SELECTION_CHANGED);
+    }
+
+    @CalledByNative
+    private void handleEditableTextChanged(int id) {
+        sendAccessibilityEvent(id, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);
+        sendAccessibilityEvent(id, AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED);

[alanv, 2013/06/04 17:50:22]

Does this also raise a TEXT_SELECTION_CHANGED event from the cursor position
moving?

[dmazzoni, 2013/06/07 20:23:15]

If only one changes, it definitely does the right thing now.

If both change, which one implies which, or is it okay to send both for a single
user action?

[alanv, 2013/06/17 22:53:28]

In the platform, TEXT_CHANGED from typing is always followed by
TEXT_SELECTION_CHANGED due to how the system updates cursor position resulting
from typed text. So yes, send both if both the text and cursor position change.
It probably won't break TalkBack if the SELECTION_CHANGED isn't sent, but it may
affect BrailleBack.

[dmazzoni, 2013/06/18 20:22:35]

Great, we're already sending the right events then.

I think the only major issue affecting BrailleBack is Chrome's
implementation of IME APIs, which we'll handle in a separate
changelist.

+    }
+
+    @CalledByNative
+    private void handleContentChanged(int id) {
+        sendAccessibilityEvent(id, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);
+    }
+
+    @CalledByNative
+    private void handleNavigate() {
+        mA11yFocusId = View.NO_ID;
+        mUserHasTouchExplored = false;
+        mFrameInfoInitialized = false;
+    }
+
+    @CalledByNative
+    private void handleScrolledToAnchor(int id) {
+        if (mA11yFocusId == id)
+            return;
+        mA11yFocusId = id;
+        sendAccessibilityEvent(id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED);
+    }
+
+    @CalledByNative
+    private void announceObjectShow(int id, boolean force_assertive) {
+        int nativeNode = nativeGetNativeNodeById(mNativeObj, id);
+        String nodeName = BrowserAccessibility.nativeGetNameJNI(nativeNode);
+        boolean assertive = force_assertive;
+        if (!force_assertive) {
+            String ariaLive = BrowserAccessibility.nativeGetAriaLiveJNI(nativeNode);
+            assertive = ariaLive == "assertive";
+        }
+        // TODO(aboxhall): handle assertive once supported by framework
+        mView.announceForAccessibility(nodeName);
+    }
+
+    private native int nativeGetRootId(int nativeBrowserAccessibilityManagerAndroid);
+    private native int nativeHitTest(int nativeBrowserAccessibilityManagerAndroid, int x, int y);
+    private native int nativeGetNativeNodeById(
+        int nativeBrowserAccessibilityManagerAndroid, int id);
+}