[WIP] Add support for Android SuggestionSpans when editing text

content/public/android/java/src/org/chromium/content/browser/input/ImeAdapter.java

 // Copyright 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package org.chromium.content.browser.input;
 
+import android.annotation.TargetApi;
+import android.content.Context;
 import android.content.res.Configuration;
+import android.graphics.Color;
+import android.graphics.Rect;
+import android.graphics.drawable.ColorDrawable;
+import android.graphics.drawable.Drawable;
 import android.os.Build;
 import android.os.ResultReceiver;
 import android.os.SystemClock;
+import android.text.Spannable;
+import android.text.Spanned;
 import android.text.SpannableString;
 import android.text.TextUtils;
 import android.text.style.BackgroundColorSpan;
 import android.text.style.CharacterStyle;
+import android.text.style.SuggestionSpan;
+import android.text.style.TextAppearanceSpan;
 import android.text.style.UnderlineSpan;
+import android.util.DisplayMetrics;
+import android.view.Gravity;
 import android.view.KeyCharacterMap;
 import android.view.KeyEvent;
+import android.view.LayoutInflater;
 import android.view.View;
+import android.view.ViewGroup;
+import android.view.ViewGroup.LayoutParams;
 import android.view.inputmethod.BaseInputConnection;
+import android.view.inputmethod.CursorAnchorInfo;
 import android.view.inputmethod.EditorInfo;
 import android.view.inputmethod.InputConnection;
+import android.view.WindowManager;
+import android.widget.AdapterView;
+import android.widget.AdapterView.OnItemClickListener;
+import android.widget.BaseAdapter;
+import android.widget.LinearLayout;
+import android.widget.ListView;
+import android.widget.PopupWindow;
+import android.widget.PopupWindow.OnDismissListener;
+import android.widget.TextView;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 
 import org.chromium.base.Log;
 import org.chromium.base.VisibleForTesting;
 import org.chromium.base.annotations.CalledByNative;
 import org.chromium.base.annotations.JNINamespace;
 import org.chromium.blink_public.web.WebInputEventModifier;
 import org.chromium.blink_public.web.WebInputEventType;
 import org.chromium.blink_public.web.WebTextInputMode;
+import org.chromium.content.R;
 import org.chromium.content.browser.RenderCoordinates;
 import org.chromium.content.browser.picker.InputDialogContainer;
 import org.chromium.ui.base.ime.TextInputType;
 
 /**
  * Adapts and plumbs android IME service onto the chrome text input API.
  * ImeAdapter provides an interface in both ways native <-> java:
  * 1. InputConnectionAdapter notifies native code of text composition state and
  *    dispatch key events from java -> WebKit.
  * 2. Native ImeAdapter notifies java side to clear composition text.
@@ skipping 57 matching lines @@
     private ChromiumBaseInputConnection mInputConnection;
     private ChromiumBaseInputConnection.Factory mInputConnectionFactory;
 
     private final ImeAdapterDelegate mViewEmbedder;
     // This holds the information necessary for constructing CursorAnchorInfo, and notifies to
     // InputMethodManager on appropriate timing, depending on how IME requested the information
     // via InputConnection. The update request is per InputConnection, hence for each time it is
     // re-created, the monitoring status will be reset.
     private final CursorAnchorInfoController mCursorAnchorInfoController;
 
+    SuggestionsPopupWindow mSuggestionsPopupWindow;
+    private SuggestionInfo[] mSuggestionInfos = new SuggestionInfo[0];
+
     private int mTextInputType = TextInputType.NONE;
     private int mTextInputFlags;
     private int mTextInputMode = WebTextInputMode.kDefault;
 
     // Keep the current configuration to detect the change when onConfigurationChanged() is called.
     private Configuration mCurrentConfig;
 
     private int mLastSelectionStart;
     private int mLastSelectionEnd;
     private String mLastText;
     private int mLastCompositionStart;
     private int mLastCompositionEnd;
 
+    private CursorAnchorInfo mLastCursorAnchorInfo;
+
     /**
      * @param wrapper InputMethodManagerWrapper that should receive all the call directed to
      *                InputMethodManager.
      * @param embedder The view that is used for callbacks from ImeAdapter.
      */
     public ImeAdapter(InputMethodManagerWrapper wrapper, ImeAdapterDelegate embedder) {
         mInputMethodManagerWrapper = wrapper;
         mViewEmbedder = embedder;
         // Deep copy newConfig so that we can notice the difference.
         mCurrentConfig = new Configuration(
                 mViewEmbedder.getAttachedView().getResources().getConfiguration());
-        // CursorAnchroInfo is supported only after L.
+        // CursorAnchorInfo is supported only after L.
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
-            mCursorAnchorInfoController = CursorAnchorInfoController.create(wrapper,
-                    new CursorAnchorInfoController.ComposingTextDelegate() {
+            mCursorAnchorInfoController = CursorAnchorInfoController.create(
+                    wrapper, new CursorAnchorInfoController.ComposingTextDelegate() {
                         @Override
                         public CharSequence getText() {
                             return mLastText;
                         }
                         @Override
                         public int getSelectionStart() {
                             return mLastSelectionStart;
                         }
                         @Override
                         public int getSelectionEnd() {
                             return mLastSelectionEnd;
                         }
                         @Override
                         public int getComposingTextStart() {
                             return mLastCompositionStart;
                         }
                         @Override
                         public int getComposingTextEnd() {
                             return mLastCompositionEnd;
                         }
-                    });
+                    }, this);
         } else {
             mCursorAnchorInfoController = null;
         }
     }
 
     private void createInputConnectionFactory() {
         if (mInputConnectionFactory != null) return;
         mInputConnectionFactory = new ThreadedInputConnectionFactory(mInputMethodManagerWrapper);
     }
 
@@ skipping 551 matching lines @@
      */
     public void onUpdateFrameInfo(RenderCoordinates renderCoordinates, boolean hasInsertionMarker,
             boolean isInsertionMarkerVisible, float insertionMarkerHorizontal,
             float insertionMarkerTop, float insertionMarkerBottom) {
         if (mCursorAnchorInfoController == null) return;
         mCursorAnchorInfoController.onUpdateFrameInfo(renderCoordinates, hasInsertionMarker,
                 isInsertionMarkerVisible, insertionMarkerHorizontal, insertionMarkerTop,
                 insertionMarkerBottom, mViewEmbedder.getAttachedView());
     }
 
+    /**
+     * Called by CursorAnchorInfoController to notify ImeAdapter when there's
+     * updated CursorAnchorInfo
+     */
+    public void updateCursorAnchorInfo(View view, CursorAnchorInfo cursorAnchorInfo) {
+        mLastCursorAnchorInfo = cursorAnchorInfo;
+        if (mSuggestionsPopupWindow != null) {
+            mSuggestionsPopupWindow.updatePosition();
+        }
+    }
+
+    @TargetApi(Build.VERSION_CODES.LOLLIPOP)
+    private class SuggestionsPopupWindow implements OnItemClickListener, OnDismissListener {

[aelias_OOO_until_Jul13, 2017/01/25 03:34:26]

Please move this into a new .java file.

+        protected PopupWindow mPopupWindow;
+        protected ViewGroup mContentView;
+        int mPositionX, mPositionY;
+        int mClippingLimitLeft, mClippingLimitRight;
+        private Rect mTempRect;
+
+        private SuggestionAdapter mSuggestionsAdapter;
+        private final TextAppearanceSpan mHighlightSpan;
+        private TextView mDeleteButton;
+        private ListView mSuggestionListView;
+        private int mContainerMarginWidth;
+        private int mContainerMarginTop;
+        private double mLastInsertionMarkerBottom = 0.0;
+        private LinearLayout mContainerView;
+
+        private boolean mDismissedByItemTap = false;
+
+        protected void createPopupWindow() {
+            mPopupWindow = new PopupWindow();
+            mPopupWindow.setInputMethodMode(PopupWindow.INPUT_METHOD_NOT_NEEDED);
+            mPopupWindow.setBackgroundDrawable(new ColorDrawable(Color.TRANSPARENT));
+
+            mPopupWindow.setFocusable(true);
+            mPopupWindow.setClippingEnabled(false);
+
+            mPopupWindow.setOnDismissListener(this);
+        }
+
+        protected void initContentView() {
+            final LayoutInflater inflater =
+                    (LayoutInflater) mInputMethodManagerWrapper.getContext().getSystemService(
+                            Context.LAYOUT_INFLATER_SERVICE);
+            mContentView =
+                    (ViewGroup) inflater.inflate(R.layout.text_edit_suggestion_container, null);
+            mContainerView =
+                    (LinearLayout) mContentView.findViewById(R.id.suggestionWindowContainer);
+            ViewGroup.MarginLayoutParams lp =
+                    (ViewGroup.MarginLayoutParams) mContainerView.getLayoutParams();
+            mContainerMarginWidth = lp.leftMargin + lp.rightMargin;
+            mContainerMarginTop = lp.topMargin;
+            mClippingLimitLeft = lp.leftMargin;
+            mClippingLimitRight = lp.rightMargin;
+
+            mSuggestionListView = (ListView) mContentView.findViewById(R.id.suggestionContainer);
+
+            mSuggestionsAdapter = new SuggestionAdapter();
+            mSuggestionListView.setAdapter(mSuggestionsAdapter);
+            mSuggestionListView.setOnItemClickListener(this);
+
+            mDeleteButton = (TextView) mContentView.findViewById(R.id.deleteButton);
+            mDeleteButton.setOnClickListener(new View.OnClickListener() {
+                public void onClick(View v) {
+                    nativeDeleteSuggestionHighlight(mNativeImeAdapterAndroid);
+                    mDismissedByItemTap = true;
+                    mPopupWindow.dismiss();
+                }
+            });
+        }
+
+        public SuggestionsPopupWindow() {
+            mHighlightSpan = new TextAppearanceSpan(
+                    mInputMethodManagerWrapper.getContext(), R.style.SuggestionHighlight);
+
+            createPopupWindow();
+
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
+                mPopupWindow.setWindowLayoutType(
+                        WindowManager.LayoutParams.TYPE_APPLICATION_SUB_PANEL);
+            }
+            mPopupWindow.setWidth(ViewGroup.LayoutParams.WRAP_CONTENT);
+            mPopupWindow.setHeight(ViewGroup.LayoutParams.WRAP_CONTENT);
+            initContentView();
+
+            LayoutParams wrapContent = new LayoutParams(
+                    ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT);
+            mContentView.setLayoutParams(wrapContent);
+            mPopupWindow.setContentView(mContentView);
+        }
+
+        public void updatePosition() {
+            if (!mPopupWindow.isShowing()) {
+                return;
+            }
+            show();
+        }
+
+        public boolean isShowing() {
+            return mPopupWindow.isShowing();
+        }
+
+        private class SuggestionAdapter extends BaseAdapter {
+            private LayoutInflater mInflater =
+                    (LayoutInflater) mInputMethodManagerWrapper.getContext().getSystemService(
+                            Context.LAYOUT_INFLATER_SERVICE);
+            @Override
+            public int getCount() {
+                return mSuggestionInfos.length;
+            }
+            @Override
+            public Object getItem(int position) {
+                return mSuggestionInfos[position];
+            }
+            @Override
+            public long getItemId(int position) {
+                return position;
+            }
+            @Override
+            public View getView(int position, View convertView, ViewGroup parent) {
+                TextView textView = (TextView) convertView;
+                if (textView == null) {
+                    textView = (TextView) mInflater.inflate(
+                            R.layout.text_edit_suggestion_item, parent, false);
+                }
+                final SuggestionInfo suggestionInfo = mSuggestionInfos[position];
+                SpannableString textToSet = new SpannableString(
+                        suggestionInfo.mPrefix + suggestionInfo.mText + suggestionInfo.mSuffix);
+                textToSet.setSpan(mHighlightSpan, suggestionInfo.mPrefix.length(),
+                        suggestionInfo.mPrefix.length() + suggestionInfo.mText.length(),
+                        Spanned.SPAN_EXCLUSIVE_EXCLUSIVE);
+                textView.setText(textToSet);
+                return textView;
+            }
+        }
+
+        protected void measureContent() {
+            final DisplayMetrics displayMetrics =
+                    mInputMethodManagerWrapper.getContext().getResources().getDisplayMetrics();
+            final int horizontalMeasure = View.MeasureSpec.makeMeasureSpec(
+                    displayMetrics.widthPixels, View.MeasureSpec.AT_MOST);
+            final int verticalMeasure = View.MeasureSpec.makeMeasureSpec(
+                    displayMetrics.heightPixels, View.MeasureSpec.AT_MOST);
+            int width = 0;
+            View view = null;
+            for (int i = 0; i < mSuggestionInfos.length; i++) {
+                view = mSuggestionsAdapter.getView(i, view, mContentView);
+                view.getLayoutParams().width = LayoutParams.WRAP_CONTENT;
+                view.measure(horizontalMeasure, verticalMeasure);
+                width = Math.max(width, view.getMeasuredWidth());
+            }
+            mDeleteButton.measure(horizontalMeasure, verticalMeasure);
+            width = Math.max(width, mDeleteButton.getMeasuredWidth());
+            width += mContainerView.getPaddingLeft() + mContainerView.getPaddingRight()
+                    + mContainerMarginWidth;
+            // Enforce the width based on actual text widths
+            mContentView.measure(View.MeasureSpec.makeMeasureSpec(width, View.MeasureSpec.EXACTLY),
+                    verticalMeasure);
+            Drawable popupBackground = mPopupWindow.getBackground();
+            if (popupBackground != null) {
+                if (mTempRect == null) mTempRect = new Rect();
+                popupBackground.getPadding(mTempRect);
+                width += mTempRect.left + mTempRect.right;
+            }
+            mPopupWindow.setWidth(width);
+        }
+
+        public void show() {
+            mSuggestionsAdapter.notifyDataSetChanged();
+            // We get the insertion point coords relative to the WebView bounds.
+            // We need to render the popup relative to the display
+            int[] embedderViewCoords = new int[2];
+            mViewEmbedder.getAttachedView().getLocationOnScreen(embedderViewCoords);
+
+            final DisplayMetrics displayMetrics =
+                    mInputMethodManagerWrapper.getContext().getResources().getDisplayMetrics();
+
+            float density = displayMetrics.density;
+
+            measureContent();
+            int width = mContentView.getMeasuredWidth();
+
+            int positionX = 0;
+            int positionY = 0;
+
+            positionX = Math.round(
+                    mLastCursorAnchorInfo.getInsertionMarkerHorizontal() * density - width / 2.0f);
+            double insertionMarkerBottom = mLastCursorAnchorInfo.getInsertionMarkerBottom();
+            if (Double.isNaN(insertionMarkerBottom)) {
+                // certain operations, e.g. rotating the device while the menu

[aelias_OOO_until_Jul13, 2017/01/25 03:34:26]

Seems this is no longer needed after changwan@'s fix?

+                // is open, can cause getInsertionMarkerBottom() to start
+                // returning NaN, just keep using the previous value in this case
+                insertionMarkerBottom = mLastInsertionMarkerBottom;
+            } else {
+                mLastInsertionMarkerBottom = insertionMarkerBottom;
+            }
+            positionY = (int) Math.round(
+                    (embedderViewCoords[1] + insertionMarkerBottom - 24) * density);
+
+            // horizontal clipping
+            width = mContentView.getMeasuredWidth();
+            positionX =
+                    Math.min(displayMetrics.widthPixels - width + mClippingLimitRight, positionX);
+            positionX = Math.max(-mClippingLimitLeft, positionX);
+
+            // vertical clipping
+            final int height = mContentView.getMeasuredHeight();
+            positionY = Math.min(positionY, displayMetrics.heightPixels - height);
+
+            if (isShowing()) {
+                mPopupWindow.update(positionX, positionY, -1, -1);
+            } else {
+                mPopupWindow.showAtLocation(
+                        mViewEmbedder.getAttachedView(), Gravity.NO_GRAVITY, positionX, positionY);
+            }
+        }
+
+        @Override
+        public void onItemClick(AdapterView<?> parent, View view, int position, long id) {
+            SuggestionInfo suggestionInfo = mSuggestionInfos[position];
+            nativeApplySuggestionReplacement(mNativeImeAdapterAndroid,
+                    suggestionInfo.mDocumentMarkerID, suggestionInfo.mSuggestionIndex);
+            // replaceWithSuggestion(suggestionInfo);
+            // hideWithCleanUp();
+            mDismissedByItemTap = true;

[aelias_OOO_until_Jul13, 2017/01/25 03:34:26]

Can we instead not dismiss at all at just yet?  We can replace it when the new
menu comes in from Blink instead.

[rlanday, 2017/01/25 18:48:10]

I don't understand what you're asking here, the menu is supposed to close when
you pick a suggestion

+            mPopupWindow.dismiss();
+        }
+
+        public void closeMenu() {

[aelias_OOO_until_Jul13, 2017/01/25 03:34:26]

This has no callsites, please remove.

[rlanday, 2017/01/25 18:48:10]

ah, I added this when I was thinking we might want to close the menu on
backgrounding the app/rotation before I got the menu tracking the cursor
properly in those cases

+            mPopupWindow.dismiss();
+        }
+
+        @Override
+        public void onDismiss() {
+            if (!mDismissedByItemTap) {
+                nativeCloseSuggestionMenu(mNativeImeAdapterAndroid);
+            }
+            mDismissedByItemTap = false;
+        }
+    }
+
     @CalledByNative
     private void populateUnderlinesFromSpans(CharSequence text, long underlines) {
         if (DEBUG_LOGS) {
             Log.w(TAG, "populateUnderlinesFromSpans: text [%s], underlines [%d]", text, underlines);
         }
         if (!(text instanceof SpannableString)) return;
 
         SpannableString spannableString = ((SpannableString) text);
         CharacterStyle spans[] =
                 spannableString.getSpans(0, text.length(), CharacterStyle.class);
         for (CharacterStyle span : spans) {
             if (span instanceof BackgroundColorSpan) {
                 nativeAppendBackgroundColorSpan(underlines, spannableString.getSpanStart(span),
                         spannableString.getSpanEnd(span),
                         ((BackgroundColorSpan) span).getBackgroundColor());
+            } else if (span instanceof SuggestionSpan) {
+                if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
+                    // The suggestion menu is only supported on Lollipop and newer
+                    continue;
+                }
+
+                int underlineColor = 0;
+                // Use getUnderlineColor method by reflection to avoid having to reimplement

[aelias_OOO_until_Jul13, 2017/01/25 03:34:26]

The method is public, it's just @hide.  You can probably just call it directly
and suppress the build warning if you get one.

[rlanday, 2017/01/25 18:48:10]

Hmm I think I tried that but I'll look into it again

[rlanday, 2017/01/26 00:23:05]

Trying to call span.getUnderlineColor() directly results in the compiler saying
"error: cannot find symbol"...the actual implementation is here:
https://android.googlesource.com/platform/frameworks/base/+/master/core/java/...

Since I didn't end up implementing any of the flags, I think we can just hard
code the gray color and avoid having to call this

+                try {
+                    Method getUnderlineColor = SuggestionSpan.class.getMethod("getUnderlineColor");
+                    underlineColor = (int) getUnderlineColor.invoke((SuggestionSpan) span);
+                } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException
+                        | RuntimeException e) {
+                    continue;
+                }
+
+                SuggestionSpan suggestionSpan = (SuggestionSpan) span;
+                nativeAppendSuggestionSpan(underlines, spannableString.getSpanStart(suggestionSpan),
+                        spannableString.getSpanEnd(suggestionSpan), underlineColor,
+                        suggestionSpan.getFlags(), suggestionSpan.getSuggestions());
+
             } else if (span instanceof UnderlineSpan) {
                 nativeAppendUnderlineSpan(underlines, spannableString.getSpanStart(span),
                         spannableString.getSpanEnd(span));
             }
         }
     }
 
     @CalledByNative
     private void cancelComposition() {
         if (DEBUG_LOGS) Log.w(TAG, "cancelComposition");
         if (mInputConnection != null) restartInput();
     }
 
     @CalledByNative
     private void setCharacterBounds(float[] characterBounds) {
         if (mCursorAnchorInfoController == null) return;
         mCursorAnchorInfoController.setCompositionCharacterBounds(characterBounds,
                 mViewEmbedder.getAttachedView());
     }
 
+    public class SuggestionInfo {
+        int mDocumentMarkerID;
+        int mSuggestionIndex;
+        String mPrefix;
+        String mText;
+        String mSuffix;
+
+        SuggestionInfo(int documentMarkerID, int suggestionIndex, String prefix, String text,
+                String suffix) {
+            mDocumentMarkerID = documentMarkerID;
+            mSuggestionIndex = suggestionIndex;
+            mPrefix = prefix;
+            mText = text;
+            mSuffix = suffix;
+        }
+    }
+
+    @CalledByNative
+    private void showSuggestionMenu(SuggestionInfo[] suggestionInfos) {
+        mSuggestionInfos = suggestionInfos;
+        if (mSuggestionsPopupWindow == null) {
+            mSuggestionsPopupWindow = new SuggestionsPopupWindow();
+        }
+
+        mSuggestionsPopupWindow.show();
+    }
+
     @CalledByNative
     private void detach() {
         if (DEBUG_LOGS) Log.w(TAG, "detach");
         mNativeImeAdapterAndroid = 0;
         if (mCursorAnchorInfoController != null) {
             mCursorAnchorInfoController.focusedNodeChanged(false);
         }
     }
 
     private native boolean nativeSendKeyEvent(long nativeImeAdapterAndroid, KeyEvent event,
             int type, int modifiers, long timestampMs, int keyCode, int scanCode,
             boolean isSystemKey, int unicodeChar);
-    private static native void nativeAppendUnderlineSpan(long underlinePtr, int start, int end);
     private static native void nativeAppendBackgroundColorSpan(long underlinePtr, int start,
             int end, int backgroundColor);
+    private static native void nativeAppendSuggestionSpan(long underlinePtr, int start, int end,
+            int underlineColor, int flags, String[] suggestions);
+    private static native void nativeAppendUnderlineSpan(long underlinePtr, int start, int end);
     private native void nativeSetComposingText(long nativeImeAdapterAndroid, CharSequence text,
             String textStr, int newCursorPosition);
     private native void nativeCommitText(
             long nativeImeAdapterAndroid, CharSequence text, String textStr, int newCursorPosition);
     private native void nativeFinishComposingText(long nativeImeAdapterAndroid);
     private native void nativeAttachImeAdapter(long nativeImeAdapterAndroid);
     private native void nativeSetEditableSelectionOffsets(long nativeImeAdapterAndroid,
             int start, int end);
     private native void nativeSetComposingRegion(long nativeImeAdapterAndroid, int start, int end);
     private native void nativeDeleteSurroundingText(long nativeImeAdapterAndroid,
             int before, int after);
     private native void nativeResetImeAdapter(long nativeImeAdapterAndroid);
     private native boolean nativeRequestTextInputStateUpdate(long nativeImeAdapterAndroid);
     private native void nativeRequestCursorUpdate(long nativeImeAdapterAndroid,
             boolean immediateRequest, boolean monitorRequest);
+    private native void nativeApplySuggestionReplacement(
+            long nativeImeAdapterAndroid, int documentMarkerID, int suggestionIndex);
+    private native void nativeDeleteSuggestionHighlight(long nativeImeAdapterAndroid);
+    private native void nativeCloseSuggestionMenu(long nativeImeAdapterAndroid);
 }