Reimplement automation API on top of C++-backed AXTree.

chrome/common/extensions/api/automation.idl

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // The <code>chrome.automation</code> API allows developers to access the
 // automation (accessibility) tree for the browser. The tree resembles the DOM
 // tree, but only exposes the <em>semantic</em> structure of a page. It can be
 // used to programmatically interact with a page by examining names, roles, and
 // states, listening for events, and performing actions on nodes.
 [nocompile] namespace automation {
@@ skipping 299 matching lines @@
   [nocompile, noinline_doc] dictionary AutomationNode {
     // The root node of the tree containing this AutomationNode.
     AutomationNode root;
 
     // Whether this AutomationNode is root node.
     boolean isRootNode;
 
     // The role of this node.
     automation.RoleType role;
 
-    // TODO(aboxhall): expose states as mixins instead
-
     // The $(ref:automation.StateType)s describing this node.
     object state;
 
     // The rendered location (as a bounding box) of this node within the frame.
     automation.Rect location;
 
     // The purpose of the node, other than the role, if any.
     DOMString description;
 
     // The help text for the node, if any.
@@ skipping 30 matching lines @@
     // one. See
     // <a href="http://www.w3.org/TR/wai-aria/states_and_properties#aria-flowto">
     // <code>aria-flowto</code></a>.
     AutomationNode[] flowTo;
 
     // The nodes, if any, which form a label for this element. Generally, the
     // text from these elements will also be exposed as the element's accessible
     // name, via the $(ref:automation.AutomationNode.name) attribute.
     AutomationNode[] labelledBy;
 
-    // The nodes, if any, which are to be considered children of this node but
-    // are not children in the DOM tree.
-    AutomationNode[] owns;
-
-    // TODO(aboxhall): Make this private?
-
-    // A collection of this node's other attributes.
-    object? attributes;
-
-    // The index of this node in its parent node's list of children. If this is
-    // the root node, this will be undefined.
-    long? indexInParent;
-
-    AutomationNode[] children;
-    AutomationNode parent;
-    AutomationNode firstChild;
-    AutomationNode lastChild;
-    AutomationNode previousSibling;
-    AutomationNode nextSibling;
-
-    // Does the default action based on this node's role. This is generally
-    // the same action that would result from clicking the node such as
-    // expanding a treeitem, toggling a checkbox, selecting a radiobutton,
-    // or activating a button.
-    static void doDefault();
-
-    // Places focus on this node.
-    static void focus();
-
-    // Scrolls this node to make it visible.
-    static void makeVisible();
-
-    // Sets selection within a text field.
-    static void setSelection(long startIndex, long endIndex);
-
-    // Shows the context menu resulting from a right click on this node.
-    static void showContextMenu();
-
-    // Adds a listener for the given event type and event phase.
-    static void addEventListener(
-        EventType eventType, AutomationListener listener, boolean capture);
-
-    // Removes a listener for the given event type and event phase.
-    static void removeEventListener(
-        EventType eventType, AutomationListener listener, boolean capture);
-
-    // Gets the first node in this node's subtree which matches the given CSS
-    // selector and is within the same DOM context.
-    //
-    // If this node doesn't correspond directly with an HTML node in the DOM,
-    // querySelector will be run on this node's nearest HTML node ancestor. Note
-    // that this may result in the query returning a node which is not a
-    // descendant of this node.
-    //
-    // If the selector matches a node which doesn't directly correspond to an
-    // automation node (for example an element within an ARIA widget, where the
-    // ARIA widget forms one node of the automation tree, or an element which
-    // is hidden from accessibility via hiding it using CSS or using
-    // aria-hidden), this will return the nearest ancestor which does correspond
-    // to an automation node.
-    static void domQuerySelector(DOMString selector, QueryCallback callback);
-
-    // Finds the first AutomationNode in this node's subtree which matches the
-    // given search parameters.
-    static AutomationNode find(FindParams params);
-
-    // Finds all the AutomationNodes in this node's subtree which matches the
-    // given search parameters.
-    static AutomationNode[] findAll(FindParams params);
-
-    // Returns whether this node matches the given $(ref:automation.FindParams).
-    static boolean matches(FindParams params);
-  };
-
-  dictionary ActiveDescendantMixin {
     // The node referred to by <code>aria-activedescendant</code>, where
     // applicable
     AutomationNode activedescendant;
-  };
 
-  // Attributes which are mixed in to an AutomationNode if it is a link.
-  dictionary LinkMixins {
-    // TODO(aboxhall): Add visited state
+    //
+    // Link attributes.
+    //
 
     // The URL that this link will navigate to.
     DOMString url;
-  };
 
-  // Attributes which are mixed in to an AutomationNode if it is a document.
-  dictionary DocumentMixins {
+    //
+    // Document attributes.
+    //
+
     // The URL of this document.
     DOMString docUrl;
 
     // The title of this document.
     DOMString docTitle;
 
     // Whether this document has finished loading.
     boolean docLoaded;
 
     // The proportion (out of 1.0) that this doc has completed loading.
     double docLoadingProgress;
-  };
 
-  // TODO(aboxhall): document ScrollableMixins (e.g. what is scrollXMin? is it
-  // ever not 0?)
+    //
+    // Scrollable container attributes.
+    //
 
-  // Attributes which are mixed in to an AutomationNode if it is scrollable.
-  dictionary ScrollableMixins {
     long scrollX;

[aboxhall, 2015/06/11 17:42:45]

One thing we were trying to avoid with mixins was the API explosion of having
every possible attribute exposed on every automation node, and having a long
list of attributes to parse at once in the documentation.

Is there any way we can keep this "mixin" style interface, but have it
implemented more efficiently?

[dmazzoni, 2015/06/11 19:09:12]

Rather than mixins, I think we could bundle attributes together in sub-objects.

For example, instead of node.scrollXMin, etc. you'd have node.scroll.xMin.

If you access node.scroll, it'd return an object if it has scrollable
interfaces, or undefined if it doesn't.

So you could write code like:

if (node.scroll) {
  outputScrollPosition(node.scroll.x, node.scroll.y);
}

Something like that sound good as a follow-up?

[aboxhall, 2015/06/11 22:22:34]

Sorry, missed this in my responses earlier.

Sure, something like this should work. As long as we have a solution to the two
problems I mentioned, I don't mind that much what it looks like. When do you
think you might get to that follow-up?

[dmazzoni, 2015/06/12 17:51:36]

I'll send out a proposal for just one set of attributes (former mixin) right
away, to follow this change.

     long scrollXMin;
     long scrollXMax;
     long scrollY;
     long scrollYMin;
     long scrollYMax;
-  };
 
-  // Attributes which are mixed in to an AutomationNode if it is editable text.
-  dictionary EditableTextMixins {
+    //
+    // Editable text field attributes.
+    //
+
     // The character index of the start of the selection within this editable
     // text element; -1 if no selection.
     long textSelStart;
 
     // The character index of the end of the selection within this editable
     // text element; -1 if no selection.
     long textSelEnd;
 
     // The input type, like email or number.
     DOMString textInputType;
-  };
 
-  // Attributes which are mixed in to an AutomationNode if it is a range.
-  dictionary RangeMixins {
+    //
+    // Range attributes.
+    //
+
     // The current value for this range.
     double valueForRange;
 
     // The minimum possible value for this range.
     double minValueForRange;
 
     // The maximum possible value for this range.
     double maxValueForRange;
-  };
 
-  // TODO(aboxhall): live region mixins.
+    //
+    // Table attributes.
+    //
 
-  // Attributes which are mixed in to an AutomationNode if it is a table.
-  dictionary TableMixins {
     // The number of rows in this table.
     long tableRowCount;
 
     // The number of columns in this table.
     long tableColumnCount;
-  };
 
-  // Attributes which are mixed in to an AutomationNode if it is a table cell.
-  dictionary TableCellMixins {
+    //
+    // Table cell attributes.
+    //
+
     // The zero-based index of the column that this cell is in.
     long tableCellColumnIndex;
 
     // The number of columns that this cell spans (default is 1).
     long tableCellColumnSpan;
 
     // The zero-based index of the row that this cell is in.
     long tableCellRowIndex;
 
     // The number of rows that this cell spans (default is 1).
     long tableCellRowSpan;
+
+    //
+    // Walking the tree.
+    //
+
+    AutomationNode[] children;
+    AutomationNode parent;
+    AutomationNode firstChild;
+    AutomationNode lastChild;
+    AutomationNode previousSibling;
+    AutomationNode nextSibling;
+
+    // The index of this node in its parent node's list of children. If this is
+    // the root node, this will be undefined.
+    long? indexInParent;
+
+    //
+    // Actions.
+    //
+
+    // Does the default action based on this node's role. This is generally
+    // the same action that would result from clicking the node such as
+    // expanding a treeitem, toggling a checkbox, selecting a radiobutton,
+    // or activating a button.
+    static void doDefault();
+
+    // Places focus on this node.
+    static void focus();
+
+    // Scrolls this node to make it visible.
+    static void makeVisible();
+
+    // Sets selection within a text field.
+    static void setSelection(long startIndex, long endIndex);
+
+    // Adds a listener for the given event type and event phase.
+    static void addEventListener(
+        EventType eventType, AutomationListener listener, boolean capture);
+
+    // Removes a listener for the given event type and event phase.
+    static void removeEventListener(
+        EventType eventType, AutomationListener listener, boolean capture);
+
+    // Gets the first node in this node's subtree which matches the given CSS
+    // selector and is within the same DOM context.
+    //
+    // If this node doesn't correspond directly with an HTML node in the DOM,
+    // querySelector will be run on this node's nearest HTML node ancestor. Note
+    // that this may result in the query returning a node which is not a
+    // descendant of this node.
+    //
+    // If the selector matches a node which doesn't directly correspond to an
+    // automation node (for example an element within an ARIA widget, where the
+    // ARIA widget forms one node of the automation tree, or an element which
+    // is hidden from accessibility via hiding it using CSS or using
+    // aria-hidden), this will return the nearest ancestor which does correspond
+    // to an automation node.
+    static void domQuerySelector(DOMString selector, QueryCallback callback);
+
+    // Finds the first AutomationNode in this node's subtree which matches the
+    // given search parameters.
+    static AutomationNode find(FindParams params);
+
+    // Finds all the AutomationNodes in this node's subtree which matches the
+    // given search parameters.
+    static AutomationNode[] findAll(FindParams params);
+
+    // Returns whether this node matches the given $(ref:automation.FindParams).
+    static boolean matches(FindParams params);
   };
 
   // Called when the <code>AutomationNode</code> for the page is available.
   callback RootCallback = void(AutomationNode rootNode);
 
   interface Functions {
     // Get the automation tree for the tab with the given tabId, or the current
     // tab if no tabID is given, enabling automation if necessary. Returns a
     // tree with a placeholder root node; listen for the "loadComplete" event to
     // get a notification that the tree has fully loaded (the previous root node
     // reference will stop working at or before this point).
     [nocompile] static void getTree(optional long tabId, RootCallback callback);
 
     // Get the automation tree for the whole desktop which consists of all on
     // screen views. Note this API is currently only supported on Chrome OS.
     [nocompile] static void getDesktop(RootCallback callback);
 
     // Add a tree change observer. Tree change observers are static/global, they
     // listen to changes across all trees.
     [nocompile] static void addTreeChangeObserver(
         TreeChangeObserver observer);
 
     // Remove a tree change observer.
     [nocompile] static void removeTreeChangeObserver(
         TreeChangeObserver observer);
   };
 };