Select-to-speak extension code

chrome/browser/resources/chromeos/select_to_speak/background.js

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
+
+var RoleType = chrome.automation.RoleType;
+
+/**
+ * @constructor

[David Tseng, 2016/11/22 19:01:09]

Are you planning on compiling this file (at least for type cechking on upload)?

[dmazzoni, 2016/11/28 18:51:12]

Yes. I finished adding that to this change now. To type check
manually, run:

third_party/closure_compiler/run_compiler select_to_speak

+ */
+SelectToSpeak = function() {
+  this.node_ = null;
+  this.down_ = false;
+
+  chrome.automation.getDesktop(function(d) {
+    d.addEventListener('mousePressed', this.onMousePressed_.bind(this), true);

[David Tseng, 2016/11/22 19:01:10]

chrome.automation.EventType

[dmazzoni, 2016/11/28 18:51:12]

Done.

+    d.addEventListener('mouseDragged', this.onMouseDragged_.bind(this), true);
+    d.addEventListener('mouseMoved', this.onMouseDragged_.bind(this), true);

[David Tseng, 2016/11/22 19:01:09]

onMouseDraggedOrMoved? (or were these supposed to be handled separately).

[dmazzoni, 2016/11/28 18:51:13]

Actually we don't need to handle mouseMoved. Fixed.

+    d.addEventListener('mouseReleased', this.onMouseReleased_.bind(this), true);
+    d.addEventListener('mouseCanceled', this.onMouseCanceled_.bind(this), true);
+  }.bind(this));
+};
+
+SelectToSpeak.prototype = {
+  /**
+   * Called when the mouse is pressed and the user is in a mode where
+   * select-to-speak is capturing mouse events (for example holding down
+   * Search).
+   *
+   * @param {!AutomationEvent} evt
+   */
+  onMousePressed_: function(evt) {
+    this.down_ = true;
+    this.mouseStart_ = {x: evt.mouseX, y: evt.mouseY};
+    this.startNode_ = evt.target;
+    this.speechQueue_ = [];
+    chrome.tts.stop();
+    this.onMouseDragged_(evt);
+  },
+
+  /**
+   * Called when the mouse is moved or dragged and the user is in a
+   * mode where select-to-speak is capturing mouse events (for example

[David Tseng, 2016/11/22 19:01:10]

Select to speak only captures if search is down, right?

[dmazzoni, 2016/11/28 18:51:13]

Currently yes but I wanted to write the comment in a way
that allows for future expansion, there may be multiple
ways to trigger it.

+   * holding down Search).
+   *
+   * @param {!AutomationEvent} evt
+   */
+  onMouseDragged_: function(evt) {
+    if (!this.down_)

[David Tseng, 2016/11/22 19:01:10]

Doesn't a mouse drag already mean a mouse down? I thought that was what
distinguished drag from move?

[dmazzoni, 2016/11/28 18:51:12]

This is still needed to handle a race condition where
the drag starts before the extension loads. We should
only handle the drag if we got the mouse down event
first and initialized our state properly.

+      return;
+
+    var rect = {left: Math.floor(this.mouseStart_.x),
+                top: Math.floor(this.mouseStart_.y),

[David Tseng, 2016/11/22 19:01:10]

How does this work if I drag up or to the left? Does the focus indicator support
negative width/height?

[dmazzoni, 2016/11/28 18:51:12]

Good point. Fixed.

+                width: Math.floor(evt.mouseX - this.mouseStart_.x),
+                height: Math.floor(evt.mouseY - this.mouseStart_.y)};
+    chrome.accessibilityPrivate.setFocusRing([rect]);
+  },
+
+  /**
+   * Called when the mouse is moved or dragged and the user is in a
+   * mode where select-to-speak is capturing mouse events (for example

[David Tseng, 2016/11/22 19:01:10]

Update the comment

[dmazzoni, 2016/11/28 18:51:12]

Done.

+   * holding down Search).
+   *
+   * @param {!AutomationEvent} evt
+   */
+  onMouseReleased_: function(evt) {
+    this.onMouseDragged_(evt);
+    this.down_ = false;
+
+    chrome.accessibilityPrivate.setFocusRing([]);

[David Tseng, 2016/11/22 19:01:10]

I would have expected this to happen when s2s goes inactive (e.g. search is
lifted).

[dmazzoni, 2016/11/28 18:51:12]

That's handled in onMouseCanceled.

For onMouseReleased, we clear the focus ring and then highlight
each object as it's spoken.

I'll probably refine it later to show the rect in one color
and highlight each object in another color.

+
+    // Walk up to the nearest window, web area, or dialog that the
+    // hit node is contained inside. Only speak objects within that
+    // container. In the future we might include other container-like
+    // roles here.
+    var root = this.startNode_;
+    while (root.parent &&
+        root.role != RoleType.window &&
+        root.role != RoleType.rootWebArea &&
+        root.role != RoleType.desktop &&
+        root.role != RoleType.dialog) {
+      root = root.parent;
+    }
+
+    var rect = {left: Math.floor(this.mouseStart_.x),
+                top: Math.floor(this.mouseStart_.y),
+                width: Math.floor(evt.mouseX - this.mouseStart_.x),
+                height: Math.floor(evt.mouseY - this.mouseStart_.y)};
+    var nodes = [];
+    this.findAllMatching_(root, rect, nodes);
+
+    this.speechQueue_ = nodes;
+    this.speakNextInQueue_();
+  },
+
+  /**
+   * Called when the user cancels select-to-speak's capturing of mouse
+   * events (for example by releasing Search while the mouse is still down).
+   *
+   * @param {!AutomationEvent} evt
+   */
+  onMouseCanceled_: function(evt) {
+    this.down_ = false;
+    chrome.accessibilityPrivate.setFocusRing([]);
+    this.speechQueue_ = [];
+    chrome.tts.stop();
+  },
+
+  /**
+   * Returns true if |rect1| and |rect2| overlap. The rects must define
+   * left, top, width, and height.
+   * @param {Object} rect1
+   * @param {Object} rect2
+   * @return {boolean} True if the rects overlap.
+   */
+  overlaps_: function(rect1, rect2) {

[David Tseng, 2016/11/22 19:01:09]

No need to define this function in this class; make it a function outside the
prototype.

[dmazzoni, 2016/11/28 18:51:12]

Done.

+    var l1 = rect1.left;
+    var r1 = rect1.left + rect1.width;
+    var t1 = rect1.top;
+    var b1 = rect1.top + rect1.height;
+    var l2 = rect2.left;
+    var r2 = rect2.left + rect2.width;
+    var t2 = rect2.top;
+    var b2 = rect2.top + rect2.height;
+    return (l1 < r2 && r1 > l2 && t1 < b2 && b1 > t2);

[David Tseng, 2016/11/22 19:01:09]

Doesn't look right. What happens if the two rects are equal? Or, if one is
contained within another?

[dmazzoni, 2016/11/28 18:51:13]

Both of those cases work as long as the rects have nonzero
width and height.

Think about just the x coordinates. If the rects are
equal (and nonempty), l1 will be less than r1, so
l1 will be less than r2 too.

If the first rect is contained in the second, l1
will still be less than r2, and r1 will still be
greater than r2.

+  },
+
+  /**
+   * Finds all nodes within the subtree rooted at |node| that overlap
+   * a given rectangle.
+   * @param {AutomationNode} The starting node.
+   * @param {Object} rect The bounding box to search.
+   * @param {Array<AutomationNode>} nodes The matching node array to be
+   *     populated.
+   * @return {boolean} True if any matches are found.
+   */
+  findAllMatching_: function(node, rect, nodes) {
+    var found = false;
+    for (var c = node.firstChild; c; c = c.nextSibling) {
+      if (this.findAllMatching_(c, rect, nodes))
+        found = true;
+    }
+
+    if (found)
+      return true;
+
+    if (!node.name)

[David Tseng, 2016/11/22 19:01:10]

This is going to give some strange results I think because there are places in
the tree where nodes have a name but it isn't visually rendered. Why not grab
the static text and read that?

[dmazzoni, 2016/11/28 18:51:13]

I think it's actually useful that it reads the alt text for
icons and images. You can click on an extension icon and it
reads the name of the extension, for example.

Happy to refine it after we try it out and get some feedback,
but this isn't meant to be perfect, just a reasonable
first draft. I don't want to overengineer it.

+      return false;
+
+    if (this.overlaps_(node.location, rect)) {
+      nodes.push(node);
+      return true;
+    }
+
+    return false;
+  },
+
+  /**
+   * Pop the next matching node from the queue and speak and highlight it.
+   */
+  speakNextInQueue_: function() {
+    if (!this.speechQueue_ || this.speechQueue_.length == 0) {

[David Tseng, 2016/11/22 19:01:10]

Looks like type checking would help here. Set this.speechQueue_ to [] in the
constructor.

[dmazzoni, 2016/11/28 18:51:12]

Done.

+      chrome.accessibilityPrivate.setFocusRing([]);
+      return;
+    }
+
+    var node = this.speechQueue_.shift();
+    chrome.accessibilityPrivate.setFocusRing([node.location]);
+    chrome.tts.speak(node.name, {
+      lang: 'en-US',

[David Tseng, 2016/11/22 19:01:10]

Maybe we should think about starting some code sharing between ChromeVox and s2s
(for the future).

[dmazzoni, 2016/11/28 18:51:13]

Agreed, perhaps some common utility libraries.

+      'enqueue': false,

[David Tseng, 2016/11/22 19:01:09]

Can we let the native tts queue handle things?

[dmazzoni, 2016/11/28 18:51:12]

Done, it just means slightly trickier logic inside
onEvent.

+      onEvent: (function(event) {
+        if (event.type == 'end') {
+          this.speakNextInQueue_();
+        } else if (event.type == 'interrupted' ||
+                   event.type == 'cancelled') {
+          chrome.accessibilityPrivate.setFocusRing([]);
+        }
+      }).bind(this)
+    });
+  }
+};
+
+new SelectToSpeak();