Refactor Select-to-speak so that mouse events are forwarded to the extension.

chrome/browser/resources/chromeos/select_to_speak/select_to_speak.js

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 var AutomationEvent = chrome.automation.AutomationEvent;
 var AutomationNode = chrome.automation.AutomationNode;
 var EventType = chrome.automation.EventType;
 var RoleType = chrome.automation.RoleType;
 
 /**
@@ skipping 35 matching lines @@
 }
 
 /**
  * @constructor
  */
 var SelectToSpeak = function() {
   /** @private { AutomationNode } */
   this.node_ = null;
 
   /** @private { boolean } */
-  this.down_ = false;
+  this.trackingMouse_ = false;
+
+  /** @private { boolean } */
+  this.didTrackMouse_ = false;
+
+  /** @private { boolean } */
+  this.searchKeyDown_ = false;

[David Tseng, 2017/05/16 15:54:39]

nit: isSearchKeyDown_

[dmazzoni, 2017/05/16 18:23:48]

Done.

+
+  /** @private { Set } */

[David Tseng, 2017/05/16 15:54:42]

nit: !Set<number>

[dmazzoni, 2017/05/16 18:23:48]

Done.

+  this.keysCurrentlyDown_ = new Set();
+
+  /** @private { Set } */
+  this.keysPressedTogether_ = new Set();
 
   /** @private {{x: number, y: number}} */
   this.mouseStart_ = {x: 0, y: 0};
 
+  /** @private {{x: number, y: number}} */
+  this.mouseEnd_ = {x: 0, y: 0};
+
+  /** @private {AutomationRootNode} */
   chrome.automation.getDesktop(function(desktop) {
+    this.desktop_ = desktop;
+
+    // After the user selects a region of the screen, we do a hit test at
+    // the center of that box using the automation API. The result of the
+    // hit test is a MOUSE_RELEASED accessibility event.
     desktop.addEventListener(
-        EventType.MOUSE_PRESSED, this.onMousePressed_.bind(this), true);
-    desktop.addEventListener(
-        EventType.MOUSE_DRAGGED, this.onMouseDragged_.bind(this), true);
-    desktop.addEventListener(
-        EventType.MOUSE_RELEASED, this.onMouseReleased_.bind(this), true);
-    desktop.addEventListener(
-        EventType.MOUSE_CANCELED, this.onMouseCanceled_.bind(this), true);
+        EventType.MOUSE_RELEASED, this.onAutomationHitTest_.bind(this),
+        true);
   }.bind(this));
 
   /** @private { ?string } */
   this.voiceNameFromPrefs_ = null;
 
   /** @private { ?string } */
   this.voiceNameFromLocale_ = null;
 
   /** @private { Set<string> } */
   this.validVoiceNames_ = new Set();
 
   /** @private { number } */
   this.speechRate_ = 1.0;
 
   /** @const { string } */
   this.color_ = "#f73a98";
 
   this.initPreferences_();
+
+  this.setUpEventListeners_();
 };
 
+/** @const {number} */
+SelectToSpeak.SEARCH_KEY_CODE = 91;
+
+/** @const {number} */
+SelectToSpeak.CONTROL_KEY_CODE = 17;
+
 SelectToSpeak.prototype = {
   /**
    * Called when the mouse is pressed and the user is in a mode where
    * select-to-speak is capturing mouse events (for example holding down
    * Search).
    *
-   * @param {!AutomationEvent} evt
+   * @param {!Event} evt The DOM event
+   * @return {boolean} True if the default action should be performed.
    */
-  onMousePressed_: function(evt) {
-    this.down_ = true;
-    this.mouseStart_ = {x: evt.mouseX, y: evt.mouseY};
-    this.startNode_ = evt.target;
+  onMouseDown_: function(evt) {
+    if (!this.searchKeyDown_)
+      return false;

[David Tseng, 2017/05/16 15:54:43]

return true

[dmazzoni, 2017/05/16 18:23:48]

Leaving as false since we're not reinjecting

+
+    this.trackingMouse_ = true;
+    this.didTrackMouse_ = true;
+    this.mouseStart_ = {x: evt.screenX, y: evt.screenY};
     chrome.tts.stop();
-    this.onMouseDragged_(evt);
+
+    // Fire a hit test event on click to warm up the cache.
+    this.desktop_.hitTest(evt.screenX, evt.screenY, EventType.MOUSE_PRESSED);
+
+    this.onMouseMove_(evt);
+    return false;
   },
 
   /**
    * Called when the mouse is moved or dragged and the user is in a
    * mode where select-to-speak is capturing mouse events (for example
    * holding down Search).
    *
-   * @param {!AutomationEvent} evt
+   * @param {!Event} evt The DOM event
+   * @return {boolean} True if the default action should be performed.
    */
-  onMouseDragged_: function(evt) {
-    if (!this.down_)
-      return;
+  onMouseMove_: function(evt) {
+    if (!this.trackingMouse_)
+      return false;

[David Tseng, 2017/05/16 15:54:39]

return true

 
     var rect = rectFromPoints(
         this.mouseStart_.x, this.mouseStart_.y,
-        evt.mouseX, evt.mouseY);
+        evt.screenX, evt.screenY);
     chrome.accessibilityPrivate.setFocusRing([rect], this.color_);
+    return false;
   },
 
   /**
    * Called when the mouse is released and the user is in a
    * mode where select-to-speak is capturing mouse events (for example
    * holding down Search).
    *
-   * @param {!AutomationEvent} evt
+   * @param {!Event} evt
+   * @return {boolean} True if the default action should be performed.
    */
-  onMouseReleased_: function(evt) {
-    this.onMouseDragged_(evt);
-    this.down_ = false;
+  onMouseUp_: function(evt) {
+    this.onMouseMove_(evt);
+    this.trackingMouse_ = false;
 
     chrome.accessibilityPrivate.setFocusRing([]);
 
-    // Walk up to the nearest window, web area, or dialog that the
+    this.mouseEnd_ = {x: evt.screenX, y: evt.screenY};
+    var ctrX = Math.floor((this.mouseStart_.x + this.mouseEnd_.x) / 2);
+    var ctrY = Math.floor((this.mouseStart_.y + this.mouseEnd_.y) / 2);
+
+    // Do a hit test at the center of the area the user dragged over.
+    // This will give us some context when searching the accessibility tree.
+    // The hit test will result in a EventType.MOUSE_RELEASED event being
+    // fired on the result of that hit test, which will trigger
+    // onAutomationHitTest_.
+    this.desktop_.hitTest(ctrX, ctrY, EventType.MOUSE_RELEASED);
+    return false;
+  },
+
+  /**
+   * Called in response to our hit test after the mouse is released,
+   * when the user is in a mode where select-to-speak is capturing
+   * mouse events (for example holding down Search).
+   *
+   * @param {!AutomationEvent} evt The automation event.
+   */
+  onAutomationHitTest_: function(evt) {
+    // Walk up to the nearest window, web area, toolbar, or dialog that the
     // hit node is contained inside. Only speak objects within that
     // container. In the future we might include other container-like
     // roles here.
-    var root = this.startNode_;
+    var root = evt.target;
     while (root.parent &&
         root.role != RoleType.WINDOW &&
         root.role != RoleType.ROOT_WEB_AREA &&
         root.role != RoleType.DESKTOP &&
-        root.role != RoleType.DIALOG) {
+        root.role != RoleType.DIALOG &&
+        root.role != RoleType.ALERT_DIALOG &&
+        root.role != RoleType.TOOLBAR) {
       root = root.parent;
     }
 
     var rect = rectFromPoints(
         this.mouseStart_.x, this.mouseStart_.y,
-        evt.mouseX, evt.mouseY);
+        this.mouseEnd_.x, this.mouseEnd_.y);
     var nodes = [];
     this.findAllMatching_(root, rect, nodes);
     this.startSpeechQueue_(nodes);
   },
 
   /**
-   * Called when the user cancels select-to-speak's capturing of mouse
-   * events (for example by releasing Search while the mouse is still down).
-   *
-   * @param {!AutomationEvent} evt
+   * @param {!Event} evt
    */
-  onMouseCanceled_: function(evt) {
-    this.down_ = false;
-    chrome.accessibilityPrivate.setFocusRing([]);
-    chrome.tts.stop();
+  onKeyDown_: function(evt) {
+    if (this.keysPressedTogether_.size == 0 &&
+        evt.keyCode == SelectToSpeak.SEARCH_KEY_CODE) {
+      this.searchKeyDown_ = true;
+    } else if (!this.trackingMouse_) {
+      this.searchKeyDown_ = false;
+    }
+
+    this.keysCurrentlyDown_.add(evt.keyCode);
+    this.keysPressedTogether_.add(evt.keyCode);
   },
 
   /**
+   * @param {!Event} evt
+   */
+  onKeyUp_: function(evt) {
+    if (evt.keyCode == SelectToSpeak.SEARCH_KEY_CODE) {
+      this.searchKeyDown_ = false;
+
+      // If we were in the middle of tracking the mouse, cancel it.
+      if (this.trackingMouse_) {
+        this.trackingMouse_ = false;
+        chrome.accessibilityPrivate.setFocusRing([]);
+        chrome.tts.stop();
+      }
+    }
+
+    // Stop speech when the user taps and releases Control or Search
+    // without using the mouse or pressing any other keys along the way.
+    if (!this.didTrackMouse_ &&
+        (evt.keyCode == SelectToSpeak.SEARCH_KEY_CODE ||
+         evt.keyCode == SelectToSpeak.CONTROL_KEY_CODE) &&
+        this.keysPressedTogether_.has(evt.keyCode) &&
+        this.keysPressedTogether_.size == 1) {
+      this.trackingMouse_ = false;
+      chrome.accessibilityPrivate.setFocusRing([]);
+      chrome.tts.stop();
+    }
+
+    this.keysCurrentlyDown_.delete(evt.keyCode);
+    if (this.keysCurrentlyDown_.size == 0) {
+      this.keysPressedTogether_.clear();
+      this.didTrackMouse_ = false;
+    }
+  },
+
+  /**
+   * Set up event listeners for mouse ane keyboard events. These are

[David Tseng, 2017/05/16 15:54:41]

s/ane/and

[dmazzoni, 2017/05/16 18:23:48]

Done.

+   * forwarded to us from the SelectToSpeakEventHandler so they should
+   * be interpreted as global events on the whole screen, not local to
+   * any particular window.
+   */
+  setUpEventListeners_: function() {
+    document.addEventListener('keydown', this.onKeyDown_.bind(this));
+    document.addEventListener('keyup', this.onKeyUp_.bind(this));
+    document.addEventListener('mousedown', this.onMouseDown_.bind(this));
+    document.addEventListener('mousemove', this.onMouseMove_.bind(this));
+    document.addEventListener('mouseup', this.onMouseUp_.bind(this));
+  },
+
+  /**
    * Finds all nodes within the subtree rooted at |node| that overlap
    * a given rectangle.
    * @param {AutomationNode} node The starting node.
    * @param {{left: number, top: number, width: number, height: number}} rect
    *     The bounding box to search.
    * @param {Array<AutomationNode>} nodes The matching node array to be
    *     populated.
    * @return {boolean} True if any matches are found.
    */
   findAllMatching_: function(node, rect, nodes) {
@@ skipping 96 matching lines @@
     this.updateDefaultVoice_();
     window.speechSynthesis.onvoiceschanged = (function() {
       this.updateDefaultVoice_();
     }).bind(this);
   },
 
   /**
    * Get the list of TTS voices, and set the default voice if not already set.
    */
   updateDefaultVoice_: function() {
-    console.log('updateDefaultVoice_ ' + this.down_);
     var uiLocale = chrome.i18n.getMessage('@@ui_locale');
     uiLocale = uiLocale.replace('_', '-').toLowerCase();

[David Tseng, 2017/05/16 15:54:40]

For future cl. An extjs test would be appropriate for select to speak to get
some coverage for specific html snippets similar to the mock ffedback style of
tests in ChromeVox. 

[dmazzoni, 2017/05/16 18:23:48]

Sounds good

 
     chrome.tts.getVoices((function(voices) {
       console.log('updateDefaultVoice_ voices: ' + voices.length);
       this.validVoiceNames_ = new Set();
 
       if (voices.length == 0)
         return;
 
       voices.forEach((function(voice) {
         this.validVoiceNames_.add(voice.voiceName);
@@ skipping 15 matching lines @@
       this.voiceNameFromLocale_ = voices[0].voiceName;
 
       chrome.storage.sync.get(['voice'], (function(prefs) {
         if (!prefs['voice']) {
           chrome.storage.sync.set({'voice': voices[0].voiceName});
         }
       }).bind(this));
     }).bind(this));
   }
 };