Corner Passthrough for Accessibility

ui/chromeos/touch_exploration_controller.cc

Index: ui/chromeos/touch_exploration_controller.cc
diff --git a/ui/chromeos/touch_exploration_controller.cc b/ui/chromeos/touch_exploration_controller.cc
index 3670f1e948f213908d1884d9f844eacf35916a5c..451f6abce29568c0b426e4ffe26c08282991bb65 100644
--- a/ui/chromeos/touch_exploration_controller.cc
+++ b/ui/chromeos/touch_exploration_controller.cc
@@ -25,6 +25,10 @@ namespace {
 // Delay between adjustment sounds.
 const base::TimeDelta kSoundDelay = base::TimeDelta::FromMilliseconds(150);
 
+// Delay before corner passthrough activates.
+const base::TimeDelta kCornerPassthroughDelay =
+    base::TimeDelta::FromMilliseconds(700);
+
 // In ChromeOS, VKEY_LWIN is synonymous for the search key.
 const ui::KeyboardCode kChromeOSSearchKey = ui::VKEY_LWIN;
 }  // namespace
@@ -38,7 +42,8 @@ TouchExplorationController::TouchExplorationController(
       gesture_provider_(this),
       prev_state_(NO_FINGERS_DOWN),
       VLOG_on_(true),
-      tick_clock_(NULL) {
+      tick_clock_(NULL),
+      waiting_for_corner_passthrough_(false) {
   CHECK(root_window);
   root_window->GetHost()->GetEventSource()->AddEventRewriter(this);
 }
@@ -68,6 +73,13 @@ ui::EventRewriteStatus TouchExplorationController::RewriteEvent(
     // this event under this new state.
   }
 
+  if (long_press_timer_.IsRunning() &&
+      event.time_stamp() - initial_press_->time_stamp() >
+          gesture_detector_config_.longpress_timeout) {
+    long_press_timer_.Stop();
+    OnLongPressTimerFired();
+  }
+
   const ui::EventType type = touch_event.type();
   const gfx::PointF& location = touch_event.location_f();
   const int touch_id = touch_event.touch_id();
@@ -75,17 +87,24 @@ ui::EventRewriteStatus TouchExplorationController::RewriteEvent(
   // Always update touch ids and touch locations, so we can use those
   // no matter what state we're in.
   if (type == ui::ET_TOUCH_PRESSED) {
+    // If the user enters the screen from the edge then send an earcon.
+    gfx::Point edges = touch_event.location();
+    if (FindEdgesWithinBounds(edges, kLeavingScreenEdge) != NO_EDGE)

[aboxhall, 2014/08/06 18:10:45]

Do we want to not play this if a user already has a finger down? I'm wondering
whether this logic belongs in InNoFingersDown instead.

[lisayin, 2014/08/06 20:37:35]

Hmmm. Good point. I'll move it to InNoFingersDown.

+      delegate_->PlayEnterScreenEarcon();
     current_touch_ids_.push_back(touch_id);
     touch_locations_.insert(std::pair<int, gfx::PointF>(touch_id, location));
   } else if (type == ui::ET_TOUCH_RELEASED || type == ui::ET_TOUCH_CANCELLED) {
     // In order to avoid accidentally double tapping when moving off the edge of
     // the screen, the state will be rewritten to NoFingersDown.
     TouchEvent touch_event = static_cast<const TouchEvent&>(event);
-    if (FindEdgesWithinBounds(touch_event.location(), kLeavingScreenEdge) !=
-        NO_EDGE) {
-      if (current_touch_ids_.size() == 0) {
-        ResetToNoFingersDown();
-      }
+    gfx::Point edges = touch_event.location();
+    if (FindEdgesWithinBounds(edges, kLeavingScreenEdge) != NO_EDGE) {

[aboxhall, 2014/08/06 18:10:45]

Perhaps this needs to check for state as well: I imagine there are several
states for which it doesn't make sense (e.g. mid-swipe gesture? Do we want
someone to know they swiped off the screen if a swipe has already been
successfully registered?)

[lisayin, 2014/08/06 20:37:35]

I actually think it makes sense to notify the user that their swipe has left the
screen since it'll still process the gesture even if they've left the screen.
Swiping off the screen in the middle of a gesture will cause it to process since
we're only checking the type of the event and not where it's located.

On 2014/08/06 18:10:45, aboxhall wrote:

[aboxhall, 2014/08/06 21:03:12]

I guess it could be handy knowing that you'll consistently hear a beep if you
swipe off the edge, but I could imagine that if you're simply performing a swipe
gesture you don't care whether you've swiped off the edge: chances are you'll
lift your finger very shortly afterwards anyway (or else swipe off the screen
altogether) so it'll just be noise. Is that not the case?

[lisayin, 2014/08/06 21:48:22]

So another thing that kind of keeps the sound from playing is the fact that it
only plays if you're within a very small number of pixels. I played around with
gestures on the pixel and even when I leave the screen, the official "release"
location is actually a lot farther away from the edge. Should I compensate for
this discrepancy?

On 2014/08/06 21:03:12, aboxhall wrote:

+        // Indicates to the user that they are leaving the screen.
+      if (VLOG_on_)
+        VLOG(0) << "Leaving the Screen";
+      delegate_->PlayExitScreenEarcon();
+      if (current_touch_ids_.size() == 0)
+          ResetToNoFingersDown();
     }
 
     std::vector<int>::iterator it = std::find(
@@ -128,6 +147,8 @@ ui::EventRewriteStatus TouchExplorationController::RewriteEvent(
       return InGestureInProgress(touch_event, rewritten_event);
     case TOUCH_EXPLORE_SECOND_PRESS:
       return InTouchExploreSecondPress(touch_event, rewritten_event);
+    case CORNER_PASSTHROUGH:
+      return InCornerPassthrough(touch_event, rewritten_event);
     case SLIDE_GESTURE:
       return InSlideGesture(touch_event, rewritten_event);
     case ONE_FINGER_PASSTHROUGH:
@@ -148,25 +169,60 @@ ui::EventRewriteStatus TouchExplorationController::NextDispatchEvent(
 ui::EventRewriteStatus TouchExplorationController::InNoFingersDown(
     const ui::TouchEvent& event, scoped_ptr<ui::Event>* rewritten_event) {
   const ui::EventType type = event.type();
-  if (type == ui::ET_TOUCH_PRESSED) {
-    initial_press_.reset(new TouchEvent(event));
-    last_unused_finger_event_.reset(new TouchEvent(event));
-    StartTapTimer();
-    gesture_provider_.OnTouchEvent(event);
-    gesture_provider_.OnTouchEventAck(false);
-    ProcessGestureEvents();
-    state_ = SINGLE_TAP_PRESSED;
-    VLOG_STATE();
-    return ui::EVENT_REWRITE_DISCARD;
+  if (type != ui::ET_TOUCH_PRESSED) {
+    NOTREACHED() << "Unexpected event type received: " << event.name();
+    return ui::EVENT_REWRITE_CONTINUE;
   }
-  NOTREACHED() << "Unexpected event type received: " << event.name();
-  return ui::EVENT_REWRITE_CONTINUE;
+  int location = FindEdgesWithinBounds(event.location(), kSlopDistanceFromEdge);
+  base::TimeDelta timeout;
+
+  // If the press was at a corner, the user might go into corner passthrough
+  // instead.
+  bool in_a_bottom_corner =
+      (BOTTOM_LEFT_CORNER == location) || (BOTTOM_RIGHT_CORNER == location);
+  if (in_a_bottom_corner) {
+    long_press_timer_.Start(FROM_HERE,
+                            gesture_detector_config_.longpress_timeout,
+                            this,
+                            &TouchExplorationController::OnLongPressTimerFired);
+    waiting_for_corner_passthrough_ = true;

[aboxhall, 2014/08/06 18:10:45]

Why not just use the long_press_timer_ (or corner_passthrough_timer_) to track
whether we're waiting for corner passthrough? When does it make sense to set
waiting_for_corner_passthrough_ to false but not stop the timer?

[lisayin, 2014/08/06 20:37:35]

I'm hoping to use this timer for evy's passthrough implementation as well so
that they both will fire an earcon if activated, but I have removed
waiting_for_corner_passthrough_ and renamed the timer to passthrough_timer_. 

+  } else {
+    waiting_for_corner_passthrough_ = false;
+  }
+  tap_timer_.Start(FROM_HERE,
+                   gesture_detector_config_.double_tap_timeout,
+                   this,
+                   &TouchExplorationController::OnTapTimerFired);
+  initial_press_.reset(new TouchEvent(event));
+  last_unused_finger_event_.reset(new TouchEvent(event));
+  gesture_provider_.OnTouchEvent(event);
+  gesture_provider_.OnTouchEventAck(false);
+  ProcessGestureEvents();
+  state_ = SINGLE_TAP_PRESSED;
+  VLOG_STATE();
+  return ui::EVENT_REWRITE_DISCARD;
 }
 
 ui::EventRewriteStatus TouchExplorationController::InSingleTapPressed(
     const ui::TouchEvent& event, scoped_ptr<ui::Event>* rewritten_event) {
   const ui::EventType type = event.type();
 
+  int location = FindEdgesWithinBounds(event.location(), kMaxDistanceFromEdge);
+  bool in_a_bottom_corner =
+      (location == BOTTOM_LEFT_CORNER) || (location == BOTTOM_RIGHT_CORNER);
+  // If the event is from the initial press and the location is no longer in the
+  // corner, then we are not waiting for a corner passthrough anymore.
+  if (event.touch_id() == initial_press_->touch_id() && !in_a_bottom_corner) {
+    waiting_for_corner_passthrough_ = false;
+    if (long_press_timer_.IsRunning()) {

[aboxhall, 2014/08/06 18:10:44]

If we're not going to use this for anything other than corner passthrough,
perhaps it would be better named 'corner_passthrough_timer_'.

[lisayin, 2014/08/06 20:37:35]

Done.

+      long_press_timer_.Stop();
+      if (event.time_stamp() - initial_press_->time_stamp() >
+          gesture_detector_config_.double_tap_timeout) {
+        OnTapTimerFired();

[aboxhall, 2014/08/06 18:10:44]

Why do this? Could you add a comment?

[lisayin, 2014/08/06 20:37:35]

This is in case the finger moves out of the corner before the long press timer
has fired but after the tap timer has fired to make sure you can still go into
touch exploration.

[aboxhall, 2014/08/06 21:03:12]

Ah, I see now. But won't the tap timer have been running (due to line 190) and
caused OnTapTimerFired() to be executed already?

[lisayin, 2014/08/06 21:48:22]

In TapTimerFired, if the long press timer is running, I don't change the state.
So now that the long press timer has been stopped, I need to check to see if the
tap timer should have been fired or not.

On 2014/08/06 21:03:12, aboxhall wrote:

[aboxhall, 2014/08/06 22:01:24]

Oh, duh! Makes sense, thanks for the explanation :)

+      }
+    }
+  }
+
   if (type == ui::ET_TOUCH_PRESSED) {
     // TODO (evy, lisayin) : add support for multifinger swipes.
     // For now, we wait for there to be only one finger down again.
@@ -200,7 +256,7 @@ ui::EventRewriteStatus TouchExplorationController::InSingleTapPressed(
     }
     // Change to slide gesture if the slide occurred at the right edge.
     int edge = FindEdgesWithinBounds(event.location(), kMaxDistanceFromEdge);
-    if (edge & RIGHT_EDGE) {
+    if (edge & RIGHT_EDGE && edge != BOTTOM_RIGHT_CORNER) {
       state_ = SLIDE_GESTURE;
       VLOG_STATE();
       return InSlideGesture(event, rewritten_event);
@@ -393,6 +449,40 @@ ui::EventRewriteStatus TouchExplorationController::InGestureInProgress(
   return ui::EVENT_REWRITE_DISCARD;
 }
 
+ui::EventRewriteStatus TouchExplorationController::InCornerPassthrough(
+    const ui::TouchEvent& event,
+    scoped_ptr<ui::Event>* rewritten_event) {
+  ui::EventType type = event.type();
+
+  // If the first finger has left the corner, then exit passthrough.
+  if (event.touch_id() == initial_press_->touch_id()) {
+    int edges = FindEdgesWithinBounds(event.location(), kSlopDistanceFromEdge);
+    bool in_a_bottom_corner = (edges == BOTTOM_LEFT_CORNER) ||
+                              (edges == BOTTOM_RIGHT_CORNER);
+    if (type == ui::ET_TOUCH_MOVED && in_a_bottom_corner)
+      return ui::EVENT_REWRITE_DISCARD;
+
+    if (current_touch_ids_.size() == 0) {
+      ResetToNoFingersDown();
+      return ui::EVENT_REWRITE_DISCARD;
+    }
+
+    waiting_for_corner_passthrough_ = false;
+    state_ = WAIT_FOR_ONE_FINGER;
+    VLOG_STATE();
+    return ui::EVENT_REWRITE_DISCARD;
+  }
+
+  rewritten_event->reset(new ui::TouchEvent(
+      type, event.location(), event.touch_id(), event.time_stamp()));
+  (*rewritten_event)->set_flags(event.flags());
+
+  if (current_touch_ids_.size() == 0)
+    ResetToNoFingersDown();
+
+  return ui::EVENT_REWRITE_REWRITTEN;
+}
+
 ui::EventRewriteStatus TouchExplorationController::InOneFingerPassthrough(
     const ui::TouchEvent& event,
     scoped_ptr<ui::Event>* rewritten_event) {
@@ -468,6 +558,7 @@ ui::EventRewriteStatus TouchExplorationController::InTouchExploreSecondPress(
 ui::EventRewriteStatus TouchExplorationController::InWaitForOneFinger(
     const ui::TouchEvent& event,
     scoped_ptr<ui::Event>* rewritten_event) {
+  waiting_for_corner_passthrough_ = false;
   ui::EventType type = event.type();
   if (!(type == ui::ET_TOUCH_PRESSED || type == ui::ET_TOUCH_MOVED ||
         type == ui::ET_TOUCH_RELEASED || type == ui::ET_TOUCH_CANCELLED)) {
@@ -486,7 +577,7 @@ ui::EventRewriteStatus TouchExplorationController::InWaitForOneFinger(
 }
 
 void TouchExplorationController::PlaySoundForTimer() {
-  delegate_->PlayVolumeAdjustSound();
+  delegate_->PlayVolumeAdjustEarcon();
 }
 
 ui::EventRewriteStatus TouchExplorationController::InSlideGesture(
@@ -526,7 +617,7 @@ ui::EventRewriteStatus TouchExplorationController::InSlideGesture(
                        kSoundDelay,
                        this,
                        &ui::TouchExplorationController::PlaySoundForTimer);
-    delegate_->PlayVolumeAdjustSound();
+    delegate_->PlayVolumeAdjustEarcon();
   }
 
   // There should not be more than one finger down.
@@ -589,6 +680,8 @@ void TouchExplorationController::OnTapTimerFired() {
       return;
     }
     case SINGLE_TAP_PRESSED:
+      if (waiting_for_corner_passthrough_)
+        return;
     case GESTURE_IN_PROGRESS:
       // Discard any pending gestures.
       delete gesture_provider_.GetAndResetPendingGestures();
@@ -605,6 +698,18 @@ void TouchExplorationController::OnTapTimerFired() {
   last_touch_exploration_.reset(new TouchEvent(*initial_press_));
 }
 
+void TouchExplorationController::OnLongPressTimerFired() {
+  if (waiting_for_corner_passthrough_) {

[aboxhall, 2014/08/06 18:10:44]

Invert this logic:
if (!waiting_for_corner_passthrough_)
  return;
...

[lisayin, 2014/08/06 20:37:35]

Done.

+    if (sound_timer_.IsRunning())
+      sound_timer_.Stop();
+    delegate_->PlayPassthroughEarcon();
+    delete gesture_provider_.GetAndResetPendingGestures();
+    state_ = CORNER_PASSTHROUGH;
+    VLOG_STATE();
+    return;
+  }
+}
+
 void TouchExplorationController::DispatchEvent(ui::Event* event) {
   ui::EventDispatchDetails result ALLOW_UNUSED =
       root_window_->GetHost()->dispatcher()->OnEventFromSource(event);
@@ -646,13 +751,13 @@ void TouchExplorationController::SideSlideControl(ui::GestureEvent* gesture) {
     return;
 
   if (type == ET_GESTURE_SCROLL_BEGIN) {
-    delegate_->PlayVolumeAdjustSound();
+    delegate_->PlayVolumeAdjustEarcon();
   }
 
   if (type == ET_GESTURE_SCROLL_END) {
     if (sound_timer_.IsRunning())
       sound_timer_.Stop();
-    delegate_->PlayVolumeAdjustSound();
+    delegate_->PlayVolumeAdjustEarcon();
   }
 
   // If the user is in the corner of the right side of the screen, the volume
@@ -799,6 +904,7 @@ void TouchExplorationController::EnterTouchToMouseMode() {
 }
 
 void TouchExplorationController::ResetToNoFingersDown() {
+  waiting_for_corner_passthrough_ = false;
   ProcessGestureEvents();
   if (sound_timer_.IsRunning())
     sound_timer_.Stop();
@@ -869,6 +975,8 @@ const char* TouchExplorationController::EnumStateToString(State state) {
       return "GESTURE_IN_PROGRESS";
     case TOUCH_EXPLORE_SECOND_PRESS:
       return "TOUCH_EXPLORE_SECOND_PRESS";
+    case CORNER_PASSTHROUGH:
+      return "CORNER_PASSTHROUGH";
     case SLIDE_GESTURE:
       return "SLIDE_GESTURE";
     case ONE_FINGER_PASSTHROUGH: