Side Slide Gestures for Accessibility

ui/chromeos/touch_exploration_controller.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "ui/chromeos/touch_exploration_controller.h"
 
-#include "base/logging.h"
 #include "base/strings/string_number_conversions.h"
 #include "ui/aura/client/cursor_client.h"
 #include "ui/aura/window.h"
 #include "ui/aura/window_event_dispatcher.h"
 #include "ui/aura/window_tree_host.h"
 #include "ui/events/event.h"
 #include "ui/events/event_processor.h"
+#include "ui/gfx/geometry/rect.h"
 
 #define VLOG_STATE() if (VLOG_IS_ON(0)) VlogState(__func__)
 #define VLOG_EVENT(event) if (VLOG_IS_ON(0)) VlogEvent(event, __func__)
 
 namespace ui {
 
 namespace {
+
+// Delay between adjustment sounds.
+const base::TimeDelta kSoundDelay = base::TimeDelta::FromMilliseconds(150);
+
+const float kLeavingScreenEdge = 6;

[dmazzoni, 2014/07/16 16:54:40]

Need comment for this too.

[lisayin, 2014/07/17 16:56:17]

Done.

+
+// Swipe/scroll gestures within these bounds (in dips) will change preset
+// settings.
+const float kMaxDistanceFromEdge = 75;
+
+// After a slide gesture has been triggered, if the finger is still within these
+// bounds, the preset settings will still change.

[dmazzoni, 2014/07/16 16:54:40]

Note that this is in Dips too.

[lisayin, 2014/07/17 16:56:17]

Done.

+const float kSlopDistanceFromEdge = kMaxDistanceFromEdge + 40;
+
 // In ChromeOS, VKEY_LWIN is synonymous for the search key.
 const ui::KeyboardCode kChromeOSSearchKey = ui::VKEY_LWIN;
 }  // namespace
 
 TouchExplorationController::TouchExplorationController(
-    aura::Window* root_window)
+    aura::Window* root_window,
+    TouchExplorationControllerDelegate* delegate)
     : root_window_(root_window),
+      delegate_(delegate),
       state_(NO_FINGERS_DOWN),
       event_handler_for_testing_(NULL),
       gesture_provider_(this),
       prev_state_(NO_FINGERS_DOWN),
       VLOG_on_(true) {
   CHECK(root_window);
   root_window->GetHost()->GetEventSource()->AddEventRewriter(this);
 }
 
 
@@ skipping 21 matching lines @@
 
 bool TouchExplorationController::IsInNoFingersDownStateForTesting() const {
   return state_ == NO_FINGERS_DOWN;
 }
 
 bool TouchExplorationController::IsInGestureInProgressStateForTesting() const {
   return state_ == GESTURE_IN_PROGRESS;
 }
 
 void TouchExplorationController::SuppressVLOGsForTesting(bool suppress) {
-    VLOG_on_ = !suppress;
+  VLOG_on_ = !suppress;
+}
+
+gfx::Rect TouchExplorationController::BoundsOfRootWindowInDIPForTesting() {
+  return root_window_->GetBoundsInScreen();
+}
+
+bool TouchExplorationController::IsInSlideGestureStateForTesting() const {
+  return state_ == SLIDE_GESTURE;
 }
 
 ui::EventRewriteStatus TouchExplorationController::RewriteEvent(
     const ui::Event& event,
     scoped_ptr<ui::Event>* rewritten_event) {
   if (!event.IsTouchEvent()) {
     if (event.IsKeyEvent()) {
       const ui::KeyEvent& key_event = static_cast<const ui::KeyEvent&>(event);
       VLOG(0) << "\nKeyboard event: " << key_event.name()
               << "\n Key code: " << key_event.key_code()
@@ skipping 20 matching lines @@
   const ui::EventType type = touch_event.type();
   const gfx::PointF& location = touch_event.location_f();
   const int touch_id = touch_event.touch_id();
 
   // Always update touch ids and touch locations, so we can use those
   // no matter what state we're in.
   if (type == ui::ET_TOUCH_PRESSED) {
     current_touch_ids_.push_back(touch_id);
     touch_locations_.insert(std::pair<int, gfx::PointF>(touch_id, location));
   } else if (type == ui::ET_TOUCH_RELEASED || type == ui::ET_TOUCH_CANCELLED) {
+    // If the release is too close to the edge.
+    TouchEvent touch_event = static_cast<const TouchEvent&>(event);
+    if (WithinBoundsOfEdge(touch_event.location(), kLeavingScreenEdge) !=
+        SCREEN_CENTER) {
+      if (current_touch_ids_.size() == 0)
+        ResetToNoFingersDown();
+      else
+        state_ = WAIT_FOR_RELEASE;
+    }
+
     std::vector<int>::iterator it = std::find(
         current_touch_ids_.begin(), current_touch_ids_.end(), touch_id);
 
     // Can happen if touch exploration is enabled while fingers were down.
     if (it == current_touch_ids_.end())
       return ui::EVENT_REWRITE_CONTINUE;
 
     current_touch_ids_.erase(it);
     touch_locations_.erase(touch_id);
   } else if (type == ui::ET_TOUCH_MOVED) {
@@ skipping 24 matching lines @@
     case GESTURE_IN_PROGRESS:
       return InGestureInProgress(touch_event, rewritten_event);
     case TOUCH_EXPLORE_SECOND_PRESS:
       return InTouchExploreSecondPress(touch_event, rewritten_event);
     case TWO_TO_ONE_FINGER:
       return InTwoToOneFinger(touch_event, rewritten_event);
     case PASSTHROUGH:
       return InPassthrough(touch_event, rewritten_event);
     case WAIT_FOR_RELEASE:
       return InWaitForRelease(touch_event, rewritten_event);
+    case SLIDE_GESTURE:
+      return InSlideGesture(touch_event, rewritten_event);
   }
   NOTREACHED();
   return ui::EVENT_REWRITE_CONTINUE;
 }
 
 ui::EventRewriteStatus TouchExplorationController::NextDispatchEvent(
     const ui::Event& last_event, scoped_ptr<ui::Event>* new_event) {
   NOTREACHED();
   return ui::EVENT_REWRITE_CONTINUE;
 }
@@ skipping 50 matching lines @@
 
     float delta_time =
         (event.time_stamp() - initial_press_->time_stamp()).InSecondsF();
     float velocity = distance / delta_time;
     VLOG(0) << "\n Delta time: " << delta_time
             << "\n Distance: " << distance
             << "\n Velocity of click: " << velocity
             << "\n Minimum swipe velocity: "
             << gesture_detector_config_.minimum_swipe_velocity;
 
+    // Change to slide gesture if the slide occurred at the right edge.
+    int where = WithinBoundsOfEdge(event.location(), kMaxDistanceFromEdge);
+    if ((where | RIGHT_EDGE) == where) {

[dmazzoni, 2014/07/16 16:54:40]

This took me a second - the more typical way I'd see a test for a bit mask is:

if (where & RIGHT_EDGE) {

[lisayin, 2014/07/22 16:22:46]

Done.

+      state_ = SLIDE_GESTURE;
+      VLOG_STATE();
+      return InSlideGesture(event, rewritten_event);
+    }
+
     // If the user moves fast enough from the initial touch location, start
     // gesture detection. Otherwise, jump to the touch exploration mode early.
     if (velocity > gesture_detector_config_.minimum_swipe_velocity) {
       state_ = GESTURE_IN_PROGRESS;
       VLOG_STATE();
       return InGestureInProgress(event, rewritten_event);
     }
     EnterTouchToMouseMode();
     state_ = TOUCH_EXPLORATION;
     VLOG_STATE();
@@ skipping 260 matching lines @@
     if (current_touch_ids_.size() != 1)
       return EVENT_REWRITE_DISCARD;
 
     // Rewrite at location of last touch exploration.
     rewritten_event->reset(
         new ui::TouchEvent(ui::ET_TOUCH_RELEASED,
                            last_touch_exploration_->location(),
                            initial_press_->touch_id(),
                            event.time_stamp()));
     (*rewritten_event)->set_flags(event.flags());
+    EnterTouchToMouseMode();
     state_ = TOUCH_EXPLORATION;
     EnterTouchToMouseMode();
     VLOG_STATE();
     return ui::EVENT_REWRITE_REWRITTEN;
   }
   NOTREACHED() << "Unexpected event type received: " << event.name();
   return ui::EVENT_REWRITE_CONTINUE;
 }
 
 ui::EventRewriteStatus TouchExplorationController::InWaitForRelease(
     const ui::TouchEvent& event,
     scoped_ptr<ui::Event>* rewritten_event) {
   ui::EventType type = event.type();
   if (!(type == ui::ET_TOUCH_PRESSED || type == ui::ET_TOUCH_MOVED ||
         type == ui::ET_TOUCH_RELEASED || type == ui::ET_TOUCH_CANCELLED)) {
     NOTREACHED() << "Unexpected event type received: " << event.name();
     return ui::EVENT_REWRITE_CONTINUE;
   }
   if (current_touch_ids_.size() == 0) {
     state_ = NO_FINGERS_DOWN;
     VLOG_STATE();
     ResetToNoFingersDown();
   }
   return EVENT_REWRITE_DISCARD;
 }
 
+void TouchExplorationController::PlaySoundForTimer() {
+  delegate_->PlayVolumeAdjustSound();
+}
+
+ui::EventRewriteStatus TouchExplorationController::InSlideGesture(
+    const ui::TouchEvent& event,
+    scoped_ptr<ui::Event>* rewritten_event) {
+  ui::EventType type = event.type();
+  // If additional fingers are added before a swipe gesture has been registered,
+  // then go into two-to-one passthrough.
+  if (type == ui::ET_TOUCH_PRESSED ||
+      event.touch_id() != initial_press_->touch_id()) {

[dmazzoni, 2014/07/16 16:54:40]

I'm not sure you should switch to the two-to-one passthrough in this second
case, if you get a touch event with a different id.

In general, we've been ignoring "surprise" events (like a move or release with a
touch id we weren't tracking), not trying to handle them.

[lisayin, 2014/07/17 16:56:17]

Done.

+    if (tap_timer_.IsRunning())
+      tap_timer_.Stop();
+    if (sound_timer_.IsRunning())
+      sound_timer_.Stop();
+    // Discard any pending gestures.
+    ignore_result(gesture_provider_.GetAndResetPendingGestures());
+    state_ = TWO_TO_ONE_FINGER;
+    last_two_to_one_.reset(new TouchEvent(event));
+    rewritten_event->reset(new ui::TouchEvent(ui::ET_TOUCH_PRESSED,
+                                              event.location(),
+                                              event.touch_id(),
+                                              event.time_stamp()));
+    (*rewritten_event)->set_flags(event.flags());
+    return EVENT_REWRITE_REWRITTEN;
+  }
+
+  VLOG(0) << "Location " << event.location().ToString();

[dmazzoni, 2014/07/16 16:54:40]

Delete this log or add a bit more context so that 3 months from now you'll know
the purpose of this log

[lisayin, 2014/07/17 16:56:17]

Done.

+  // Allows user to return to the edge to adjust the sound if they have left the
+  // boundaries.
+  int where = WithinBoundsOfEdge(event.location(), kSlopDistanceFromEdge);
+  if (((where | RIGHT_EDGE) != where) && (type != ui::ET_TOUCH_RELEASED)) {

[dmazzoni, 2014/07/16 16:54:40]

same here with bit testing

[lisayin, 2014/07/17 16:56:17]

Done.

+    if (sound_timer_.IsRunning()) {
+      sound_timer_.Stop();
+    }
+    return EVENT_REWRITE_DISCARD;
+  }
+
+  // This can occur if the user leaves the screen edge and then returns to it to
+  // continue adjusting the sound.
+  if (!sound_timer_.IsRunning()) {
+    sound_timer_.Start(FROM_HERE,
+                       kSoundDelay,
+                       this,
+                       &ui::TouchExplorationController::PlaySoundForTimer);
+    delegate_->PlayVolumeAdjustSound();
+  }
+  // The timer should not fire when sliding.
+  if (tap_timer_.IsRunning())

[dmazzoni, 2014/07/16 16:54:40]

You shouldn't need to stop it every time. Can you stop it once when you enter
the state and not again?

[lisayin, 2014/07/17 16:56:17]

Done.

+    tap_timer_.Stop();
+
+  // There should not be more than one finger down.
+  DCHECK(current_touch_ids_.size() <= 1);
+  if (type == ui::ET_TOUCH_MOVED) {
+    gesture_provider_.OnTouchEvent(event);
+    gesture_provider_.OnTouchEventAck(false);
+  }
+  if (type == ui::ET_TOUCH_RELEASED || type == ui::ET_TOUCH_CANCELLED) {
+    gesture_provider_.OnTouchEvent(event);
+    gesture_provider_.OnTouchEventAck(false);
+    ignore_result(gesture_provider_.GetAndResetPendingGestures());
+    if (current_touch_ids_.size() == 0)
+      ResetToNoFingersDown();
+    return ui::EVENT_REWRITE_DISCARD;
+  }
+
+  ProcessGestureEvents();
+  return ui::EVENT_REWRITE_DISCARD;
+}
+
 void TouchExplorationController::OnTapTimerFired() {
   switch (state_) {
     case SINGLE_TAP_RELEASED:
       ResetToNoFingersDown();
       break;
     case TOUCH_EXPLORE_RELEASED:
       ResetToNoFingersDown();
       last_touch_exploration_.reset(new TouchEvent(*initial_press_));
       return;
     case SINGLE_TAP_PRESSED:
     case GESTURE_IN_PROGRESS:
       // Discard any pending gestures.
       ignore_result(gesture_provider_.GetAndResetPendingGestures());
       EnterTouchToMouseMode();
       state_ = TOUCH_EXPLORATION;
       VLOG_STATE();
       break;
     default:
       return;
   }
   scoped_ptr<ui::Event> mouse_move =
       CreateMouseMoveEvent(initial_press_->location(), initial_press_->flags());
   DispatchEvent(mouse_move.get());
   last_touch_exploration_.reset(new TouchEvent(*initial_press_));
- }
+}
 
 void TouchExplorationController::DispatchEvent(ui::Event* event) {
   if (event_handler_for_testing_) {
     event_handler_for_testing_->OnEvent(event);
     return;
   }
   ui::EventDispatchDetails result ALLOW_UNUSED =
       root_window_->GetHost()->dispatcher()->OnEventFromSource(event);
 }
 
-void TouchExplorationController::OnGestureEvent(ui::GestureEvent* gesture) {
+void TouchExplorationController::OnGestureEvent(
+    ui::GestureEvent* gesture) {
   CHECK(gesture->IsGestureEvent());
+  ui::EventType type = gesture->type();
   VLOG(0) << " \n Gesture Triggered: " << gesture->name();
-  if (gesture->type() == ui::ET_GESTURE_SWIPE) {
-    if (tap_timer_.IsRunning())
-      tap_timer_.Stop();
+  if (type == ui::ET_GESTURE_SWIPE && state_ != SLIDE_GESTURE) {
+    VLOG(0) << "Swipe!";
+    ignore_result(gesture_provider_.GetAndResetPendingGestures());
     OnSwipeEvent(gesture);
     return;
   }
 }
 
 void TouchExplorationController::ProcessGestureEvents() {
   scoped_ptr<ScopedVector<ui::GestureEvent> > gestures(
       gesture_provider_.GetAndResetPendingGestures());
   if (gestures) {
     for (ScopedVector<GestureEvent>::iterator i = gestures->begin();
          i != gestures->end();
          ++i) {
-      OnGestureEvent(*i);
+      if (state_ == SLIDE_GESTURE)
+        SideSlideControl(*i);
+      else
+        OnGestureEvent(*i);
     }
   }
 }
 
+void TouchExplorationController::SideSlideControl(ui::GestureEvent* gesture) {
+  ui::EventType type = gesture->type();
+  if (!gesture->IsScrollGestureEvent())
+    return;
+
+  gfx::Point location = gesture->location();
+  VLOG(0) << "Location " << location.ToString();
+  root_window_->GetHost()->ConvertPointToNativeScreen(&location);
+  if (WithinBoundsOfEdge(location, kSlopDistanceFromEdge) == SCREEN_CENTER) {
+    VLOG(0) << "No more Slide gesture ";
+    // Discard any pending gestures.
+     ignore_result(gesture_provider_.GetAndResetPendingGestures());
+    if (current_touch_ids_.size() > 1) {
+      state_ = WAIT_FOR_RELEASE;
+    } else if (current_touch_ids_.size() == 0) {
+      ResetToNoFingersDown();
+    } else {
+      EnterTouchToMouseMode();
+      state_ = TOUCH_EXPLORATION;

[dmazzoni, 2014/07/16 16:54:39]

Can you explain this one? When do you go from the Side Slide state to Touch
Exploration state?

[lisayin, 2014/07/17 16:56:17]

Done.

+    }
+    VLOG_STATE();
+    return;
+  }
+
+  if (type == ET_GESTURE_SCROLL_BEGIN) {
+    delegate_->PlayVolumeAdjustSound();
+  }
+
+  if (type == ET_GESTURE_SCROLL_END) {
+    if (sound_timer_.IsRunning())
+      sound_timer_.Stop();
+    delegate_->PlayVolumeAdjustSound();
+  }
+
+  location = gesture->location();
+  int where = WithinBoundsOfEdge(location, kSlopDistanceFromEdge);
+
+  // If the user is in the corner of the right side of the screen, the volume
+  // will be automatically set to 100% or muted depending on which corner they
+  // are in. Otherwise, the user will be able to adjust the volume by sliding
+  // their finger along the right side of the screen. Volume is relative to
+  // where they are on the right side of the screen.
+  if ((where | RIGHT_EDGE) != where)
+    return;
+
+  if (where == RIGHT_EDGE) {
+    location = gesture->location();
+    root_window_->GetHost()->ConvertPointFromNativeScreen(&location);
+    float volume =
+        100 -
+        ((float)location.y() - kMaxDistanceFromEdge) /
+            (root_window_->bounds().bottom() - 2 * kMaxDistanceFromEdge) * 100;

[dmazzoni, 2014/07/16 16:54:40]

I think the root window bounds top may not be zero, so you should check
height(), or subtract (bottom - top).

Maybe break this up into a few lines?

float volume_adjust_height = root_window_->bounds().bottom() - 2 *
kMaxDistanceFromEdge;
float ratio = (location.y - kMaxDistanceFromEdge) / volume_adjust_height;
float volume = 100 - 100 * ratio;

[lisayin, 2014/07/22 16:22:46]

Done.

+    VLOG(0) << "\n Volume = " << volume
+            << "\n Location = " << location.ToString()
+            << "\n Bounds = " << root_window_->bounds().right();
+
+    delegate_->SetOutputLevel(volume);
+    return;
+  }
+  else if ((where | TOP_EDGE) == where) {
+    delegate_->SetOutputLevel(100);
+    return;
+  } else if ((where | BOTTOM_EDGE) == where) {
+    delegate_->SetOutputLevel(0);
+    return;
+  }
+  NOTREACHED() << "Invalid location " << where;
+}
+
+
 void TouchExplorationController::OnSwipeEvent(ui::GestureEvent* swipe_gesture) {
   // A swipe gesture contains details for the direction in which the swipe
   // occurred.
   GestureEventDetails event_details = swipe_gesture->details();
   if (event_details.swipe_left()) {
     DispatchShiftSearchKeyEvent(ui::VKEY_LEFT);
     return;
   } else if (event_details.swipe_right()) {
     DispatchShiftSearchKeyEvent(ui::VKEY_RIGHT);
     return;
   } else if (event_details.swipe_up()) {
     DispatchShiftSearchKeyEvent(ui::VKEY_UP);
     return;
   } else if (event_details.swipe_down()) {
     DispatchShiftSearchKeyEvent(ui::VKEY_DOWN);
     return;
   }
 }
 
+int TouchExplorationController::WithinBoundsOfEdge(gfx::Point point,
+                                                   float bounds) {
+  // Since GetBoundsInScreen is in DIPs but p is not, then p needs to be

[dmazzoni, 2014/07/16 16:54:40]

What is "p"?

[lisayin, 2014/07/17 16:56:17]

Done.

+  // converted.
+  root_window_->GetHost()->ConvertPointFromNativeScreen(&point);
+  gfx::Rect window= root_window_->GetBoundsInScreen();

[dmazzoni, 2014/07/16 16:54:40]

nit: spaces around = sign

[lisayin, 2014/07/17 16:56:17]

Done.

+
+  float left_edge_limit = window.x() + bounds;
+  float right_edge_limit = window.right() - bounds;
+  float top_edge_limit = window.y() + bounds;
+  float bottom_edge_limit = window.bottom() - bounds;
+
+  // Bitwise manipulation in order to determine where on the screen the point
+  // lies. If more than one bit is turned on, then it is a corner where the two
+  // bit/edges intersect. Otherwise, if no bits are turned on, the point must be
+  // in the center of the screen.
+  int result = SCREEN_CENTER;
+  if (point.x() < left_edge_limit)
+    result = result | LEFT_EDGE;

[dmazzoni, 2014/07/16 16:54:40]

nit: result |= LEFT_EDGE

[lisayin, 2014/07/22 16:22:46]

Done.

+  if (point.x() > right_edge_limit)
+    result = result | RIGHT_EDGE;
+  if (point.y() < top_edge_limit)
+    result = result | TOP_EDGE;
+  if (point.y() > bottom_edge_limit)
+    result = result | BOTTOM_EDGE;
+  return result;
+}
+
 void TouchExplorationController::DispatchShiftSearchKeyEvent(
     const ui::KeyboardCode direction) {
   // In order to activate the shortcut shift+search+<arrow key>
   // three KeyPressed events must be dispatched in succession along
   // with three KeyReleased events.
   ui::KeyEvent shift_down = ui::KeyEvent(
       ui::ET_KEY_PRESSED, ui::VKEY_SHIFT, ui::EF_SHIFT_DOWN, false);
   ui::KeyEvent search_down = ui::KeyEvent(
       ui::ET_KEY_PRESSED, kChromeOSSearchKey, ui::EF_SHIFT_DOWN, false);
   ui::KeyEvent direction_down =
@@ skipping 29 matching lines @@
 void TouchExplorationController::EnterTouchToMouseMode() {
   aura::client::CursorClient* cursor_client =
       aura::client::GetCursorClient(root_window_);
   if (cursor_client && !cursor_client->IsMouseEventsEnabled())
     cursor_client->EnableMouseEvents();
   if (cursor_client && cursor_client->IsCursorVisible())
     cursor_client->HideCursor();
 }
 
 void TouchExplorationController::ResetToNoFingersDown() {
+  ProcessGestureEvents();
+  if (sound_timer_.IsRunning())
+    sound_timer_.Stop();
   state_ = NO_FINGERS_DOWN;
   VLOG_STATE();
   if (tap_timer_.IsRunning())
     tap_timer_.Stop();
 }
 
 void TouchExplorationController::VlogState(const char* function_name) {
   if (!VLOG_on_)
     return;
   if (prev_state_ == state_)
@@ skipping 51 matching lines @@
     case GESTURE_IN_PROGRESS:
       return "GESTURE_IN_PROGRESS";
     case TOUCH_EXPLORE_SECOND_PRESS:
       return "TOUCH_EXPLORE_SECOND_PRESS";
     case TWO_TO_ONE_FINGER:
       return "TWO_TO_ONE_FINGER";
     case PASSTHROUGH:
       return "PASSTHROUGH";
     case WAIT_FOR_RELEASE:
       return "WAIT_FOR_RELEASE";
+    case SLIDE_GESTURE:
+      return "SLIDE_GESTURE";
   }
   return "Not a state";
 }
 
 }  // namespace ui