Side Slide Gestures for Accessibility

ui/chromeos/touch_exploration_controller.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "ui/chromeos/touch_exploration_controller.h"
 
-#include "base/logging.h"
 #include "base/strings/string_number_conversions.h"
 #include "ui/aura/client/cursor_client.h"
 #include "ui/aura/window.h"
 #include "ui/aura/window_event_dispatcher.h"
 #include "ui/aura/window_tree_host.h"
 #include "ui/events/event.h"
 #include "ui/events/event_processor.h"
+#include "ui/gfx/geometry/rect.h"
 
 #define VLOG_STATE() if (VLOG_IS_ON(0)) VlogState(__func__)
 #define VLOG_EVENT(event) if (VLOG_IS_ON(0)) VlogEvent(event, __func__)
 
 namespace ui {
 
 namespace {
+
+// Delay between adjustment sounds.
+const base::TimeDelta kSoundDelay = base::TimeDelta::FromMilliseconds(150);
+
+// Within these bounds, the release event generated will reset the state to

[dmazzoni, 2014/07/22 17:14:54]

Nit: something like "within this many dips of the screen edge"

[lisayin, 2014/07/22 18:06:26]

Done.

+// NoFingersDown.
+const float kLeavingScreenEdge = 6;
+
+// Swipe/scroll gestures within these bounds (in dips) will change preset
+// settings.
+const float kMaxDistanceFromEdge = 75;

[aboxhall, 2014/07/22 17:20:41]

Where do these numbers come from? What are the units? Will they work on all
devices?

[lisayin, 2014/07/22 18:06:25]

Dip stands for device independent pixels so they should work for all devices.
The numbers are from letting Peter and David try it out and tweaking the numbers
from there.

+
+// After a slide gesture has been triggered, if the finger is still within these
+// bounds (in DIPs), the preset settings will still change.
+const float kSlopDistanceFromEdge = kMaxDistanceFromEdge + 40;
+
 // In ChromeOS, VKEY_LWIN is synonymous for the search key.
 const ui::KeyboardCode kChromeOSSearchKey = ui::VKEY_LWIN;
 }  // namespace
 
 TouchExplorationController::TouchExplorationController(
-    aura::Window* root_window)
+    aura::Window* root_window,
+    TouchExplorationControllerDelegate* delegate)
     : root_window_(root_window),
+      delegate_(delegate),
       state_(NO_FINGERS_DOWN),
       event_handler_for_testing_(NULL),
       gesture_provider_(this),
       prev_state_(NO_FINGERS_DOWN),
       VLOG_on_(true) {
   CHECK(root_window);
   root_window->GetHost()->GetEventSource()->AddEventRewriter(this);
 }
 
 TouchExplorationController::~TouchExplorationController() {
@@ skipping 31 matching lines @@
   const ui::EventType type = touch_event.type();
   const gfx::PointF& location = touch_event.location_f();
   const int touch_id = touch_event.touch_id();
 
   // Always update touch ids and touch locations, so we can use those
   // no matter what state we're in.
   if (type == ui::ET_TOUCH_PRESSED) {
     current_touch_ids_.push_back(touch_id);
     touch_locations_.insert(std::pair<int, gfx::PointF>(touch_id, location));
   } else if (type == ui::ET_TOUCH_RELEASED || type == ui::ET_TOUCH_CANCELLED) {
+    // In order to avoid accidentally double tapping when moving off the edge of
+    // the screen, the state will be rewritten to NoFingersDown.
+    TouchEvent touch_event = static_cast<const TouchEvent&>(event);
+    if (WithinBoundsOfEdge(touch_event.location(), kLeavingScreenEdge) !=
+        SCREEN_CENTER) {
+      if (current_touch_ids_.size() == 0)
+        ResetToNoFingersDown();
+    }
+
     std::vector<int>::iterator it = std::find(
         current_touch_ids_.begin(), current_touch_ids_.end(), touch_id);
 
     // Can happen if touch exploration is enabled while fingers were down.
     if (it == current_touch_ids_.end())
       return ui::EVENT_REWRITE_CONTINUE;
 
     current_touch_ids_.erase(it);
     touch_locations_.erase(touch_id);
   } else if (type == ui::ET_TOUCH_MOVED) {
@@ skipping 24 matching lines @@
     case GESTURE_IN_PROGRESS:
       return InGestureInProgress(touch_event, rewritten_event);
     case TOUCH_EXPLORE_SECOND_PRESS:
       return InTouchExploreSecondPress(touch_event, rewritten_event);
     case TWO_TO_ONE_FINGER:
       return InTwoToOneFinger(touch_event, rewritten_event);
     case PASSTHROUGH:
       return InPassthrough(touch_event, rewritten_event);
     case WAIT_FOR_RELEASE:
       return InWaitForRelease(touch_event, rewritten_event);
+    case SLIDE_GESTURE:
+      return InSlideGesture(touch_event, rewritten_event);
   }
   NOTREACHED();
   return ui::EVENT_REWRITE_CONTINUE;
 }
 
 ui::EventRewriteStatus TouchExplorationController::NextDispatchEvent(
     const ui::Event& last_event, scoped_ptr<ui::Event>* new_event) {
   NOTREACHED();
   return ui::EVENT_REWRITE_CONTINUE;
 }
@@ skipping 50 matching lines @@
 
     float delta_time =
         (event.time_stamp() - initial_press_->time_stamp()).InSecondsF();
     float velocity = distance / delta_time;
     VLOG(0) << "\n Delta time: " << delta_time
             << "\n Distance: " << distance
             << "\n Velocity of click: " << velocity
             << "\n Minimum swipe velocity: "
             << gesture_detector_config_.minimum_swipe_velocity;
 
+    // Change to slide gesture if the slide occurred at the right edge.
+    int where = WithinBoundsOfEdge(event.location(), kMaxDistanceFromEdge);

[aboxhall, 2014/07/22 17:20:41]

Similarly, I think 'where' here is a little misleading. How about 'edges'?

[lisayin, 2014/07/22 18:06:25]

Done.

+    if (where & RIGHT_EDGE) {
+      state_ = SLIDE_GESTURE;
+      VLOG_STATE();
+      return InSlideGesture(event, rewritten_event);
+    }
+
     // If the user moves fast enough from the initial touch location, start
     // gesture detection. Otherwise, jump to the touch exploration mode early.
     if (velocity > gesture_detector_config_.minimum_swipe_velocity) {
       state_ = GESTURE_IN_PROGRESS;
       VLOG_STATE();
       return InGestureInProgress(event, rewritten_event);
     }
     EnterTouchToMouseMode();
     state_ = TOUCH_EXPLORATION;
     VLOG_STATE();
@@ skipping 286 matching lines @@
     return ui::EVENT_REWRITE_CONTINUE;
   }
   if (current_touch_ids_.size() == 0) {
     state_ = NO_FINGERS_DOWN;
     VLOG_STATE();
     ResetToNoFingersDown();
   }
   return EVENT_REWRITE_DISCARD;
 }
 
+void TouchExplorationController::PlaySoundForTimer() {
+  delegate_->PlayVolumeAdjustSound();
+}
+
+ui::EventRewriteStatus TouchExplorationController::InSlideGesture(
+    const ui::TouchEvent& event,
+    scoped_ptr<ui::Event>* rewritten_event) {
+  // The timer should not fire when sliding.
+  if (tap_timer_.IsRunning())
+    tap_timer_.Stop();
+
+  ui::EventType type = event.type();
+  // If additional fingers are added before a swipe gesture has been registered,
+  // then wait until all fingers have been lifted.
+  if (type == ui::ET_TOUCH_PRESSED ||
+      event.touch_id() != initial_press_->touch_id()) {
+    if (sound_timer_.IsRunning())
+      sound_timer_.Stop();
+    // Discard any pending gestures.
+    ignore_result(gesture_provider_.GetAndResetPendingGestures());
+    state_ = WAIT_FOR_RELEASE;
+    return EVENT_REWRITE_DISCARD;
+  }
+
+  // Allows user to return to the edge to adjust the sound if they have left the
+  // boundaries.
+  int where = WithinBoundsOfEdge(event.location(), kSlopDistanceFromEdge);
+  if (!(where & RIGHT_EDGE) && (type != ui::ET_TOUCH_RELEASED)) {
+    if (sound_timer_.IsRunning()) {
+      sound_timer_.Stop();
+    }
+    return EVENT_REWRITE_DISCARD;
+  }
+
+  // This can occur if the user leaves the screen edge and then returns to it to
+  // continue adjusting the sound.
+  if (!sound_timer_.IsRunning()) {
+    sound_timer_.Start(FROM_HERE,
+                       kSoundDelay,
+                       this,
+                       &ui::TouchExplorationController::PlaySoundForTimer);
+    delegate_->PlayVolumeAdjustSound();
+  }
+
+  // There should not be more than one finger down.
+  DCHECK(current_touch_ids_.size() <= 1);
+  if (type == ui::ET_TOUCH_MOVED) {
+    gesture_provider_.OnTouchEvent(event);
+    gesture_provider_.OnTouchEventAck(false);
+  }
+  if (type == ui::ET_TOUCH_RELEASED || type == ui::ET_TOUCH_CANCELLED) {
+    gesture_provider_.OnTouchEvent(event);
+    gesture_provider_.OnTouchEventAck(false);
+    ignore_result(gesture_provider_.GetAndResetPendingGestures());
+    if (current_touch_ids_.size() == 0)
+      ResetToNoFingersDown();
+    return ui::EVENT_REWRITE_DISCARD;
+  }
+
+  ProcessGestureEvents();
+  return ui::EVENT_REWRITE_DISCARD;
+}
+
 void TouchExplorationController::OnTapTimerFired() {
   switch (state_) {
     case SINGLE_TAP_RELEASED:
       ResetToNoFingersDown();
       break;
     case TOUCH_EXPLORE_RELEASED:
       ResetToNoFingersDown();
       last_touch_exploration_.reset(new TouchEvent(*initial_press_));
       return;
     case SINGLE_TAP_PRESSED:
     case GESTURE_IN_PROGRESS:
       // Discard any pending gestures.
       ignore_result(gesture_provider_.GetAndResetPendingGestures());
       EnterTouchToMouseMode();
       state_ = TOUCH_EXPLORATION;
       VLOG_STATE();
       break;
     default:
       return;
   }
   scoped_ptr<ui::Event> mouse_move =
       CreateMouseMoveEvent(initial_press_->location(), initial_press_->flags());
   DispatchEvent(mouse_move.get());
   last_touch_exploration_.reset(new TouchEvent(*initial_press_));
- }
+}
 
 void TouchExplorationController::DispatchEvent(ui::Event* event) {
   if (event_handler_for_testing_) {
     event_handler_for_testing_->OnEvent(event);
     return;
   }
   ui::EventDispatchDetails result ALLOW_UNUSED =
       root_window_->GetHost()->dispatcher()->OnEventFromSource(event);
 }
 
-void TouchExplorationController::OnGestureEvent(ui::GestureEvent* gesture) {
+void TouchExplorationController::OnGestureEvent(
+    ui::GestureEvent* gesture) {
   CHECK(gesture->IsGestureEvent());
+  ui::EventType type = gesture->type();
   VLOG(0) << " \n Gesture Triggered: " << gesture->name();
-  if (gesture->type() == ui::ET_GESTURE_SWIPE) {
-    if (tap_timer_.IsRunning())
-      tap_timer_.Stop();
+  if (type == ui::ET_GESTURE_SWIPE && state_ != SLIDE_GESTURE) {
+    VLOG(0) << "Swipe!";
+    ignore_result(gesture_provider_.GetAndResetPendingGestures());
     OnSwipeEvent(gesture);
     return;
   }
 }
 
 void TouchExplorationController::ProcessGestureEvents() {
   scoped_ptr<ScopedVector<ui::GestureEvent> > gestures(
       gesture_provider_.GetAndResetPendingGestures());
   if (gestures) {
     for (ScopedVector<GestureEvent>::iterator i = gestures->begin();
          i != gestures->end();
          ++i) {
-      OnGestureEvent(*i);
+      if (state_ == SLIDE_GESTURE)
+        SideSlideControl(*i);
+      else
+        OnGestureEvent(*i);
     }
   }
 }
 
+void TouchExplorationController::SideSlideControl(ui::GestureEvent* gesture) {
+  ui::EventType type = gesture->type();
+  if (!gesture->IsScrollGestureEvent())
+    return;
+
+  if (type == ET_GESTURE_SCROLL_BEGIN) {
+    delegate_->PlayVolumeAdjustSound();
+  }
+
+  if (type == ET_GESTURE_SCROLL_END) {
+    if (sound_timer_.IsRunning())
+      sound_timer_.Stop();
+    delegate_->PlayVolumeAdjustSound();
+  }
+
+  // If the user is in the corner of the right side of the screen, the volume
+  // will be automatically set to 100% or muted depending on which corner they
+  // are in. Otherwise, the user will be able to adjust the volume by sliding
+  // their finger along the right side of the screen. Volume is relative to
+  // where they are on the right side of the screen.
+  gfx::Point location = gesture->location();
+  int where = WithinBoundsOfEdge(location, kSlopDistanceFromEdge);
+  if (!(where & RIGHT_EDGE))
+    return;
+
+  if (where == RIGHT_EDGE) {
+    location = gesture->location();
+    root_window_->GetHost()->ConvertPointFromNativeScreen(&location);
+    float volume_adjust_height =
+        root_window_->bounds().height() - 2 * kMaxDistanceFromEdge;
+    float ratio = (location.y() - kMaxDistanceFromEdge) / volume_adjust_height;
+    float volume = 100 - 100 * ratio;
+    VLOG(0) << "\n Volume = " << volume
+            << "\n Location = " << location.ToString()
+            << "\n Bounds = " << root_window_->bounds().right();
+
+    delegate_->SetOutputLevel(int(volume));
+    return;
+  }
+  else if (where & TOP_EDGE) {

[dmazzoni, 2014/07/22 17:14:54]

nit: belongs on previous line after }

[aboxhall, 2014/07/22 17:20:41]

Trivial, optional suggestion: move this and the below if blocks above the above
if block; since you can't be on the top and bottom at once, you can early out
and won't need any elses, then the above block becomes the default case and
doesn't need to go in an if block at all (unless you want to assert that it's
not in some impossible state).

[lisayin, 2014/07/22 18:06:25]

Done.

+    delegate_->SetOutputLevel(100);
+    return;
+  } else if (where & BOTTOM_EDGE) {
+    delegate_->SetOutputLevel(0);
+    return;
+  }
+  NOTREACHED() << "Invalid location " << where;
+}
+
+
 void TouchExplorationController::OnSwipeEvent(ui::GestureEvent* swipe_gesture) {
   // A swipe gesture contains details for the direction in which the swipe
   // occurred.
   GestureEventDetails event_details = swipe_gesture->details();
   if (event_details.swipe_left()) {
     DispatchShiftSearchKeyEvent(ui::VKEY_LEFT);
     return;
   } else if (event_details.swipe_right()) {
     DispatchShiftSearchKeyEvent(ui::VKEY_RIGHT);
     return;
   } else if (event_details.swipe_up()) {
     DispatchShiftSearchKeyEvent(ui::VKEY_UP);
     return;
   } else if (event_details.swipe_down()) {
     DispatchShiftSearchKeyEvent(ui::VKEY_DOWN);
     return;
   }
 }
 
+int TouchExplorationController::WithinBoundsOfEdge(gfx::Point point,
+                                                   float bounds) {
+  // Since GetBoundsInScreen is in DIPs but point is not, then point needs to be
+  // converted.
+  root_window_->GetHost()->ConvertPointFromNativeScreen(&point);
+  gfx::Rect window = root_window_->GetBoundsInScreen();
+
+  float left_edge_limit = window.x() + bounds;
+  float right_edge_limit = window.right() - bounds;
+  float top_edge_limit = window.y() + bounds;
+  float bottom_edge_limit = window.bottom() - bounds;
+
+  // Bitwise manipulation in order to determine where on the screen the point
+  // lies. If more than one bit is turned on, then it is a corner where the two
+  // bit/edges intersect. Otherwise, if no bits are turned on, the point must be
+  // in the center of the screen.
+  int result = SCREEN_CENTER;
+  if (point.x() < left_edge_limit)
+    result |= LEFT_EDGE;
+  if (point.x() > right_edge_limit)
+    result |= RIGHT_EDGE;
+  if (point.y() < top_edge_limit)
+    result |= TOP_EDGE;
+  if (point.y() > bottom_edge_limit)
+    result |= BOTTOM_EDGE;
+  return result;
+}
+
 void TouchExplorationController::DispatchShiftSearchKeyEvent(
     const ui::KeyboardCode direction) {
   // In order to activate the shortcut shift+search+<arrow key>
   // three KeyPressed events must be dispatched in succession along
   // with three KeyReleased events.
   ui::KeyEvent shift_down = ui::KeyEvent(
       ui::ET_KEY_PRESSED, ui::VKEY_SHIFT, ui::EF_SHIFT_DOWN, false);
   ui::KeyEvent search_down = ui::KeyEvent(
       ui::ET_KEY_PRESSED, kChromeOSSearchKey, ui::EF_SHIFT_DOWN, false);
   ui::KeyEvent direction_down =
@@ skipping 29 matching lines @@
 void TouchExplorationController::EnterTouchToMouseMode() {
   aura::client::CursorClient* cursor_client =
       aura::client::GetCursorClient(root_window_);
   if (cursor_client && !cursor_client->IsMouseEventsEnabled())
     cursor_client->EnableMouseEvents();
   if (cursor_client && cursor_client->IsCursorVisible())
     cursor_client->HideCursor();
 }
 
 void TouchExplorationController::ResetToNoFingersDown() {
+  ProcessGestureEvents();
+  if (sound_timer_.IsRunning())
+    sound_timer_.Stop();
   state_ = NO_FINGERS_DOWN;
   VLOG_STATE();
   if (tap_timer_.IsRunning())
     tap_timer_.Stop();
 }
 
 void TouchExplorationController::VlogState(const char* function_name) {
   if (!VLOG_on_)
     return;
   if (prev_state_ == state_)
@@ skipping 51 matching lines @@
     case GESTURE_IN_PROGRESS:
       return "GESTURE_IN_PROGRESS";
     case TOUCH_EXPLORE_SECOND_PRESS:
       return "TOUCH_EXPLORE_SECOND_PRESS";
     case TWO_TO_ONE_FINGER:
       return "TWO_TO_ONE_FINGER";
     case PASSTHROUGH:
       return "PASSTHROUGH";
     case WAIT_FOR_RELEASE:
       return "WAIT_FOR_RELEASE";
+    case SLIDE_GESTURE:
+      return "SLIDE_GESTURE";
   }
   return "Not a state";
 }
 
 }  // namespace ui