Draw underlay behind android apps using talkback

components/exo/shell_surface.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/exo/shell_surface.h"
 
+#include <algorithm>
+
 #include "ash/aura/wm_window_aura.h"
+#include "ash/common/accessibility_delegate.h"
 #include "ash/common/shell_window_ids.h"
+#include "ash/common/system/tray/system_tray_notifier.h"
 #include "ash/common/wm/window_resizer.h"
 #include "ash/common/wm/window_state.h"
 #include "ash/common/wm/window_state_delegate.h"
+#include "ash/common/wm_shell.h"
 #include "ash/shell.h"
 #include "ash/wm/window_state_aura.h"
 #include "ash/wm/window_util.h"
 #include "base/logging.h"
 #include "base/macros.h"
 #include "base/memory/ptr_util.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/trace_event/trace_event.h"
 #include "base/trace_event/trace_event_argument.h"
 #include "components/exo/surface.h"
 #include "ui/aura/client/aura_constants.h"
 #include "ui/aura/client/cursor_client.h"
 #include "ui/aura/window.h"
 #include "ui/aura/window_event_dispatcher.h"
 #include "ui/aura/window_property.h"
 #include "ui/aura/window_targeter.h"
 #include "ui/aura/window_tree_host.h"
 #include "ui/base/accelerators/accelerator.h"
 #include "ui/gfx/path.h"
 #include "ui/views/widget/widget.h"
 #include "ui/views/widget/widget_observer.h"
 #include "ui/wm/core/coordinate_conversion.h"
 #include "ui/wm/core/shadow.h"
 #include "ui/wm/core/shadow_controller.h"
 #include "ui/wm/core/shadow_types.h"
 #include "ui/wm/core/window_util.h"
 
+#if defined(OS_CHROMEOS)
+#include "chromeos/audio/chromeos_sounds.h"
+#endif
+
 DECLARE_WINDOW_PROPERTY_TYPE(std::string*)
 
 namespace exo {
 namespace {
 
 // This is a struct for accelerator keys used to close ShellSurfaces.
 const struct Accelerator {
   ui::KeyboardCode keycode;
   int modifiers;
 } kCloseWindowAccelerators[] = {
@@ skipping 30 matching lines @@
   void SizeConstraintsChanged() override {}
 
  private:
   views::Widget* const widget_;
 
   DISALLOW_COPY_AND_ASSIGN(CustomFrameView);
 };
 
 class CustomWindowTargeter : public aura::WindowTargeter {
  public:
-  CustomWindowTargeter() {}
+  CustomWindowTargeter() : shadow_underlay_(nullptr) {}
   ~CustomWindowTargeter() override {}
 
   // Overridden from aura::WindowTargeter:
   bool EventLocationInsideBounds(aura::Window* window,
                                  const ui::LocatedEvent& event) const override {
     Surface* surface = ShellSurface::GetMainSurface(window);
     if (!surface)
       return false;
 
     gfx::Point local_point = event.location();
-    if (window->parent())

[reveman, 2016/09/30 05:59:10]

where did this check go? can you instead keep this code as is and just add the
shadow_underlay_ check below but before the surface hit test?

[erosky, 2016/09/30 18:38:40]

ConvertPointToTarget already does the check, but I'll put it back to reduce
changes.

-      aura::Window::ConvertPointToTarget(window->parent(), window,
+
+    // If the underlay is accepting events, test against it's bounds instead
+    // since it will be larger than (and contain) the surface's bounds.
+    if (shadow_underlay_ && !shadow_underlay_->ignore_events()) {
+      aura::Window::ConvertPointToTarget(window->parent(), shadow_underlay_,
                                          &local_point);
+      return gfx::Rect(shadow_underlay_->layer()->size()).Contains(local_point);
+    }
+
+    aura::Window::ConvertPointToTarget(window->parent(), window, &local_point);
 
     aura::Window::ConvertPointToTarget(window, surface->window(), &local_point);
     return surface->HitTestRect(gfx::Rect(local_point, gfx::Size(1, 1)));
   }
 
+  ui::EventTarget* FindTargetForEvent(ui::EventTarget* root,

[reveman, 2016/09/30 05:59:10]

Why do we need to override this? What would it take to have the default
implementation return the right thing?

[erosky, 2016/09/30 18:38:40]

The default implementation clips the event at the window bounds. We're
explicitly looking for events outside the window bounds here (shadow_surface_ is
a child window which is larger than its parent)

+                                      ui::Event* event) override {
+    aura::Window* window = static_cast<aura::Window*>(root);
+    Surface* surface = ShellSurface::GetMainSurface(window);
+
+    // Send events which are outside of the surface's bounds to the underlay.
+    if (surface && event->IsLocatedEvent() && shadow_underlay_ &&
+        !shadow_underlay_->ignore_events()) {
+      gfx::Point local_point = event->AsLocatedEvent()->location();
+      aura::Window::ConvertPointToTarget(window, surface->window(),
+                                         &local_point);
+      if (!surface->HitTestRect(gfx::Rect(local_point, gfx::Size(1, 1))))
+        return shadow_underlay_;
+    }
+    return aura::WindowTargeter::FindTargetForEvent(root, event);
+  }
+
+  void set_shadow_underlay(aura::Window* shadow_underlay) {
+    shadow_underlay_ = shadow_underlay;
+  }
+
  private:
+  aura::Window* shadow_underlay_;

[reveman, 2016/09/30 05:59:10]

Instead of tracking this state here and having to worry about it potentially
getting out of sync and doing the static_casts to set it. Can you just rename
ShellSurface::shadow_underlay_for_test to ShellSurface::shadow_underlay and use
that to get the shadow underlay?

You can replace this with "views::Widget* const widget_;" and initialize it in
the ctor in a way that's consistent with CustomFrameView above. And the
following static_cast can be used to get the shell surface from the widget:

static_cast<ShellSurface*>(widget_->widget_delegate()->GetContentsView())

[erosky, 2016/09/30 18:38:40]

It can't get out of sync because there is a 1-to-1-to-1 relationship between
ShellSurface, CustomWidgetDelegate, and shadow_underlay_.

I'll do what you suggest though. It seems like a roundabout way to get
shadow_underlay_, but it is more consistent with the other classes.

+
   DISALLOW_COPY_AND_ASSIGN(CustomWindowTargeter);
 };
 
 // Handles a user's fullscreen request (Shift+F4/F4).
 class CustomWindowStateDelegate : public ash::wm::WindowStateDelegate,
                                   public views::WidgetObserver {
  public:
   explicit CustomWindowStateDelegate(views::Widget* widget) : widget_(widget) {
     widget_->AddObserver(this);
   }
@@ skipping 175 matching lines @@
   }
   WMHelper::GetInstance()->RemoveActivationObserver(this);
   if (parent_)
     parent_->RemoveObserver(this);
   if (surface_) {
     if (scale_ != 1.0)
       surface_->window()->SetTransform(gfx::Transform());
     surface_->SetSurfaceDelegate(nullptr);
     surface_->RemoveSurfaceObserver(this);
   }
+  ash::WmShell::Get()->system_tray_notifier()->RemoveAccessibilityObserver(
+      this);
 }
 
 void ShellSurface::AcknowledgeConfigure(uint32_t serial) {
   TRACE_EVENT1("exo", "ShellSurface::AcknowledgeConfigure", "serial", serial);
 
   // Apply all configs that are older or equal to |serial|. The result is that
   // the origin of the main surface will move and the resize direction will
   // change to reflect the acknowledgement of configure request with |serial|
   // at the next call to Commit().
   while (!pending_configs_.empty()) {
@@ skipping 405 matching lines @@
 // views::Views overrides:
 
 gfx::Size ShellSurface::GetPreferredSize() const {
   if (!geometry_.IsEmpty())
     return geometry_.size();
 
   return surface_ ? surface_->window()->layer()->size() : gfx::Size();
 }
 
 ////////////////////////////////////////////////////////////////////////////////
+// ash::AccessibilityObserver overrides:
+
+void ShellSurface::OnAccessibilityModeChanged(
+    ash::AccessibilityNotificationVisibility) {
+  UpdateShadow();
+}
+
+////////////////////////////////////////////////////////////////////////////////
 // ash::wm::WindowStateObserver overrides:
 
 void ShellSurface::OnPreWindowStateTypeChange(
     ash::wm::WindowState* window_state,
     ash::wm::WindowStateType old_type) {
   ash::wm::WindowStateType new_type = window_state->GetStateType();
   if (ash::wm::IsMaximizedOrFullscreenOrPinnedWindowStateType(old_type) ||
       ash::wm::IsMaximizedOrFullscreenOrPinnedWindowStateType(new_type)) {
     // When transitioning in/out of maximized or fullscreen mode we need to
     // make sure we have a configure callback before we allow the default
@@ skipping 75 matching lines @@
 void ShellSurface::OnWindowActivated(
     aura::Window* gained_active,
     aura::Window* lost_active) {
   if (!widget_)
     return;
 
   if (gained_active == widget_->GetNativeWindow() ||
       lost_active == widget_->GetNativeWindow()) {
     DCHECK(activatable_);
     Configure();
+    UpdateShadow();
   }
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 // ui::EventHandler overrides:
 
+void ShellSurface::OnEvent(ui::Event* event) {
+  // If the event is targeted at the underlay, it means the user has made an
+  // interaction that is outside the surface's bounds and we want to capture
+  // it (usually when in spoken_feedback mode). Handle the event (to prevent

[reveman, 2016/09/30 05:59:10]

nit: s/spoken_feedback/spoken feedback/

[erosky, 2016/09/30 18:38:40]

Done.

+  // behind-windows from receiving it) and play an earcon to notify the user.
+  if (event->IsLocatedEvent() && event->target() == shadow_underlay_ &&
+      shadow_underlay_ && !shadow_underlay_->ignore_events()) {
+#if defined(OS_CHROMEOS)
+    const ui::EventType kEarconEventTypes[] = {

[reveman, 2016/09/30 05:59:10]

nit: are these elements sorted in a way that I'm failing to see? maybe sort them
by EventType enum order or alphabetically if not

[erosky, 2016/09/30 18:38:40]

Done.

+        ui::ET_TOUCH_PRESSED,         ui::ET_POINTER_DOWN,
+        ui::ET_POINTER_WHEEL_CHANGED, ui::ET_SCROLL,
+        ui::ET_SCROLL_FLING_START,    ui::ET_GESTURE_BEGIN,
+        ui::ET_MOUSE_PRESSED,         ui::ET_MOUSEWHEEL};
+    bool is_earcon_event_type =
+        std::find(std::begin(kEarconEventTypes), std::end(kEarconEventTypes),
+                  event->type()) != std::end(kEarconEventTypes);
+    if (is_earcon_event_type)
+      ash::WmShell::Get()->accessibility_delegate()->PlayEarcon(
+          chromeos::SOUND_VOLUME_ADJUST);
+#endif
+    event->SetHandled();
+    return;
+  }
+  views::View::OnEvent(event);
+}
+
 void ShellSurface::OnKeyEvent(ui::KeyEvent* event) {
   if (!resizer_) {
     views::View::OnKeyEvent(event);
     return;
   }
 
   if (event->type() == ui::ET_KEY_PRESSED &&
       event->key_code() == ui::VKEY_ESCAPE) {
     EndDrag(true /* revert */);
   }
@@ skipping 135 matching lines @@
   for (const auto& entry : kCloseWindowAccelerators) {
     focus_manager->RegisterAccelerator(
         ui::Accelerator(entry.keycode, entry.modifiers),
         ui::AcceleratorManager::kNormalPriority, this);
   }
 
   // Set delegate for handling of fullscreening.
   window_state->SetDelegate(std::unique_ptr<ash::wm::WindowStateDelegate>(
       new CustomWindowStateDelegate(widget_)));
 
+  // Receive accessibility changes to update shadow underlay.
+  ash::WmShell::Get()->system_tray_notifier()->AddAccessibilityObserver(this);
+
   // Show widget next time Commit() is called.
   pending_show_widget_ = true;
 }
 
 void ShellSurface::Configure() {
   DCHECK(widget_);
 
   // Delay configure callback if |scoped_configure_| is set.
   if (scoped_configure_) {
     scoped_configure_->set_needs_configure();
@@ skipping 239 matching lines @@
     // TODO(oshima): Adjust the coordinates from client screen to
     // chromeos screen when multi displays are supported.
     gfx::Point origin = window->bounds().origin();
     gfx::Point shadow_origin = shadow_content_bounds_.origin();
     shadow_origin -= origin.OffsetFromOrigin();
     gfx::Rect shadow_bounds(shadow_origin, shadow_content_bounds_.size());
 
     // Always create and show the underlay, even in maximized/fullscreen.
     if (!shadow_underlay_) {
       shadow_underlay_ = new aura::Window(nullptr);
+      shadow_underlay_->SetTargetHandler(this);

[reveman, 2016/09/30 05:59:10]

Can we use a different aura::WindowDelegate for this instead? ie. add a
UnderlayWindowDelegate class above to keep the event handling separate from the
native window?

[erosky, 2016/09/30 18:38:40]

Done.

       DCHECK(shadow_underlay_->owned_by_parent());
-      shadow_underlay_->set_ignore_events(true);
       // Ensure the background area inside the shadow is solid black.
       // Clients that provide translucent contents should not be using
       // rectangular shadows as this method requires opaque contents to
       // cast a shadow that represent it correctly.
       shadow_underlay_->Init(ui::LAYER_SOLID_COLOR);
       shadow_underlay_->layer()->SetColor(SK_ColorBLACK);
       DCHECK(shadow_underlay_->layer()->fills_bounds_opaquely());
       window->AddChild(shadow_underlay_);
       window->StackChildAtBottom(shadow_underlay_);
+      static_cast<CustomWindowTargeter*>(
+          static_cast<ui::EventTarget*>(window)->GetEventTargeter())
+          ->set_shadow_underlay(shadow_underlay_);
     }
 
+    bool underlay_capture_events = ash::WmShell::Get()
+                                       ->accessibility_delegate()
+                                       ->IsSpokenFeedbackEnabled() &&
+                                   widget_->IsActive();
+    shadow_underlay_->set_ignore_events(!underlay_capture_events);
+
     float shadow_underlay_opacity = rectangular_shadow_background_opacity_;
     // Put the black background layer behind the window if
-    // 1) the window is in immersive fullscreen.
+    // 1) the window is in immersive fullscreen or is active with
+    //    spoken feedback enabled.
     // 2) the window can control the bounds of the window in fullscreen (
     //    thus the background can be visible).
     // 3) the window has no transform (the transformed background may
     //    not cover the entire background, e.g. overview mode).
-    if (widget_->IsFullscreen() &&
+    if ((widget_->IsFullscreen() || underlay_capture_events) &&
         ash::wm::GetWindowState(window)->allow_set_bounds_in_maximized() &&
-        window->layer()->transform().IsIdentity()) {
+        window->layer()->GetTargetTransform().IsIdentity()) {
       gfx::Point origin;
       origin -= window->bounds().origin().OffsetFromOrigin();
       shadow_bounds.set_origin(origin);
       shadow_bounds.set_size(window->parent()->bounds().size());
       shadow_underlay_opacity = 1.0f;
     }
 
     shadow_underlay_->SetBounds(shadow_bounds);
 
     // TODO(oshima): Setting to the same value should be no-op.
@@ skipping 18 matching lines @@
       shadow_overlay_->layer()->Add(shadow->layer());
       window->AddChild(shadow_overlay_);
       shadow_overlay_->Show();
     }
     shadow_overlay_->SetBounds(shadow_bounds);
     shadow->SetContentBounds(gfx::Rect(shadow_bounds.size()));
   }
 }
 
 }  // namespace exo