Draw underlay behind android apps using talkback

components/exo/shell_surface.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/exo/shell_surface.h"
 
 #include "ash/aura/wm_window_aura.h"
+#include "ash/common/accessibility_delegate.h"
 #include "ash/common/shell_window_ids.h"
+#include "ash/common/system/tray/system_tray_notifier.h"
 #include "ash/common/wm/window_resizer.h"
 #include "ash/common/wm/window_state.h"
 #include "ash/common/wm/window_state_delegate.h"
+#include "ash/common/wm_shell.h"
 #include "ash/shell.h"
 #include "ash/wm/window_state_aura.h"
 #include "ash/wm/window_util.h"
 #include "base/logging.h"
 #include "base/macros.h"
 #include "base/memory/ptr_util.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/trace_event/trace_event.h"
 #include "base/trace_event/trace_event_argument.h"
 #include "components/exo/surface.h"
 #include "ui/aura/client/aura_constants.h"
 #include "ui/aura/client/cursor_client.h"
 #include "ui/aura/window.h"
 #include "ui/aura/window_event_dispatcher.h"
 #include "ui/aura/window_property.h"
 #include "ui/aura/window_targeter.h"
 #include "ui/aura/window_tree_host.h"
 #include "ui/base/accelerators/accelerator.h"
 #include "ui/gfx/path.h"
 #include "ui/views/widget/widget.h"
 #include "ui/views/widget/widget_observer.h"
 #include "ui/wm/core/coordinate_conversion.h"
 #include "ui/wm/core/shadow.h"
 #include "ui/wm/core/shadow_controller.h"
 #include "ui/wm/core/shadow_types.h"
 #include "ui/wm/core/window_util.h"
 
+#if defined(OS_CHROMEOS)
+#include "chromeos/audio/chromeos_sounds.h"
+#endif
+
 DECLARE_WINDOW_PROPERTY_TYPE(std::string*)
 
 namespace exo {
 namespace {
 
 // This is a struct for accelerator keys used to close ShellSurfaces.
 const struct Accelerator {
   ui::KeyboardCode keycode;
   int modifiers;
 } kCloseWindowAccelerators[] = {
@@ skipping 28 matching lines @@
   void UpdateWindowIcon() override {}
   void UpdateWindowTitle() override {}
   void SizeConstraintsChanged() override {}
 
  private:
   views::Widget* const widget_;
 
   DISALLOW_COPY_AND_ASSIGN(CustomFrameView);
 };
 
 class CustomWindowTargeter : public aura::WindowTargeter {

[reveman, 2016/09/28 16:11:50]

Does this really need to know about the shadow underlay? Can we instead pass
widget as ctor argument and just fallback to |window| as target if spoken
feedback is enabled, widget is active and
ash::wm::GetWindowState(window)->allow_set_bounds_in_maximized() is true? And
maybe move these checks into a helper function so you can also use it below..

[erosky, 2016/09/28 19:21:12]

This still needs to know about the shadow_underlay_ in order to return it as an
alternate event target (in FindTargetForEvent) (it'd be a child of |window|,
among others, so finding it from just |window| wouldn't be trivial).

  public:
-  CustomWindowTargeter() {}
+  CustomWindowTargeter() : shadow_underlay_(nullptr) {}
   ~CustomWindowTargeter() override {}
 
   // Overridden from aura::WindowTargeter:
   bool EventLocationInsideBounds(aura::Window* window,
                                  const ui::LocatedEvent& event) const override {
     Surface* surface = ShellSurface::GetMainSurface(window);
     if (!surface)
       return false;
 
     gfx::Point local_point = event.location();
-    if (window->parent())
-      aura::Window::ConvertPointToTarget(window->parent(), window,
+
+    if (shadow_underlay_ && !shadow_underlay_->ignore_events()) {
+      aura::Window::ConvertPointToTarget(window->parent(), shadow_underlay_,
                                          &local_point);
+      return gfx::Rect(shadow_underlay_->layer()->size()).Contains(local_point);
+    }
+
+    aura::Window::ConvertPointToTarget(window->parent(), window, &local_point);
 
     aura::Window::ConvertPointToTarget(window, surface->window(), &local_point);
     return surface->HitTestRect(gfx::Rect(local_point, gfx::Size(1, 1)));
   }
 
+  ui::EventTarget* FindTargetForEvent(ui::EventTarget* root,
+                                      ui::Event* event) override {
+    aura::Window* window = static_cast<aura::Window*>(root);
+    Surface* surface = ShellSurface::GetMainSurface(window);
+    if (surface && event->IsLocatedEvent() && shadow_underlay_ &&
+        !shadow_underlay_->ignore_events()) {
+      gfx::Point local_point = event->AsLocatedEvent()->location();
+      aura::Window::ConvertPointToTarget(window, surface->window(),
+                                         &local_point);
+      if (!surface->HitTestRect(gfx::Rect(local_point, gfx::Size(1, 1))))
+        return shadow_underlay_;
+    }
+    return aura::WindowTargeter::FindTargetForEvent(root, event);
+  }
+
+  void set_shadow_underlay(aura::Window* shadow_underlay) {
+    shadow_underlay_ = shadow_underlay;
+  }
+
  private:
+  aura::Window* shadow_underlay_;

[reveman, 2016/09/28 16:11:51]

nit: blank line before DISALLOW_..

[erosky, 2016/09/28 19:21:12]

Done.

   DISALLOW_COPY_AND_ASSIGN(CustomWindowTargeter);
 };
 
 // Handles a user's fullscreen request (Shift+F4/F4).
 class CustomWindowStateDelegate : public ash::wm::WindowStateDelegate,
                                   public views::WidgetObserver {
  public:
   explicit CustomWindowStateDelegate(views::Widget* widget) : widget_(widget) {
     widget_->AddObserver(this);
   }
@@ skipping 175 matching lines @@
   }
   WMHelper::GetInstance()->RemoveActivationObserver(this);
   if (parent_)
     parent_->RemoveObserver(this);
   if (surface_) {
     if (scale_ != 1.0)
       surface_->window()->SetTransform(gfx::Transform());
     surface_->SetSurfaceDelegate(nullptr);
     surface_->RemoveSurfaceObserver(this);
   }
+  if (ash::WmShell::HasInstance() &&
+      ash::WmShell::Get()->system_tray_notifier()) {

[reveman, 2016/09/28 16:11:50]

Do we have to check ash::WmShell::HasInstance() &&
ash::WmShell::Get()->system_tray_notifier() here? Why are they null when this is
called?

[erosky, 2016/09/28 19:21:13]

Done.

+    ash::WmShell::Get()->system_tray_notifier()->RemoveAccessibilityObserver(
+        this);
+  }
 }
 
 void ShellSurface::AcknowledgeConfigure(uint32_t serial) {
   TRACE_EVENT1("exo", "ShellSurface::AcknowledgeConfigure", "serial", serial);
 
   // Apply all configs that are older or equal to |serial|. The result is that
   // the origin of the main surface will move and the resize direction will
   // change to reflect the acknowledgement of configure request with |serial|
   // at the next call to Commit().
   while (!pending_configs_.empty()) {
@@ skipping 404 matching lines @@
 // views::Views overrides:
 
 gfx::Size ShellSurface::GetPreferredSize() const {
   if (!geometry_.IsEmpty())
     return geometry_.size();
 
   return surface_ ? surface_->window()->layer()->size() : gfx::Size();
 }
 
 ////////////////////////////////////////////////////////////////////////////////
+// ash::AccessibilityObserver overrides:
+
+void ShellSurface::OnAccessibilityModeChanged(
+    ash::AccessibilityNotificationVisibility) {
+  audio_feedback_ =
+      ash::WmShell::Get()->accessibility_delegate()->IsSpokenFeedbackEnabled();
+  UpdateShadow();
+}
+
+////////////////////////////////////////////////////////////////////////////////
 // ash::wm::WindowStateObserver overrides:
 
 void ShellSurface::OnPreWindowStateTypeChange(
     ash::wm::WindowState* window_state,
     ash::wm::WindowStateType old_type) {
   ash::wm::WindowStateType new_type = window_state->GetStateType();
   if (ash::wm::IsMaximizedOrFullscreenOrPinnedWindowStateType(old_type) ||
       ash::wm::IsMaximizedOrFullscreenOrPinnedWindowStateType(new_type)) {
     // When transitioning in/out of maximized or fullscreen mode we need to
     // make sure we have a configure callback before we allow the default
@@ skipping 75 matching lines @@
 void ShellSurface::OnWindowActivated(
     aura::Window* gained_active,
     aura::Window* lost_active) {
   if (!widget_)
     return;
 
   if (gained_active == widget_->GetNativeWindow() ||
       lost_active == widget_->GetNativeWindow()) {
     DCHECK(activatable_);
     Configure();
+    UpdateShadow();
   }
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 // ui::EventHandler overrides:
 
+void ShellSurface::OnEvent(ui::Event* event) {
+  if (event->IsLocatedEvent() && event->target() == shadow_underlay_ &&

[reveman, 2016/09/28 16:11:50]

Same here. Can this use "event->target() == widget_->GetNativeWindow() &&
HelperIsActiveCannotChangeSizeAndSpokenFeedbackIsEnabled" instead of relying on
the shadow underlay?

[erosky, 2016/09/28 19:21:12]

widget_->GetNativeWindow() represents the actual app which means we wouldn't
want to intercept the event (event would probably not even get here since it'd
be taken by Touch or Pointer. This condition is only to catch if we've touched
outside the window (on the shadow_underlay).

Checking for IsActive() here is unnecessary because the shadow_underlay is only
"visible" when the widget is active.

+      shadow_underlay_ && !shadow_underlay_->ignore_events()) {
+#if defined(OS_CHROMEOS)
+    ui::EventType sound_types[] = {

[reveman, 2016/09/28 16:11:51]

nit: s/ui::EventType sound_types/const ui::EventType kEarconEventTypes/

[erosky, 2016/09/28 19:21:13]

Done.

+        ui::ET_TOUCH_PRESSED,         ui::ET_POINTER_DOWN,
+        ui::ET_POINTER_WHEEL_CHANGED, ui::ET_SCROLL,
+        ui::ET_SCROLL_FLING_START,    ui::ET_GESTURE_BEGIN,
+        ui::ET_MOUSE_PRESSED,         ui::ET_MOUSEWHEEL};
+    for (ui::EventType e : sound_types) {

[reveman, 2016/09/28 16:11:50]

nit: I think this would be a good place to use std::find instead of a loop to
make the code a bit easier to read. E.g.

bool is_earcon_event_type = std::find(std::begin(kEarconEventTypes),            
 
                                      std::end(kEarconEventTypes),
                                      event->type()) !=
std::end(kEarconEventTypes);
if (is_earcon_event_type) {
  ...
}

[erosky, 2016/09/28 19:21:13]

Done.

+      if (event->type() == e) {
+        ash::WmShell::Get()->accessibility_delegate()->PlayEarcon(
+            chromeos::SOUND_VOLUME_ADJUST);
+        break;
+      }
+    }
+#endif
+    event->SetHandled();
+    return;
+  }
+  views::View::OnEvent(event);
+}
+
 void ShellSurface::OnKeyEvent(ui::KeyEvent* event) {
   if (!resizer_) {
     views::View::OnKeyEvent(event);
     return;
   }
 
   if (event->type() == ui::ET_KEY_PRESSED &&
       event->key_code() == ui::VKEY_ESCAPE) {
     EndDrag(true /* revert */);
   }
@@ skipping 135 matching lines @@
   for (const auto& entry : kCloseWindowAccelerators) {
     focus_manager->RegisterAccelerator(
         ui::Accelerator(entry.keycode, entry.modifiers),
         ui::AcceleratorManager::kNormalPriority, this);
   }
 
   // Set delegate for handling of fullscreening.
   window_state->SetDelegate(std::unique_ptr<ash::wm::WindowStateDelegate>(
       new CustomWindowStateDelegate(widget_)));
 
+  // Receive accessibility changes to update shadow underlay.
+  ash::WmShell::Get()->system_tray_notifier()->AddAccessibilityObserver(this);
+  audio_feedback_ =
+      ash::WmShell::Get()->accessibility_delegate()->IsSpokenFeedbackEnabled();
+
   // Show widget next time Commit() is called.
   pending_show_widget_ = true;
 }
 
 void ShellSurface::Configure() {
   DCHECK(widget_);
 
   // Delay configure callback if |scoped_configure_| is set.
   if (scoped_configure_) {
     scoped_configure_->set_needs_configure();
@@ skipping 239 matching lines @@
     // TODO(oshima): Adjust the coordinates from client screen to
     // chromeos screen when multi displays are supported.
     gfx::Point origin = window->bounds().origin();
     gfx::Point shadow_origin = shadow_content_bounds_.origin();
     shadow_origin -= origin.OffsetFromOrigin();
     gfx::Rect shadow_bounds(shadow_origin, shadow_content_bounds_.size());
 
     // Always create and show the underlay, even in maximized/fullscreen.
     if (!shadow_underlay_) {
       shadow_underlay_ = new aura::Window(nullptr);
+      shadow_underlay_->SetTargetHandler(this);
       DCHECK(shadow_underlay_->owned_by_parent());
-      shadow_underlay_->set_ignore_events(true);
       // Ensure the background area inside the shadow is solid black.
       // Clients that provide translucent contents should not be using
       // rectangular shadows as this method requires opaque contents to
       // cast a shadow that represent it correctly.
       shadow_underlay_->Init(ui::LAYER_SOLID_COLOR);
       shadow_underlay_->layer()->SetColor(SK_ColorBLACK);
       DCHECK(shadow_underlay_->layer()->fills_bounds_opaquely());
       window->AddChild(shadow_underlay_);
       window->StackChildAtBottom(shadow_underlay_);
+      static_cast<CustomWindowTargeter*>(
+          static_cast<ui::EventTarget*>(window)->GetEventTargeter())
+          ->set_shadow_underlay(shadow_underlay_);
     }
 
+    bool underlay_capture_events = audio_feedback_ && widget_->IsActive();
+    shadow_underlay_->set_ignore_events(!underlay_capture_events);
+
     float shadow_underlay_opacity = rectangular_shadow_background_opacity_;
     // Put the black background layer behind the window if
-    // 1) the window is in immersive fullscreen.
+    // 1) the window is in immersive fullscreen or spoken_feedback/talkback

[reveman, 2016/09/28 16:11:50]

nit: s/spoken_feedback/spoken feedback/ and remove reference to "talkback" as
this code is not arc specific

[erosky, 2016/09/28 19:21:13]

Done.

+    //    is enabled.
     // 2) the window can control the bounds of the window in fullscreen (
     //    thus the background can be visible).
     // 3) the window has no transform (the transformed background may
     //    not cover the entire background, e.g. overview mode).
-    if (widget_->IsFullscreen() &&
+    if ((widget_->IsFullscreen() || underlay_capture_events) &&

[reveman, 2016/09/28 16:11:50]

use helper function?

[erosky, 2016/09/28 19:21:13]

This is the only place where the check happens, so I'm not sure breaking it into
a helper would be cleaner.

         ash::wm::GetWindowState(window)->allow_set_bounds_in_maximized() &&
-        window->layer()->transform().IsIdentity()) {
+        window->layer()->GetTargetTransform().IsIdentity()) {

[reveman, 2016/09/28 16:11:50]

is this needed as part of this patch?

[erosky, 2016/09/28 19:21:13]

Yeah, otherwise the shadow won't initially show-up if the app is started while
chromevox is already on. This happened because android doesn't know about the
animation and therefore doesn't commit the surface after the animation finishes.

       gfx::Point origin;
       origin -= window->bounds().origin().OffsetFromOrigin();
       shadow_bounds.set_origin(origin);
       shadow_bounds.set_size(window->parent()->bounds().size());
       shadow_underlay_opacity = 1.0f;
     }
 
     shadow_underlay_->SetBounds(shadow_bounds);
 
     // TODO(oshima): Setting to the same value should be no-op.
@@ skipping 18 matching lines @@
       shadow_overlay_->layer()->Add(shadow->layer());
       window->AddChild(shadow_overlay_);
       shadow_overlay_->Show();
     }
     shadow_overlay_->SetBounds(shadow_bounds);
     shadow->SetContentBounds(gfx::Rect(shadow_bounds.size()));
   }
 }
 
 }  // namespace exo