Implements computation of touch calibration transform using user provided data

ash/touch/touch_transformer_controller.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "ash/touch/touch_transformer_controller.h"
 
 #include "ash/display/window_tree_host_manager.h"
 #include "ash/host/ash_window_tree_host.h"
 #include "ash/root_window_controller.h"
 #include "ash/shell.h"
+#include "third_party/skia/include/core/SkMatrix44.h"
 #include "ui/aura/window_tree_host.h"
 #include "ui/display/display_layout.h"
 #include "ui/display/manager/chromeos/display_configurator.h"
 #include "ui/display/manager/display_manager.h"
 #include "ui/display/types/display_constants.h"
 #include "ui/display/types/display_snapshot.h"
 #include "ui/events/devices/device_data_manager.h"
 
 namespace ash {
 
 namespace {
 
 display::DisplayManager* GetDisplayManager() {
   return Shell::GetInstance()->display_manager();
 }
 
 ui::TouchscreenDevice FindTouchscreenById(int id) {
   const std::vector<ui::TouchscreenDevice>& touchscreens =
       ui::DeviceDataManager::GetInstance()->GetTouchscreenDevices();
   for (const auto& touchscreen : touchscreens) {
     if (touchscreen.id == id)
       return touchscreen;
   }
 
   return ui::TouchscreenDevice();
 }
 
+// Given an array of touch point and display point pairs, this function computes
+// and returns the constants(defined below) using a least fit algorithm.
+// If (xt, yt) is a touch point then its corresponding (xd, yd) would be defined
+// by the following 2 equations:
+// xd = xt * A + yt * B + C
+// yd = xt * D + yt * E + F
+// This function computes A, B, C, D, E and F and returns the corresponding
+// transform matrix.
+// See http://crbug.com/672293
+gfx::Transform GetCalibratedTransform(
+    std::array<std::pair<gfx::Point, gfx::Point>, 4> touch_point_pairs,

[sadrul, 2016/12/20 17:31:05]

const&

[malaykeshav, 2016/12/20 19:16:56]

They are being modified on line #57

+    const gfx::Transform& pre_calibration_tm) {
+  // Transform the display points before solving the equation.
+  // If the calibration was performed at a resolution that is 0.5 times the
+  // current resolution, then the display points (x, y) for a given touch point
+  // now represents a display point at (2 * x, 2 * y). This and other kinds of
+  // similar tranforms can be applied using |pre_calibration_tm|.
+  for (int row = 0; row < 4; row++)
+    pre_calibration_tm.TransformPoint(&touch_point_pairs[row].first);
+
+  // Vector of the X-coordinate of display points corresponding to each of the
+  // touch points.
+  SkVector4 display_points_x(
+      touch_point_pairs[0].first.x(), touch_point_pairs[1].first.x(),
+      touch_point_pairs[2].first.x(), touch_point_pairs[3].first.x());
+  // Vector of the Y-coordinate of display points corresponding to each of the
+  // touch points.
+  SkVector4 display_points_y(
+      touch_point_pairs[0].first.y(), touch_point_pairs[1].first.y(),
+      touch_point_pairs[2].first.y(), touch_point_pairs[3].first.y());
+
+  // Initialize |touch_point_matrix|
+  // If {(xt_1, yt_1), (xt_2, yt_2), (xt_3, yt_3)....} are a set of touch points
+  // received during calibration, then the |touch_point_matrix| would be defined
+  // as:
+  // |xt_1  yt_1  1  0|
+  // |xt_2  yt_2  1  0|
+  // |xt_3  yt_3  1  0|
+  // |xt_4  yt_4  1  0|
+  SkMatrix44 touch_point_matrix;
+  for (int row = 0; row < 4; row++) {
+    touch_point_matrix.set(row, 0, touch_point_pairs[row].second.x());
+    touch_point_matrix.set(row, 1, touch_point_pairs[row].second.y());
+    touch_point_matrix.set(row, 2, 1);
+    touch_point_matrix.set(row, 3, 0);
+  }
+  SkMatrix44 touch_point_matrix_transpose(touch_point_matrix);
+  touch_point_matrix_transpose.transpose();
+
+  SkMatrix44 product_matrix = touch_point_matrix_transpose * touch_point_matrix;
+
+  // Set (3, 3) = 1 so that |determinent| of the matrix is != 0 and the inverse
+  // can be calculated.
+  product_matrix.set(3, 3, 1);
+
+  SkMatrix44 product_matrix_inverse;
+  // #NOTE: If the determinent is zero then the inverse cannot be computed. The

[sadrul, 2016/12/20 17:31:05]

Just // NOTE:

[malaykeshav, 2016/12/20 19:16:56]

Done

+  // only solution is to restart touch calibration and get new points from user.
+  CHECK(product_matrix.invert(&product_matrix_inverse));

[sadrul, 2016/12/20 17:31:05]

How does a user recover from this error? i.e. how do you get different
calibration values after the crash+restart?

[malaykeshav, 2016/12/20 19:16:56]

The user has to go to settings>display>touch calibration

https://drive.google.com/a/google.com/file/d/0B_WkX8bSkoT1QVR0WFJYdUY2SmZTdWd...

+
+  product_matrix_inverse.set(3, 3, 0);
+
+  product_matrix = product_matrix_inverse * touch_point_matrix_transpose;
+
+  // Constants [A, B, C, 0] used to calibrate the x-coordinate of touch input.
+  // x_new = x_old * A + y_old * B + C;
+  SkVector4 x_constants = product_matrix * display_points_x;
+  // Constants [D, E, F, 0] used to calibrate the y-coordinate of touch input.
+  // y_new = x_old * D + y_old * E + F;
+  SkVector4 y_constants = product_matrix * display_points_y;
+
+  // Create a transform matrix using the touch calibration data.
+  return gfx::Transform(x_constants.fData[0], x_constants.fData[1], 0,
+                        x_constants.fData[2], y_constants.fData[0],
+                        y_constants.fData[1], 0, y_constants.fData[2], 0, 0, 1,
+                        0, 0, 0, 0, 1);
+}
+
+// Returns an uncalibrated touch transform.
+gfx::Transform GetUncalibratedTransform(
+    gfx::Transform ctm,

[sadrul, 2016/12/20 17:31:05]

const &

Make a copy in the function body that you then return.

[malaykeshav, 2016/12/20 19:16:56]

Done

+    const display::ManagedDisplayInfo& display,
+    const display::ManagedDisplayInfo& touch_display,
+    const gfx::SizeF& touch_area,
+    const gfx::SizeF& touch_native_size) {
+  auto current_size = gfx::SizeF(display.bounds_in_native().size());

[sadrul, 2016/12/20 17:31:05]

gfx::SizeF current_size(display.bounds_in_native().size())

[malaykeshav, 2016/12/20 19:16:56]

done

+
+  // Take care of panel fitting only if supported. Panel fitting is emulated
+  // in software mirroring mode (display != touch_display).
+  // If panel fitting is enabled then the aspect ratio is preserved and the
+  // display is scaled acordingly. In this case blank regions would be present
+  // in order to center the displayed area.
+  if (display.is_aspect_preserving_scaling() ||
+      display.id() != touch_display.id()) {
+    float touch_calib_ar =
+        touch_native_size.width() / touch_native_size.height();
+    float current_ar = current_size.width() / current_size.height();
+
+    if (current_ar > touch_calib_ar) {  // Letterboxing
+      ctm.Translate(
+          0, (1 - current_ar / touch_calib_ar) * 0.5 * current_size.height());
+      ctm.Scale(1, current_ar / touch_calib_ar);
+    } else if (touch_calib_ar > current_ar) {  // Pillarboxing
+      ctm.Translate(
+          (1 - touch_calib_ar / current_ar) * 0.5 * current_size.width(), 0);
+      ctm.Scale(touch_calib_ar / current_ar, 1);
+    }
+  }
+  // Take care of scaling between touchscreen area and display resolution.
+  ctm.Scale(current_size.width() / touch_area.width(),
+            current_size.height() / touch_area.height());
+  return ctm;
+}
+
 }  // namespace
 
 // This is to compute the scale ratio for the TouchEvent's radius. The
 // configured resolution of the display is not always the same as the touch
 // screen's reporting resolution, e.g. the display could be set as
 // 1920x1080 while the touchscreen is reporting touch position range at
 // 32767x32767. Touch radius is reported in the units the same as touch position
 // so we need to scale the touch radius to be compatible with the display's
 // resolution. We compute the scale as
 // sqrt of (display_area / touchscreen_area)
@@ skipping 30 matching lines @@
   auto touch_area = gfx::SizeF(framebuffer_size);
 #endif
 
   gfx::Transform ctm;
 
   if (current_size.IsEmpty() || touch_native_size.IsEmpty() ||
       touch_area.IsEmpty() || touchscreen.id == ui::InputDevice::kInvalidId)
     return ctm;
 
 #if defined(USE_OZONE)
-  // Translate the touch so that it falls within the display bounds.
-  ctm.Translate(display.bounds_in_native().x(), display.bounds_in_native().y());
+  // Translate the touch so that it falls within the display bounds. This should
+  // not be performed if the displays are mirrored.
+  if (display.id() == touch_display.id())
+    ctm.Translate(display.bounds_in_native().x(),
+                  display.bounds_in_native().y());

[sadrul, 2016/12/20 17:31:05]

{}

[malaykeshav, 2016/12/20 19:16:56]

done

 #endif
 
-  // Take care of panel fitting only if supported. Panel fitting is emulated in
-  // software mirroring mode (display != touch_display).
-  // If panel fitting is enabled then the aspect ratio is preserved and the
-  // display is scaled acordingly. In this case blank regions would be present
-  // in order to center the displayed area.
-  if (display.is_aspect_preserving_scaling() ||
-      display.id() != touch_display.id()) {
-    float touch_native_ar =
-        touch_native_size.width() / touch_native_size.height();
+  // If the device is currently under calibration, then do not return any
+  // transform as we want to use the raw native touch input data for calibration
+  if (is_calibrating_)
+    return ctm;
+
+  // If touch calibration data is unavailable, use naive approach.
+  if (!touch_display.has_touch_calibration_data()) {
+    return GetUncalibratedTransform(ctm, display, touch_display, touch_area,
+                                    touch_native_size);
+  }
+
+  // The resolution at which the touch calibration was performed.
+  gfx::SizeF touch_calib_size(touch_display.GetTouchCalibrationData().bounds);
+
+  // Any additional transfomration that needs to be applied to the display
+  // points, before we solve for the final transform.
+  gfx::Transform pre_transform;
+
+  if (display.id() != touch_display.id() ||
+      display.is_aspect_preserving_scaling()) {
+    // Case of displays being mirrored or in panel fitting mode.
+    // Aspect ratio of the touch display's resolution during calibration.
+    float calib_ar = touch_calib_size.width() / touch_calib_size.height();
+    // Aspect ratio of the display that is being mirrored.
     float current_ar = current_size.width() / current_size.height();
 
-    if (current_ar > touch_native_ar) {  // Letterboxing
-      ctm.Translate(
-          0, (1 - current_ar / touch_native_ar) * 0.5 * current_size.height());
-      ctm.Scale(1, current_ar / touch_native_ar);
-    } else if (touch_native_ar > current_ar) {  // Pillarboxing
-      ctm.Translate(
-          (1 - touch_native_ar / current_ar) * 0.5 * current_size.width(), 0);
-      ctm.Scale(touch_native_ar / current_ar, 1);
+    if (current_ar < calib_ar) {
+      pre_transform.Scale(current_size.height() / touch_calib_size.height(),
+                          current_size.height() / touch_calib_size.height());
+      pre_transform.Translate(
+          (current_ar / calib_ar - 1.f) * touch_calib_size.width() * 0.5f, 0);
+    } else {
+      pre_transform.Scale(current_size.width() / touch_calib_size.width(),
+                          current_size.width() / touch_calib_size.width());
+      pre_transform.Translate(
+          0, (calib_ar / current_ar - 1.f) * touch_calib_size.height() * 0.5f);
     }
+  } else {
+    // Case of current resolution being different from the resolution when the
+    // touch calibration was performed.
+    pre_transform.Scale(current_size.width() / touch_calib_size.width(),
+                        current_size.height() / touch_calib_size.height());
   }
+  // Solve for coefficients and compute transform matrix.
+  gfx::Transform stored_ctm = GetCalibratedTransform(
+      touch_display.GetTouchCalibrationData().point_pairs, pre_transform);
 
-  // Take care of scaling between touchscreen area and display resolution.
-  ctm.Scale(current_size.width() / touch_area.width(),
-            current_size.height() / touch_area.height());
-  return ctm;
+  stored_ctm.ConcatTransform(ctm);
+  return stored_ctm;
 }
 
 TouchTransformerController::TouchTransformerController() {
   Shell::GetInstance()->window_tree_host_manager()->AddObserver(this);
 }
 
 TouchTransformerController::~TouchTransformerController() {
   Shell::GetInstance()->window_tree_host_manager()->RemoveObserver(this);
 }
 
@@ skipping 91 matching lines @@
 }
 
 void TouchTransformerController::OnDisplaysInitialized() {
   UpdateTouchTransformer();
 }
 
 void TouchTransformerController::OnDisplayConfigurationChanged() {
   UpdateTouchTransformer();
 }
 
+void TouchTransformerController::SetForCalibration(bool is_calibrating) {
+  is_calibrating_ = is_calibrating;
+  UpdateTouchTransformer();
+}
+
 }  // namespace ash