Display images in multiline Braille

chrome/browser/resources/chromeos/chromevox/braille/braille_display_manager.js

Index: chrome/browser/resources/chromeos/chromevox/braille/braille_display_manager.js
diff --git a/chrome/browser/resources/chromeos/chromevox/braille/braille_display_manager.js b/chrome/browser/resources/chromeos/chromevox/braille/braille_display_manager.js
index ab17296192861b8d25991ed2463da568109a1638..b1dfadcde002194c37e60baeab896bb1abf3c833 100644
--- a/chrome/browser/resources/chromeos/chromevox/braille/braille_display_manager.js
+++ b/chrome/browser/resources/chromeos/chromevox/braille/braille_display_manager.js
@@ -57,8 +57,8 @@ cvox.BrailleDisplayManager = function(translatorManager) {
    * @type {!cvox.BrailleDisplayState}
    * @private
    */
-  this.displayState_ = {available: false, textRowCount: undefined,
-      textColumnCount: undefined};
+  this.displayState_ = {available: false, textRowCount: 0,
+      textColumnCount: 0};
   /**
    * State reported from the chrome api, reflecting a real hardware
    * display.
@@ -67,6 +67,8 @@ cvox.BrailleDisplayManager = function(translatorManager) {
    */
   this.realDisplayState_ = this.displayState_;
 
+  this.displayingImage_ = false;

[David Tseng, 2017/02/22 18:15:24]

js doc

[dmazzoni, 2017/02/24 00:19:41]

Done.

+
   translatorManager.addChangeListener(function() {
     this.translateContent_(this.content_, this.expansionType_);
   }.bind(this));
@@ -118,11 +120,78 @@ cvox.BrailleDisplayManager.CURSOR_DOTS_ = 1 << 6 | 1 << 7;
  */
 cvox.BrailleDisplayManager.prototype.setContent = function(
     content, expansionType) {
+  this.displayingImage_ = false;
   this.translateContent_(content, expansionType);
 };
 
 
 /**
+ * Takes an image, in the form of a data url, and displays it in braille
+ * onto the physical braille display and the virtual braille captions display.
+ * @param {!string} imageUrl The image, in the form of a data url.
+ */
+cvox.BrailleDisplayManager.prototype.setImageContent = function(imageUrl) {
+  this.displayingImage_ = true;
+  if (!this.displayState_.available) {
+    return;
+  }
+
+  var rows = this.displayState_.textRowCount;
+  var columns = this.displayState_.textColumnCount;
+  var imageDataUrl = imageUrl;
+  var imgElement = document.createElement('img');
+  imgElement.src = imageDataUrl;
+  var canvas = document.createElement('canvas');
+  var context = canvas.getContext('2d');
+  canvas.width = columns * 2;
+  canvas.height = rows * 4;
+  context.drawImage(imgElement, 0, 0, canvas.width, canvas.height);
+  var imageData = context.getImageData(0, 0, canvas.width, canvas.height);
+  var data = imageData.data;
+  var outputData = [];
+
+  // Convert image to black and white.
+  for (var i = 0; i < data.length; i += 4) {
+    // Show if the alpha value is visible (above 20) and the pixel is not
+    // white (under 600).
+    var show = (data[i] +
+                data[i + 1] +
+                data[i + 2] <= 200 * 3) &&
+                (data[i + 3]) == 255;
+    outputData.push(show);
+  }
+
+  // Convert to Braille.
+  var buf = new ArrayBuffer(rows * columns);
+  var view = new Uint8Array(buf);
+  var coordsToBrailleDot = [0x1, 0x2, 0x4, 0x40, 0x8, 0x10, 0x20, 0x80];
+  for (var i = 0; i < rows; i++) {
+    for (var j = 0; j < columns; j++) {
+      //Index in braille array
+      var index = i * columns + j;
+      view[index] = 0;
+      for (var x = 0; x < 2; x++) {
+        for (var y = 0; y < 4; y++) {
+          if (outputData[((i * columns * 4 + j + y * columns) * 2 + x)])
+            view[index] += coordsToBrailleDot[x * 4 + y];
+        }
+      }
+    }
+  }
+
+  if (this.realDisplayState_.available) {
+    chrome.brailleDisplayPrivate.writeDots(
+        buf,
+        this.displayState_.textColumnCount,
+        this.displayState_.textRowCount);
+  }
+  if (cvox.BrailleCaptionsBackground.isEnabled()) {
+    cvox.BrailleCaptionsBackground.setImageContent(buf, rows, columns);
+  }
+};
+
+
+/**
  * Sets the command listener.  When a command is invoked, the listener will be
  * called with the BrailleKeyEvent corresponding to the command and the content
  * that was present on the display when the command was invoked.  The content
@@ -136,7 +205,16 @@ cvox.BrailleDisplayManager.prototype.setCommandListener = function(func) {
 
 
 /**
- * @param {!cvox.BrailleDisplayState} newState Display state reported
+ * @return {!cvox.BrailleDisplayState} The current display state.
+ */
+cvox.BrailleDisplayManager.prototype.getDisplayState = function() {
+  return this.displayState_;
+};
+
+
+/**
+ * @param {{available: boolean, textRowCount: (number|undefined),
+ *     textColumnCount: (number|undefined)}} newState Display state reported

[David Tseng, 2017/02/22 18:15:24]

This should be cvox.BrailleDisplayState.

[dmazzoni, 2017/02/24 00:19:41]

I did this on purpose. The extension API allows
textRowCount and textColumnCount to be undefined, but
I changed cvox.BrailleDisplayState to require them to be
defined because it helps catch error, like a place in the
code where we were still using textCellCount.

I added this to the jsdoc to be clear.

  *     by the extension API.
  * @private
  */
@@ -154,7 +232,11 @@ cvox.BrailleDisplayManager.prototype.refreshDisplayState_ = function(
     }
     this.refresh_();
   }.bind(this);
-  this.realDisplayState_ = newState;
+  this.realDisplayState_ = {
+    available: newState.available,
+    textRowCount: newState.textRowCount || 0,
+    textColumnCount: newState.textColumnCount || 0
+  };
   if (newState.available) {
     processDisplayState(newState);
     // Update the dimensions of the virtual braille captions display to those
@@ -183,7 +265,7 @@ cvox.BrailleDisplayManager.prototype.onCaptionsStateChanged_ = function() {
  * @private
  */
 cvox.BrailleDisplayManager.prototype.refresh_ = function() {
-  if (!this.displayState_.available) {
+  if (!this.displayState_.available || this.displayingImage_) {

[David Tseng, 2017/02/22 18:15:24]

Seems like you should get this to work without keeping explicit track of whether
an image is displayed or not.

[dmazzoni, 2017/02/24 00:19:41]

They will have pretty different panning behavior, though, right?

     return;
   }
   var brailleBuf = this.panStrategy_.getCurrentBrailleViewportContents();