Display images in multiline Braille

chrome/browser/resources/chromeos/chromevox/braille/braille_display_manager.js

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 /**
  * @fileoverview Puts text on a braille display.
  *
  */
 
 goog.provide('cvox.BrailleDisplayManager');
@@ skipping 69 matching lines @@
    * @type {!Array<number>}
    * @private
    */
   this.textToBraille_ = [];
   /**
    * @type {!Array<number>}
    * @private
    */
   this.brailleToText_ = [];
 
+  this.displayingImage_ = false;
+
   translatorManager.addChangeListener(function() {
     this.translateContent_(this.content_, this.expansionType_);
   }.bind(this));
 
   chrome.storage.onChanged.addListener(function(changes, area) {
     if (area == 'local' && 'brailleWordWrap' in changes) {
       this.updatePanStrategy_(changes.brailleWordWrap.newValue);
     }
   }.bind(this));
   chrome.storage.local.get({brailleWordWrap: true}, function(items) {
@@ skipping 27 matching lines @@
 
 /**
  * @param {!cvox.NavBraille} content Content to send to the braille display.
  * @param {!cvox.ExpandingBrailleTranslator.ExpansionType} expansionType
  *     If the text has a {@code ValueSpan}, this indicates how that part
  *     of the display content is expanded when translating to braille.
  *     (See {@code cvox.ExpandingBrailleTranslator}).
  */
 cvox.BrailleDisplayManager.prototype.setContent = function(
     content, expansionType) {
+  this.displayingImage_ = false;
   this.translateContent_(content, expansionType);
 };
 
 
 /**
+ * Takes an image, in the form of a data url, and displays it in braille
+ * onto the physical braille display and the virtual braille captions display.
+ * @param {!string} imageUrl The image, in the form of a data url.

[David Tseng, 2016/12/08 16:44:22]

primitive type doesn't require nullable i.e. @param {string} is the same as
@param {!string}

[ultimatedbz, 2016/12/08 19:29:05]

Done.

+ */
+cvox.BrailleDisplayManager.prototype.setImageContent = function(imageUrl) {
+  this.displayingImage_ = true;
+  if (!this.displayState_.available) {
+    return;
+  }
+
+  var rows = this.displayState_.textRowCount;
+  var columns = this.displayState_.textColumnCount;
+  var imageDataUrl = imageUrl;
+  var imgElement = document.createElement('img');
+  imgElement.src = imageDataUrl;
+  var canvas = document.createElement('canvas');
+  var context = canvas.getContext('2d');
+  canvas.width = columns * 2;
+  canvas.height = rows * 4;
+  context.drawImage(imgElement, 0, 0, canvas.width, canvas.height);
+  var imageData = context.getImageData(0, 0, canvas.width, canvas.height);
+  var data = imageData.data;
+  var outputData = [];
+
+  // Convert image to black and white.
+  for (var i = 0; i < data.length; i += 4) {
+    // Show if the alpha value is visible (above 20) and the pixel is not
+    // white (under 600).
+    var show = (data[i] +
+                data[i + 1] +
+                data[i + 2] <= 200 * 3) &&
+                (data[i + 3]) == 255;
+    outputData.push(show);
+  }
+
+  // Convert to Braille.
+  var buf = new ArrayBuffer(rows * columns);
+  var view = new Uint8Array(buf);
+  var coordsToBrailleDot = [0x1, 0x2, 0x4, 0x40, 0x8, 0x10, 0x20, 0x80];
+  for (var i = 0; i < rows; i++) {
+    for (var j = 0; j < columns; j++) {
+      //Index in braille array

[David Tseng, 2016/12/08 16:44:22]

nit: period at end.

[ultimatedbz, 2016/12/08 19:29:05]

Done.

+      var index = i * columns + j;
+      view[index] = 0;
+      for (var x = 0; x < 2; x++) {
+        for (var y = 0; y < 4; y++) {
+          if (outputData[((i * columns * 4 + j + y * columns) * 2 + x)])
+            view[index] += coordsToBrailleDot[x * 4 + y];
+        }
+      }
+    }
+  }
+
+  if (this.realDisplayState_.available) {
+    chrome.brailleDisplayPrivate.writeDots(buf, buf.byteLength, 1);
+  }
+  if (cvox.BrailleCaptionsBackground.isEnabled()) {
+    cvox.BrailleCaptionsBackground.setImageContent(buf, rows, columns);
+  }
+};
+
+
+/**
  * Sets the command listener.  When a command is invoked, the listener will be
  * called with the BrailleKeyEvent corresponding to the command and the content
  * that was present on the display when the command was invoked.  The content
  * is guaranteed to be identical to an object previously used as the parameter
  * to cvox.BrailleDisplayManager.setContent, or null if no content was set.
  * @param {function(!cvox.BrailleKeyEvent, !cvox.NavBraille)} func The listener.
  */
 cvox.BrailleDisplayManager.prototype.setCommandListener = function(func) {
   this.commandListener_ = func;
 };
@@ skipping 31 matching lines @@
 cvox.BrailleDisplayManager.prototype.onCaptionsStateChanged_ = function() {
   // Force reevaluation of the display state based on our stored real
   // hardware display state, meaning that if a real display is connected,
   // that takes precedence over the state from the captions 'virtual' display.
   this.refreshDisplayState_(this.realDisplayState_);
 };
 
 
 /** @private */
 cvox.BrailleDisplayManager.prototype.refresh_ = function() {
-  if (!this.displayState_.available) {
+  if (!this.displayState_.available || this.displayingImage_) {

[David Tseng, 2016/12/08 16:44:22]

Panning support in v2? TODO(dmazzoni) perhaps? You could just override by using
fixed panning and make this work for an actual braille display.

     return;
   }
   var viewPort = this.panStrategy_.viewPort;
   var buf = this.displayedContent_.slice(viewPort.start, viewPort.end);
   if (this.realDisplayState_.available) {
     chrome.brailleDisplayPrivate.writeDots(buf, buf.byteLength, 1);
   }
   if (cvox.BrailleCaptionsBackground.isEnabled()) {
     var start = this.brailleToTextPosition_(viewPort.start);
     var end = this.brailleToTextPosition_(viewPort.end);
@@ skipping 186 matching lines @@
  */
 cvox.BrailleDisplayManager.prototype.updatePanStrategy_ = function(wordWrap) {
   var newStrategy = wordWrap ? new cvox.WrappingPanStrategy() :
       new cvox.FixedPanStrategy();
   newStrategy.setDisplaySize(this.displayState_.textColumnCount || 0);
   newStrategy.setContent(this.translatedContent_,
                          this.panStrategy_.viewPort.start);
   this.panStrategy_ = newStrategy;
   this.refresh_();
 };