Display images in multiline Braille

chrome/browser/resources/chromeos/chromevox/braille/braille_display_manager.js

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 /**
  * @fileoverview Puts text on a braille display.
  *
  */
 
 goog.provide('cvox.BrailleDisplayManager');
@@ skipping 57 matching lines @@
   this.realDisplayState_ = this.displayState_;
 
   translatorManager.addChangeListener(function() {
     this.translateContent_(this.content_, this.expansionType_);
   }.bind(this));
 
   chrome.storage.onChanged.addListener(function(changes, area) {
     if (area == 'local' && 'brailleWordWrap' in changes) {
       this.updatePanStrategy_(changes.brailleWordWrap.newValue);
     }
+    if (area == 'local' && ('virtualBrailleRows' in changes ||

[dmazzoni, 2016/12/05 17:29:40]

This should probably be part of the previous change, or
a separate change

[ultimatedbz, 2016/12/05 19:37:38]

Done.

+          'virtualBrailleColumns' in changes)) {
+      this.onCaptionsStateChanged_();
+    }
   }.bind(this));
   chrome.storage.local.get({brailleWordWrap: true}, function(items) {
     this.updatePanStrategy_(items.brailleWordWrap);
   }.bind(this));
 
   cvox.BrailleCaptionsBackground.init(goog.bind(
       this.onCaptionsStateChanged_, this));
   if (goog.isDef(chrome.brailleDisplayPrivate)) {
     var onDisplayStateChanged = goog.bind(this.refreshDisplayState_, this);
     chrome.brailleDisplayPrivate.getDisplayState(onDisplayStateChanged);
@@ skipping 24 matching lines @@
  *     of the display content is expanded when translating to braille.
  *     (See {@code cvox.ExpandingBrailleTranslator}).
  */
 cvox.BrailleDisplayManager.prototype.setContent = function(
     content, expansionType) {
   this.translateContent_(content, expansionType);
 };
 
 
 /**
+ * Takes a imageDataUrl and displays it in braille onto the physical braille

[dmazzoni, 2016/12/05 17:29:40]

Takes an image, in the form of a data url,

[ultimatedbz, 2016/12/05 19:32:00]

Done.

+ * display and the virtual braille captions display.
+ * @param {!string} imageUrl The imageDataUrl of the image to display.

[dmazzoni, 2016/12/05 17:29:40]

Same, "The image, in the form of a data url" or something like that

[ultimatedbz, 2016/12/05 19:32:01]

Done.

+ */
+cvox.BrailleDisplayManager.prototype.setImageContent = function(imageUrl) {
+  var rows = this.displayState_.textRowCount;
+  var columns = this.displayState_.textColumnCount;
+  var imageDataUrl = imageUrl;
+  var imgElement = document.createElement('img');
+  imgElement.src = imageDataUrl;
+  var c1 = document.createElement('canvas');

[dmazzoni, 2016/12/05 17:29:40]

var canvas

[ultimatedbz, 2016/12/05 19:32:01]

Done.

+  var context = c1.getContext('2d');
+  c1.width = imgElement.width;
+  c1.height = imgElement.height;
+  context.drawImage(imgElement, 0, 0);
+  var originalImageData = context.getImageData(0, 0, c1.width, c1.height);
+  var originalData = originalImageData.data;
+
+  // Convert image to black and white.
+  var c2 = document.createElement('canvas');

[dmazzoni, 2016/12/05 17:29:40]

You probably don't need two canvases for this, right? I
think you can do it in-place

[ultimatedbz, 2016/12/05 19:32:01]

Done.

+  var context2 = c2.getContext('2d');
+  c2.width = imgElement.width;
+  c2.height = imgElement.height;
+  var blackWhiteData = [];
+  for (var i = 0; i < originalData.length; i += 4) {
+    // If the alpha value is transparent (less than 20) or the pixel is
+    // white(over 600), don't show.
+    var value = (originalData[i] +
+                originalData[i + 1] +
+                originalData[i + 2] > 200 * 3) ||
+                (originalData[i + 3]) < 20 ? 0 : 255;
+    blackWhiteData[i] = value;
+    blackWhiteData[i + 1] = value;
+    blackWhiteData[i + 2] = value;
+  }
+  var blackWhiteImageData = context2.getImageData(0, 0, c2.width, c2.height);
+  blackWhiteImageData.data = blackWhiteData;
+  context2.putImageData(blackWhiteImageData, 0, 0, 0, 0, c2.width, c2.height);
+
+  // Scaled down to braille dimensions.

[dmazzoni, 2016/12/05 17:29:40]

I think it'd look better to rescale first, then
convert to black and white. You can rescale up above
where you call context.drawImage() - basically you
can just call scale() on the canvas context if you
want the image to be drawn smaller or larger. So I
think you can skip this step entirely.

[ultimatedbz, 2016/12/05 19:32:01]

That's awesome! You can actually specify how big you want to draw the image in
context.drawImage(), so 
'  canvas1.width = columns * 2;
  canvas1.height = rows * 4;
  context.drawImage(imgElement, 0, 0, canvas1.width, canvas1.height);
'
Did the trick!

+  var c3 = document.createElement('canvas');
+  var context3 = c3.getContext('2d');
+  c3.width = columns * 2;
+  c3.height = rows * 4;
+  var imageData3 = context3.createImageData(c3.width, c3.height);
+  var data3 = imageData3.data;
+  var data = blackWhiteData;
+  var xScale = c2.width / c3.width;
+  var yScale = c2.height / c3.height;
+  for (var i = 0; i < c3.height; i++) {
+    for (var j = 0; j < c3.width; j++) {
+      var newX = Math.floor(j * xScale);
+      var newY = Math.floor(i * yScale);
+      var index = newX * 4 + newY * 4 * c2.width;
+      var index3 = i * 4 * c3.width + 4 * j;
+
+      data3[index3] = data[index];
+      data3[index3 + 1] = data[index + 1];
+      data3[index3 + 2] = data[index + 2];
+      data3[index3 + 3] = data[index + 3];
+    }
+  }
+  context3.putImageData(imageData3, 0, 0);
+
+  // Covert to Braille.

[dmazzoni, 2016/12/05 17:29:40]

Convert

[ultimatedbz, 2016/12/05 19:32:01]

Done.

+  var buf = new ArrayBuffer(rows * columns);
+  var view = new Uint8Array(buf);
+  var dot1 = 0x1;
+  var dot2 = 0x2;
+  var dot3 = 0x4;
+  var dot4 = 0x8;
+  var dot5 = 0x10;
+  var dot6 = 0x20;
+  var dot7 = 0x40;
+  var dot8 = 0x80;
+
+  for (var i = 0; i < rows; i++) {
+    for (var j = 0; j < columns; j++) {
+      //Index in braille array
+      var index = i * columns + j;
+
+      view[index] = 0;
+      if (data3[(i * columns * 4 + j) * 2 * 4] == 255)

[dmazzoni, 2016/12/05 17:29:40]

I'd do something like this:

coordsToBrailleDot[0][0] = 0x1;
...

for (int x = 0; x < 2; x++) {
  for (int y = 0; y < 4; y++) {
    if (data3[((i * columns * 4 + j + y * columns) * 2 + x) * 4] == 255)
      view[index] += coordsToBrailleDot[x][y];
  }
}

[ultimatedbz, 2016/12/05 19:32:01]

This was really clever. On top of this, I made some other optimizations too!

+        view[index] += dot1;
+      if (data3[(i * columns * 4 + j + columns) * 2 * 4] == 255)
+        view[index] += dot2;
+      if (data3[(i * columns * 4 + j + 2 * columns) * 2 * 4] == 255)
+        view[index] += dot3;
+      if (data3[(i * columns * 4 + j + 3 * columns) * 2 * 4] == 255)
+        view[index] += dot7;
+
+      if (data3[((i * columns * 4 + j) * 2 + 1) * 4] == 255)
+        view[index] += dot4;
+      if (data3[((i * columns * 4 + j + columns) * 2 + 1) * 4] == 255)
+        view[index] += dot5;
+      if (data3[((i * columns * 4 + j + 2 * columns) * 2 + 1) * 4] == 255)
+        view[index] += dot6;
+      if (data3[((i * columns * 4 + j + 3 * columns) * 2 + 1) * 4] == 255)
+        view[index] += dot8;
+    }
+  }
+
+  if (!this.displayState_.available) {

[dmazzoni, 2016/12/05 17:29:40]

Put this check at the top

[ultimatedbz, 2016/12/05 19:32:01]

Done.

+    return;
+  }
+  if (this.realDisplayState_.available) {
+    chrome.brailleDisplayPrivate.writeDots(buf, buf.byteLength, 1);
+  }
+  if (cvox.BrailleCaptionsBackground.isEnabled()) {
+    cvox.BrailleCaptionsBackground.setImageContent(buf, rows, columns);
+  }
+};
+
+
+/**
  * Sets the command listener.  When a command is invoked, the listener will be
  * called with the BrailleKeyEvent corresponding to the command and the content
  * that was present on the display when the command was invoked.  The content
  * is guaranteed to be identical to an object previously used as the parameter
  * to cvox.BrailleDisplayManager.setContent, or null if no content was set.
  * @param {function(!cvox.BrailleKeyEvent, !cvox.NavBraille)} func The listener.
  */
 cvox.BrailleDisplayManager.prototype.setCommandListener = function(func) {
   this.commandListener_ = func;
 };
@@ skipping 230 matching lines @@
 };
 
 /**
  * @param {boolean} wordWrap
  * @private
  */
 cvox.BrailleDisplayManager.prototype.updatePanStrategy_ = function(wordWrap) {
   this.panStrategy_.setPanStrategy(wordWrap);
   this.refresh_();
 };