Add support for retrieving image thumbnails as part of the accessibility tree.

third_party/WebKit/Source/modules/accessibility/AXLayoutObject.cpp

 /*
 * Copyright (C) 2008 Apple Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1.  Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 * 2.  Redistributions in binary form must reproduce the above copyright
@@ skipping 25 matching lines @@
 #include "core/dom/shadow/ShadowRoot.h"
 #include "core/editing/EditingUtilities.h"
 #include "core/editing/FrameSelection.h"
 #include "core/editing/RenderedPosition.h"
 #include "core/editing/TextAffinity.h"
 #include "core/editing/VisibleUnits.h"
 #include "core/editing/iterators/CharacterIterator.h"
 #include "core/editing/iterators/TextIterator.h"
 #include "core/frame/FrameOwner.h"
 #include "core/frame/FrameView.h"
+#include "core/frame/ImageBitmap.h"
 #include "core/frame/LocalFrame.h"
 #include "core/frame/Settings.h"
+#include "core/html/HTMLCanvasElement.h"
 #include "core/html/HTMLFrameOwnerElement.h"
 #include "core/html/HTMLImageElement.h"
 #include "core/html/HTMLInputElement.h"
 #include "core/html/HTMLLabelElement.h"
 #include "core/html/HTMLOptionElement.h"
 #include "core/html/HTMLSelectElement.h"
 #include "core/html/HTMLTextAreaElement.h"
+#include "core/html/HTMLVideoElement.h"
+#include "core/html/ImageData.h"
 #include "core/html/LabelsNodeList.h"
 #include "core/html/shadow/ShadowElementNames.h"
+#include "core/imagebitmap/ImageBitmapOptions.h"
 #include "core/layout/HitTestResult.h"
 #include "core/layout/LayoutFileUploadControl.h"
 #include "core/layout/LayoutHTMLCanvas.h"
 #include "core/layout/LayoutImage.h"
 #include "core/layout/LayoutInline.h"
 #include "core/layout/LayoutListMarker.h"
 #include "core/layout/LayoutMenuList.h"
 #include "core/layout/LayoutTextControl.h"
 #include "core/layout/LayoutView.h"
 #include "core/layout/api/LayoutAPIShim.h"
@@ skipping 780 matching lines @@
   if (!getLayoutObject())
     return AXNodeObject::fontSize();
 
   const ComputedStyle* style = getLayoutObject()->style();
   if (!style)
     return AXNodeObject::fontSize();
 
   return style->computedFontSize();
 }
 
+String AXLayoutObject::imageDataUrl(const IntSize& maxSize) const {
+  Node* node = getNode();
+  if (!node)
+    return String();
+
+  ImageBitmapOptions options;
+  ImageBitmap* imageBitmap = nullptr;
+  Document* document = &node->document();
+  if (isHTMLImageElement(node)) {
+    imageBitmap = ImageBitmap::create(toHTMLImageElement(node),
+                                      Optional<IntRect>(), document, options);
+  } else if (isHTMLCanvasElement(node)) {
+    imageBitmap = ImageBitmap::create(toHTMLCanvasElement(node),
+                                      Optional<IntRect>(), options);
+  } else if (isHTMLVideoElement(node)) {
+    imageBitmap = ImageBitmap::create(toHTMLVideoElement(node),
+                                      Optional<IntRect>(), document, options);
+  }
+  if (!imageBitmap)

[aboxhall, 2016/11/22 19:06:31]

It would be neat at some point if we could render arbitrary elements.

[dmazzoni, 2016/11/22 22:00:13]

True. Not sure if it'd make sense to render them in isolation, or
snip out part of the webpage, or what.

+    return String();
+
+  sk_sp<SkImage> image = imageBitmap->bitmapImage()->imageForCurrentFrame();
+  if (!image || image->width() <= 0 || image->height() <= 0)
+    return String();
+
+  // Determine the width and height of the output image, using a proportional
+  // scale factor such that it's no larger than |maxSize|.
+  float xScale = maxSize.width() ? maxSize.width() * 1.0 / image->width() : 1.0;
+  float yScale =
+      maxSize.height() ? maxSize.height() * 1.0 / image->height() : 1.0;
+  float scale = std::min(xScale, yScale);
+  if (scale >= 1.0)

[aboxhall, 2016/11/22 19:06:31]

Does this mean you can't scale the image up? I can't imagine why you'd want to,
but maybe worth documenting.

[dmazzoni, 2016/11/22 22:00:13]

Yes, that's how I've designed it. Doesn't the comment above
cover it? It says that it ensures the output image is no
larger than |maxSize|.

My intent is to set it to the size of a braille display.

If we wanted to display this in dev tools we could set it
to a fixed thumbnail size or something like that.

[aboxhall, 2016/11/22 22:04:51]

I mean you can't pass something larger than the size of the image for |maxSize|
and have it return something larger than the original image. That's not
explicitly mentioned in the comment - it's strictly true that it won't be larger
than |maxSize|, but it also won't be any larger than the original image, and
won't scale.

[dmazzoni, 2016/11/22 22:27:03]

Clarified in comment.

+    scale = 1.0;
+  int width = std::round(image->width() * scale);
+  int height = std::round(image->height() * scale);
+
+  // Draw the scaled image into a bitmap in native format.
+  SkBitmap bitmap;
+  bitmap.allocPixels(SkImageInfo::MakeN32(width, height, kPremul_SkAlphaType));
+  SkCanvas canvas(bitmap);
+  canvas.clear(SK_ColorTRANSPARENT);
+  canvas.drawImageRect(image, SkRect::MakeIWH(width, height), nullptr);
+
+  // Copy the bits into a buffer in RGBA_8888 unpremultiplied format
+  // for encoding.
+  SkImageInfo info = SkImageInfo::Make(width, height, kRGBA_8888_SkColorType,
+                                       kUnpremul_SkAlphaType);
+  size_t rowBytes = info.minRowBytes();
+  Vector<char> pixelStorage(info.getSafeSize(rowBytes));
+  SkPixmap pixmap(info, pixelStorage.data(), rowBytes);
+  if (!SkImage::MakeFromBitmap(bitmap)->readPixels(pixmap, 0, 0))
+    return String();
+
+  // Encode as a PNG and return as a data url.
+  String dataUrl =
+      ImageDataBuffer(
+          IntSize(width, height),
+          reinterpret_cast<const unsigned char*>(pixelStorage.data()))
+          .toDataURL("image/png", 1.0);
+  return dataUrl;
+}
+
 String AXLayoutObject::text() const {
   if (isPasswordFieldAndShouldHideValue()) {
     if (!getLayoutObject())
       return String();
 
     const ComputedStyle* style = getLayoutObject()->style();
     if (!style)
       return String();
 
     unsigned unmaskedTextLength = AXNodeObject::text().length();
@@ skipping 1535 matching lines @@
 
 bool AXLayoutObject::elementAttributeValue(
     const QualifiedName& attributeName) const {
   if (!m_layoutObject)
     return false;
 
   return equalIgnoringCase(getAttribute(attributeName), "true");
 }
 
 }  // namespace blink