FaceDetection: add support for <video> input

third_party/WebKit/Source/modules/shapedetection/FaceDetector.cpp

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "modules/shapedetection/FaceDetector.h"
 
 #include "core/dom/DOMException.h"
 #include "core/dom/DOMRect.h"
 #include "core/dom/Document.h"
 #include "core/fetch/ImageResource.h"
 #include "core/frame/ImageBitmap.h"
 #include "core/frame/LocalDOMWindow.h"
 #include "core/frame/LocalFrame.h"
 #include "core/html/HTMLImageElement.h"
+#include "core/html/HTMLVideoElement.h"
 #include "core/html/canvas/CanvasImageSource.h"
 #include "platform/graphics/Image.h"
 #include "public/platform/InterfaceProvider.h"
 #include "third_party/skia/include/core/SkImage.h"
 #include "third_party/skia/include/core/SkImageInfo.h"
 #include "wtf/CheckedNumeric.h"
 
 namespace blink {
 
 namespace {
 
 static CanvasImageSource* toImageSourceInternal(
     const CanvasImageSourceUnion& value) {
   if (value.isHTMLImageElement())
     return value.getAsHTMLImageElement();
 
   if (value.isImageBitmap()) {
     if (static_cast<ImageBitmap*>(value.getAsImageBitmap())->isNeutered())
       return nullptr;

[xianglu, 2016/10/31 23:07:52]

I wonder if we can put them in the same if statement like this?

if (value.isImageBitmap() &&
!static_cast<ImageBitmap*>(value.getAsImageBitmap())->isNeutered())
  return value.getAsImageBitmap();

[mcasas, 2016/11/01 01:10:58]

Done.

     return value.getAsImageBitmap();
   }
 
+  if (value.isHTMLVideoElement())
+    return value.getAsHTMLVideoElement();
+
   return nullptr;
 }
 
 }  // anonymous namespace
 
 FaceDetector* FaceDetector::create(ScriptState* scriptState) {
   return new FaceDetector(*scriptState->domWindow()->frame());
 }
 
 FaceDetector::FaceDetector(LocalFrame& frame) {
@@ skipping 12 matching lines @@
   if (!imageSourceInternal) {
     // TODO(mcasas): Implement more CanvasImageSources, https://crbug.com/659138
     NOTIMPLEMENTED() << "Unsupported CanvasImageSource";
     resolver->reject(
         DOMException::create(NotFoundError, "Unsupported source."));
     return promise;
   }
 
   if (imageSourceInternal->wouldTaintOrigin(
           scriptState->getExecutionContext()->getSecurityOrigin())) {
-    resolver->reject(DOMException::create(SecurityError,
-                                          "Image source would taint origin."));
+    resolver->reject(
+        DOMException::create(SecurityError, "Source would taint origin."));
     return promise;
   }
 
   if (imageSource.isHTMLImageElement()) {
     return detectFacesOnImageElement(
         resolver, static_cast<HTMLImageElement*>(imageSourceInternal));
   }
   if (imageSourceInternal->isImageBitmap()) {
     return detectFacesOnImageBitmap(
         resolver, static_cast<ImageBitmap*>(imageSourceInternal));
   }
+  if (imageSourceInternal->isVideoElement()) {
+    return detectFacesOnVideoElement(
+        resolver, static_cast<HTMLVideoElement*>(imageSourceInternal));
+  }
+
   NOTREACHED();
   return promise;
 }
 
 ScriptPromise FaceDetector::detectFacesOnImageElement(
     ScriptPromiseResolver* resolver,
     const HTMLImageElement* img) {
   ScriptPromise promise = resolver->promise();
   if (img->bitmapSourceSize().isZero()) {
     resolver->resolve(HeapVector<Member<DOMRect>>());
@@ skipping 96 matching lines @@
     allocationSize = pixmap.getSafeSize();
   } else {
     pixelData = imageBitmap->copyBitmapData(imageBitmap->isPremultiplied()
                                                 ? PremultiplyAlpha
                                                 : DontPremultiplyAlpha,
                                             N32ColorType);
     pixelDataPtr = pixelData->data();
     allocationSize = imageBitmap->size().area() * 4 /* bytes per pixel */;
   }
 
+  return detectFacesOnData(resolver, pixelDataPtr,
+                           allocationSize.ValueOrDefault(0),
+                           imageBitmap->width(), imageBitmap->height());
+}
+
+ScriptPromise FaceDetector::detectFacesOnVideoElement(
+    ScriptPromiseResolver* resolver,
+    const HTMLVideoElement* video) {
+  ScriptPromise promise = resolver->promise();
+
+  // TODO(mcasas): Check if |video| is actually playing a MediaStream by using
+  // HTMLMediaElement::isMediaStreamURL(video->currentSrc().getString()); if
+  // there is a local WebCam associated, there might be sophisticated ways to
+  // detect faces on it. Until then, treat as a normal <video> element.
+
+  // !hasAvailableVideoFrame() is a bundle of invalid states.
+  if (!video->hasAvailableVideoFrame()) {
+    resolver->reject(DOMException::create(
+        InvalidStateError, "Invalid HTMLVideoElement or state."));
+    return promise;
+  }
+
+  const FloatSize videoSize(video->videoWidth(), video->videoHeight());
+  SourceImageStatus sourceImageStatus = InvalidSourceImageStatus;
+  RefPtr<Image> image =
+      video->getSourceImageForCanvas(&sourceImageStatus, PreferNoAcceleration,
+                                     SnapshotReasonDrawImage, videoSize);
+
+  DCHECK_EQ(NormalSourceImageStatus, sourceImageStatus);
+
+  SkPixmap pixmap;
+  RefPtr<Uint8Array> pixelData;
+  uint8_t* pixelDataPtr = nullptr;
+  WTF::CheckedNumeric<int> allocationSize = 0;
+  // Use |skImage|'s pixels if it has direct access to them.
+  sk_sp<SkImage> skImage = image->imageForCurrentFrame();
+  if (skImage->peekPixels(&pixmap)) {
+    pixelDataPtr = static_cast<uint8_t*>(pixmap.writable_addr());
+    allocationSize = pixmap.getSafeSize();
+  } else {
+    // TODO(mcasas): retrieve the pixels from elsewhere.
+    NOTREACHED();
+    resolver->reject(DOMException::create(
+        InvalidStateError, "Failed to get pixels for current frame."));
+    return promise;
+  }
+
+  return detectFacesOnData(resolver, pixelDataPtr,
+                           allocationSize.ValueOrDefault(0), image->width(),
+                           image->height());
+}
+
+ScriptPromise FaceDetector::detectFacesOnData(ScriptPromiseResolver* resolver,
+                                              uint8_t* data,
+                                              int size,
+                                              int width,
+                                              int height) {
+  DCHECK(data);
+  DCHECK(size);
+  ScriptPromise promise = resolver->promise();
+
   mojo::ScopedSharedBufferHandle sharedBufferHandle =
-      mojo::SharedBufferHandle::Create(allocationSize.ValueOrDefault(0));
-
-  if (!pixelDataPtr || !sharedBufferHandle->is_valid()) {
+      mojo::SharedBufferHandle::Create(size);
+  if (!sharedBufferHandle->is_valid()) {
     resolver->reject(
         DOMException::create(InvalidStateError, "Internal allocation error"));
     return promise;
   }
 
   if (!m_service) {
     resolver->reject(DOMException::create(
-        NotFoundError, "Face detection service unavailable."));
+        NotSupportedError, "Face detection service unavailable."));
     return promise;
   }
 
   const mojo::ScopedSharedBufferMapping mappedBuffer =
-      sharedBufferHandle->Map(allocationSize.ValueOrDefault(0));
+      sharedBufferHandle->Map(size);
   DCHECK(mappedBuffer.get());
 
-  memcpy(mappedBuffer.get(), pixelDataPtr, allocationSize.ValueOrDefault(0));
+  memcpy(mappedBuffer.get(), data, size);
 
   m_serviceRequests.add(resolver);
   DCHECK(m_service.is_bound());
-  m_service->DetectFace(std::move(sharedBufferHandle), imageBitmap->width(),
-                        imageBitmap->height(),
+  m_service->DetectFace(std::move(sharedBufferHandle), width, height,
                         convertToBaseCallback(WTF::bind(
                             &FaceDetector::onDetectFace, wrapPersistent(this),
                             wrapPersistent(resolver))));
   sharedBufferHandle.reset();
   return promise;
 }
 
 void FaceDetector::onDetectFace(
     ScriptPromiseResolver* resolver,
     mojom::blink::FaceDetectionResultPtr faceDetectionResult) {
   if (!m_serviceRequests.contains(resolver))
     return;
 
   HeapVector<Member<DOMRect>> detectedFaces;
   for (const auto& boundingBox : faceDetectionResult->boundingBoxes) {
     detectedFaces.append(DOMRect::create(boundingBox->x, boundingBox->y,
                                          boundingBox->width,
                                          boundingBox->height));
   }
 
   resolver->resolve(detectedFaces);
   m_serviceRequests.remove(resolver);
 }
 
 DEFINE_TRACE(FaceDetector) {
   visitor->trace(m_serviceRequests);
 }
 
 }  // namespace blink