Mac Video Capture: Add support for pixel format NV12 for AVFoundation

media/video/capture/mac/video_capture_device_avfoundation_mac.mm

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #import "media/video/capture/mac/video_capture_device_avfoundation_mac.h"
 
 #import <CoreVideo/CoreVideo.h>
 
 #include "base/logging.h"
 #include "base/mac/foundation_util.h"
 #include "media/video/capture/mac/video_capture_device_mac.h"
 #include "ui/gfx/geometry/size.h"
 
 // Prefer MJPEG if frame width or height is larger than this.
 static const int kMjpegWidthThreshold = 640;
 static const int kMjpegHeightThreshold = 480;
 
 // This function translates Mac Core Video pixel formats to Chromium pixel
 // formats. Chromium pixel formats are sorted in order of preference.
 media::VideoPixelFormat FourCCToChromiumPixelFormat(FourCharCode code) {

[mcasas, 2015/01/28 20:17:01]

This shouldn't be FourCharCode, that's too generic. We should use 
CMPixelFormatType instead [1] but that doesn't cover the kCVPixel...
coming from CVPixelBuffer.h, sigh :( we might need to stick to it.

[1]
https://developer.apple.com/library/ios/documentation/CoreMedia/Reference/CMF...

[magjed_chromium, 2015/01/29 13:42:17]

Acknowledged.

   switch (code) {
     case kCVPixelFormatType_422YpCbCr8:
       return media::PIXEL_FORMAT_UYVY;
     case CoreMediaGlue::kCMPixelFormat_422YpCbCr8_yuvs:
       return media::PIXEL_FORMAT_YUY2;
     case CoreMediaGlue::kCMVideoCodecType_JPEG_OpenDML:
       return media::PIXEL_FORMAT_MJPEG;
+    case CoreMediaGlue::kCMPixelFormat_420YpCbCr8BiPlanarVideoRange:
+    case CoreMediaGlue::kCMPixelFormat_420YpCbCr8BiPlanarFullRange:

[mcasas, 2015/01/28 20:17:01]

Correct naming as per earlier comments.

[magjed_chromium, 2015/01/29 13:42:17]

Done.

+      return media::PIXEL_FORMAT_NV12;
     default:
       return media::PIXEL_FORMAT_UNKNOWN;
   }
 }
 
 @implementation VideoCaptureDeviceAVFoundation
 
 #pragma mark Class methods
 
 + (void)getDeviceNames:(NSMutableDictionary*)deviceNames {
@@ skipping 146 matching lines @@
   // VideoCaptureDeviceMac::ReceiveFrame() is calling here, depending on the
   // running state. VCDM::ReceiveFrame() calls here to change aspect ratio.
   DCHECK((![captureSession_ isRunning] &&
       main_thread_checker_.CalledOnValidThread()) ||
       callback_thread_checker_.CalledOnValidThread());
 
   frameWidth_ = width;
   frameHeight_ = height;
   frameRate_ = frameRate;
 
   FourCharCode best_fourcc = kCVPixelFormatType_422YpCbCr8;

[mcasas, 2015/01/28 20:17:01]

See comments above for FourCharCode.

[magjed_chromium, 2015/01/29 13:42:17]

Acknowledged.

   const bool prefer_mjpeg =
       width > kMjpegWidthThreshold || height > kMjpegHeightThreshold;
   for (CrAVCaptureDeviceFormat* format in captureDevice_.formats) {
     const FourCharCode fourcc =
         CoreMediaGlue::CMFormatDescriptionGetMediaSubType(
             [format formatDescription]);
     if (prefer_mjpeg &&
         fourcc == CoreMediaGlue::kCMVideoCodecType_JPEG_OpenDML) {
       best_fourcc = fourcc;
       break;
     }
     // Compare according to Chromium preference.
     if (FourCCToChromiumPixelFormat(fourcc) <
         FourCCToChromiumPixelFormat(best_fourcc)) {

[mcasas, 2015/01/28 20:17:01]

Hmm here the semantics are somehow cumbersome because we
translate, with an implicit priority, CVPixel format to chromium,
which is itself non-evidently enum-ordered by priority.

Syntactic sugar proposal? Surprised you didn't come up
with this Magnus, this is your specialty :)

best_fourcc = s std::max(fourcc, best_fourcc,
                         [](const FourCharCode& c1, const FourCharCode& c2) {
                           return FourCCToChromiumPixelFormat(c1) <
                                  FourCCToChromiumPixelFormat(c2);
                         })


Then a concern: AFAIK all cameras support I422 formats, and they
are higher in the FourCCToChromiumPixelFormat() prio scheme than
I420 formats. How can we be sure this code path is used?

[magjed_chromium, 2015/01/29 13:42:17]

I will certainly consider a lambda :), but in a different CL. Regarding
I420/I422 order, all I420 are above I422 in media::VideoPixelFormat:
https://code.google.com/p/chromium/codesearch#chromium/src/media/video/captur...

       best_fourcc = fourcc;
     }
   }
 
   // The capture output has to be configured, despite Mac documentation
   // detailing that setting the sessionPreset would be enough. The reason for
   // this mismatch is probably because most of the AVFoundation docs are written
   // for iOS and not for MacOsX. AVVideoScalingModeKey() refers to letterboxing
   // yes/no and preserve aspect ratio yes/no when scaling. Currently we set
   // cropping and preservation.
@@ skipping 86 matching lines @@
           blockBuffer, 0, &lengthAtOffset, &frameSize, &baseAddress);
       // Expect the MJPEG data to be available as a contiguous reference, i.e.
       // not covered by multiple memory blocks.
       CHECK_EQ(lengthAtOffset, frameSize);
     }
   } else {
     videoFrame = CoreMediaGlue::CMSampleBufferGetImageBuffer(sampleBuffer);
     // Lock the frame and calculate frame size.
     if (CVPixelBufferLockBaseAddress(videoFrame, kCVPixelBufferLock_ReadOnly) ==
         kCVReturnSuccess) {
-      baseAddress = static_cast<char*>(CVPixelBufferGetBaseAddress(videoFrame));
-      frameSize = CVPixelBufferGetHeight(videoFrame) *
-                  CVPixelBufferGetBytesPerRow(videoFrame);
+      baseAddress = static_cast<char*>(
+          CVPixelBufferGetPlaneCount(videoFrame) == 0
+              ? CVPixelBufferGetBaseAddress(videoFrame)
+              : CVPixelBufferGetBaseAddressOfPlane(videoFrame, 0));
+      // Chromium expects the data to be tightly packed, i.e. no row padding
+      // and the planes should follow consecutively.
+      switch (captureFormat.pixel_format) {
+        case media::PIXEL_FORMAT_YUY2:
+        case media::PIXEL_FORMAT_UYVY:
+          CHECK_EQ(CVPixelBufferGetBytesPerRow(videoFrame),
+                   static_cast<UInt>(2 * captureFormat.frame_size.width()));
+          frameSize = captureFormat.frame_size.GetArea() * 16 / 8;  // 16bpp.
+          break;
+        case media::PIXEL_FORMAT_NV12:
+          CHECK_EQ(CVPixelBufferGetBytesPerRowOfPlane(videoFrame, 0),
+                   static_cast<UInt>(captureFormat.frame_size.width()));

[mcasas, 2015/01/28 20:17:01]

|UInt| looks funny. What about |uint32| or even better |size_t|?

[magjed_chromium, 2015/01/29 13:42:17]

Done.

+          CHECK_EQ(CVPixelBufferGetBytesPerRowOfPlane(videoFrame, 1),
+                   static_cast<UInt>(captureFormat.frame_size.width()));
+          CHECK_EQ(CVPixelBufferGetBaseAddressOfPlane(videoFrame, 1),
+                   baseAddress + captureFormat.frame_size.GetArea());
+          frameSize = captureFormat.frame_size.GetArea() * 12 / 8;  // 12bpp.
+          break;
+        default:
+          NOTREACHED();
+      }
     } else {
       videoFrame = nil;
     }
   }
 
   {
     base::AutoLock lock(lock_);
     if (frameReceiver_ && baseAddress) {
       frameReceiver_->ReceiveFrame(reinterpret_cast<uint8_t*>(baseAddress),
                                    frameSize, captureFormat, 0, 0);
@@ skipping 14 matching lines @@
 }
 
 - (void)sendErrorString:(NSString*)error {
   DLOG(ERROR) << [error UTF8String];
   base::AutoLock lock(lock_);
   if (frameReceiver_)
     frameReceiver_->ReceiveError([error UTF8String]);
 }
 
 @end