Android video capture: use new libyuv::Android420ToI420 API for format converting.

media/capture/video/android/video_capture_device_android.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/capture/video/android/video_capture_device_android.h"
 
 #include <stdint.h>
 #include <utility>
 
 #include "base/android/jni_android.h"
 #include "base/android/jni_array.h"
 #include "base/android/jni_string.h"
 #include "base/strings/string_number_conversions.h"
 #include "jni/VideoCapture_jni.h"
 #include "media/capture/video/android/photo_capabilities.h"
 #include "media/capture/video/android/video_capture_device_factory_android.h"
 #include "mojo/public/cpp/bindings/string.h"
+#include "third_party/libyuv/include/libyuv.h"
 
 using base::android::AttachCurrentThread;
 using base::android::CheckException;
 using base::android::GetClass;
 using base::android::MethodID;
 using base::android::JavaRef;
 using base::android::ScopedJavaLocalRef;
 
 namespace media {
 
@@ skipping 174 matching lines @@
   if (state_ != kCapturing || !client_.get())
     return;
 
   jbyte* buffer = env->GetByteArrayElements(data, NULL);
   if (!buffer) {
     LOG(ERROR) << "VideoCaptureDeviceAndroid::OnFrameAvailable: "
                   "failed to GetByteArrayElements";
     return;
   }
 
-  base::TimeTicks current_time = base::TimeTicks::Now();
+  const base::TimeTicks current_time = base::TimeTicks::Now();
   if (!got_first_frame_) {
     // Set aside one frame allowance for fluctuation.
     expected_next_frame_time_ = current_time - frame_interval_;
     first_ref_time_ = current_time;
     got_first_frame_ = true;
   }
 
   // Deliver the frame when it doesn't arrive too early.
   if (expected_next_frame_time_ <= current_time) {
     expected_next_frame_time_ += frame_interval_;
 
     // TODO(qiangchen): Investigate how to get raw timestamp for Android,
     // rather than using reference time to calculate timestamp.
     client_->OnIncomingCapturedData(reinterpret_cast<uint8_t*>(buffer), length,
                                     capture_format_, rotation, current_time,
                                     current_time - first_ref_time_);
   }
 
   env->ReleaseByteArrayElements(data, buffer, JNI_ABORT);
 }
 
+void VideoCaptureDeviceAndroid::OnI420FrameAvailable(JNIEnv* env,
+                                                     jobject obj,
+                                                     jobject y_buffer,
+                                                     jint y_stride,
+                                                     jobject u_buffer,
+                                                     jobject v_buffer,
+                                                     jint uv_row_stride,
+                                                     jint uv_pixel_stride,
+                                                     jint width,
+                                                     jint height,
+                                                     jint rotation) {
+  const base::TimeTicks current_time = base::TimeTicks::Now();
+  if (!got_first_frame_) {
+    // Set aside one frame allowance for fluctuation.
+    expected_next_frame_time_ = current_time - frame_interval_;
+    first_ref_time_ = current_time;
+    got_first_frame_ = true;
+  }
+
+  uint8_t* const y_src =
+      reinterpret_cast<uint8_t*>(env->GetDirectBufferAddress(y_buffer));
+  CHECK(y_src);
+  uint8_t* const u_src =
+      reinterpret_cast<uint8_t*>(env->GetDirectBufferAddress(u_buffer));
+  CHECK(u_src);
+  uint8_t* const v_src =
+      reinterpret_cast<uint8_t*>(env->GetDirectBufferAddress(v_buffer));
+  CHECK(v_src);
+
+  const int y_plane_length = width * height;
+  const int uv_plane_length = y_plane_length / 4;
+  const int buffer_length = y_plane_length + uv_plane_length * 2;
+  std::unique_ptr<uint8_t> buffer(new uint8_t[buffer_length]);
+
+  libyuv::Android420ToI420(y_src, y_stride, u_src, uv_row_stride, v_src,

[miu, 2016/07/20 19:34:22]

Sanity-check question: Is client code allowed to modify this memory? You might
want to check the Android documentation (or contact Android devs, if the
documentation is not sufficient).

[braveyao, 2016/07/20 22:02:04]

Not quite understand the concern here. Planes have been copied to local 'buffer'
here, so what client code gets should have nothing to do with the input buffer
any more (not like the above OnFrameAvailable() ). 

[miu, 2016/07/22 20:44:24]

Oh, never mind. I read this wrong. No problem here! :)

+                           uv_row_stride, uv_pixel_stride, buffer.get(), width,
+                           buffer.get() + y_plane_length, width / 2,
+                           buffer.get() + y_plane_length + uv_plane_length,
+                           width / 2, width, height);
+
+  // Deliver the frame when it doesn't arrive too early.
+  if (expected_next_frame_time_ <= current_time) {
+    expected_next_frame_time_ += frame_interval_;
+
+    // TODO(qiangchen): Investigate how to get raw timestamp for Android,

[miu, 2016/07/20 19:34:22]

We solved this problem in the screen capture (part I) change. You can get the
timestamp from the Image object:
https://developer.android.com/reference/android/media/Image.html#getTimestamp()

[braveyao, 2016/07/20 22:02:04]

Screen capture doesn't care about A/V sync (no audio capture in
MediaProjection). So the difference of drift between capture timestamp and
system timestamp has no impact.
Video capture has to consider it. Anyway it's something big for a separate cl.
:)

+    // rather than using reference time to calculate timestamp.
+    client_->OnIncomingCapturedData(buffer.get(), buffer_length,
+                                    capture_format_, rotation, current_time,
+                                    current_time - first_ref_time_);
+  }
+}
+
 void VideoCaptureDeviceAndroid::OnError(JNIEnv* env,
                                         const JavaParamRef<jobject>& obj,
                                         const JavaParamRef<jstring>& message) {
   SetErrorState(FROM_HERE,
                 base::android::ConvertJavaStringToUTF8(env, message));
 }
 
 void VideoCaptureDeviceAndroid::OnPhotoTaken(
     JNIEnv* env,
     const base::android::JavaParamRef<jobject>& obj,
@@ skipping 46 matching lines @@
     const tracked_objects::Location& from_here,
     const std::string& reason) {
   {
     base::AutoLock lock(lock_);
     state_ = kError;
   }
   client_->OnError(from_here, reason);
 }
 
 }  // namespace media