ScreenCapture for Android phase1, part I

media/capture/content/android/screen_capture_machine_android.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/capture/content/android/screen_capture_machine_android.h"
+
+#include "base/android/context_utils.h"
+#include "base/android/jni_android.h"
+#include "base/android/scoped_java_ref.h"
+#include "jni/ScreenCapture_jni.h"
+#include "media/base/video_capture_types.h"
+#include "media/base/yuv_convert.h"
+#include "media/capture/content/video_capture_oracle.h"
+#include "third_party/libyuv/include/libyuv.h"
+
+using base::android::AttachCurrentThread;
+
+namespace media {
+
+// static
+bool ScreenCaptureMachineAndroid::RegisterScreenCaptureMachine(JNIEnv* env) {
+  return RegisterNativesImpl(env);
+}
+
+ScreenCaptureMachineAndroid::ScreenCaptureMachineAndroid() {}
+
+ScreenCaptureMachineAndroid::~ScreenCaptureMachineAndroid() {}
+
+// static
+ScopedJavaLocalRef<jobject>
+ScreenCaptureMachineAndroid::createScreenCaptureMachineAndroid(
+    jlong nativeScreenCaptureMachineAndroid) {
+  return (Java_ScreenCapture_createScreenCaptureMachine(
+      AttachCurrentThread(), base::android::GetApplicationContext(),
+      nativeScreenCaptureMachineAndroid));
+}
+
+void ScreenCaptureMachineAndroid::OnRGBAFrameAvailable(JNIEnv* env,
+                                                       jobject obj,
+                                                       jobject buf,
+                                                       jint width,
+                                                       jint height,
+                                                       jint rowStride,
+                                                       jlong timestamp) {
+  scoped_refptr<VideoFrame> i420_frame = VideoFrame::CreateFrame(
+      PIXEL_FORMAT_I420, gfx::Size(width, height), gfx::Rect(width, height),
+      gfx::Size(width, height), base::TimeDelta());
+
+  // ABGR little endian (rgba in memory) to I420.
+  libyuv::ABGRToI420(
+      reinterpret_cast<uint8_t*>(env->GetDirectBufferAddress(buf)), rowStride,
+      i420_frame->visible_data(VideoFrame::kYPlane),
+      i420_frame->stride(VideoFrame::kYPlane),
+      i420_frame->visible_data(VideoFrame::kUPlane),
+      i420_frame->stride(VideoFrame::kUPlane),
+      i420_frame->visible_data(VideoFrame::kVPlane),
+      i420_frame->stride(VideoFrame::kVPlane),
+      i420_frame->visible_rect().width(), i420_frame->visible_rect().height());
+
+  OnIncomingFrameAvailable(i420_frame->visible_data(VideoFrame::kYPlane),
+                           i420_frame->stride(VideoFrame::kYPlane),
+                           i420_frame->visible_data(VideoFrame::kUPlane),
+                           i420_frame->stride(VideoFrame::kUPlane),
+                           i420_frame->visible_data(VideoFrame::kVPlane),
+                           i420_frame->stride(VideoFrame::kVPlane),
+                           i420_frame->visible_rect().width(),
+                           i420_frame->visible_rect().height(),
+                           static_cast<int64_t>(timestamp));
+}
+
+void ScreenCaptureMachineAndroid::OnI420FrameAvailable(JNIEnv* env,
+                                                       jobject obj,
+                                                       jobject y_buffer,
+                                                       jint y_stride,
+                                                       jobject u_buffer,
+                                                       jint u_stride,
+                                                       jobject v_buffer,
+                                                       jint v_stride,
+                                                       jint width,
+                                                       jint height,
+                                                       jlong timestamp) {
+  uint8 *y_src, *u_src, *v_src;
+  y_src = reinterpret_cast<uint8_t*>(env->GetDirectBufferAddress(y_buffer));

[mcasas, 2016/05/23 18:19:55]

uint8* const y_src = ...
(here and for |u_src|, |v_src|, obviously).

In Chromium, the '*' goes with the type. 
Always prefer initialization over assignment
(e.g. https://www.doc.ic.ac.uk/lab/cplus/c++.rules/chap11.html)

[braveyao, 2016/05/24 00:03:26]

Done.

+  CHECK(y_src);
+  u_src = reinterpret_cast<uint8_t*>(env->GetDirectBufferAddress(u_buffer));
+  CHECK(u_src);
+  v_src = reinterpret_cast<uint8_t*>(env->GetDirectBufferAddress(v_buffer));
+  CHECK(v_src);
+
+  // U and V planes have |pixelStride| = 2, unpack them here.
+  int uv_plane_len = (int)env->GetDirectBufferCapacity(u_buffer);

[mcasas, 2016/05/23 18:19:55]

Hmm if you have to unpack, then incoming data
is NV12 (or NV21) [1], so please update the method
names accordingly, and instead of unpacking 
yourself, use, again, libyuv [2], which is never
worse and every so often way faster.


[1] http://www.fourcc.org/yuv.php#NV12
[2]
https://code.google.com/p/chromium/codesearch#chromium/src/third_party/libyuv...

[braveyao, 2016/05/24 00:03:26]

Nope. libyuv::NV21ToI420 doesn't apply here. Is the camera capture output also
NV12/NV21? Since they should be unpacked in the same way, so they are the same
format.
Anyway, any format of 4:2:0 should have a frame size as W x H x 3 / 2. But the
output of camera capture and media projection is actually W x H x 4 / 2. It is
confusing... 

[mcasas, 2016/05/25 17:42:58]

What do you mean it doesn't apply here?

I imagine you mean that you are unpacking in-place,
reusing |u_src| and |v_src|, and libyuv::NV21ToI420 
doesn't like that. Am I right?

Then, since you'll be doing a resizing in 
OnIncomingFrameAvailable(), you should use 
libyuv::ConvertToI420 to change format and 
resize at once, indicate as origin pixel format 
libyuv::FOURCC_NV12 (or NV21).
Let me use code search for you:

https://code.google.com/p/chromium/codesearch#chromium/src/media/capture/vide...

And

https://code.google.com/p/chromium/codesearch#chromium/src/media/capture/vide...

the latter implying that pixel formats on Android
seems to be a long standing issue.
I don't understand this. 
We need to dig deeper. If we're requesting I420
triplanar from the Android API and it's providing
a biplanar NV12/NV21, then you have to, at least,
write a comment on Java side detailing this 
circumstance. 

What I'm seeing reading 
https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV...
is that there are no warranties on the packing of
the Chrome planes. Which is probably to 
accommodate different MIPI connections. It's quite
likely that you'll have to support both I420 and 
NV12/NV21 here, by looking at the pixelStrides.

[braveyao, 2016/05/25 19:00:15]

I don't understand how it could be NV12/NV21, which suppose to have one y plane
and one interleaved uv plane. What we get is a triplanar image from Android, and
no matter you put the unpacked U or V plane as the UV plane into NV21ToI420(),
no correct I420 frame can be got. 

[mcasas, 2016/06/07 22:23:19]

NV12:

2 pointers:
y_plane_nv12 => yyyyyyyyyyy...
uv_plane_nv12 => uvuvuvuvuv...

what you're getting (possibly) is:
y_plane_yuv == y_plane_nv12
u_plane_yuv == uv_plane_nv12
v_plane_yuvv == uv_plane_nv12 +1
 and then you get told that the pixel stride (not the
row stride) for |u_plane_yuv| and |v_plane_yuv| is 2, so 
you skip 1 pixel-channel every two. 

Have you checked the addresses for u and v planes
to either verify or refute this theory?

Also I commented elsewhere that doing the
deinterlacing UV here by hand and then an
I420Scale is wasteful, please dig deeper in the
libyuv library, or consult fbarchard@.

+  for (int index = 0; index * 2 <= uv_plane_len; index++) {
+    u_src[index] = u_src[index * 2];
+    v_src[index] = v_src[index * 2];
+  }
+  OnIncomingFrameAvailable(y_src, y_stride, u_src, u_stride, v_src, v_stride,
+                           width, height, static_cast<int64_t>(timestamp));
+}
+
+void ScreenCaptureMachineAndroid::OnActivityResult(JNIEnv* env,
+                                                   jobject obj,
+                                                   jboolean result) {
+  if (!result) {
+    oracle_proxy_->ReportError(FROM_HERE, "The user denied screen capture");
+    return;
+  }
+
+  Java_ScreenCapture_startCapture(env, obj);
+}
+
+void ScreenCaptureMachineAndroid::Start(
+    const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
+    const VideoCaptureParams& params,
+    const base::Callback<void(bool)> callback) {
+  DCHECK(oracle_proxy.get());
+  oracle_proxy_ = oracle_proxy;
+
+  j_capture_.Reset(
+      createScreenCaptureMachineAndroid(reinterpret_cast<intptr_t>(this)));
+
+  if (j_capture_.obj() == nullptr) {
+    LOG(ERROR) << "Failed to createScreenCaptureAndroid,"
+                  " Maybe android version is too low";
+    callback.Run(false);
+    return;
+  }
+
+  DCHECK(params.requested_format.frame_size.GetArea());
+  DCHECK(!(params.requested_format.frame_size.width() % 2));
+  DCHECK(!(params.requested_format.frame_size.height() % 2));
+
+  const jboolean ret = Java_ScreenCapture_startPrompt(
+      AttachCurrentThread(), j_capture_.obj(),
+      params.requested_format.frame_size.width(),
+      params.requested_format.frame_size.height());
+
+  callback.Run(ret);
+}
+
+void ScreenCaptureMachineAndroid::Stop(const base::Closure& callback) {
+  Java_ScreenCapture_stopCapture(AttachCurrentThread(), j_capture_.obj());
+
+  callback.Run();
+}
+
+// ScreenCapture on Android works in passive way and there will be no captured
+// frame when there is no update to the screen. When oracle asks capturing for
+// refresh, the cached last frame will be delivered.
+void ScreenCaptureMachineAndroid::MaybeCaptureForRefresh() {
+  if (lastFrame_.get() == nullptr)
+    return;
+
+  OnIncomingFrameAvailable(lastFrame_->visible_data(VideoFrame::kYPlane),
+                           lastFrame_->stride(VideoFrame::kYPlane),
+                           lastFrame_->visible_data(VideoFrame::kUPlane),
+                           lastFrame_->stride(VideoFrame::kUPlane),
+                           lastFrame_->visible_data(VideoFrame::kVPlane),
+                           lastFrame_->stride(VideoFrame::kVPlane),
+                           lastFrame_->visible_rect().width(),
+                           lastFrame_->visible_rect().height(),
+                           base::TimeTicks::Now().ToInternalValue());
+}
+
+void ScreenCaptureMachineAndroid::OnIncomingFrameAvailable(const uint8_t* y_src,
+                                                           int y_stride,
+                                                           const uint8_t* u_src,
+                                                           int u_stride,
+                                                           const uint8_t* v_src,
+                                                           int v_stride,
+                                                           int width,
+                                                           int height,
+                                                           int64_t timestamp) {
+  const VideoCaptureOracle::Event event = VideoCaptureOracle::kCompositorUpdate;
+  const uint64_t absolute_micro =
+      timestamp / base::Time::kNanosecondsPerMicrosecond;
+  const base::TimeTicks start_time =
+      base::TimeTicks::FromInternalValue(absolute_micro);
+  scoped_refptr<VideoFrame> frame;
+  ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb;
+
+  if (!oracle_proxy_->ObserveEventAndDecideCapture(
+          event, gfx::Rect(), start_time, &frame, &capture_frame_cb)) {
+    return;
+  }
+
+  DCHECK(frame->format() == PIXEL_FORMAT_I420 ||
+         frame->format() == PIXEL_FORMAT_YV12);
+
+  libyuv::I420Scale(y_src, y_stride, u_src, u_stride, v_src, v_stride, width,
+                    height, frame->visible_data(VideoFrame::kYPlane),
+                    frame->stride(VideoFrame::kYPlane),
+                    frame->visible_data(VideoFrame::kUPlane),
+                    frame->stride(VideoFrame::kUPlane),
+                    frame->visible_data(VideoFrame::kVPlane),
+                    frame->stride(VideoFrame::kVPlane),
+                    frame->visible_rect().width(),
+                    frame->visible_rect().height(), libyuv::kFilterBilinear);
+
+  // Deliver the populated VideoFrame.
+  capture_frame_cb.Run(frame, start_time, true);
+
+  lastFrame_ = frame;
+}
+
+}  // namespace media