ScreenCapture for Android phase1, part I

media/capture/content/android/java/src/org/chromium/media/ScreenCapture.java

Index: media/capture/content/android/java/src/org/chromium/media/ScreenCapture.java
diff --git a/media/capture/content/android/java/src/org/chromium/media/ScreenCapture.java b/media/capture/content/android/java/src/org/chromium/media/ScreenCapture.java
new file mode 100644
index 0000000000000000000000000000000000000000..c4e2df8454cf828b3b0480308623e4a8d2b74674
--- /dev/null
+++ b/media/capture/content/android/java/src/org/chromium/media/ScreenCapture.java
@@ -0,0 +1,348 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.media;
+
+import android.annotation.TargetApi;
+import android.app.Activity;
+import android.app.Fragment;
+import android.app.FragmentManager;
+import android.app.FragmentTransaction;
+import android.content.Context;
+import android.content.Intent;
+import android.graphics.ImageFormat;
+import android.graphics.PixelFormat;
+import android.hardware.display.DisplayManager;
+import android.hardware.display.VirtualDisplay;
+import android.media.Image;
+import android.media.ImageReader;
+import android.media.projection.MediaProjection;
+import android.media.projection.MediaProjectionManager;
+import android.os.Build;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.util.DisplayMetrics;
+import android.view.Display;
+import android.view.Surface;
+
+import org.chromium.base.ApplicationStatus;
+import org.chromium.base.Log;
+import org.chromium.base.annotations.CalledByNative;
+import org.chromium.base.annotations.JNINamespace;
+
+import java.nio.ByteBuffer;
+
+/**
+ * This class implements Screen Capture using projection API, introduced in Android
+ * API 21 (L Release). Capture takes place in the current Looper, while pixel
+ * download takes place in another thread used by ImageReader.
+ **/
+@JNINamespace("media")
+@TargetApi(Build.VERSION_CODES.LOLLIPOP)
+public class ScreenCapture extends Fragment {
+    // Internal class implementing the ImageReader listener. Gets pinged when a
+    // new frame is been captured and downloaded to memory-backed buffers.
+    // TODO(braveyao): This is very similar as the one in VideoCaptureCamera2. Try to see
+    // if we can extend from it.

[mcasas, 2016/05/19 19:11:40]

Add a bug.

[braveyao, 2016/05/20 22:27:25]

Done.
I'll eliminate all the TODOs in phase 2 for crbug/487935. So just add it for all
the TODOs here.

+    private class CrImageReaderListener implements ImageReader.OnImageAvailableListener {
+        @Override
+        public void onImageAvailable(ImageReader reader) {
+            synchronized (mCaptureStateLock) {
+                if (mCaptureState != CaptureState.STARTED) {
+                    Log.e(TAG, "Get captured frame in unexpected state.");
+                    return;
+                }
+            }
+
+            try (Image image = reader.acquireLatestImage()) {
+                if (image == null) return;

[mcasas, 2016/05/19 19:11:41]

Also check that |mFormat == image.getFormat()| ?

[braveyao, 2016/05/20 22:27:25]

I move it to "switch()" below.

+                if (reader.getWidth() != image.getWidth()
+                        || reader.getHeight() != image.getHeight()) {
+                    Log.e(TAG, "ImageReader size (" + reader.getWidth() + "x" + reader.getHeight()
+                                    + ") did not match Image size (" + image.getWidth() + "x"
+                                    + image.getHeight() + ")");
+                    throw new IllegalStateException();
+                }
+                // TODO(braveyao): check if we need to consider corping.

[mcasas, 2016/05/19 19:11:40]

s/corping/cropping/
All TODO()s should have a bug.

[braveyao, 2016/05/20 22:27:25]

Done.

+                switch (image.getFormat()) {
+                    case PixelFormat.RGBA_8888:
+                        if (image.getPlanes().length != 1) {
+                            Log.e(TAG, "Unexpected image planes for RGBA_8888 format: "
+                                            + image.getPlanes().length);
+                            throw new IllegalStateException();
+                        }
+                        nativeOnRGBAFrameAvailable(mNativeScreenCaptureMachineAndroid,
+                                image.getPlanes()[0].getBuffer(), image.getWidth(),
+                                image.getHeight(), image.getPlanes()[0].getRowStride(),
+                                image.getTimestamp());
+                        break;
+                    case ImageFormat.YUV_420_888:
+                        if (image.getPlanes().length != 3) {
+                            Log.e(TAG, "Unexpected image planes for YUV_420_888 format: "
+                                            + image.getPlanes().length);
+                            throw new IllegalStateException();
+                        }
+                        nativeOnI420FrameAvailable(mNativeScreenCaptureMachineAndroid,
+                                image.getPlanes()[0].getBuffer(),
+                                image.getPlanes()[0].getRowStride(),
+                                image.getPlanes()[1].getBuffer(),
+                                image.getPlanes()[1].getRowStride() / 2,

[mcasas, 2016/05/19 19:11:41]

Why / 2 ? The stride of a subsampled plane should 
already be subsampled, right? Is this based on 
observation? Seeing the treatment you do of the
data in the .cc file, I'd say you're getting NV12,
where U and V planes are interlaced in a single 
buffer.

[braveyao, 2016/05/20 22:27:25]

U/V plane should have half width/stride of Y plane. But actually the rowStride
of all 3 planes are same. I guess the reason is the pixelStride of U/V plane is
2 from OS. See the comments in VideoCaptureCamera2.java, https://goo.gl/8OhYUg.
So I have to unpack them in the .cc file since libyuv can't handle it. 
Also I forgot where I read it that android getRowStride() always returns the
stride of the first plane. So doing '/2' here, also avoid the trouble of
operating jint in cc file.

+                                image.getPlanes()[2].getBuffer(),
+                                image.getPlanes()[2].getRowStride() / 2, image.getWidth(),
+                                image.getHeight(), image.getTimestamp());
+                        break;
+                    default:
+                        Log.e(TAG, "Unexpected image format: " + image.getFormat());
+                        throw new IllegalStateException();
+                }
+            } catch (IllegalStateException ex) {
+                Log.e(TAG, "acquireLatestImage():" + ex);
+            } catch (UnsupportedOperationException ex) {
+                Log.i(TAG, "acquireLatestImage():" + ex);
+                // YUV_420_888 is the preference, but not all devices support it,
+                // fall-back to RGBA_8888 then.
+                mImageReader.close();
+                mVirtualDisplay.release();
+
+                mFormat = PixelFormat.RGBA_8888;
+                createImageReaderWithFormat(mFormat);
+                createVirtualDisplay();
+            }
+        }
+    }
+
+    private class MediaProjectionCallback extends MediaProjection.Callback {
+        @Override
+        public void onStop() {
+            changeCaptureStateAndNotify(CaptureState.STOPPED);
+            mMediaProjection = null;
+            if (mVirtualDisplay == null) return;
+            mVirtualDisplay.release();
+            mVirtualDisplay = null;
+        }
+    }
+
+    private final Context mContext;
+
+    private MediaProjection mMediaProjection;
+    private MediaProjectionManager mMediaProjectionManager;
+    private VirtualDisplay mVirtualDisplay;
+    private static final int REQUEST_MEDIA_PROJECTION = 1;
+
+    private Surface mSurface;
+    private ImageReader mImageReader = null;
+    private HandlerThread mThread;
+    private Handler mBackgroundHandler;
+
+    private int mScreenDensity;
+    private int mWidth;
+    private int mHeight;
+    private int mFormat;
+
+    private int mResultCode;
+    private Intent mResultData;
+
+    // Native callback context variable.
+    private final long mNativeScreenCaptureMachineAndroid;
+
+    private static enum CaptureState { ATTACHED, ALLOWED, STARTED, STOPPING, STOPPED }
+    private CaptureState mCaptureState = CaptureState.STOPPED;
+    private final Object mCaptureStateLock = new Object();
+
+    private static final String TAG = "ScreenCaptureMachine";
+
+    ScreenCapture(Context context, long nativeScreenCaptureMachineAndroid) {
+        mContext = context;
+        mNativeScreenCaptureMachineAndroid = nativeScreenCaptureMachineAndroid;
+
+        Activity activity = ApplicationStatus.getLastTrackedFocusedActivity();
+        if (activity == null) {
+            Log.e(TAG, "activity is null");
+            return;
+        }
+
+        try {
+            FragmentManager fragmentManager = activity.getFragmentManager();
+            FragmentTransaction fragmentTransaction = fragmentManager.beginTransaction();
+            fragmentTransaction.add(this, "screencapture");
+            fragmentTransaction.commit();
+        } catch (Exception e) {

[mcasas, 2016/05/19 19:11:41]

Again, no catching generic Exceptions. Also, the try {}
is too large: I see e.g. getFragmentManager() does not
throw [1] (and if it does, then add a comment that the
docs are incorrect and isolate it into a try-catch on its
own). I only see commit() throwing [2]. If you cannot
find docs on a particular exception being thrown,
chances are it's a RuntimeExceptio.

[1]
https://developer.android.com/reference/android/app/Activity.html#getFragment...
[2]
https://developer.android.com/reference/android/app/FragmentTransaction.html#...

[braveyao, 2016/05/20 22:27:25]

Yes, that's what I understand too. Catching generic Exception can also catch
RuntimeException. That's why I used it here. Anyway correct it to be more clear.

[mcasas, 2016/05/23 18:19:55]

RuntimeException <is a> Exception, and you 
should catch the most narrow exception 
available, hence catching a concrete 
exception is better than catching a 
RuntimeException is better than catching an
Exception (which you should never do).

[braveyao, 2016/05/24 00:03:26]

Acknowledged.

+            Log.e(TAG, "ScreenCaptureExcaption " + e);
+        }
+
+        try {
+            DisplayMetrics metrics = new DisplayMetrics();
+            Display display = activity.getWindowManager().getDefaultDisplay();
+            display.getMetrics(metrics);
+            mScreenDensity = metrics.densityDpi;
+        } catch (Exception e) {

[mcasas, 2016/05/19 19:11:40]

Ditto, narrow down the Exception and the
try{} block. From the docs, I don't see any 
of the statements in l.175-178 throwing. 
Which one do you see throwing?

[braveyao, 2016/05/20 22:27:25]

Removed.
No, I saw no specific exception in doc. Since doc can't list all the exceptions,
as the UnsupportedOperationException for ImageReader is undocumented, so I
thought it might be better to try to catch generic one :)

[mcasas, 2016/05/23 18:19:55]

What do you mean, that the Android documentation
"can't list all the exceptions"? Then how are we 
supposed to develop? Cr Java code cannot catch
Exceptions perfunctorily.

If you think you have found an Exception being
thrown and not documented, then catch it _and_
add a comment:
// Code blabla throws Exception despite not being
// documented, see [1]
and [1] _must_ be a link to the AOSP code line(s) 
where the Exception is thrown. http://androidxref.com/

[braveyao, 2016/05/24 00:03:26]

Acknowledged.

+            Log.e(TAG, "ScreenCaptureExcaption " + e);
+        }
+    }
+
+    // Factory methods.

[mcasas, 2016/05/19 19:11:40]

s/methods/method/ ?

[braveyao, 2016/05/20 22:27:25]

Done.

+    @CalledByNative
+    static ScreenCapture createScreenCaptureMachine(
+            Context context, long nativeScreenCaptureMachineAndroid) {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
+            return new ScreenCapture(context, nativeScreenCaptureMachineAndroid);
+        }
+        return null;
+    }
+
+    @Override
+    public void onAttach(Context context) {
+        super.onAttach(context);
+        Log.d(TAG, "onAttach");
+        changeCaptureStateAndNotify(CaptureState.ATTACHED);
+    }
+
+    // This method was deprecated in API level 23 by onAttach(Context).
+    // TODO(braveyao): remove this method after the minSdkVersion of chromium is 23.

[mcasas, 2016/05/19 19:11:40]

s/chromium/chrome/ [1]
bug for this.

[1] https://www.chromium.org/developers/coding-style#TOC-Naming

[braveyao, 2016/05/20 22:27:25]

Thanks for reminding! Read it before but forgot apparently...

+    @SuppressWarnings("deprecation")
+    @Override
+    public void onAttach(Activity activity) {
+        super.onAttach(activity);
+        Log.d(TAG, "onAttach");
+        changeCaptureStateAndNotify(CaptureState.ATTACHED);
+    }
+
+    @Override
+    public void onDetach() {
+        super.onDetach();
+        Log.d(TAG, "onDetach");
+        stopCapture();
+    }
+
+    @CalledByNative
+    public boolean startPrompt(int width, int height) {
+        Log.d(TAG, "startPrompt");
+        synchronized (mCaptureStateLock) {
+            while (mCaptureState != CaptureState.ATTACHED) {
+                try {
+                    mCaptureStateLock.wait();
+                } catch (InterruptedException ex) {
+                    Log.e(TAG, "ScreenCaptureException: " + ex);
+                }
+            }
+        }
+
+        mWidth = width;
+        mHeight = height;
+
+        mMediaProjectionManager = (MediaProjectionManager) mContext.getSystemService(
+                Context.MEDIA_PROJECTION_SERVICE);
+        if (mMediaProjectionManager == null) {
+            Log.e(TAG, "mMediaProjectionManager is null");
+            return false;
+        }
+
+        try {
+            startActivityForResult(
+                    mMediaProjectionManager.createScreenCaptureIntent(), REQUEST_MEDIA_PROJECTION);
+        } catch (android.content.ActivityNotFoundException e) {
+            Log.e(TAG, "ScreenCaptureException " + e);
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public void onActivityResult(int requestCode, int resultCode, Intent data) {
+        if (requestCode == REQUEST_MEDIA_PROJECTION) {

[mcasas, 2016/05/19 19:11:40]

Prefer early return:

if (requestCode != REQUEST_MEDIA_PROJECTION) return;
// the rest...

[braveyao, 2016/05/20 22:27:25]

Done.

+            boolean result;

[mcasas, 2016/05/19 19:11:40]

Unused variable? Remove.

[braveyao, 2016/05/20 22:27:25]

Done.

+            if (resultCode == Activity.RESULT_OK) {
+                mResultCode = resultCode;
+                mResultData = data;
+                changeCaptureStateAndNotify(CaptureState.ALLOWED);
+            }
+            nativeOnActivityResult(
+                    mNativeScreenCaptureMachineAndroid, resultCode == Activity.RESULT_OK);
+        }
+    }
+
+    @CalledByNative
+    public void startCapture() {
+        Log.d(TAG, "startCapture");
+        synchronized (mCaptureStateLock) {
+            if (mCaptureState != CaptureState.ALLOWED) {
+                Log.e(TAG, "startCapture() invoked without user permission.");
+                return;
+            }
+        }
+        mMediaProjection = mMediaProjectionManager.getMediaProjection(mResultCode, mResultData);
+        if (mMediaProjection == null) {
+            Log.e(TAG, "mMediaProjection is null");
+            return;
+        }
+        mMediaProjection.registerCallback(new MediaProjectionCallback(), null);
+
+        mThread = new HandlerThread("Screen");

[mcasas, 2016/05/19 19:11:40]

s/Screen/ScreenCapture/ ?

[braveyao, 2016/05/20 22:27:25]

Done.

+        mThread.start();
+        mBackgroundHandler = new Handler(mThread.getLooper());
+
+        // On Android M and above, YUV420 is prefered. Some Android L devices will silently
+        // fail with YUV420, so keep with RGBA_8888 on L.
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) {
+            mFormat = PixelFormat.RGBA_8888;
+        } else {
+            mFormat = ImageFormat.YUV_420_888;
+        }
+        createImageReaderWithFormat(mFormat);
+        createVirtualDisplay();
+
+        changeCaptureStateAndNotify(CaptureState.STARTED);
+    }
+
+    @CalledByNative
+    public void stopCapture() {
+        Log.d(TAG, "stopCapture");
+        synchronized (mCaptureStateLock) {
+            if (mMediaProjection != null && mCaptureState == CaptureState.STARTED) {
+                mMediaProjection.stop();
+                changeCaptureStateAndNotify(CaptureState.STOPPING);
+            }
+
+            while (mCaptureState != CaptureState.STOPPED) {
+                try {
+                    mCaptureStateLock.wait();
+                } catch (InterruptedException ex) {
+                    Log.e(TAG, "ScreenCaptureEvent: " + ex);
+                }
+            }
+        }
+    }
+
+    private void createImageReaderWithFormat(int format) {
+        final int maxImages = 2;
+        mImageReader = ImageReader.newInstance(mWidth, mHeight, format, maxImages);
+        mSurface = mImageReader.getSurface();
+        final CrImageReaderListener imageReaderListener = new CrImageReaderListener();
+        mImageReader.setOnImageAvailableListener(imageReaderListener, mBackgroundHandler);
+    }
+
+    private void createVirtualDisplay() {
+        mVirtualDisplay = mMediaProjection.createVirtualDisplay("ScreenCapture", mWidth, mHeight,
+                mScreenDensity, DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, mSurface, null,
+                null);
+    }
+
+    private void changeCaptureStateAndNotify(CaptureState state) {
+        synchronized (mCaptureStateLock) {
+            mCaptureState = state;
+            mCaptureStateLock.notifyAll();
+        }
+    }
+
+    // Method for ScreenCapture implementations to call back native code.
+    private native void nativeOnRGBAFrameAvailable(long nativeScreenCaptureMachineAndroid,
+            ByteBuffer buf, int width, int height, int rowStride, long timestamp);
+    // Method for ScreenCapture implementations to call back native code.
+    private native void nativeOnI420FrameAvailable(long nativeScreenCaptureMachineAndroid,
+            ByteBuffer yBuffer, int yStride, ByteBuffer uBuffer, int uStride, ByteBuffer vBuffer,
+            int vStride, int width, int height, long timestamp);
+    // Method for ScreenCapture implementations to call back native code.
+    private native void nativeOnActivityResult(
+            long nativeScreenCaptureMachineAndroid, boolean result);
+}