ScreenCapture for Android phase1, part I

media/base/android/java/src/org/chromium/media/ScreenCapture.java

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.media;
+
+import android.annotation.TargetApi;
+import android.app.Activity;
+import android.app.Fragment;
+import android.app.FragmentManager;
+import android.app.FragmentTransaction;
+import android.content.Context;
+import android.content.Intent;
+import android.graphics.PixelFormat;
+import android.hardware.display.DisplayManager;
+import android.hardware.display.VirtualDisplay;
+import android.media.Image;
+import android.media.ImageReader;
+import android.media.projection.MediaProjection;
+import android.media.projection.MediaProjectionManager;
+import android.os.Build;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.util.DisplayMetrics;
+import android.view.Display;
+import android.view.Surface;
+
+import org.chromium.base.ApplicationStatus;
+import org.chromium.base.Log;
+import org.chromium.base.annotations.CalledByNative;
+import org.chromium.base.annotations.JNINamespace;
+
+import java.nio.ByteBuffer;
+
+/**
+ * This class implements Screen Capture using projection API, introduced in Android
+ * API 21 (L Release). Capture takes place in the current Looper, while pixel
+ * download takes place in another thread used by ImageReader.
+ **/
+@JNINamespace("media")
+@TargetApi(Build.VERSION_CODES.LOLLIPOP)
+public class ScreenCapture extends Fragment {
+    // Internal class implementing the ImageReader listener. Gets pinged when a
+    // new frame is been captured and downloaded to memory-backed buffers.
+    private class CrImageReaderListener implements ImageReader.OnImageAvailableListener {
+        @Override
+        public void onImageAvailable(ImageReader reader) {
+            synchronized (mCaptureStateLock) {
+                if (mCaptureState != CaptureState.STARTED) {
+                    Log.e(TAG, "Get captured frame in unexpected state.");
+                    return;
+                }
+            }
+
+            Image image = null;
+            try {
+                image = reader.acquireLatestImage();
+                if (image == null) return;
+                if (image.getFormat() != PixelFormat.RGBA_8888) {
+                    Log.e(TAG, "Unexpected image format: " + image.getFormat() + " or #planes: "
+                                    + image.getPlanes().length);
+                    return;
+                }
+
+                Image.Plane[] planes = image.getPlanes();
+                ByteBuffer buffer = (ByteBuffer) planes[0].getBuffer();
+                buffer.get(mCapturedData);

[miu, 2016/05/10 00:29:00]

This is copying the image data into a separate byte array, which is expensive.
Instead, just pass the ByteBuffer and have the C++ impl read directly from that.

[braveyao, 2016/05/18 00:41:46]

Done.

+
+                nativeOnFrameAvailable(mNativeScreenCaptureMachineAndroid, mCapturedData,
+                        image.getCropRect().width(), image.getCropRect().height(),

[miu, 2016/05/10 00:29:00]

More information needs to be passed to the native impl:

1. What if the crop rect does not originate at coordinates (0,0)? The native
impl needs to know it'll have to start reading at an offset from the start of
the buffer.

2. What if the stride is different than the width of the image (see
documentation for Plane.getRowStride())? The native impl needs to know how many
bytes to skip to get to the start of the next row. Images are not always
guaranteed to be "packed."

[braveyao, 2016/05/18 00:41:46]

Yes you're right. I meet a case that stride is different than the width of the
image when I tried more capture size. Thanks!
Most probably there isn't cropping here, at least for now. Add a TODO for future
investigation.

+                        image.getTimestamp());
+            } catch (IllegalStateException ex) {
+                Log.e(TAG, "acquireLatestImage():" + ex);
+                return;
+            } finally {
+                if (image != null) image.close();
+            }
+        }
+    }
+
+    private class MediaProjectionCallback extends MediaProjection.Callback {
+        @Override
+        public void onStop() {
+            changeCaptureStateAndNotify(CaptureState.STOPPED);
+            mMediaProjection = null;
+            if (mVirtualDisplay == null) return;
+            mVirtualDisplay.release();
+            mVirtualDisplay = null;
+        }
+    }
+
+    private byte[] mCapturedData;
+    private final Context mContext;
+
+    private MediaProjection mMediaProjection;
+    private MediaProjectionManager mMediaProjectionManager;
+    private VirtualDisplay mVirtualDisplay;
+    private static final int REQUEST_MEDIA_PROJECTION = 1;
+
+    private Surface mSurface;
+    private ImageReader mImageReader = null;
+    private int mScreenDensity;
+    private int mWidth;
+    private int mHeight;
+    private int mResultCode;
+    private Intent mResultData;
+
+    // Native callback context variable.
+    private final long mNativeScreenCaptureMachineAndroid;
+
+    private static enum CaptureState { ATTACHED, ALLOWED, STARTED, STOPPING, STOPPED }
+    private CaptureState mCaptureState = CaptureState.STOPPED;
+    private final Object mCaptureStateLock = new Object();
+
+    private static final String TAG = "ScreenCaptureMachine";
+
+    ScreenCapture(Context context, long nativeScreenCaptureMachineAndroid) {
+        mContext = context;
+        mNativeScreenCaptureMachineAndroid = nativeScreenCaptureMachineAndroid;
+
+        Activity activity = ApplicationStatus.getLastTrackedFocusedActivity();
+        if (activity == null) {
+            Log.e(TAG, "activity is null");
+            return;
+        }
+
+        try {
+            FragmentManager fragmentManager = activity.getFragmentManager();
+            FragmentTransaction fragmentTransaction = fragmentManager.beginTransaction();
+            fragmentTransaction.add(this, "screencapture");
+            fragmentTransaction.commit();
+        } catch (Exception e) {
+            Log.e(TAG, "ScreenCaptureExcaption " + e);
+        }
+
+        try {
+            DisplayMetrics metrics = new DisplayMetrics();
+            Display display = activity.getWindowManager().getDefaultDisplay();
+            display.getMetrics(metrics);
+            mScreenDensity = metrics.densityDpi;
+        } catch (Exception e) {
+            Log.e(TAG, "ScreenCaptureExcaption " + e);
+        }
+    }
+
+    // Factory methods.
+    @CalledByNative
+    static ScreenCapture createScreenCaptureMachine(
+            Context context, long nativeScreenCaptureMachineAndroid) {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
+            return new ScreenCapture(context, nativeScreenCaptureMachineAndroid);
+        }
+        return null;
+    }
+
+    @Override
+    public void onAttach(Context context) {
+        super.onAttach(context);
+        Log.d(TAG, "onAttach");
+        changeCaptureStateAndNotify(CaptureState.ATTACHED);
+    }
+
+    // This method was deprecated in API level 23 by onAttach(Context).
+    // TODO(braveyao): remove this method after the minSdkVersion of chromium is 23.
+    @SuppressWarnings("deprecation")
+    @Override
+    public void onAttach(Activity activity) {
+        super.onAttach(activity);
+        Log.d(TAG, "onAttach");
+        changeCaptureStateAndNotify(CaptureState.ATTACHED);
+    }
+
+    @Override
+    public void onDetach() {
+        super.onDetach();
+        Log.d(TAG, "onDetach");
+        stopCapture();
+    }
+
+    @CalledByNative
+    public boolean startPrompt(int width, int height) {
+        Log.d(TAG, "startPrompt");
+        synchronized (mCaptureStateLock) {
+            while (mCaptureState != CaptureState.ATTACHED) {
+                try {
+                    mCaptureStateLock.wait();
+                } catch (InterruptedException ex) {
+                    Log.e(TAG, "ScreenCaptureException: " + ex);
+                }
+            }
+        }
+
+        try {
+            mWidth = width;
+            mHeight = height;
+            int expectedFrameSize = mWidth * mHeight * 4;
+            mCapturedData = new byte[expectedFrameSize];
+
+            mMediaProjectionManager = (MediaProjectionManager) mContext.getSystemService(
+                    Context.MEDIA_PROJECTION_SERVICE);
+            if (mMediaProjectionManager == null) {
+                Log.e(TAG, "mMediaProjectionManager is null");
+                return false;
+            }
+
+            startActivityForResult(
+                    mMediaProjectionManager.createScreenCaptureIntent(), REQUEST_MEDIA_PROJECTION);
+        } catch (Exception e) {
+            Log.e(TAG, "ScreenCaptureException " + e);
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public void onActivityResult(int requestCode, int resultCode, Intent data) {
+        if (requestCode == REQUEST_MEDIA_PROJECTION) {
+            boolean result;
+            if (resultCode != Activity.RESULT_OK) {
+                result = false;
+            } else {
+                result = true;
+                mResultCode = resultCode;
+                mResultData = data;
+                changeCaptureStateAndNotify(CaptureState.ALLOWED);
+            }
+            nativeOnActivityResult(mNativeScreenCaptureMachineAndroid, result);

[mcasas, 2016/05/04 19:56:48]

s/result/resultCode == Activity.RESULT_OK/ ?

[braveyao, 2016/05/18 00:41:46]

Done.

+        }
+    }
+
+    @CalledByNative
+    public void startCapture() {
+        Log.d(TAG, "startCapture");
+        synchronized (mCaptureStateLock) {
+            if (mCaptureState != CaptureState.ALLOWED) {
+                Log.e(TAG, "startCapture() invoked without user permission.");
+                return;
+            }
+        }
+        mMediaProjection = mMediaProjectionManager.getMediaProjection(mResultCode, mResultData);
+        if (mMediaProjection == null) {
+            Log.e(TAG, "mMediaProjection is null");
+            return;
+        }
+        mMediaProjection.registerCallback(new MediaProjectionCallback(), null);
+
+        final int maxImages = 2;
+        mImageReader = ImageReader.newInstance(mWidth, mHeight, PixelFormat.RGBA_8888, maxImages);
+        mSurface = mImageReader.getSurface();
+        HandlerThread thread = new HandlerThread("Screen");
+        thread.start();
+        final Handler backgroundHandler = new Handler(thread.getLooper());
+        final CrImageReaderListener imageReaderListener = new CrImageReaderListener();
+        mImageReader.setOnImageAvailableListener(imageReaderListener, backgroundHandler);
+
+        mVirtualDisplay = mMediaProjection.createVirtualDisplay("ScreenCapture", mWidth, mHeight,
+                mScreenDensity, DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, mSurface, null,
+                null);
+
+        changeCaptureStateAndNotify(CaptureState.STARTED);
+    }
+
+    @CalledByNative
+    public void stopCapture() {
+        Log.d(TAG, "stopCapture");
+        synchronized (mCaptureStateLock) {
+            if (mMediaProjection != null && mCaptureState == CaptureState.STARTED) {
+                mMediaProjection.stop();
+                changeCaptureStateAndNotify(CaptureState.STOPPING);
+            }
+
+            while (mCaptureState != CaptureState.STOPPED) {
+                try {
+                    mCaptureStateLock.wait();
+                } catch (InterruptedException ex) {
+                    Log.e(TAG, "ScreenCaptureEvent: " + ex);
+                }
+            }
+        }
+    }
+
+    private void changeCaptureStateAndNotify(CaptureState state) {
+        synchronized (mCaptureStateLock) {
+            mCaptureState = state;
+            mCaptureStateLock.notifyAll();
+        }
+    }
+
+    // Method for ScreenCapture implementations to call back native code.
+    private native void nativeOnFrameAvailable(long nativeScreenCaptureMachineAndroid, byte[] data,
+            int cropWidth, int cropHeight, long timestamp);
+
+    // Method for ScreenCapture implementations to call back native code.
+    private native void nativeOnActivityResult(
+            long nativeScreenCaptureMachineAndroid, boolean result);
+}