Implement screen capture for android

media/base/android/java/src/org/chromium/media/ScreenCapture.java

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.media;
+
+import android.app.Activity;
+import android.app.Fragment;
+import android.app.FragmentManager;
+import android.app.FragmentTransaction;
+import android.content.Context;
+import android.content.Intent;
+import android.graphics.PixelFormat;
+import android.graphics.Point;
+import android.hardware.display.DisplayManager;
+import android.hardware.display.VirtualDisplay;
+import android.media.Image;
+import android.media.ImageReader;
+import android.media.projection.MediaProjection;
+import android.media.projection.MediaProjectionManager;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.util.DisplayMetrics;
+import android.view.Display;
+import android.view.Surface;
+
+import org.chromium.base.BaseChromiumApplication;
+import org.chromium.base.CalledByNative;
+import org.chromium.base.JNINamespace;
+import org.chromium.base.Log;
+
+import java.nio.ByteBuffer;
+
+/**
+ * This class implements Screen Capture using projection API, introduced in Android
+ * API 21 (L Release). Capture takes place in the current Looper, while pixel
+ * download takes place in another thread used by ImageReader.
+ **/
+@JNINamespace("media")
+public class ScreenCapture extends Fragment {
+
+    // Internal class implementing the ImageReader listener. Gets pinged when a
+    // new frame is been captured and downloaded to memory-backed buffers.
+    private class CrImageReaderListener implements ImageReader.OnImageAvailableListener {
+        @Override
+        public void onImageAvailable(ImageReader reader) {
+            Image image = null;
+            try {
+                image = reader.acquireLatestImage();
+                if (image == null) return;
+                if (image.getFormat() != PixelFormat.RGBA_8888) {
+                    Log.e(TAG, "Unexpected image format: " + image.getFormat()
+                            + " or #planes: " + image.getPlanes().length);
+                    return;
+                }
+
+                Image.Plane[] planes = image.getPlanes();
+                ByteBuffer buffer = (ByteBuffer) planes[0].getBuffer();
+                buffer.get(mCapturedData);
+
+                nativeOnFrameAvailable(mNativeScreenCapturerAndroid,

[miu, 2015/05/20 03:33:27]

You need to pass extra values here:

1. image.getCropRect() is needed since you cannot guarantee the ByteBuffer
contains tightly-packed pixels.

2. image.getTimestamp() should be passed instead of just timestamping the
media::VideoFrames with "TickTime::Now()" in the C++ module.  Why throw away the
perfect timestamp value?  ;-)

+                                       mCapturedData,
+                                       mWidth,
+                                       mHeight);
+
+            } catch (IllegalStateException ex) {
+                Log.e(TAG, "acquireLatestImage():" + ex);
+                return;
+            } finally {
+                if (image != null) {
+                    image.close();
+                }
+            }
+        }
+    };
+
+    private byte[] mCapturedData;
+    private final Context mContext;
+
+    private MediaProjection mMediaProjection;
+    private MediaProjectionManager mMediaProjectionManager;
+    private VirtualDisplay mVirtualDisplay;
+    private static final int REQUEST_MEDIA_PROJECTION = 1;
+
+    private Surface mSurface;
+    private ImageReader mImageReader = null;
+    private int mScreenDensity;
+    private int mWidth;
+    private int mHeight;
+    private int mResultCode;
+    private Intent mResultData;
+    // Native callback context variable.
+    private final long mNativeScreenCapturerAndroid;
+
+    private static final String TAG = "ScreenCaptureAndroid";
+
+    ScreenCapture(Context context, long nativeScreenCapturerAndroid) {
+        mContext = context;
+        mNativeScreenCapturerAndroid = nativeScreenCapturerAndroid;
+    }
+
+    @CalledByNative
+    public void start() {
+        Log.i(TAG, "start");
+        try {
+            BaseChromiumApplication ba = (BaseChromiumApplication) mContext;
+            Activity activity = ba.getActivity();
+            if (activity == null) {
+                Log.e(TAG, "activity is null");
+                return;
+            }
+
+            FragmentManager fragmentManager = activity.getFragmentManager();
+            FragmentTransaction fragmentTransaction = fragmentManager.beginTransaction();
+            fragmentTransaction.add(this, "screencapture");
+            fragmentTransaction.commit();
+
+            DisplayMetrics metrics = new DisplayMetrics();
+            Point size = new Point();
+            Display display = activity.getWindowManager().getDefaultDisplay();
+            display.getMetrics(metrics);
+            display.getSize(size);
+            mScreenDensity = metrics.densityDpi;
+            //mWidth = size.x;

[miu, 2015/05/20 03:33:27]

Did you mean to fix these LOC rather than hard-code the width and height?

+            //mHeight = size.y;
+            // Fixeme: If use size.x and size.y as the width and height, the screen image
+            // will be disordered.
+            mWidth = 640;
+            mHeight = 480;
+            int expectedFrameSize = mWidth * mHeight * 4;
+            mCapturedData = new byte[expectedFrameSize];
+        } catch (Exception e) {
+            Log.e(TAG, "ScreenCapture " + e);
+            return;
+        }
+
+        mMediaProjectionManager = (MediaProjectionManager)
+            mContext.getSystemService(Context.MEDIA_PROJECTION_SERVICE);
+        if (mMediaProjectionManager == null) {
+            Log.e(TAG, "mMediaProjectionManager is null");
+            return;
+        }
+
+        startActivityForResult(mMediaProjectionManager.createScreenCaptureIntent(),
+                               REQUEST_MEDIA_PROJECTION);
+    }
+
+    @Override
+    public void onActivityResult(int requestCode, int resultCode, Intent data) {
+        if (requestCode == REQUEST_MEDIA_PROJECTION) {
+            int result;
+            if (resultCode != Activity.RESULT_OK) {
+                result = 0;
+            } else {
+                result = 1;
+                mResultCode = resultCode;
+                mResultData = data;
+            }
+            nativeOnActivityResult(mNativeScreenCapturerAndroid, result);
+        }
+    }
+
+    // start screen capture.
+    @CalledByNative
+    public void startCapture() {
+        Log.i(TAG, "startCapture");
+        mMediaProjection = mMediaProjectionManager.getMediaProjection(mResultCode, mResultData);
+        if (mMediaProjection == null) {
+            Log.i(TAG, "mMediaProjection is null");
+            return;
+        }
+
+        mImageReader = ImageReader.newInstance(mWidth, mHeight, PixelFormat.RGBA_8888, 2);

[miu, 2015/05/20 03:33:27]

Could you add a comment here about why "2" is a reasonable value?  I suspect the
point is to allow the Chrome thread to memcpy the image data inside the call to
nativeOnFrameAvailable() while the image reader thread is busy capturing the
next frame.

+        mSurface = mImageReader.getSurface();
+        HandlerThread thread = new HandlerThread("ScreenPreview");

[miu, 2015/05/20 03:33:27]

ScreenPreview?

+        thread.start();
+        final Handler backgroundHandler = new Handler(thread.getLooper());
+        final CrImageReaderListener imageReaderListener = new CrImageReaderListener();
+        mImageReader.setOnImageAvailableListener(imageReaderListener,
+                                                 backgroundHandler);
+
+        mVirtualDisplay = mMediaProjection.createVirtualDisplay("ScreenCapture",
+            mWidth, mHeight, mScreenDensity,
+            DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
+            mSurface, null, null);
+    }
+
+    // Stops screen capture.
+    @CalledByNative
+    public void stopCapture() {
+        Log.i(TAG, "stopCapture");
+        if (mImageReader != null) mImageReader.close();
+
+        if (mVirtualDisplay != null) {
+            mVirtualDisplay.release();
+            mVirtualDisplay = null;
+        }
+        if (mMediaProjection != null) {
+            mMediaProjection.stop();
+            mMediaProjection = null;
+        }
+    }
+
+    // Method for ScreenCapture implementations to call back native code.
+    public native void nativeOnFrameAvailable(long nativeScreenCapturerAndroid,
+                                              byte[] data,
+                                              int width,
+                                              int height);
+
+    // Method for ScreenCapture implementations to call back native code.
+    public native void nativeOnActivityResult(long nativeScreenCapturerAndroid,
+                                              int result);
+
+}