Android: Add new renderer SurfaceViewRenderer

talk/app/webrtc/java/android/org/webrtc/SurfaceViewRenderer.java

+/*
+ * libjingle
+ * Copyright 2015 Google Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.webrtc;
+
+import java.nio.ByteBuffer;
+
+import android.content.Context;
+import android.graphics.Point;
+import android.graphics.SurfaceTexture;
+import android.opengl.EGLContext;
+import android.opengl.GLES20;
+import android.opengl.Matrix;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.util.AttributeSet;
+import android.util.Log;
+import android.view.SurfaceHolder;
+import android.view.SurfaceView;
+
+/**
+ * Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream on a SurfaceView.
+ * renderFrame() is asynchronous to avoid blocking the calling thread.
+ * This class is thread safe and handles access from potentially four different threads:
+ * Interaction from the main app in init, release, setMirror, and setScalingtype.
+ * Interaction from C++ webrtc::VideoRendererInterface in renderFrame and canApplyRotation.
+ * Interaction from the Activity lifecycle in surfaceCreated, surfaceChanged, and surfaceDestroyed.
+ * Interaction with the layout framework in onMeasure and onSizeChanged.
+ */
+public class SurfaceViewRenderer extends SurfaceView
+    implements SurfaceHolder.Callback, VideoRenderer.Callbacks {
+  private static final String TAG = "SurfaceViewRenderer";
+
+  // These variables are synchronized on |threadLock|.
+  private final Object threadLock = new Object();
+  // Dedicated render thread.
+  private HandlerThread renderThread;
+  // Handler for inter-thread communication.
+  private Handler renderThreadHandler;
+
+  // EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed
+  // from the render thread.
+  private EglBase eglBase;
+  private GlRectDrawer drawer;
+  // Texture ids for YUV frames. Allocated on first arrival of a YUV frame.
+  private int[] yuvTextures = null;
+
+  // Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|.
+  private final Object frameLock = new Object();
+  private VideoRenderer.I420Frame pendingFrame;
+
+  // These variables are synchronized on |layoutLock|.
+  private final Object layoutLock = new Object();
+  // These three different dimension values are used to keep track of the state in these functions:
+  // requestLayout() -> onMeasure() -> onLayout() -> surfaceChanged().
+  // requestLayout() is triggered internally by frame size changes, but can also be triggered
+  // externally by layout update requests.
+  // Most recent measurement specification from onMeasure().
+  private int widthSpec;
+  private int heightSpec;
+  // Current size on screen in pixels. Updated in onLayout(), and should be consistent with
+  // |widthSpec|/|heightSpec| after that.
+  private int layoutWidth;
+  private int layoutHeight;
+  // Current surface size of the underlying Surface. Updated in surfaceChanged(), and should be
+  // consistent with |layoutWidth|/|layoutHeight| after that.
+  // TODO(magjed): Enable hardware scaler with SurfaceHolder.setFixedSize(). This will decouple
+  // layout and surface size.
+  private int surfaceWidth;
+  private int surfaceHeight;
+  // Last rendered frame dimensions, or 0 if no frame has been rendered yet.
+  private int frameWidth;
+  private int frameHeight;
+  private int frameRotation;
+  // |scalingType| determines how the video will fill the allowed layout area in onMeasure().
+  private RendererCommon.ScalingType scalingType = RendererCommon.ScalingType.SCALE_ASPECT_BALANCED;
+  // If true, mirrors the video stream horizontally.
+  private boolean mirror;
+  // Callback for reporting renderer events.
+  private RendererCommon.RendererEvents rendererEvents;
+
+  // These variables are synchronized on |statisticsLock|.
+  private final Object statisticsLock = new Object();
+  // Total number of video frames received in renderFrame() call.
+  private int framesReceived;
+  // Number of video frames dropped by renderFrame() because previous frame has not been rendered
+  // yet.
+  private int framesDropped;
+  // Number of rendered video frames.
+  private int framesRendered;
+  // Time in ns when the first video frame was rendered.
+  private long firstFrameTimeNs;
+  // Time in ns spent in renderFrameOnRenderThread() function.
+  private long renderTimeNs;
+
+  // Runnable for posting frames to render thread..
+  private final Runnable renderFrameRunnable = new Runnable() {
+    @Override public void run() {
+      renderFrameOnRenderThread();
+    }
+  };
+
+  /**
+   * Standard View constructor. In order to render something, you must first call init().
+   */
+  public SurfaceViewRenderer(Context context) {
+    super(context);
+  }
+
+  /**
+   * Standard View constructor. In order to render something, you must first call init().
+   */
+  public SurfaceViewRenderer(Context context, AttributeSet attrs) {
+    super(context, attrs);
+  }
+
+  /**
+   * Initialize this class, sharing resources with |sharedContext|.
+   */
+  public void init(
+      EGLContext sharedContext, RendererCommon.RendererEvents rendererEvents) {
+    if (renderThreadHandler != null) {
+      throw new IllegalStateException("Already initialized");
+    }
+    Log.d(TAG, "Initializing");
+    this.rendererEvents = rendererEvents;
+    renderThread = new HandlerThread(TAG);
+    renderThread.start();
+    renderThreadHandler = new Handler(renderThread.getLooper());
+    eglBase = new EglBase(sharedContext, EglBase.ConfigType.PLAIN);
+    drawer = new GlRectDrawer();
+    getHolder().addCallback(this);
+  }
+
+  /**
+   * Release all resources. This needs to be done manually, otherwise the resources are leaked. You
+   * should call this before the Activity is destroyed, while the EGLContext is still valid.
+   */
+  public void release() {
+    synchronized (threadLock) {
+      if (renderThreadHandler == null) {
+        Log.d(TAG, "Already released");
+        return;
+      }
+      // Release EGL and GL resources on render thread.
+      // TODO(magjed): This might not be necessary - all OpenGL resources are automatically deleted
+      // when the EGL context is lost. It might be dangerous to delete them manually in
+      // Activity.onDestroy().
+      renderThreadHandler.post(new Runnable() {
+        @Override public void run() {
+          drawer.release();
+          drawer = null;
+          if (yuvTextures != null) {
+            GLES20.glDeleteTextures(3, yuvTextures, 0);
+            yuvTextures = null;
+          }
+          eglBase.release();
+          eglBase = null;
+        }
+      });
+      // Don't accept any more messages to the render thread.
+      renderThreadHandler = null;
+      // Quit safely to make sure the EGL/GL cleanup posted above is executed.
+      renderThread.quitSafely();
+      renderThread = null;
+    }
+    getHolder().removeCallback(this);
+    synchronized (frameLock) {
+      if (pendingFrame != null) {
+        VideoRenderer.renderFrameDone(pendingFrame);
+        pendingFrame = null;
+      }
+    }
+  }
+
+  /**
+   * Set if the video stream should be mirrored or not.
+   */
+  public void setMirror(final boolean mirror) {
+    synchronized (layoutLock) {
+      this.mirror = mirror;
+    }
+  }
+
+  /**
+   * Set how the video will fill the allowed layout area.
+   */
+  public void setScalingType(RendererCommon.ScalingType scalingType) {
+    synchronized (layoutLock) {
+      this.scalingType = scalingType;
+    }
+  }
+
+  // VideoRenderer.Callbacks interface.
+  @Override
+  public void renderFrame(VideoRenderer.I420Frame frame) {
+    synchronized (statisticsLock) {
+      ++framesReceived;
+    }
+    synchronized (threadLock) {
+      if (renderThreadHandler == null) {
+        Log.d(TAG, "Dropping frame - SurfaceViewRenderer not initialized or already released.");
+      } else {
+        synchronized (frameLock) {
+          if (pendingFrame == null) {
+            updateFrameDimensionsAndReportEvents(frame);
+            pendingFrame = frame;
+            renderThreadHandler.post(renderFrameRunnable);
+            return;
+          }
+        }
+      }
+    }
+    // Drop frame.
+    synchronized (statisticsLock) {
+      ++framesDropped;
+    }
+    VideoRenderer.renderFrameDone(frame);
+  }
+
+  // Returns desired layout size given current measure specification and video aspect ratio.
+  private Point getDesiredLayoutSize() {
+    synchronized (layoutLock) {
+      final int maxWidth = getDefaultSize(Integer.MAX_VALUE, widthSpec);
+      final int maxHeight = getDefaultSize(Integer.MAX_VALUE, heightSpec);
+      final Point size =
+          RendererCommon.getDisplaySize(scalingType, frameAspectRatio(), maxWidth, maxHeight);
+      if (MeasureSpec.getMode(widthSpec) == MeasureSpec.EXACTLY) {
+        size.x = maxWidth;
+      }
+      if (MeasureSpec.getMode(heightSpec) == MeasureSpec.EXACTLY) {
+        size.y = maxHeight;
+      }
+      return size;
+    }
+  }
+
+  // View layout interface.
+  @Override
+  protected void onMeasure(int widthSpec, int heightSpec) {
+    synchronized (layoutLock) {
+      this.widthSpec = widthSpec;
+      this.heightSpec = heightSpec;
+    }
+    final Point size = getDesiredLayoutSize();
+    setMeasuredDimension(size.x, size.y);
+  }
+
+  @Override
+  protected void onLayout(boolean changed, int left, int top, int right, int bottom) {
+    synchronized (layoutLock) {
+      layoutWidth = right - left;
+      layoutHeight = bottom - top;
+    }
+    // Might have a pending frame waiting for a layout of correct size.
+    runOnRenderThread(renderFrameRunnable);
+  }
+
+  // SurfaceHolder.Callback interface.
+  @Override
+  public void surfaceCreated(final SurfaceHolder holder) {
+    Log.d(TAG, "Surface created");
+    runOnRenderThread(new Runnable() {
+      @Override public void run() {
+        eglBase.createSurface(holder.getSurface());
+        eglBase.makeCurrent();
+        // Necessary for YUV frames with odd width.
+        GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
+      }
+    });
+  }
+
+  @Override
+  public void surfaceDestroyed(SurfaceHolder holder) {
+    Log.d(TAG, "Surface destroyed");
+    synchronized (layoutLock) {
+      surfaceWidth = 0;
+      surfaceHeight = 0;
+    }
+    runOnRenderThread(new Runnable() {
+      @Override public void run() {
+        eglBase.releaseSurface();
+      }
+    });
+  }
+
+  @Override
+  public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
+    Log.d(TAG, "Surface changed: " + width + "x" + height);
+    synchronized (layoutLock) {
+      surfaceWidth = width;
+      surfaceHeight = height;
+    }
+    // Might have a pending frame waiting for a surface of correct size.
+    runOnRenderThread(renderFrameRunnable);
+  }
+
+  /**
+   * Private helper function to post tasks safely.
+   */
+  private void runOnRenderThread(Runnable runnable) {
+    synchronized (threadLock) {
+      if (renderThreadHandler != null) {
+        renderThreadHandler.post(runnable);
+      }
+    }
+  }
+
+  /**
+   * Requests new layout if necessary. Returns true if layout and surface size are consistent.
+   */
+  private boolean checkConsistentLayout() {
+    synchronized (layoutLock) {
+      final Point desiredLayoutSize = getDesiredLayoutSize();
+      if (desiredLayoutSize.x != layoutWidth || desiredLayoutSize.y != layoutHeight) {
+        Log.d(TAG, "Requesting new layout with size: "
+            + desiredLayoutSize.x + "x" + desiredLayoutSize.y);
+        // Request layout update on UI thread.
+        post(new Runnable() {
+          @Override public void run() {
+            requestLayout();
+          }
+        });
+        return false;
+      }
+      // Wait for requestLayout() to propagate through this sequence before returning true:
+      // requestLayout() -> onMeasure() -> onLayout() -> surfaceChanged().
+      return surfaceWidth == layoutWidth && surfaceHeight == layoutHeight;
+    }
+  }
+
+  /**
+   * Renders and releases |pendingFrame|.
+   */
+  private void renderFrameOnRenderThread() {
+    if (eglBase == null || !eglBase.hasSurface()) {
+      Log.d(TAG, "No surface to draw on");
+      return;
+    }
+    if (!checkConsistentLayout()) {
+      // Output intermediate black frames while the layout is updated.
+      GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
+      eglBase.swapBuffers();
+      return;
+    }
+    // After a surface size change, the EGLSurface might still have a buffer of the old size in the
+    // pipeline. Querying the EGLSurface will show if the underlying buffer dimensions haven't yet
+    // changed. Such a buffer will be rendered incorrectly, so flush it with a black frame.
+    synchronized (layoutLock) {
+      if (eglBase.surfaceWidth() != surfaceWidth || eglBase.surfaceHeight() != surfaceHeight) {
+        GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
+        eglBase.swapBuffers();
+      }
+    }
+    // Fetch and render |pendingFrame|.
+    final VideoRenderer.I420Frame frame;
+    synchronized (frameLock) {
+      if (pendingFrame == null) {
+        return;
+      }
+      frame = pendingFrame;
+      pendingFrame = null;
+    }
+
+    final long startTimeNs = System.nanoTime();
+    if (!frame.yuvFrame) {
+      // TODO(magjed): Move updateTexImage() to the video source instead.
+      SurfaceTexture surfaceTexture = (SurfaceTexture) frame.textureObject;
+      surfaceTexture.updateTexImage();
+    }
+
+    final float[] texMatrix = new float[16];
+    synchronized (layoutLock) {
+      final float[] samplingMatrix = RendererCommon.getSamplingMatrix(
+          (SurfaceTexture) frame.textureObject, frame.rotationDegree);
+      final float[] layoutMatrix = RendererCommon.getLayoutMatrix(
+          mirror, frameAspectRatio(), (float) layoutWidth / layoutHeight);
+      Matrix.multiplyMM(texMatrix, 0, samplingMatrix, 0, layoutMatrix, 0);
+    }
+
+    GLES20.glViewport(0, 0, surfaceWidth, surfaceHeight);
+    if (frame.yuvFrame) {
+      // Make sure YUV textures are allocated.
+      if (yuvTextures == null) {
+        yuvTextures = new int[3];
+        for (int i = 0; i < 3; i++)  {
+          yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D);
+        }
+      }
+      drawer.uploadYuvData(
+          yuvTextures, frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes);
+      drawer.drawYuv(yuvTextures, texMatrix);
+    } else {
+      drawer.drawOes(frame.textureId, texMatrix);
+    }
+
+    eglBase.swapBuffers();
+    VideoRenderer.renderFrameDone(frame);
+    synchronized (statisticsLock) {
+      if (framesRendered == 0) {
+        firstFrameTimeNs = startTimeNs;
+      }
+      ++framesRendered;
+      renderTimeNs += (System.nanoTime() - startTimeNs);
+      if (framesRendered % 300 == 0) {
+        logStatistics();
+      }
+    }
+  }
+
+  // Return current frame aspect ratio, taking rotation into account.
+  private float frameAspectRatio() {
+    synchronized (layoutLock) {
+      if (frameWidth == 0 || frameHeight == 0) {
+        return 0.0f;
+      }
+      return (frameRotation % 180 == 0) ? (float) frameWidth / frameHeight
+                                        : (float) frameHeight / frameWidth;
+    }
+  }
+
+  // Update frame dimensions and report any changes to |rendererEvents|.
+  private void updateFrameDimensionsAndReportEvents(VideoRenderer.I420Frame frame) {
+    synchronized (layoutLock) {
+      if (frameWidth != frame.width || frameHeight != frame.height
+          || frameRotation != frame.rotationDegree) {
+        if (rendererEvents != null) {
+          final String id = getResources().getResourceEntryName(getId());
+          if (frameWidth == 0 || frameHeight == 0) {
+            Log.d(TAG, "ID: " + id + ". Reporting first rendered frame.");
+            rendererEvents.onFirstFrameRendered();
+          }
+          Log.d(TAG, "ID: " + id + ". Reporting frame resolution changed to "
+              + frame.width + "x" + frame.height + " with rotation " + frame.rotationDegree);
+          rendererEvents.onFrameResolutionChanged(frame.width, frame.height, frame.rotationDegree);
+        }
+        frameWidth = frame.width;
+        frameHeight = frame.height;
+        frameRotation = frame.rotationDegree;
+      }
+    }
+  }
+
+  private void logStatistics() {
+    synchronized (statisticsLock) {
+      Log.d(TAG, "ID: " + getResources().getResourceEntryName(getId()) + ". Frames received: "
+          + framesReceived + ". Dropped: " + framesDropped + ". Rendered: " + framesRendered);
+      if (framesReceived > 0 && framesRendered > 0) {
+        final long timeSinceFirstFrameNs = System.nanoTime() - firstFrameTimeNs;
+        Log.d(TAG, "Duration: " + (int) (timeSinceFirstFrameNs / 1e6) +
+            " ms. FPS: " + (float) framesRendered * 1e9 / timeSinceFirstFrameNs);
+        Log.d(TAG, "Average render time: "
+            + (int) (renderTimeNs / (1000 * framesRendered)) + " us.");
+      }
+    }
+  }
+}