AndroidVDA by using Android's MediaCodec API.

media/base/android/media_codec_bridge.cc

Index: media/base/android/media_codec_bridge.cc
diff --git a/media/base/android/media_codec_bridge.cc b/media/base/android/media_codec_bridge.cc
new file mode 100644
index 0000000000000000000000000000000000000000..4c8dd037d03627b7fa30af2bd511643cdfb4e677
--- /dev/null
+++ b/media/base/android/media_codec_bridge.cc
@@ -0,0 +1,371 @@
+// Copyright (c) 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/base/android/media_codec_bridge.h"
+
+#include <jni.h>
+
+#include "base/android/jni_android.h"
+#include "base/android/jni_array.h"
+#include "base/android/jni_string.h"
+#include "base/basictypes.h"
+#include "base/logging.h"
+#include "base/stringprintf.h"
+
+#include "jni/MediaFormat_jni.h"
+#include "MediaCodec_jni.h"
+
+
+using base::android::AttachCurrentThread;
+using base::android::ConvertUTF8ToJavaString;
+using base::android::ScopedJavaLocalRef;
+
+namespace {
+
+static jclass g_MediaCodecBufferInfo_clazz = NULL;

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

Why not move this into BI_FI below, since it's never accessed elsewhere?

+
+struct BufferInfo_FieldIds {

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

I believe you can drastically simplify the code around this stuff:

static void GetBufferInfo(JNIEnv* env, jobject buffer_info, int* offset, int*
size, int64* presentation_time, int* flags) {
  static ScopedJavaLocalRef<jclass> clazz = base::android::GetClass(env,
"android/media/MediaCodec$BufferInfo");
  static jfieldID offset_id = env->GetFieldID(g_MediaCodecBufferInfo_clazz,
"offset", "I");
  ... the other 3 field id finders
  *offset = static_cast<int>(env->GetIntField(buffer_info, offset_id);
  ... the other 3 field getters
}

Done.

Those 12 lines (plus wrapping for 80-cols)  should let you drop l.24-51, and the
call-site becomes much simpler.

[dwkang1, 2013/01/28 14:54:30]

Done.

+  BufferInfo_FieldIds(JNIEnv* env) {
+    offset = env->GetFieldID(g_MediaCodecBufferInfo_clazz, "offset", "I");
+    size = env->GetFieldID(g_MediaCodecBufferInfo_clazz, "size", "I");
+    presentation_time_us = env->GetFieldID(g_MediaCodecBufferInfo_clazz,
+        "presentationTimeUs", "J");
+    flags = env->GetFieldID(g_MediaCodecBufferInfo_clazz, "flags", "I");
+  }
+  jfieldID offset;
+  jfieldID size;
+  jfieldID presentation_time_us;
+  jfieldID flags;
+};
+static BufferInfo_FieldIds* g_field_ids;

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

s/;/ = NULL;/

[dwkang1, 2013/01/28 14:54:30]

Done.

+
+const char kMediaCodecBufferInfoClassPath[] =
+    "android/media/MediaCodec$BufferInfo";
+
+static bool MediaCodecBufferInfo_RegisterNativesImpl(JNIEnv* env) {
+  g_MediaCodecBufferInfo_clazz = reinterpret_cast<jclass>(env->NewGlobalRef(
+      base::android::GetUnscopedClass(env, kMediaCodecBufferInfoClassPath)));
+  g_field_ids = new BufferInfo_FieldIds(env);
+  return true;

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

What if any of the lookups fail?  Don't you want to do something like
env->ExceptionCheck()?  (probably there's a base/android wrapper for that, I
haven't looked yet)

[dwkang1, 2013/01/28 14:54:30]

base::android::CheckException(env) is added.

+}
+
+static base::subtle::AtomicWord g_MediaCodecBufferInfo_Constructor = 0;
+static ScopedJavaLocalRef<jobject> Java_MediaCodecBufferInfo_Constructor(
+    JNIEnv* env) {
+  /* Must call RegisterNativesImpl()  */
+  DCHECK(g_MediaCodecBufferInfo_clazz);
+  jmethodID method_id =
+      base::android::MethodID::LazyGet<base::android::MethodID::TYPE_INSTANCE>(
+      env, g_MediaCodecBufferInfo_clazz,
+      "<init>", "()V",
+      &g_MediaCodecBufferInfo_Constructor);
+
+  jobject ret = env->NewObject(g_MediaCodecBufferInfo_clazz, method_id);
+  base::android::CheckException(env);
+  return ScopedJavaLocalRef<jobject>(env, ret);
+}
+
+void RegisterNativesIfNeeded(JNIEnv* env) {
+  static bool jni_initialized = false;
+  if (!jni_initialized) {
+    JNI_MediaCodec::RegisterNativesImpl(env);
+    MediaCodecBufferInfo_RegisterNativesImpl(env);
+    JNI_MediaFormat::RegisterNativesImpl(env);

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

These 3 methods can return false; shouldn't you handle that?

[dwkang1, 2013/01/28 14:54:30]

Done.

+    jni_initialized = true;
+  }
+}
+}  // namespace
+
+namespace media {
+
+MediaCodecBridge::MediaCodecBridge(const std::string& type)
+    : j_byte_array_size_(0) {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+  RegisterNativesIfNeeded(env);
+
+  ScopedJavaLocalRef<jstring> j_type = ConvertUTF8ToJavaString(env, type);
+
+  ScopedJavaLocalRef<jobject> tmp(
+      JNI_MediaCodec::Java_MediaCodec_createDecoderByType(env, j_type.obj()));
+  DCHECK(!tmp.is_null());
+  j_media_codec_.Reset(tmp);
+}
+
+MediaCodecBridge::~MediaCodecBridge() {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  // Release graphics memory.
+  JNI_MediaCodec::Java_MediaCodec_release(env, j_media_codec_.obj());
+}
+
+void MediaCodecBridge::ConfigureAudio(
+    const std::string& mime, int sample_rate, int channel_count,
+    const uint8* csd0, int csd0_size, const uint8* csd1, int csd1_size,
+    int encoder_delay, int encoder_padding, int max_input_size) {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  ScopedJavaLocalRef<jstring> j_mime = ConvertUTF8ToJavaString(env, mime);
+  ScopedJavaLocalRef<jobject> j_format(
+      JNI_MediaFormat::Java_MediaFormat_createAudioFormat(
+          env, j_mime.obj(), static_cast<jint>(sample_rate),
+          static_cast<jint>(channel_count)));
+  DCHECK(!j_format.is_null());

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

Again, I worry about a free-text mime-type causing a crash.

[dwkang1, 2013/01/28 14:54:30]

CodecToMimeType() is introduced for type checking.

+
+  if (csd0 != NULL && csd0_size > 0) {
+    ScopedJavaLocalRef<jstring> j_csd_string =
+        ConvertUTF8ToJavaString(env, "csd-0");
+    JNI_MediaFormat::Java_MediaFormat_setByteBuffer(
+        env, j_format.obj(), j_csd_string.obj(),
+        env->NewDirectByteBuffer(const_cast<uint8*>(csd0), csd0_size));
+  }
+
+  if (csd1 != NULL && csd1_size > 0) {
+    ScopedJavaLocalRef<jstring> j_csd_string =
+        ConvertUTF8ToJavaString(env, "csd-1");
+    JNI_MediaFormat::Java_MediaFormat_setByteBuffer(
+        env, j_format.obj(), j_csd_string.obj(),
+        env->NewDirectByteBuffer(const_cast<uint8*>(csd1), csd1_size));
+  }
+
+  if (encoder_delay > 0) {
+    ScopedJavaLocalRef<jstring> j_encoder_delay_string =
+        ConvertUTF8ToJavaString(env, "encoder-delay");

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

Where does this string come from?  I don't see it documented in
Media{Format,Codec}'s javadocs.
Ditto for the string literal keys elsewhere in this file.

[dwkang1, 2013/01/28 14:54:30]

Removed. Will add when these are needed.

+    JNI_MediaFormat::Java_MediaFormat_setInteger(
+        env, j_format.obj(), j_encoder_delay_string.obj(),
+        static_cast<jint>(encoder_delay));
+  }
+
+  if (encoder_padding > 0) {
+    ScopedJavaLocalRef<jstring> j_encoder_padding_string =
+        ConvertUTF8ToJavaString(env, "encoder-padding");
+    JNI_MediaFormat::Java_MediaFormat_setInteger(
+        env, j_format.obj(), j_encoder_padding_string.obj(),
+        static_cast<jint>(encoder_padding));
+  }
+
+  if (max_input_size > 0) {
+    ScopedJavaLocalRef<jstring> j_max_input_size_string =
+        ConvertUTF8ToJavaString(env, "max-input-size");
+    JNI_MediaFormat::Java_MediaFormat_setInteger(
+        env, j_format.obj(), j_max_input_size_string.obj(),
+        static_cast<jint>(max_input_size));
+  }

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

There's an aweful lot of copy/pasta above; why not use a helper function?

[dwkang1, 2013/01/28 14:54:30]

Removed.

+
+  ScopedJavaLocalRef<jstring> j_is_adts_string =
+      ConvertUTF8ToJavaString(env, "is-adts");
+  JNI_MediaFormat::Java_MediaFormat_setInteger(
+      env, j_format.obj(), j_is_adts_string.obj(), static_cast<jint>(1));

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

Why is this always true?

[dwkang1, 2013/01/28 14:54:30]

I believe this is added while testing AAC codec and not properly handled later.
Removed and will be added when needed.

+
+  JNI_MediaCodec::Java_MediaCodec_configure(
+      env, j_media_codec_.obj(), j_format.obj(), NULL, NULL, 0);
+}
+
+void MediaCodecBridge::ConfigureVideo(
+    const std::string& mime, int width, int height,

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

prefer to use gfx::Size instead of width/height params to avoid mixing up the
order (and reduce arities/increase readability)

[dwkang1, 2013/01/28 14:54:30]

Done.

+    const uint8* csd0, int csd0_size, const uint8* csd1, int csd1_size,
+    jobject surface) {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  ScopedJavaLocalRef<jstring> j_mime = ConvertUTF8ToJavaString(env, mime);
+  ScopedJavaLocalRef<jobject> j_format(
+      JNI_MediaFormat::Java_MediaFormat_createVideoFormat(
+          env, j_mime.obj(), static_cast<jint>(width), static_cast<jint>(height)));

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

80-cols

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

Is it really necessary to static_cast an int to a jint??

[dwkang1, 2013/01/28 14:54:30]

Done.

[dwkang1, 2013/01/28 14:54:30]

Done.

+  DCHECK(!j_format.is_null());
+
+  if (csd0 != NULL && csd0_size > 0) {
+    ScopedJavaLocalRef<jstring> j_csd_string =
+        ConvertUTF8ToJavaString(env, "csd-0");
+    JNI_MediaFormat::Java_MediaFormat_setByteBuffer(
+        env, j_format.obj(), j_csd_string.obj(),
+        env->NewDirectByteBuffer(const_cast<uint8*>(csd0), csd0_size));
+  }
+
+  if (csd1 != NULL && csd1_size > 0) {
+    ScopedJavaLocalRef<jstring> j_csd_string =
+        ConvertUTF8ToJavaString(env, "csd-1");
+    JNI_MediaFormat::Java_MediaFormat_setByteBuffer(
+        env, j_format.obj(), j_csd_string.obj(),
+        env->NewDirectByteBuffer(const_cast<uint8*>(csd1), csd1_size));
+  }
+
+  JNI_MediaCodec::Java_MediaCodec_configure(
+      env, j_media_codec_.obj(), j_format.obj(), surface, NULL, 0);
+}
+
+void MediaCodecBridge::Start() {

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

Why not make all configuration (audio/video) a single call and then always
Start() at the end of it?

[dwkang1, 2013/01/28 14:54:30]

I guess it was because they have difference argument types. But, I think it can
be done by passing MediaFormat as an argument. This is your intention, right?
Added XXX for this refactoring. Please let me know if I misunderstood your
suggestion.

+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  JNI_MediaCodec::Java_MediaCodec_start(env, j_media_codec_.obj());
+}
+
+void MediaCodecBridge::Flush() {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  JNI_MediaCodec::Java_MediaCodec_flush(env, j_media_codec_.obj());
+}
+
+void MediaCodecBridge::Stop() {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  JNI_MediaCodec::Java_MediaCodec_stop(env, j_media_codec_.obj());
+}
+
+void MediaCodecBridge::Release() {

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

Hopefully you can drop this method in favor of the dtor.

[dwkang1, 2013/01/28 14:54:30]

Done.

+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  JNI_MediaCodec::Java_MediaCodec_release(env, j_media_codec_.obj());
+}
+
+void MediaCodecBridge::GetOutputFormat(int* format, int* width, int* height) {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  ScopedJavaLocalRef<jobject> j_format(
+     JNI_MediaCodec::Java_MediaCodec_getOutputFormat(
+         env, j_media_codec_.obj()));
+  if (!j_format.is_null()) {
+    ScopedJavaLocalRef<jstring> j_key_format =
+        ConvertUTF8ToJavaString(env, "color-format");
+    *format = static_cast<int>(JNI_MediaFormat::Java_MediaFormat_getInteger(
+        env, j_format.obj(), j_key_format.obj()));
+
+    ScopedJavaLocalRef<jstring> j_key_width =
+        ConvertUTF8ToJavaString(env, "width");
+    *width = static_cast<int>(JNI_MediaFormat::Java_MediaFormat_getInteger(
+        env, j_format.obj(), j_key_width.obj()));
+
+    ScopedJavaLocalRef<jstring> j_key_height =
+        ConvertUTF8ToJavaString(env, "height");
+    *height = static_cast<int>(JNI_MediaFormat::Java_MediaFormat_getInteger(
+        env, j_format.obj(), j_key_height.obj()));
+  }
+}
+
+void MediaCodecBridge::QueueInputBuffer(
+    int index, int offset, int size, int64 presentation_time_us, int flags) {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  JNI_MediaCodec::Java_MediaCodec_queueInputBuffer(
+      env, j_media_codec_.obj(),
+      static_cast<jint>(index), static_cast<jint>(offset),
+      static_cast<jint>(size), static_cast<jlong>(presentation_time_us),
+      static_cast<jint>(flags));
+}
+
+int MediaCodecBridge::DequeueInputBuffer(int64 timeout_us) {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  return JNI_MediaCodec::Java_MediaCodec_dequeueInputBuffer(
+      env, j_media_codec_.obj(), static_cast<jlong>(timeout_us));
+}
+
+int MediaCodecBridge::DequeueOutputBuffer(
+    int64 timeout_us, int* offset, int* size, int64* presentation_time_us,
+    int* flags) {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  ScopedJavaLocalRef<jobject> j_info(
+      Java_MediaCodecBufferInfo_Constructor(env));
+  jint j_buffer = JNI_MediaCodec::Java_MediaCodec_dequeueOutputBuffer(
+      env, j_media_codec_.obj(), j_info.obj(), static_cast<jlong>(timeout_us));
+
+  if (j_buffer >= 0) {
+    *offset =
+        static_cast<int>(env->GetIntField(j_info.obj(), g_field_ids->offset));
+    *size = static_cast<int>(env->GetIntField(j_info.obj(), g_field_ids->size));
+    *presentation_time_us = static_cast<int64>(env->GetLongField(
+        j_info.obj(), g_field_ids->presentation_time_us));
+    *flags =
+        static_cast<int>(env->GetIntField(j_info.obj(), g_field_ids->flags));
+  }
+  return static_cast<int>(j_buffer);
+}
+
+void MediaCodecBridge::ReleaseOutputBuffer(int index, bool render) {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  JNI_MediaCodec::Java_MediaCodec_releaseOutputBuffer(
+      env, j_media_codec_.obj(), static_cast<jint>(index),
+      static_cast<jboolean>(render));
+}
+
+int MediaCodecBridge::GetInputBuffers() {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  j_input_buffers_.Reset(
+      JNI_MediaCodec::Java_MediaCodec_getInputBuffers(
+          env, j_media_codec_.obj()));
+  if (j_input_buffers_.is_null()) return -1;

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

I don't see this in the javadoc.

[dwkang1, 2013/01/28 14:54:30]

Removed.

+
+  return static_cast<int>(env->GetArrayLength(j_input_buffers_.obj()));
+}
+
+int MediaCodecBridge::GetOutputBuffers() {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  j_output_buffers_.Reset(
+      JNI_MediaCodec::Java_MediaCodec_getOutputBuffers(
+          env, j_media_codec_.obj()));
+  if (j_output_buffers_.is_null()) return -1;

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

again, javadoc doesn't support this.

[dwkang1, 2013/01/28 14:54:30]

Removed.

+
+  return static_cast<int>(env->GetArrayLength(j_output_buffers_.obj()));
+}
+
+void MediaCodecBridge::PutToInputBuffer(
+    int index, const uint8* data, int size) {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  ScopedJavaLocalRef<jobject> j_buffer(
+      env, env->GetObjectArrayElement(j_input_buffers_.obj(), index));
+
+  uint8* direct_buffer =
+      static_cast<uint8*>(env->GetDirectBufferAddress(j_buffer.obj()));

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

What guarantees that the bytebuffers MediaCodec returns are "direct"?

[ycheo (away), 2013/01/27 07:34:48]

Actually, we refereed the underlying implementation.
https://cs.corp.google.com/#googletv/frameworks/base/media/jni/android_media_...

Do you think that we should check if the buffer is direct?

[Ami GONE FROM CHROMIUM, 2013/01/27 17:50:42]

If all known impls are direct then probably not worth adding in code that'll
never run.
Are you in a position to get the MediaCodec spec changed to specify that
returned buffers are direct (at least for future SDK releases)?

[ycheo (away), 2013/01/27 19:47:22]

We can ask them, but I don't expect that they try to change this actively.

+  memcpy(direct_buffer, data, size);

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

what if |size| is too big?

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

Is it not necessary to set the bytebuffer's mark/position/limit?

[ycheo (away), 2013/01/27 07:34:48]

Yes, the underlying implementation doesn't care about them.
That's why the original queueInputBuffer() has separate offset and size fields.
http://developer.android.com/reference/android/media/MediaCodec.html#queueInp...,
int, int, long, int)

+}
+
+void MediaCodecBridge::GetFromOutputBuffer(
+    int index, int offset, uint8* data, int size) {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  ScopedJavaLocalRef<jobject> j_buffer(
+      env, env->GetObjectArrayElement(j_output_buffers_.obj(), index));
+
+  uint8* direct_buffer =
+      static_cast<uint8*>(env->GetDirectBufferAddress(j_buffer.obj()));

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

ditto what if not direct?

+  memcpy(data, direct_buffer + offset, size);
+}
+
+void MediaCodecBridge::GetFromOutputBuffer(int index,
+    uint8* data0, int size0, uint8* data1, int size1, uint8* data2, int size2) {
+  JNIEnv* env = AttachCurrentThread();
+  CHECK(env);
+
+  ScopedJavaLocalRef<jobject> j_buffer(
+      env, env->GetObjectArrayElement(j_output_buffers_.obj(), index));
+
+  uint8* direct_buffer =
+      static_cast<uint8*>(env->GetDirectBufferAddress(j_buffer.obj()));
+  memcpy(data0, direct_buffer, size0);
+  memcpy(data1, direct_buffer + size0, size1);

[Ami GONE FROM CHROMIUM, 2013/01/23 01:32:32]

What guarantees that the planes are mashed together like this, without, e.g.
padding to a word or power-of-two boundary?

[dwkang1, 2013/01/28 14:54:30]

This method will be removed and added when this is needed. Sorry for not
cleaning up this in advance.

+  memcpy(data2, direct_buffer + size0 + size1, size2);
+}
+
+}  // namespace media
+