Add video capture on Android.

media/video/capture/android/video_capture_device_android.cc

+// Copyright (c) 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/video/capture/android/video_capture_device_android.h"
+
+#include <string>
+
+#include "base/android/jni_android.h"
+#include "base/android/scoped_java_ref.h"
+#include "base/string_number_conversions.h"
+#include "base/stringprintf.h"
+#include "jni/Camera_jni.h"
+#include "jni/VideoCapture_jni.h"
+
+using base::android::AttachCurrentThread;
+using base::android::CheckException;
+using base::android::GetClass;
+using base::android::MethodID;
+using base::android::JavaRef;
+using base::android::ScopedJavaLocalRef;
+
+namespace {
+
+// TODO(wjia): add stride as part of buffer parameter.
+void ResetBufferI420(uint8* buffer, int width, int height) {
+  int y_size = width * height;
+  memset(buffer, 0, y_size);
+  buffer += y_size;
+  memset(buffer, 128, y_size / 2);
+}
+
+void RotatePlaneByPixels(
+    uint8* src,
+    uint8* dest,
+    int width,
+    int height,
+    int rotation,

[Ami GONE FROM CHROMIUM, 2013/02/06 20:13:42]

clockwise?  Counter?  Should have a comment.

[wjia(left Chromium), 2013/02/06 23:20:39]

Done.

+    bool flip_vert,
+    bool flip_horiz) {
+  // Consolidate cases. Only 0 and 90 are left.
+  if (rotation == 180 || rotation == 270) {
+    rotation -= 180;
+    flip_vert = !flip_vert;
+    flip_horiz = !flip_horiz;
+  }
+
+  int num_rows = height;
+  int num_cols = width;
+  int src_stride = width;
+  int dest_row_step = width;
+  int dest_col_step = 1;
+
+  if (rotation == 0) {
+    if (flip_horiz) {
+      // Use pixel copying.
+      dest_col_step = -1;
+      if (flip_vert) {
+        // Rotation 180.
+        dest_row_step = -width;
+        dest += height * width - 1;
+      } else {
+        dest += width - 1;
+      }
+    } else {
+      if (flip_vert) {
+        // Fast copy by rows.
+        dest += width * (height - 1);
+        for (int row = 0; row < height; ++row) {
+          memcpy(dest, src, width);
+          src += width;
+          dest -= width;
+        }
+      } else {
+        memcpy(dest, src, width * height);
+      }
+      return;
+    }
+  } else if (rotation == 90) {
+    int offset = (width - height) / 2;
+    if (width > height) {
+      src += offset;
+      num_rows = num_cols = height;
+    } else {
+      src += width * offset;

[Ami GONE FROM CHROMIUM, 2013/02/06 20:13:42]

So in this case width <= height, so offset is <= 0, so you're considering moving
src to *before* the beginning of the plane?  I'm confused.

[wjia(left Chromium), 2013/02/06 23:20:39]

hrmm, the offset is calculated 2 lines behind. That's apparently a mistake.
Thanks for catching that!

+      num_rows = num_cols = width;

[Ami GONE FROM CHROMIUM, 2013/02/06 20:13:42]

I don't get this; doesn't this result in only rotating a square subrect of |src|
into |dest|?

[wjia(left Chromium), 2013/02/06 23:20:39]

Yes, a square is the overlapped area of src and dest. The src has width x
height. After rotated by 90 degree, it becomes heigh x width. But we still want
to deliver width x height frame to renderer process. So we have to crop src
either horizontally or vertically, then pad dest on the opposite.

The reason for always delivering width x height frames is that we don't support
multiple OnFrameInfo yet.

+      offset = (height - width) / 2;
+    }
+
+    dest_col_step = (flip_vert ? -width : width);
+    if (flip_horiz) {
+      dest_row_step = 1;

[Ami GONE FROM CHROMIUM, 2013/02/06 20:13:42]

Set this like dest_col_step at l.90?

[wjia(left Chromium), 2013/02/06 23:20:39]

This one is correct. I have added some comments about meaning of dest_row_step
and dest_col_step.

+      if (flip_vert) {
+        dest += (width > height ? width * (height - 1) + offset :
+                                  width * (height - offset - 1));
+      } else {
+        dest += (width > height ? offset : width * offset);

[Ami GONE FROM CHROMIUM, 2013/02/06 20:13:42]

I'm not 100% sure I understand how offset is being used, but I *think* that if
you pre-multiply it by width at l.87 then the two cases in the ?: in each of the
4 occurrences in l.94-107 have their "else" clause look a lot more like the
"then" clause.

[wjia(left Chromium), 2013/02/06 23:20:39]

"offset" is the cropped/padded pixels on one line (actually half of them) when
width != height. It's used to calculate the starting pointer of src and dest for
pixel copying. The pixel copying always follows src pixel order, traverse row by
row. Therefore, we need calculate dest_row_step and dest_col_step corresponding
to row and column change in src pointer, respectively.

+      }
+    } else {
+      dest_row_step = -1;
+      if (flip_vert) {
+        dest += (width > height ?  width * height - offset - 1 :
+                                   width * (height - offset) - 1);
+      } else {
+        dest += (width > height ? width - offset - 1 :
+                                  width * (offset + 1) - 1);
+      }
+    }

[Ami GONE FROM CHROMIUM, 2013/02/06 20:13:42]

else NOTREACHED?

[wjia(left Chromium), 2013/02/06 23:20:39]

l99 is the "else".

+  }
+
+  // Copy pixels.
+  for (int row = 0; row < num_rows; ++row) {
+    uint8* src_ptr = src;
+    uint8* dest_ptr = dest;
+    for (int col = 0; col < num_cols; ++col) {
+      *dest_ptr = *src_ptr++;
+      dest_ptr += dest_col_step;
+    }
+    src += src_stride;
+    dest += dest_row_step;
+  }
+}

[Ami GONE FROM CHROMIUM, 2013/02/06 20:13:42]

This is a crazy function to have no test for.
Especially given how easy it would be to write that test; fill a small
(assymetric) plane with uint8's 0..23 and test the 16 rotations/flips by typing
out the matrices as literals.

[wjia(left Chromium), 2013/02/06 23:20:39]

Agree that we need a test for this function. I have manually tested some cases
(when src's width > height). I will follow up a patch to add the test. Need to
move it to some util file.

+
+int GetIntField(JNIEnv* env,
+                const JavaRef<jclass>& clazz,
+                const JavaRef<jobject>& instance,
+                const char* field_name) {
+  jfieldID field = GetFieldID(env, clazz, field_name, "I");
+  jint int_value = env->GetIntField(instance.obj(), field);
+  return int_value;
+}
+
+}  // namespace
+
+namespace media {
+
+// static
+void VideoCaptureDevice::GetDeviceNames(Names* device_names) {
+  device_names->clear();
+
+  JNIEnv* env = AttachCurrentThread();
+
+  int num_cameras = JNI_Camera::Java_Camera_getNumberOfCameras(env);
+  DVLOG(1) << "VideoCaptureDevice::GetDeviceNames: num_cameras=" << num_cameras;
+  if (num_cameras <= 0)
+    return;
+
+  // TODO(wjia): switch to using same approach as Camera when
+  // jar_file_jni_generator.gypi supports system inner classes.
+  std::string camera_info_string("android/hardware/Camera$CameraInfo");
+
+  ScopedJavaLocalRef<jclass> camera_info_class(
+      GetClass(env, camera_info_string.c_str()));
+  jmethodID constructor = MethodID::Get<MethodID::TYPE_INSTANCE>(
+      env, camera_info_class.obj(), "<init>", "()V");
+
+  ScopedJavaLocalRef<jobject> object_camera_info(
+      env, env->NewObject(camera_info_class.obj(), constructor));
+
+  jfieldID field_facing = GetFieldID(env, camera_info_class, "facing", "I");
+  jfieldID field_facing_front = GetStaticFieldID(
+      env, camera_info_class, "CAMERA_FACING_FRONT", "I");
+
+  for (int camera_id = num_cameras - 1; camera_id >= 0; --camera_id) {
+    JNI_Camera::Java_Camera_getCameraInfo(
+        env, camera_id, object_camera_info.obj());
+    CheckException(env);

[Yaron, 2013/02/06 19:25:51]

This is unnecessary (handled by the binding)

[wjia(left Chromium), 2013/02/06 23:20:39]

Done.

+
+    Name name;
+    name.unique_id = StringPrintf("%d", camera_id);
+    std::string facing_string;
+    if (env->GetIntField(object_camera_info.obj(), field_facing) ==
+        env->GetStaticIntField(camera_info_class.obj(), field_facing_front)) {
+      facing_string = "front";
+    } else {
+      facing_string = "back";
+    }
+    name.device_name = StringPrintf(
+        "camera %d, facing %s", camera_id, facing_string.c_str());
+    device_names->push_back(name);
+    jfieldID field_orientation = GetFieldID(
+        env, camera_info_class, "orientation", "I");
+    jint orientation = env->GetIntField(object_camera_info.obj(),
+                                        field_orientation);
+    DVLOG(1) << "VideoCaptureDevice::GetDeviceNames: camera device_name="
+             << name.device_name
+             << ", unique_id="
+             << name.unique_id
+             << ", orientation "
+             << orientation;
+  }
+}
+
+// static
+VideoCaptureDevice* VideoCaptureDevice::Create(const Name& device_name) {
+  return VideoCaptureDeviceAndroid::Create(device_name);
+}
+
+// static
+VideoCaptureDevice* VideoCaptureDeviceAndroid::Create(const Name& device_name) {
+  scoped_ptr<VideoCaptureDeviceAndroid> ret(
+      new VideoCaptureDeviceAndroid(device_name));
+  if (ret->Init())
+    return ret.release();
+  return NULL;
+}
+
+// static
+bool VideoCaptureDeviceAndroid::RegisterVideoCaptureDevice(JNIEnv* env) {
+  return RegisterNativesImpl(env) && JNI_Camera::RegisterNativesImpl(env);
+}
+
+VideoCaptureDeviceAndroid::VideoCaptureDeviceAndroid(const Name& device_name)
+    : state_(kIdle),
+      observer_(NULL),
+      device_name_(device_name),
+      current_settings_(),
+      rotation_(0) {
+}
+
+VideoCaptureDeviceAndroid::~VideoCaptureDeviceAndroid() {
+  DeAllocate();
+}
+
+bool VideoCaptureDeviceAndroid::Init() {
+  int id;
+  if (!base::StringToInt(device_name_.unique_id, &id))
+    return false;
+
+  JNIEnv* env = AttachCurrentThread();
+
+  j_capture_.Reset(Java_VideoCapture_createVideoCapture(
+      env, base::android::GetApplicationContext(), id,
+      reinterpret_cast<jlong>(this)));
+
+  return true;
+}
+
+const VideoCaptureDevice::Name& VideoCaptureDeviceAndroid::device_name() {
+  return device_name_;
+}
+
+void VideoCaptureDeviceAndroid::Allocate(
+    int width,
+    int height,
+    int frame_rate,
+    EventHandler* observer) {
+  {
+    base::AutoLock lock(lock_);
+    if (state_ != kIdle)
+      return;
+    observer_ = observer;
+    state_ = kAllocated;
+  }
+
+  JNIEnv* env = AttachCurrentThread();
+
+  jboolean ret = Java_VideoCapture_allocate(env,
+                                            j_capture_.obj(),
+                                            width,
+                                            height,
+                                            frame_rate);
+  if (!ret) {
+    SetErrorState("failed to allocate");
+    return;
+  }
+
+  // Store current width and height.
+  current_settings_.width =
+      Java_VideoCapture_queryWidth(env, j_capture_.obj());
+  current_settings_.height =
+      Java_VideoCapture_queryHeight(env, j_capture_.obj());
+  current_settings_.frame_rate =
+      Java_VideoCapture_queryFrameRate(env, j_capture_.obj());
+  current_settings_.color = VideoCaptureCapability::kYV12;
+  CHECK(current_settings_.width > 0 && !(current_settings_.width % 2));
+  CHECK(current_settings_.height > 0 && !(current_settings_.height % 2));
+
+  DVLOG(1) << "VideoCaptureDeviceAndroid::Allocate: queried width="
+           << current_settings_.width
+           << ", height="
+           << current_settings_.height
+           << ", frame_rate="
+           << current_settings_.frame_rate;
+  // Report the frame size to the observer.
+  observer_->OnFrameInfo(current_settings_);
+
+  int y_size = current_settings_.width * current_settings_.height;
+  rotation_buffer_.reset(new uint8[y_size * 3 / 2]);
+  ResetBufferI420(rotation_buffer_.get(),
+                  current_settings_.width,
+                  current_settings_.height);
+}
+
+void VideoCaptureDeviceAndroid::Start() {
+  DVLOG(1) << "VideoCaptureDeviceAndroid::Start";
+  {
+    base::AutoLock lock(lock_);
+    DCHECK_EQ(state_, kAllocated);
+  }
+
+  JNIEnv* env = AttachCurrentThread();
+
+  jint ret = Java_VideoCapture_startCapture(env, j_capture_.obj());
+  if (ret < 0) {
+    SetErrorState("failed to start capture");
+    return;
+  }
+
+  {
+    base::AutoLock lock(lock_);
+    state_ = kCapturing;
+  }
+}
+
+void VideoCaptureDeviceAndroid::Stop() {
+  DVLOG(1) << "VideoCaptureDeviceAndroid::Stop";
+  {
+    base::AutoLock lock(lock_);
+    if (state_ != kCapturing && state_ != kError)
+      return;
+    if (state_ == kCapturing)
+      state_ = kAllocated;
+  }
+
+  JNIEnv* env = AttachCurrentThread();
+
+  jint ret = Java_VideoCapture_stopCapture(env, j_capture_.obj());
+  if (ret < 0) {
+    SetErrorState("failed to stop capture");
+    return;
+  }
+}
+
+void VideoCaptureDeviceAndroid::DeAllocate() {
+  DVLOG(1) << "VideoCaptureDeviceAndroid::DeAllocate";
+  {
+    base::AutoLock lock(lock_);
+    if (state_ == kIdle)
+      return;
+
+    if (state_ == kCapturing) {
+      base::AutoUnlock unlock(lock_);
+      Stop();
+    }
+
+    if (state_ == kAllocated)
+      state_ = kIdle;
+
+    observer_ = NULL;
+  }
+
+  JNIEnv* env = AttachCurrentThread();
+
+  Java_VideoCapture_deallocate(env, j_capture_.obj());
+}
+
+void VideoCaptureDeviceAndroid::OnFrameAvailable(
+    JNIEnv* env,
+    jobject obj,
+    jbyteArray data,
+    jint length,
+    jint rotation,
+    jboolean flip_vert,
+    jboolean flip_horiz) {
+  DVLOG(3) << "VideoCaptureDeviceAndroid::OnFrameAvailable: length =" << length;
+
+  base::AutoLock lock(lock_);
+  if (state_ != kCapturing || !observer_)
+    return;
+
+  jbyte* buffer = env->GetByteArrayElements(data, NULL);
+  if (!buffer) {
+    LOG(ERROR) << "VideoCaptureDeviceAndroid::OnFrameAvailable: "
+                  "failed to GetByteArrayElements";
+    return;
+  }
+
+  // TODO(wjia): move rotation into VideoCaptureController to remove
+  // one buffer copying.
+  // Rotate the buffer when needed.
+  int width = current_settings_.width;
+  int height = current_settings_.height;
+  if (rotation_ != rotation) {
+    rotation_ = rotation;
+    ResetBufferI420(rotation_buffer_.get(), width, height);

[Ami GONE FROM CHROMIUM, 2013/02/06 20:13:42]

Is there no need to call observer_->OnFrameInfo() here in case width/height got
swapped as a result of the rotation?

[wjia(left Chromium), 2013/02/06 23:20:39]

No, we don't support dimension change in one capture session yet. It requires
re-allocate shared memory buffers. That's why we can only copy a square area
when width != height.

To think about it further, even if we support multiple OnFrameInfo, when
renderer process receives the new frames with different size, it has to blit the
frame into the same display size. In this case, in order to keep aspect ratio,
renderer process has to chop/pad anyway.

[Ami GONE FROM CHROMIUM, 2013/02/07 05:37:34]

See above; I think we need to honor rotation from the beginning to have a
reasonable UX when the device starts rotated/flipped.

[wjia(left Chromium), 2013/02/12 00:23:45]

To support resolution change on the fly, we need add quite some thing in the
video capture frame work. There seems desire to do that. After resolution change
is supported in video capture framework, we could do OnFrameInfo here.

+  }
+
+  uint8* src = reinterpret_cast<uint8*>(buffer);
+  uint8* dest = rotation_buffer_.get();
+
+  RotatePlaneByPixels(src, dest, width, height, rotation, flip_vert,
+                      flip_horiz);
+  int y_size = width * height;
+  src += y_size;
+  dest += y_size;
+  RotatePlaneByPixels(src, dest, width/2, height/2, rotation, flip_vert,
+                      flip_horiz);
+  src += y_size/4;
+  dest += y_size/4;
+  RotatePlaneByPixels(src, dest, width/2, height/2, rotation, flip_vert,
+                      flip_horiz);
+  observer_->OnIncomingCapturedFrame(rotation_buffer_.get(), length,
+                                     base::Time::Now());
+
+  env->ReleaseByteArrayElements(data, buffer, JNI_ABORT);
+}
+
+void VideoCaptureDeviceAndroid::SetErrorState(const std::string& reason) {
+  LOG(ERROR) << "VideoCaptureDeviceAndroid::SetErrorState: " << reason;
+  {
+    base::AutoLock lock(lock_);
+    state_ = kError;
+  }
+  observer_->OnError();
+}
+
+}  // namespace media