media: add video capture device for ARC++ camera HAL v3

media/capture/video/chromeos/camera_device_delegate.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/capture/video/chromeos/camera_device_delegate.h"
+
+#include <libdrm/drm_fourcc.h>
+
+#include "media/capture/video/chromeos/camera_hal_delegate.h"
+#include "media/capture/video/chromeos/camera_metadata_utils.h"
+#include "mojo/edk/embedder/embedder.h"
+#include "mojo/edk/embedder/scoped_platform_handle.h"
+#include "third_party/libsync/include/sync/sync.h"
+
+namespace media {
+
+namespace {
+
+struct SupportedFormat {
+  VideoPixelFormat chromium_format;
+  arc::mojom::HalPixelFormat hal_format;
+  uint32_t drm_format;
+} const kSupportedFormats[] = {
+    // The Android camera HAL v3 has three types of mandatory pixel formats:
+    //
+    //   1. HAL_PIXEL_FORMAT_YCbCr_420_888 (YUV flexible format).
+    //   2. HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED (platform-specific format).
+    //   3. HAL_PIXEL_FORMAT_BLOB (for JPEG).
+    //
+    // We can't use HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED as it is highly
+    // platform specific and there is no way for Chrome to query the exact
+    // pixel layout of the implementation-defined buffer.
+    //
+    // On Android the framework requests the preview stream with the
+    // implementation-defined format, and as a result some camera HALs support
+    // only implementation-defined preview buffers.  We should use the video
+    // capture stream in Chrome VCD as it is mandatory for the camera HAL to
+    // support YUV flexbile format video streams.
+
+    // TODO(jcliang): Change NV12 to I420 after the camera HAL supports hanlding

[wuchengli, 2017/05/23 04:29:07]

handling

[jcliang, 2017/05/25 14:23:46]

Done.

+    //                I420 buffers.
+    {PIXEL_FORMAT_NV12,
+     arc::mojom::HalPixelFormat::HAL_PIXEL_FORMAT_YCbCr_420_888,
+     DRM_FORMAT_NV12},
+};
+
+}  // namespace
+
+CameraDeviceDelegate::CameraDeviceDelegate(
+    VideoCaptureDeviceDescriptor device_descriptor,
+    scoped_refptr<CameraHalDelegate> camera_hal_delegate,
+    const scoped_refptr<base::SingleThreadTaskRunner> ipc_task_runner)
+    : device_descriptor_(device_descriptor),
+      camera_hal_delegate_(camera_hal_delegate),
+      state_(kStopped),
+      rotation_(0),
+      callback_ops_(this),
+      ipc_task_runner_(ipc_task_runner),
+      frame_number_(0),
+      partial_result_count_(1),
+      first_frame_shutter_time_(base::TimeTicks::Now()) {}
+
+// static
+VideoPixelFormat CameraDeviceDelegate::PixFormatHalToChromium(
+    arc::mojom::HalPixelFormat from) {
+  auto it =
+      std::find_if(std::begin(kSupportedFormats), std::end(kSupportedFormats),
+                   [from](SupportedFormat f) { return f.hal_format == from; });
+  if (it == std::end(kSupportedFormats)) {
+    return PIXEL_FORMAT_UNKNOWN;
+  }
+  return it->chromium_format;
+}
+
+// static
+uint32_t CameraDeviceDelegate::PixFormatChromiumToDrm(VideoPixelFormat from) {
+  auto it = std::find_if(
+      std::begin(kSupportedFormats), std::end(kSupportedFormats),
+      [from](SupportedFormat f) { return f.chromium_format == from; });
+  if (it == std::end(kSupportedFormats)) {
+    return 0;
+  }
+  return it->drm_format;
+}
+
+void CameraDeviceDelegate::AllocateAndStart(
+    const VideoCaptureParams& params,
+    std::unique_ptr<VideoCaptureDevice::Client> client) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(!client_);
+  DCHECK_EQ(state_, kStopped);
+
+  chrome_capture_params_ = params;
+  client_ = std::move(client);
+  frame_number_ = 0;
+  partial_results_.clear();
+  first_frame_shutter_time_ = base::TimeTicks::Now();
+  SetState(kStarting);
+
+  int32_t camera_id = std::stoi(device_descriptor_.device_id);
+  // We need to get the static camera metadata of the camera deivce first.
+  camera_hal_delegate_->GetCameraInfo(
+      camera_id,
+      base::Bind(&CameraDeviceDelegate::OnGotCameraInfoOnModuleDelegate, this));

[wuchengli, 2017/05/23 04:29:07]

There seems to be a utility macro of function to bind the thread when calling
this.

[jcliang, 2017/05/25 14:23:46]

Found it: https://cs.chromium.org/chromium/src/media/base/bind_to_current_loop.h

+}
+
+void CameraDeviceDelegate::StopAndDeAllocate() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  // StopAndDeAllocate may be called at any state except kStopping.
+  DCHECK_NE(state_, kStopping);
+
+  if (state_ == kStopped) {
+    // In case of Mojo connection error the device may be stopped before
+    // StopAndDeAllocate is called.
+    return;
+  }
+
+  SetState(kStopping);
+  if (!device_ops_.is_bound()) {
+    // The device delegate is in the process of opening the camera device.
+    return;
+  }
+  device_ops_->Close(base::Bind(&CameraDeviceDelegate::OnClosed, this));
+}
+
+void CameraDeviceDelegate::TakePhoto(
+    VideoCaptureDevice::TakePhotoCallback callback) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  // TODO(jcliang): Implement TakePhoto.
+}
+
+void CameraDeviceDelegate::GetPhotoCapabilities(
+    VideoCaptureDevice::GetPhotoCapabilitiesCallback callback) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  // TODO(jcliang): Implement GetPhotoCapabilities.
+}
+
+void CameraDeviceDelegate::SetPhotoOptions(
+    mojom::PhotoSettingsPtr settings,
+    VideoCaptureDevice::SetPhotoOptionsCallback callback) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  // TODO(jcliang): Implement SetPhotoOptions.
+}
+
+void CameraDeviceDelegate::SetRotation(int rotation) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(rotation >= 0 && rotation < 360 && rotation % 90 == 0);
+  rotation_ = rotation;
+}
+
+void CameraDeviceDelegate::SetState(State state) {
+  state_ = state;
+}
+
+void CameraDeviceDelegate::SetErrorState(
+    const tracked_objects::Location& from_here,
+    const std::string& reason) {
+  state_ = kError;
+  LOG(ERROR) << reason;
+  client_->OnError(from_here, reason);
+}
+
+void CameraDeviceDelegate::ResetMojoInterface() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  device_ops_.reset();
+  if (callback_ops_.is_bound()) {
+    callback_ops_.Unbind();
+  }
+}
+
+void CameraDeviceDelegate::OnMojoConnectionError() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  if (state_ == kStopping) {
+    // When in stopping state the camera HAL adapter may terminate the Mojo
+    // channel before we do, in which case the OnClosed callback would not be
+    // called.
+    OnClosed(0);
+  } else {
+    // The Mojo channel terminated unexpectedly.
+    ResetMojoInterface();
+    SetState(kStopped);
+    SetErrorState(FROM_HERE, "Mojo connection error");
+  }
+}
+
+void CameraDeviceDelegate::OnClosed(int32_t result) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK_EQ(state_, kStopping);
+  if (result) {
+    client_->OnLog(std::string("Failed to close device: ") +
+                   std::string(strerror(result)));
+  }
+  ResetMojoInterface();
+  // Only after the Mojo channel is closed can we be sure that |stream_context_|
+  // is not accessed anymore.
+  stream_context_.reset();
+  client_.reset();
+  SetState(kStopped);
+}
+
+void CameraDeviceDelegate::OnGotCameraInfoOnModuleDelegate(
+    int32_t result,
+    arc::mojom::CameraInfoPtr camera_info) {
+  // This method runs on |module_task_runner_| of |camera_hal_delegate_|.
+  ipc_task_runner_->PostTask(
+      FROM_HERE, base::Bind(&CameraDeviceDelegate::OnGotCameraInfo, this,
+                            result, base::Passed(&camera_info)));
+}
+
+void CameraDeviceDelegate::OnGotCameraInfo(
+    int32_t result,
+    arc::mojom::CameraInfoPtr camera_info) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kStarting || state_ == kStopping);
+
+  if (state_ == kStopping) {
+    OnClosed(0);
+    return;
+  }
+
+  if (result) {
+    std::string error_msg = "Failed to get camera info";
+    LOG(ERROR) << error_msg;
+    client_->OnError(FROM_HERE, error_msg);
+    return;
+  }
+  static_metadata_ = std::move(camera_info->static_camera_characteristics);
+  const arc::mojom::CameraMetadataEntryPtr* partial_count = GetMetadataEntry(
+      static_metadata_,
+      arc::mojom::CameraMetadataTag::ANDROID_REQUEST_PARTIAL_RESULT_COUNT);
+  // The partial result count metadata is optional; defaults to 1 in case it
+  // is not set in the static metadata.
+  if (partial_count) {
+    partial_result_count_ =
+        *reinterpret_cast<int32_t*>((*partial_count)->data.data());
+  }
+  int32_t camera_id = std::stoi(device_descriptor_.device_id);
+  camera_hal_delegate_->OpenDevice(
+      camera_id,
+      base::Bind(&CameraDeviceDelegate::OnOpenedDeviceOnModuleDelegate, this));
+}
+
+void CameraDeviceDelegate::OnOpenedDeviceOnModuleDelegate(
+    int32_t result,
+    arc::mojom::Camera3DeviceOpsPtr device_ops) {
+  // This method runs on |module_task_runner_| of |camera_hal_delegate_|.
+  mojo::InterfacePtrInfo<arc::mojom::Camera3DeviceOps> device_ops_info =
+      device_ops.PassInterface();
+  ipc_task_runner_->PostTask(
+      FROM_HERE, base::Bind(&CameraDeviceDelegate::OnOpenedDevice, this, result,
+                            base::Passed(&device_ops_info)));
+}
+
+void CameraDeviceDelegate::OnOpenedDevice(
+    int32_t result,
+    mojo::InterfacePtrInfo<arc::mojom::Camera3DeviceOps> device_ops_info) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kStarting || state_ == kStopping);
+
+  if (state_ == kStopping) {

[wuchengli, 2017/05/23 04:29:07]

Document mojo will disconnect because we don't keep a reference to it.

[jcliang, 2017/05/25 14:23:46]

Done.

+    OnClosed(0);
+    return;
+  }
+
+  if (result) {
+    std::string error_msg = "Failed to open camera device";
+    LOG(ERROR) << error_msg;
+    client_->OnError(FROM_HERE, error_msg);
+    return;
+  }
+  if (!device_ops_info.is_valid()) {
+    std::string error_msg = "Invalid device_ops_info";
+    LOG(ERROR) << error_msg;
+    client_->OnError(FROM_HERE, error_msg);
+    return;
+  }
+  device_ops_.Bind(std::move(device_ops_info));
+  device_ops_.set_connection_error_handler(
+      base::Bind(&CameraDeviceDelegate::OnMojoConnectionError, this));
+  Initialize();
+}
+
+void CameraDeviceDelegate::Initialize() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK_EQ(state_, kStarting);
+
+  // Set up context for preview stream.
+  arc::mojom::Camera3StreamPtr preview_stream =
+      arc::mojom::Camera3Stream::New();
+  preview_stream->id = static_cast<uint64_t>(
+      arc::mojom::Camera3RequestTemplate::CAMERA3_TEMPLATE_PREVIEW);
+  preview_stream->stream_type =
+      arc::mojom::Camera3StreamType::CAMERA3_STREAM_OUTPUT;
+  preview_stream->width =
+      chrome_capture_params_.requested_format.frame_size.width();
+  preview_stream->height =
+      chrome_capture_params_.requested_format.frame_size.height();
+  preview_stream->format =
+      arc::mojom::HalPixelFormat::HAL_PIXEL_FORMAT_YCbCr_420_888;
+  preview_stream->data_space = 0;
+  preview_stream->rotation =
+      arc::mojom::Camera3StreamRotation::CAMERA3_STREAM_ROTATION_0;
+
+  stream_context_.reset(new StreamContext);
+  stream_context_->stream = std::move(preview_stream);
+
+  device_ops_->Initialize(
+      callback_ops_.CreateInterfacePtrAndBind(),
+      base::Bind(&CameraDeviceDelegate::OnInitialized, this));
+  callback_ops_.set_connection_error_handler(
+      base::Bind(&CameraDeviceDelegate::OnMojoConnectionError, this));
+}
+
+void CameraDeviceDelegate::OnInitialized(int32_t result) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kStarting || state_ == kStopping);
+
+  if (state_ == kStopping) {
+    return;
+  }
+  if (result) {
+    SetErrorState(FROM_HERE, std::string("Failed to initialize camera device") +
+                                 std::to_string(result));
+    return;
+  }
+  SetState(kInitialized);
+  ConfigureStreams();
+}
+
+void CameraDeviceDelegate::ConfigureStreams() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kInitialized);
+
+  arc::mojom::Camera3StreamConfigurationPtr stream_config =
+      arc::mojom::Camera3StreamConfiguration::New();
+  stream_config->streams.push_back(stream_context_->stream.Clone());
+  stream_config->operation_mode = arc::mojom::Camera3StreamConfigurationMode::
+      CAMERA3_STREAM_CONFIGURATION_NORMAL_MODE;
+  device_ops_->ConfigureStreams(
+      std::move(stream_config),
+      base::Bind(&CameraDeviceDelegate::OnConfiguredStreams, this));
+}
+
+void CameraDeviceDelegate::OnConfiguredStreams(
+    int32_t result,
+    arc::mojom::Camera3StreamConfigurationPtr updated_config) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kInitialized || state_ == kStopping);

[wuchengli, 2017/05/23 04:29:07]

Discussed offline. Having DCHECK depends on the implementation of HAL to
serialize IPCs. For example, ConfigureStreams and Close. So CameraDeviceDelegate
won't get OnConfigureStreams after OnClose. We can change the code to
if (state_ != kInitialized && state_ != kStopping) return;
So this doesn't depend on HAL implementation. Also, it's better not to have any
chance of crashing browser process.

[jcliang, 2017/05/25 14:23:46]

Done.

+
+  if (state_ == kStopping) {
+    return;
+  }
+  if (result) {
+    SetErrorState(FROM_HERE, std::string("Failed to configure streams") +
+                                 std::to_string(result));
+    return;
+  }
+  if (!updated_config || updated_config->streams.size() != 1) {
+    SetErrorState(FROM_HERE,
+                  std::string("Wrong number of streams configured") +
+                      std::to_string(updated_config->streams.size()));
+    return;
+  }
+  auto& updated_stream = updated_config->streams[0];
+  VideoCaptureFormat capture_format = chrome_capture_params_.requested_format;
+  // TODO(jcliang): Determine the best format from metadata.
+  capture_format.pixel_format = PIXEL_FORMAT_NV12;
+  stream_context_->capture_format = capture_format;
+  stream_context_->stream->usage = updated_stream->usage;
+  stream_context_->stream->max_buffers = updated_stream->max_buffers;

[wuchengli, 2017/05/25 07:59:06]

Let's have a limit here in case HAL has a bug and give us a very huge number.

[jcliang, 2017/05/25 14:23:46]

Done.

+
+  VLOG(2) << "Stream " << updated_stream->id
+          << " configured: usage=" << updated_stream->usage
+          << " max_buffers=" << updated_stream->max_buffers;
+
+  // Allocate buffers.
+  size_t num_buffers = stream_context_->stream->max_buffers;
+  stream_context_->buffers.resize(num_buffers);
+  for (size_t j = 0; j < num_buffers; ++j) {
+    const VideoCaptureFormat frame_format(
+        gfx::Size(stream_context_->stream->width,
+                  stream_context_->stream->height),
+        0.0, stream_context_->capture_format.pixel_format);
+    std::unique_ptr<base::SharedMemory> buffer(new base::SharedMemory());
+    base::SharedMemoryCreateOptions options;
+    options.size = frame_format.ImageAllocationSize();
+    options.share_read_only = false;
+    bool ret = buffer->Create(options);
+    if (!ret) {
+      SetErrorState(FROM_HERE, "Failed to create SharedMemory buffer");
+      return;
+    }
+    ret = buffer->Map(buffer->requested_size());
+    if (!ret) {
+      SetErrorState(FROM_HERE, "Failed to map SharedMemory buffer");
+      return;
+    }
+    stream_context_->buffers[j] = std::move(buffer);
+    stream_context_->free_buffers.push(j);
+  }
+  VLOG(2) << "Allocated " << stream_context_->stream->max_buffers << " buffers";
+  ConstructDefaultRequestSettings();
+  client_->OnStarted();
+}
+
+void CameraDeviceDelegate::ConstructDefaultRequestSettings() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(stream_context_);
+
+  device_ops_->ConstructDefaultRequestSettings(
+      arc::mojom::Camera3RequestTemplate::CAMERA3_TEMPLATE_PREVIEW,
+      base::Bind(&CameraDeviceDelegate::OnConstructedDefaultRequestSettings,
+                 this));
+}
+
+void CameraDeviceDelegate::OnConstructedDefaultRequestSettings(
+    arc::mojom::CameraMetadataPtr settings) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+
+  if (state_ == kStopping) {
+    return;
+  }
+  DCHECK(stream_context_);
+  stream_context_->request_settings = std::move(settings);
+  SetState(kStreamConfigured);
+  StartCapture();
+}
+
+void CameraDeviceDelegate::StartCapture() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kStreamConfigured);
+
+  if (state_ == kStopping) {
+    return;
+  }
+  DCHECK(stream_context_);
+  DCHECK(!stream_context_->request_settings.is_null());
+  SetState(kCapturing);

[wuchengli, 2017/05/25 07:59:06]

Document we cannot use a loop to register all the buffers here because halv3 API
says we cannot call ProcessCaptureRequest if the previous one hasn't returned
yet.

[jcliang, 2017/05/25 14:23:46]

Done.

+  RegisterBuffer();
+}
+
+void CameraDeviceDelegate::RegisterBuffer() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kCapturing || state_ == kStopping);
+
+  if (state_ == kStopping) {
+    return;
+  }
+  DCHECK(stream_context_);
+  if (stream_context_->free_buffers.empty()) {
+    return;
+  }
+
+  size_t buffer_id = stream_context_->free_buffers.front();
+  stream_context_->free_buffers.pop();
+  const base::SharedMemory* buffer = stream_context_->buffers[buffer_id].get();
+
+  VideoPixelFormat buffer_format = stream_context_->capture_format.pixel_format;
+  uint32_t drm_format = PixFormatChromiumToDrm(buffer_format);
+  if (!drm_format) {
+    SetErrorState(FROM_HERE, std::string("Unsupported video pixel format") +
+                                 VideoPixelFormatToString(buffer_format));
+    return;
+  }
+  arc::mojom::HalPixelFormat hal_pixel_format = stream_context_->stream->format;
+
+  size_t num_planes = VideoFrame::NumPlanes(buffer_format);
+  std::vector<mojo::ScopedHandle> fds(num_planes);
+  std::vector<uint32_t> strides(num_planes);
+  std::vector<uint32_t> offsets(num_planes);
+  for (size_t i = 0; i < num_planes; ++i) {
+    base::SharedMemoryHandle shm_handle = buffer->handle();
+    // Wrap the platform handle.
+    MojoHandle wrapped_handle;
+    MojoResult result = mojo::edk::CreatePlatformHandleWrapper(
+        mojo::edk::ScopedPlatformHandle(mojo::edk::PlatformHandle(
+            base::SharedMemory::DuplicateHandle(shm_handle).GetHandle())),
+        &wrapped_handle);
+    if (result != MOJO_RESULT_OK) {
+      SetErrorState(FROM_HERE, "Failed to wrap shared memory handle");
+      return;
+    }
+    fds[i].reset(mojo::Handle(wrapped_handle));
+    strides[i] = VideoFrame::RowBytes(
+        i, buffer_format,
+        chrome_capture_params_.requested_format.frame_size.width());
+    if (!i) {
+      offsets[i] = 0;
+    } else {
+      offsets[i] = offsets[i - 1] +
+                   VideoFrame::PlaneSize(
+                       buffer_format, i - 1,
+                       chrome_capture_params_.requested_format.frame_size)
+                       .GetArea();
+    }
+  }
+  device_ops_->RegisterBuffer(
+      buffer_id, arc::mojom::Camera3DeviceOps::BufferType::SHM, std::move(fds),
+      drm_format, hal_pixel_format, stream_context_->stream->width,
+      stream_context_->stream->height, std::move(strides), std::move(offsets),
+      base::Bind(&CameraDeviceDelegate::OnRegisteredBuffer, this, buffer_id));
+  VLOG(2) << "Registered buffer " << buffer_id;
+}
+
+void CameraDeviceDelegate::OnRegisteredBuffer(size_t buffer_id,
+                                              int32_t result) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kCapturing || state_ == kStopping);
+
+  if (state_ == kStopping) {
+    return;
+  }
+  if (result) {
+    SetErrorState(FROM_HERE, std::string("Failed to register buffer: ") +
+                                 std::to_string(result));
+    return;
+  }
+  ProcessCaptureRequest(buffer_id);
+}
+
+void CameraDeviceDelegate::ProcessCaptureRequest(size_t buffer_id) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kCapturing || state_ == kStopping);
+  DCHECK(stream_context_);
+
+  arc::mojom::Camera3StreamBufferPtr buffer =
+      arc::mojom::Camera3StreamBuffer::New();
+  buffer->stream_id = static_cast<uint64_t>(
+      arc::mojom::Camera3RequestTemplate::CAMERA3_TEMPLATE_PREVIEW);
+  buffer->buffer_id = buffer_id;
+  buffer->status = arc::mojom::Camera3BufferStatus::CAMERA3_BUFFER_STATUS_OK;
+
+  arc::mojom::Camera3CaptureRequestPtr request =
+      arc::mojom::Camera3CaptureRequest::New();
+  request->frame_number = frame_number_;
+  request->settings = stream_context_->request_settings.Clone();
+  request->output_buffers.push_back(std::move(buffer));
+
+  device_ops_->ProcessCaptureRequest(
+      std::move(request),
+      base::Bind(&CameraDeviceDelegate::OnProcessedCaptureRequest, this));
+  VLOG(2) << "Requested capture for frame " << frame_number_ << " with buffer "
+          << buffer_id;
+  frame_number_++;
+  // In case |frame_number_| wraps around, we start at 1 to avoid resetting
+  // |first_frame_shutter_time_|.
+  if (!frame_number_) {
+    frame_number_++;
+  }
+}
+
+void CameraDeviceDelegate::OnProcessedCaptureRequest(int32_t result) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kCapturing || state_ == kStopping);
+
+  if (state_ == kStopping) {
+    return;
+  }
+  if (result) {
+    SetErrorState(FROM_HERE, std::string("Process capture request failed") +
+                                 std::to_string(result));
+    return;
+  }
+  RegisterBuffer();
+}
+
+void CameraDeviceDelegate::ProcessCaptureResult(
+    arc::mojom::Camera3CaptureResultPtr result) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+

[wuchengli, 2017/05/23 04:29:07]

return if state is kStopping or kStopped

[jcliang, 2017/05/25 14:23:46]

Done.

+  uint32_t frame_number = result->frame_number;
+  // A new partial result may be created in either ProcessCaptureResult or
+  // Notify.
+  CaptureResult& partial_result = partial_results_[frame_number];

[wuchengli, 2017/05/25 07:59:05]

Print an error or error out if the number of partial_results_ is bigger than the
number of buffers.

[jcliang, 2017/05/25 14:23:46]

Done.

+  if (result->output_buffers) {
+    if (result->output_buffers->size() != 1) {
+      SetErrorState(
+          FROM_HERE,
+          std::string("Incorrect number of output buffers received: ") +
+              std::to_string(result->output_buffers->size()));
+      return;
+    }
+    arc::mojom::Camera3StreamBufferPtr& stream_buffer =
+        result->output_buffers.value()[0];
+    VLOG(2) << "Received capture result for frame " << frame_number
+            << " stream_id: " << stream_buffer->stream_id;
+    // The camera HAL v3 API specifies that only one capture result can carry
+    // the result buffer for any given frame number.
+    if (!partial_result.buffer.is_null()) {
+      SetErrorState(FROM_HERE,
+                    std::string("Received multiple result buffers for frame ") +
+                        std::to_string(frame_number));
+      return;
+    } else {
+      partial_result.buffer = std::move(stream_buffer);
+    }
+  }
+
+  // |result->partial_result| is set to 0 if the capture result contains only
+  // the result buffer handles and no result metadata.
+  if (result->partial_result) {
+    uint32_t result_id = result->partial_result;
+    if (result_id > partial_result_count_) {
+      SetErrorState(FROM_HERE, std::string("Invalid partial_result id: ") +
+                                   std::to_string(result_id));
+      return;
+    }
+    if (partial_result.partial_metadata_received.find(result_id) !=
+        partial_result.partial_metadata_received.end()) {
+      SetErrorState(FROM_HERE,
+                    std::string("Received duplicated partial metadata: ") +
+                        std::to_string(result_id));
+      return;
+    }
+    partial_result.partial_metadata_received.insert(result_id);
+    MergeMetadata(&partial_result.metadata, result->result);
+  }
+
+  if (partial_result.partial_metadata_received.size() ==
+          partial_result_count_ &&
+      !partial_result.buffer.is_null()) {
+    // We can only submit the result buffer after we receive the shutter time.
+    if (partial_result.reference_time != base::TimeTicks()) {
+      SubmitCaptureResult(frame_number);
+    }
+  }
+}
+
+void CameraDeviceDelegate::Notify(arc::mojom::Camera3NotifyMsgPtr message) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+

[wuchengli, 2017/05/23 04:29:07]

return if state is kStopping or kStopped

[jcliang, 2017/05/25 14:23:46]

Done.

+  if (message->type == arc::mojom::Camera3MsgType::CAMERA3_MSG_ERROR) {

[wuchengli, 2017/05/25 07:59:06]

This function is long. Line 633-695 can be moved to a function.

[jcliang, 2017/05/25 14:23:46]

Done.

+    uint32_t frame_number = message->message->get_error()->frame_number;
+    uint64_t error_stream_id = message->message->get_error()->error_stream_id;
+    arc::mojom::Camera3ErrorMsgCode error_code =
+        message->message->get_error()->error_code;
+    switch (error_code) {
+      case arc::mojom::Camera3ErrorMsgCode::CAMERA3_MSG_ERROR_DEVICE:
+        // Fatal error and no more frames will be produced by the device.
+        SetErrorState(FROM_HERE, "Fatal device error");
+        break;
+      case arc::mojom::Camera3ErrorMsgCode::CAMERA3_MSG_ERROR_REQUEST: {
+        // An error has occurred in processing the request; the request
+        // specified by |frame_number| has been dropped by the camera device.
+        // Subsequent requests are unaffected.
+        //
+        // The HAL will call ProcessCaptureResult with the buffers' state set to
+        // STATUS_ERROR.  The content of the buffers will be dropped and the
+        // buffers will be reused in SubmitCaptureResult.
+        std::string warning_msg =
+            std::string(
+                "An error occurred while processing request for frame ") +
+            std::to_string(frame_number);
+        LOG(WARNING) << warning_msg;
+        client_->OnLog(warning_msg);
+        break;
+      }
+      case arc::mojom::Camera3ErrorMsgCode::CAMERA3_MSG_ERROR_RESULT: {
+        // An error has occurred in producing the output metadata buffer for a
+        // result; the output metadata will not be available for the frame
+        // specified by |frame_number|.  Subsequent requests are unaffected.
+        std::string warning_msg =
+            std::string(
+                "An error occurred while producing result "
+                "metadata for frame ") +
+            std::to_string(frame_number);
+        LOG(WARNING) << warning_msg;
+        client_->OnLog(warning_msg);
+        // The result metadata will not be complete so we don't need to wait for
+        // partial results on frame |frame_number|.
+        partial_results_[frame_number].partial_metadata_received.clear();
+        for (uint32_t i = 0; i < partial_result_count_; ++i) {
+          partial_results_[frame_number].partial_metadata_received.insert(i);
+        }
+        break;
+      }
+      case arc::mojom::Camera3ErrorMsgCode::CAMERA3_MSG_ERROR_BUFFER:
+        // An error has occurred in placing the output buffer into a stream for
+        // a request. |frame_number| specifies the request for which the buffer
+        // was dropped, and |error_stream_id| specifies the stream that dropped
+        // the buffer.
+        //
+        // The HAL will call ProcessCaptureResult with the buffer's state set to
+        // STATUS_ERROR.  The content of the buffer will be dropped and the
+        // buffer will be reused in SubmitCaptureResult.
+        client_->OnLog(
+            std::string(
+                "An error occurred while filling output buffer of stream ") +
+            std::to_string(error_stream_id) + std::string(" in frame ") +
+            std::to_string(frame_number));
+        break;
+      default:
+        // To eliminate the warning for not handling CAMERA3_MSG_NUM_ERRORS
+        break;
+    }
+  } else {  // arc::mojom::Camera3MsgType::CAMERA3_MSG_SHUTTER
+    uint32_t frame_number = message->message->get_shutter()->frame_number;
+    uint64_t shutter_time = message->message->get_shutter()->timestamp;
+    // A new partial result may be created in either ProcessCaptureResult or
+    // Notify.
+    VLOG(2) << "Received shutter time for frame " << frame_number;
+    if (!shutter_time) {
+      SetErrorState(FROM_HERE, std::string("Received invalid shutter time: ") +
+                                   std::to_string(shutter_time));
+      return;
+    }
+    CaptureResult& partial_result = partial_results_[frame_number];

[wuchengli, 2017/05/25 07:59:06]

Print an error or error out if the number of partial_results_ is bigger than the
number of buffers.

[jcliang, 2017/05/25 14:23:46]

Done.

+    // Shutter timestamp is in ns.
+    base::TimeTicks reference_time =
+        base::TimeTicks::FromInternalValue(shutter_time / 1000);
+    partial_result.reference_time = reference_time;
+    if (!frame_number) {
+      // Record the shutter time of the first frame for calculating the
+      // timestamp.
+      first_frame_shutter_time_ = reference_time;
+    }
+    partial_result.timestamp = reference_time - first_frame_shutter_time_;
+    if (partial_result.partial_metadata_received.size() ==
+            partial_result_count_ &&
+        !partial_result.buffer.is_null()) {
+      SubmitCaptureResult(frame_number);
+    }
+  }
+}
+
+void CameraDeviceDelegate::SubmitCaptureResult(uint32_t frame_number) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+
+  if (partial_results_.begin()->first != frame_number) {
+    SetErrorState(FROM_HERE, "Received out-of-order frames from HAL");

[wuchengli, 2017/05/25 07:59:06]

print partial_results_.begin()->first and frame_number

[jcliang, 2017/05/25 14:23:46]

Done.

+    return;
+  }
+
+  VLOG(2) << "Submit capture result of frame " << frame_number;
+  CaptureResult& partial_result = partial_results_[frame_number];
+  DCHECK_EQ(partial_result.partial_metadata_received.size(),
+            partial_result_count_);
+  DCHECK(partial_result.buffer);
+  uint32_t buffer_id = partial_result.buffer->buffer_id;
+
+  // Wait on release fence before delivering the result buffer to client.
+  if (partial_result.buffer->release_fence.is_valid()) {
+    const int kSyncWaitTimeoutMs = 1000;
+    mojo::edk::ScopedPlatformHandle fence;
+    MojoResult result = mojo::edk::PassWrappedPlatformHandle(
+        partial_result.buffer->release_fence.release().value(), &fence);
+    if (result != MOJO_RESULT_OK) {
+      SetErrorState(FROM_HERE, "Failed to unwrap release fence fd");
+      return;
+    }
+    if (!sync_wait(fence.get().handle, kSyncWaitTimeoutMs)) {
+      SetErrorState(FROM_HERE, "Sync wait on release fence timed out");
+      return;
+    }
+  }
+
+  // Deliver the captured data to client and then re-queue the buffer.
+  if (partial_result.buffer->status !=
+      arc::mojom::Camera3BufferStatus::CAMERA3_BUFFER_STATUS_ERROR) {
+    const base::SharedMemory* shm_buffer =
+        stream_context_->buffers[buffer_id].get();
+    client_->OnIncomingCapturedData(
+        reinterpret_cast<uint8_t*>(shm_buffer->memory()),
+        shm_buffer->mapped_size(), stream_context_->capture_format, rotation_,
+        partial_result.reference_time, partial_result.timestamp);
+  }
+  stream_context_->free_buffers.push(buffer_id);
+  partial_results_.erase(frame_number);
+  RegisterBuffer();
+}
+
+}  // namespace media