media: add video capture device for ARC++ camera HAL v3

media/capture/video/chromeos/camera_device_delegate.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/capture/video/chromeos/camera_device_delegate.h"
+
+#include <libdrm/drm_fourcc.h>
+
+#include "media/capture/video/chromeos/camera_metadata_utils.h"
+#include "mojo/edk/embedder/embedder.h"
+#include "mojo/edk/embedder/scoped_platform_handle.h"
+#include "third_party/libsync/include/sync/sync.h"
+
+namespace media {
+
+namespace {
+
+struct SupportedFormat {
+  VideoPixelFormat chromium_format;
+  arc::mojom::HalPixelFormat hal_format;
+  uint32_t drm_format;
+} const kSupportedFormats[] = {
+    // The Android camera HAL v3 has three types of mandatory pixel formats:
+    //
+    //   1. HAL_PIXEL_FORMAT_YCbCr_420_888 (YUV flexible format).
+    //   2. HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED (platform-specific format).
+    //   3. HAL_PIXEL_FORMAT_BLOB (for JPEG).
+    //
+    // We can't use HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED as it is highly
+    // platform specific and there is no way for Chrome to query the exact
+    // pixel layout of the implementation-defined buffer.
+    //
+    // On Android the framework requests the preview stream with the
+    // implementation-defined format, and as a result some camera HALs support
+    // only implementation-defined preview buffers.  We should use the video
+    // capture stream in Chrome VCD as it is mandatory for the camera HAL to
+    // support YUV flexbile format video streams.
+    {PIXEL_FORMAT_I420,
+     arc::mojom::HalPixelFormat::HAL_PIXEL_FORMAT_YCbCr_420_888,
+     DRM_FORMAT_YUV420},
+    // TODO(jcliang): Do not use IMPLEMENTATION_DEFINED formats at all as it is
+    //                nearly impossible to get it right across all boards.
+    {PIXEL_FORMAT_NV12,
+     arc::mojom::HalPixelFormat::HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED,
+     DRM_FORMAT_NV12},
+};
+
+}  // namespace
+
+CameraDeviceDelegate::CameraDeviceDelegate(
+    VideoCaptureDeviceDescriptor device_descriptor,
+    arc::mojom::CameraMetadataPtr static_metadata,
+    mojo::InterfacePtrInfo<arc::mojom::Camera3DeviceOps> device_ops_info,
+    const scoped_refptr<base::SingleThreadTaskRunner> ipc_task_runner)
+    : device_descriptor_(device_descriptor),
+      static_metadata_(std::move(static_metadata)),
+      state_(kStopped),
+      closed_(nullptr),
+      rotation_(0),
+      device_ops_info_(std::move(device_ops_info)),
+      callback_ops_(this),
+      ipc_task_runner_(ipc_task_runner),
+      frame_number_(0),
+      partial_result_count_(1),
+      first_frame_shutter_time_(base::TimeTicks::Now()) {}
+
+// static
+VideoPixelFormat CameraDeviceDelegate::PixFormatHalToChromium(
+    arc::mojom::HalPixelFormat from) {
+  auto it =
+      std::find_if(std::begin(kSupportedFormats), std::end(kSupportedFormats),
+                   [from](SupportedFormat f) { return f.hal_format == from; });
+  if (it == std::end(kSupportedFormats)) {
+    return PIXEL_FORMAT_UNKNOWN;
+  }
+  return it->chromium_format;
+}
+
+// static
+uint32_t CameraDeviceDelegate::PixFormatChromiumToDrm(VideoPixelFormat from) {
+  auto it = std::find_if(
+      std::begin(kSupportedFormats), std::end(kSupportedFormats),
+      [from](SupportedFormat f) { return f.chromium_format == from; });
+  if (it == std::end(kSupportedFormats)) {
+    return 0;
+  }
+  return it->drm_format;
+}
+
+void CameraDeviceDelegate::AllocateAndStart(
+    const VideoCaptureParams& params,
+    std::unique_ptr<VideoCaptureDevice::Client> client) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(!client_);
+  DCHECK_EQ(state_, kStopped);
+  const arc::mojom::CameraMetadataEntryPtr* partial_count = GetMetadataEntry(
+      static_metadata_,
+      arc::mojom::CameraMetadataTag::ANDROID_REQUEST_PARTIAL_RESULT_COUNT);
+  // The partial result count metadata is optional. It defaults to 1 in case it
+  // is not set in the static metadata.
+  if (partial_count) {
+    partial_result_count_ =
+        *reinterpret_cast<int32_t*>((*partial_count)->data.data());
+  }
+
+  client_ = std::move(client);
+  device_ops_.Bind(std::move(device_ops_info_), ipc_task_runner_);
+  device_ops_.set_connection_error_handler(
+      base::Bind(&CameraDeviceDelegate::OnMojoConnectionError, this));
+  frame_number_ = 0;
+  partial_results_.clear();
+  first_frame_shutter_time_ = base::TimeTicks::Now();
+
+  // Set up context for preview stream.
+  arc::mojom::Camera3StreamPtr preview_stream =
+      arc::mojom::Camera3Stream::New();
+  preview_stream->id = static_cast<uint64_t>(
+      arc::mojom::Camera3RequestTemplate::CAMERA3_TEMPLATE_PREVIEW);
+  preview_stream->stream_type =
+      arc::mojom::Camera3StreamType::CAMERA3_STREAM_OUTPUT;
+  preview_stream->width = params.requested_format.frame_size.width();
+  preview_stream->height = params.requested_format.frame_size.height();
+  // TODO(jcliang): We should not use implementation defined format here.
+  preview_stream->format =
+      arc::mojom::HalPixelFormat::HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
+  preview_stream->data_space = 0;
+  preview_stream->rotation =
+      arc::mojom::Camera3StreamRotation::CAMERA3_STREAM_ROTATION_0;
+
+  stream_context_.reset(new StreamContext);
+  stream_context_->params = params;
+  stream_context_->stream = std::move(preview_stream);
+
+  SetState(kStarting);
+  Initialize();
+}
+
+void CameraDeviceDelegate::StopAndDeAllocate(base::WaitableEvent* closed) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  // StopAndDeAllocate may be called at any state except kStopping.
+  DCHECK_NE(state_, kStopping);
+
+  if (!device_ops_.is_bound() || state_ == kStopped) {
+    // In case of Mojo connection error |device_ops_| and |callback_ops_| are
+    // unbound.
+    closed->Signal();
+    return;
+  }
+  closed_ = closed;
+  SetState(kStopping);
+  device_ops_->Close(base::Bind(&CameraDeviceDelegate::OnClosed, this));
+}
+
+void CameraDeviceDelegate::TakePhoto(
+    VideoCaptureDevice::TakePhotoCallback callback) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  // TODO(jcliang): Implement TakePhoto.
+}
+
+void CameraDeviceDelegate::GetPhotoCapabilities(
+    VideoCaptureDevice::GetPhotoCapabilitiesCallback callback) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  // TODO(jcliang): Implement GetPhotoCapabilities.
+}
+
+void CameraDeviceDelegate::SetPhotoOptions(
+    mojom::PhotoSettingsPtr settings,
+    VideoCaptureDevice::SetPhotoOptionsCallback callback) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  // TODO(jcliang): Implement SetPhotoOptions.
+}
+
+void CameraDeviceDelegate::SetRotation(int rotation) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(rotation >= 0 && rotation < 360 && rotation % 90 == 0);
+  rotation_ = rotation;
+}
+
+void CameraDeviceDelegate::SetState(State state) {
+  state_ = state;
+}
+
+void CameraDeviceDelegate::SetErrorState(
+    const tracked_objects::Location& from_here,
+    const std::string& reason) {
+  state_ = kError;
+  LOG(ERROR) << reason;
+  client_->OnError(from_here, reason);
+}
+
+void CameraDeviceDelegate::ResetMojoInterface() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  device_ops_.reset();
+  if (callback_ops_.is_bound()) {
+    callback_ops_.Unbind();
+  }
+}
+
+void CameraDeviceDelegate::OnMojoConnectionError() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  if (state_ == kStopping) {
+    // When in stopping state the camera HAL adapter may terminate the Mojo
+    // channel before we do, in which case the OnClosed callback would not be
+    // called.
+    OnClosed(0);
+  } else {
+    // The Mojo channel terminated unexpectedly.
+    ResetMojoInterface();
+    SetState(kStopped);
+    SetErrorState(FROM_HERE, "Mojo connection error");
+  }
+}
+
+void CameraDeviceDelegate::OnClosed(int32_t result) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(closed_);
+  if (result) {
+    client_->OnLog(std::string("Failed to close device: ") +
+                   std::string(strerror(result)));
+  }

[wuchengli, 2017/05/17 03:40:12]

Clear stream_context_ here. Document why we should not clear it in
StopAndDeAllocate.

[jcliang, 2017/05/17 07:32:10]

Done.

+  ResetMojoInterface();
+  client_.reset();
+  SetState(kStopped);
+  closed_->Signal();
+  closed_ = nullptr;
+}
+
+void CameraDeviceDelegate::Initialize() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK_EQ(state_, kStarting);
+
+  device_ops_->Initialize(
+      callback_ops_.CreateInterfacePtrAndBind(),
+      base::Bind(&CameraDeviceDelegate::OnInitialized, this));
+  callback_ops_.set_connection_error_handler(
+      base::Bind(&CameraDeviceDelegate::OnMojoConnectionError, this));
+}
+
+void CameraDeviceDelegate::OnInitialized(int32_t result) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kStarting || state_ == kStopping);
+
+  if (state_ == kStopping) {
+    return;
+  }
+  if (result) {
+    SetErrorState(FROM_HERE, "Failed to initialize camera device");
+    return;
+  }
+  SetState(kInitialized);
+  ConfigureStreams();
+}
+
+void CameraDeviceDelegate::ConfigureStreams() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kInitialized || state_ == kStopping);

[wuchengli, 2017/05/17 03:40:12]

remove state_ == kStopping

[jcliang, 2017/05/17 07:32:09]

Done.

+
+  arc::mojom::Camera3StreamConfigurationPtr stream_config =
+      arc::mojom::Camera3StreamConfiguration::New();
+  stream_config->streams.push_back(stream_context_->stream.Clone());
+  stream_config->operation_mode = arc::mojom::Camera3StreamConfigurationMode::
+      CAMERA3_STREAM_CONFIGURATION_NORMAL_MODE;
+  device_ops_->ConfigureStreams(
+      std::move(stream_config),
+      base::Bind(&CameraDeviceDelegate::OnConfiguredStreams, this));
+}
+
+void CameraDeviceDelegate::OnConfiguredStreams(
+    int32_t result,
+    arc::mojom::Camera3StreamConfigurationPtr updated_config) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kInitialized || state_ == kStopping);
+
+  if (state_ == kStopping) {
+    return;
+  }
+  if (result) {
+    SetErrorState(FROM_HERE, "Failed to configure streams");
+    return;
+  }
+  if (!updated_config || updated_config->streams.size() != 1) {
+    SetErrorState(FROM_HERE, "Wrong number of streams configured");

[wuchengli, 2017/05/17 03:40:13]

print updated_config->streams.size()

[jcliang, 2017/05/17 07:32:10]

Done.

+    return;
+  }
+  auto& updated_stream = updated_config->streams[0];
+  // TODO(jcliang): Determine the best format from metadata.
+  VideoCaptureFormat capture_format = stream_context_->params.requested_format;
+  capture_format.pixel_format = PIXEL_FORMAT_NV12;
+  stream_context_->capture_format = capture_format;
+  stream_context_->stream->usage = updated_stream->usage;
+  stream_context_->stream->max_buffers = updated_stream->max_buffers;
+
+  VLOG(2) << "Stream " << updated_stream->id
+          << " configured: usage=" << updated_stream->usage
+          << " max_buffers=" << updated_stream->max_buffers;
+
+  // Allocate buffers.
+  size_t num_buffers = stream_context_->stream->max_buffers;
+  stream_context_->buffers.resize(num_buffers);
+  for (size_t j = 0; j < num_buffers; ++j) {
+    const VideoCaptureFormat frame_format(
+        gfx::Size(stream_context_->stream->width,
+                  stream_context_->stream->height),
+        0.0, stream_context_->capture_format.pixel_format);
+    std::unique_ptr<base::SharedMemory> buffer(new base::SharedMemory());
+    base::SharedMemoryCreateOptions options;
+    options.size = frame_format.ImageAllocationSize();
+    options.share_read_only = false;
+    buffer->Create(options);

[wuchengli, 2017/05/17 03:40:12]

check return value and print error. Same for buffer->Map.

[jcliang, 2017/05/17 07:32:10]

Done.

+    buffer->Map(buffer->requested_size());
+    stream_context_->buffers[j] = std::move(buffer);
+    stream_context_->free_buffers.push(j);
+  }
+  VLOG(2) << "Allocated " << stream_context_->stream->max_buffers << " buffers";
+  ConstructDefaultRequestSettings();
+  client_->OnStarted();
+}
+
+void CameraDeviceDelegate::ConstructDefaultRequestSettings() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(stream_context_);
+
+  device_ops_->ConstructDefaultRequestSettings(
+      arc::mojom::Camera3RequestTemplate::CAMERA3_TEMPLATE_PREVIEW,
+      base::Bind(&CameraDeviceDelegate::OnConstructedDefaultRequestSettings,
+                 this));
+}
+
+void CameraDeviceDelegate::OnConstructedDefaultRequestSettings(
+    arc::mojom::CameraMetadataPtr settings) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+
+  if (state_ == kStopping) {
+    return;
+  }
+  DCHECK(stream_context_);
+  stream_context_->request_settings = std::move(settings);
+  SetState(kStreamConfigured);
+  StartCapture();
+}
+
+void CameraDeviceDelegate::StartCapture() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  // We may get here when either after the streams are configured, or when we
+  // start still capture while the preview capture is running.

[wuchengli, 2017/05/17 03:40:13]

This comment can be removed because we don't support still capture

[jcliang, 2017/05/17 07:32:10]

Done.

+  DCHECK(state_ == kStreamConfigured || state_ == kCapturing ||

[wuchengli, 2017/05/17 03:40:13]

remove state_ == kCapturing

[jcliang, 2017/05/17 07:32:09]

Done.

+         state_ == kStopping);
+
+  if (state_ == kStopping) {
+    return;
+  }
+  DCHECK(stream_context_);
+  DCHECK(!stream_context_->request_settings.is_null());
+  SetState(kCapturing);
+  RegisterBuffer();
+}
+
+void CameraDeviceDelegate::RegisterBuffer() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kCapturing || state_ == kStopping);
+
+  if (state_ == kStopping) {
+    return;
+  }
+  DCHECK(stream_context_);
+  if (stream_context_->free_buffers.empty()) {
+    return;
+  }
+
+  const VideoCaptureParams& params = stream_context_->params;
+  const arc::mojom::Camera3StreamPtr& stream = stream_context_->stream;
+  size_t buffer_id = stream_context_->free_buffers.front();
+  stream_context_->free_buffers.pop();
+  const base::SharedMemory* buffer = stream_context_->buffers[buffer_id].get();
+
+  VideoPixelFormat buffer_format = stream_context_->capture_format.pixel_format;
+  uint32_t drm_format = PixFormatChromiumToDrm(buffer_format);
+  if (!drm_format) {
+    SetErrorState(FROM_HERE, "Unsupported video pixel format");

[wuchengli, 2017/05/17 03:40:12]

print buffer_format

[jcliang, 2017/05/17 07:32:09]

Done.

+    return;
+  }
+  arc::mojom::HalPixelFormat hal_pixel_format = stream->format;
+
+  size_t num_planes = VideoFrame::NumPlanes(buffer_format);
+  std::vector<mojo::ScopedHandle> fds(num_planes);
+  std::vector<uint32_t> strides(num_planes);
+  std::vector<uint32_t> offsets(num_planes);
+  for (size_t i = 0; i < num_planes; ++i) {
+    base::SharedMemoryHandle shm_handle = buffer->handle();
+    // Wrap the platform handle.
+    MojoHandle wrapped_handle;
+    MojoResult result = mojo::edk::CreatePlatformHandleWrapper(
+        mojo::edk::ScopedPlatformHandle(mojo::edk::PlatformHandle(
+            base::SharedMemory::DuplicateHandle(shm_handle).fd)),
+        &wrapped_handle);
+    if (result != MOJO_RESULT_OK) {
+      SetErrorState(FROM_HERE, "Failed to wrap shared memory handle");
+      return;
+    }
+    fds[i].reset(mojo::Handle(wrapped_handle));
+    strides[i] = VideoFrame::RowBytes(i, buffer_format, stream->width);

[wuchengli, 2017/05/17 03:40:13]

Use params.requested_format.frame_size.width to be consistent with line 406.

[jcliang, 2017/05/17 07:32:10]

Done.

+    if (!i) {
+      offsets[i] = 0;
+    } else {
+      offsets[i] = offsets[i - 1] +
+                   VideoFrame::PlaneSize(buffer_format, i,

[wuchengli, 2017/05/17 03:40:13]

i-1

[jcliang, 2017/05/17 07:32:10]

Done.

+                                         params.requested_format.frame_size)
+                       .GetArea();
+    }
+  }
+  device_ops_->RegisterBuffer(
+      buffer_id, arc::mojom::Camera3DeviceOps::BufferType::SHM, std::move(fds),
+      drm_format, hal_pixel_format, stream_context_->stream->width,
+      stream_context_->stream->height, std::move(strides), std::move(offsets),
+      base::Bind(&CameraDeviceDelegate::OnRegisteredBuffer, this, buffer_id));
+  VLOG(2) << "Registered buffer " << buffer_id;
+}
+
+void CameraDeviceDelegate::OnRegisteredBuffer(size_t buffer_index,

[wuchengli, 2017/05/17 03:40:13]

s/buffer_index/buffer_id/

[jcliang, 2017/05/17 07:32:09]

Done.

+                                              int32_t result) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kCapturing || state_ == kStopping);
+
+  if (state_ == kStopping) {
+    return;
+  }
+  if (result) {
+    SetErrorState(FROM_HERE, "Failed to register buffer");

[wuchengli, 2017/05/17 03:40:13]

print result. Same for other IPC result if it contains an error code.

[jcliang, 2017/05/17 07:32:10]

Done.

+    return;
+  }
+  ProcessCaptureRequest(buffer_index);
+}
+
+void CameraDeviceDelegate::ProcessCaptureRequest(size_t buffer_index) {

[wuchengli, 2017/05/17 03:40:13]

s/buffer_index/buffer_id/

[jcliang, 2017/05/17 07:32:10]

Done.

+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kCapturing || state_ == kStopping);
+  DCHECK(stream_context_);
+
+  arc::mojom::Camera3StreamBufferPtr buffer =
+      arc::mojom::Camera3StreamBuffer::New();
+  buffer->stream_id = static_cast<uint64_t>(
+      arc::mojom::Camera3RequestTemplate::CAMERA3_TEMPLATE_PREVIEW);
+  buffer->buffer_id = buffer_index;
+  buffer->status = arc::mojom::Camera3BufferStatus::CAMERA3_BUFFER_STATUS_OK;
+
+  arc::mojom::Camera3CaptureRequestPtr request =
+      arc::mojom::Camera3CaptureRequest::New();
+  request->frame_number = frame_number_;
+  request->settings = stream_context_->request_settings.Clone();
+  request->output_buffers.push_back(std::move(buffer));
+
+  device_ops_->ProcessCaptureRequest(
+      std::move(request),
+      base::Bind(&CameraDeviceDelegate::OnProcessedCaptureRequest, this));
+  VLOG(2) << "Requested capture for frame " << frame_number_ << " with buffer "
+          << buffer_index;
+  frame_number_++;

[wuchengli, 2017/05/17 03:40:12]

Handle wraparound. Start with 1 so the initial timestamp is not set again?

[jcliang, 2017/05/17 07:32:10]

Done.

+}
+
+void CameraDeviceDelegate::OnProcessedCaptureRequest(int32_t result) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(state_ == kCapturing || state_ == kStopping);
+
+  if (state_ == kStopping) {
+    return;
+  }
+  if (result) {
+    SetErrorState(FROM_HERE, "Process capture request failed");
+    return;
+  }
+  RegisterBuffer();
+}
+
+void CameraDeviceDelegate::ProcessCaptureResult(
+    arc::mojom::Camera3CaptureResultPtr result) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+
+  uint32_t frame_number = result->frame_number;
+  // A new partial result may be created in either ProcessCaptureResult or
+  // Notify.
+  CaptureResult& partial_result = partial_results_[frame_number];
+  if (result->output_buffers.has_value()) {
+    if (result->output_buffers->size() != 1) {
+      client_->OnError(FROM_HERE,

[wuchengli, 2017/05/17 03:40:13]

Use SetErrorState

[jcliang, 2017/05/17 07:32:10]

Done.

+                       "Incorrect number of output buffers received");

[wuchengli, 2017/05/17 03:40:13]

print output_buffers->size()

[jcliang, 2017/05/17 07:32:10]

Done.

+      return;
+    }
+    arc::mojom::Camera3StreamBufferPtr& stream_buffer =
+        result->output_buffers.value()[0];
+    VLOG(2) << "Received capture result for frame " << frame_number
+            << " stream_id: " << stream_buffer->stream_id;
+    // The camera HAL v3 API specifies that only one capture result can carry
+    // the result buffer for any given frame number.
+    if (!partial_result.buffer.is_null()) {

[wuchengli, 2017/05/17 03:40:12]

Use SetErrorState so the preview will be stopped and we can catch the bug
earlier.

[jcliang, 2017/05/17 07:32:10]

Done.

+      LOG(ERROR) << "Received multiple result buffers for frame "
+                 << frame_number;
+    } else {
+      partial_result.buffer = std::move(stream_buffer);
+    }
+  }
+
+  // |result->partial_result| is set to 0 if the capture result contains only
+  // the result buffer handles and no result metadata.
+  if (result->partial_result) {
+    partial_result.partial_stage = result->partial_result;

[wuchengli, 2017/05/17 03:40:13]

Check Android API if partial_result is increasing.

[jcliang, 2017/05/17 07:32:10]

Per camera HALv3 API it's not required for partial_result to be increasing; the
partial_result only needs to be distinct. I've changed the handling of
partial_result below.

+    MergeMetadata(&partial_result.metadata, result->result);
+  }
+
+  if (partial_result.partial_stage == partial_result_count_ &&

[wuchengli, 2017/05/17 03:40:13]

Let's do some error checking because HAL will definitely have bugs. For example,
result->partial_result <= partial_result_count_

[jcliang, 2017/05/17 07:32:10]

Done.

+      !partial_result.buffer.is_null()) {
+    // We can only submit the result buffer after we receive the shutter time.
+    if (partial_result.reference_time != base::TimeTicks()) {
+      SubmitCaptureResult(frame_number);
+    }
+  }
+}
+
+void CameraDeviceDelegate::Notify(arc::mojom::Camera3NotifyMsgPtr message) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+
+  if (message->type == arc::mojom::Camera3MsgType::CAMERA3_MSG_ERROR) {
+    uint32_t frame_number = message->message->get_error()->frame_number;
+    uint64_t error_stream_id = message->message->get_error()->error_stream_id;
+    arc::mojom::Camera3ErrorMsgCode error_code =
+        message->message->get_error()->error_code;
+    switch (error_code) {
+      case arc::mojom::Camera3ErrorMsgCode::CAMERA3_MSG_ERROR_DEVICE:
+        // Fatal error and no more frames will be produced by the device.
+        SetErrorState(FROM_HERE, "Fatal device error");
+        break;
+      case arc::mojom::Camera3ErrorMsgCode::CAMERA3_MSG_ERROR_REQUEST:
+        // An error has occurred in processing the request; the request
+        // specified by |frame_number| has been dropped by the camera device.
+        // Subsequent requests are unaffected.
+        //
+        // The HAL will call ProcessCaptureResult with the buffers' state set to
+        // STATUS_ERROR.  The content of the buffers will be dropped and the
+        // buffers will be reused in SubmitCaptureResult.

[wuchengli, 2017/05/17 03:40:13]

Make this this is printed because this shouldn't happen. Or use LOG(WARNING).
Same for OnLog below. 

[jcliang, 2017/05/17 07:32:10]

Done.

+        client_->OnLog(
+            std::string(
+                "An error occurred while processing request for frame ") +
+            std::to_string(frame_number));
+        break;
+      case arc::mojom::Camera3ErrorMsgCode::CAMERA3_MSG_ERROR_RESULT:
+        // An error has occurred in producing the output metadata buffer for a
+        // result; the output metadata will not be available for the frame
+        // specified by |frame_number|.  Subsequent requests are unaffected.
+        client_->OnLog(std::string("An error occurred while producing result "
+                                   "metadata for frame ") +
+                       std::to_string(frame_number));
+        // The result metadata will not be complete so we don't need to wait for
+        // partial results on frame |frame_number|.
+        partial_results_[frame_number].partial_stage = partial_result_count_;
+        break;
+      case arc::mojom::Camera3ErrorMsgCode::CAMERA3_MSG_ERROR_BUFFER:
+        // An error has occurred in placing the output buffer into a stream for
+        // a request. |frame_number| specifies the request for which the buffer
+        // was dropped, and |error_stream_id| specifies the stream that dropped
+        // the buffer.
+        //
+        // The HAL will call ProcessCaptureResult with the buffer's state set to
+        // STATUS_ERROR.  The content of the buffer will be dropped and the
+        // buffer will be reused in SubmitCaptureResult.
+        client_->OnLog(
+            std::string(
+                "An error occurred while filling output buffer of stream ") +
+            std::to_string(error_stream_id) + std::string(" in frame ") +
+            std::to_string(frame_number));
+        break;
+      default:
+        // To eliminate the warning for not handling CAMERA3_MSG_NUM_ERRORS
+        break;
+    }
+  } else {  // arc::mojom::Camera3MsgType::CAMERA3_MSG_SHUTTER
+    uint32_t frame_number = message->message->get_shutter()->frame_number;
+    uint64_t shutter_time = message->message->get_shutter()->timestamp;

[wuchengli, 2017/05/17 03:40:12]

Check if the timestamp is not 0. Otherwise, preview will be stuck.

[jcliang, 2017/05/17 07:32:10]

Done.

+    // A new partial result may be created in either ProcessCaptureResult or
+    // Notify.
+    VLOG(2) << "Received shutter time for frame " << frame_number;
+    CaptureResult& partial_result = partial_results_[frame_number];
+    // Shutter timestamp is in ns.
+    base::TimeTicks reference_time =
+        base::TimeTicks::FromInternalValue(shutter_time / 1000);
+    partial_result.reference_time = reference_time;
+    if (!frame_number) {
+      // Record the shutter time of the first frame for calculating the
+      // timestamp.
+      first_frame_shutter_time_ = reference_time;
+      partial_result.timestamp = base::TimeDelta::FromMicroseconds(0);
+    } else {
+      partial_result.timestamp = reference_time - first_frame_shutter_time_;

[wuchengli, 2017/05/17 03:40:13]

This can be extracted. 
reference_time - first_frame_shutter_time_ will be 0 if the frame number is 0.

[jcliang, 2017/05/17 07:32:10]

Done.

+    }
+    if (partial_result.partial_stage == partial_result_count_ &&
+        !partial_result.buffer.is_null()) {
+      SubmitCaptureResult(frame_number);
+    }
+  }
+}
+
+void CameraDeviceDelegate::SubmitCaptureResult(uint32_t frame_number) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+
+  if (partial_results_.begin()->first != frame_number) {
+    SetErrorState(FROM_HERE, "Received out-of-order frames from HAL");
+    return;
+  }
+
+  VLOG(2) << "Submit capture result of frame " << frame_number;
+  CaptureResult& partial_result = partial_results_[frame_number];
+  DCHECK_EQ(partial_result.partial_stage, partial_result_count_);
+  DCHECK(partial_result.buffer);
+  uint32_t buffer_id = partial_result.buffer->buffer_id;
+
+  // Wait on release fence before delivering the result buffer to client.
+  if (partial_result.buffer->release_fence.is_valid()) {
+    const int kSyncWaitTimeoutMs = 1000;
+    mojo::edk::ScopedPlatformHandle fence;
+    MojoResult result = mojo::edk::PassWrappedPlatformHandle(
+        partial_result.buffer->release_fence.release().value(), &fence);
+    if (result != MOJO_RESULT_OK) {
+      SetErrorState(FROM_HERE, "Failed to unwrap release fence fd");
+      return;
+    }
+    if (!sync_wait(fence.get().handle, kSyncWaitTimeoutMs)) {
+      SetErrorState(FROM_HERE, "Sync wait on release fence timed out");
+      return;
+    }
+  }
+
+  // Deliver the captured data to client and then re-queue the buffer.
+  if (partial_result.buffer->status !=
+      arc::mojom::Camera3BufferStatus::CAMERA3_BUFFER_STATUS_ERROR) {
+    const base::SharedMemory* shm_buffer =
+        stream_context_->buffers[buffer_id].get();
+    client_->OnIncomingCapturedData(
+        reinterpret_cast<uint8_t*>(shm_buffer->memory()),
+        shm_buffer->mapped_size(), stream_context_->capture_format, rotation_,
+        partial_result.reference_time, partial_result.timestamp);
+  }
+  stream_context_->free_buffers.push(buffer_id);
+  ipc_task_runner_->PostTask(

[wuchengli, 2017/05/17 03:40:13]

Call RegisterBuffer directly because it's the same thread.

[jcliang, 2017/05/17 07:32:10]

Done.

+      FROM_HERE, base::Bind(&CameraDeviceDelegate::RegisterBuffer, this));
+  partial_results_.erase(frame_number);
+}
+
+}  // namespace media