media: add video capture device for ARC++ camera HAL v3

media/capture/video/chromeos/stream_buffer_manager.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/capture/video/chromeos/camera_device_delegate.h"
+
+#include "media/capture/video/chromeos/camera_hal_delegate.h"
+#include "media/capture/video/chromeos/camera_metadata_utils.h"
+#include "media/capture/video/chromeos/pixel_format_utils.h"
+#include "mojo/edk/embedder/embedder.h"
+#include "mojo/edk/embedder/scoped_platform_handle.h"
+#include "third_party/libsync/include/sync/sync.h"
+
+namespace media {
+
+CameraDeviceDelegate::StreamBufferManager::StreamBufferManager(
+    arc::mojom::Camera3CallbackOpsRequest callback_ops_request,
+    scoped_refptr<CameraDeviceDelegate> camera_device_delegate,
+    scoped_refptr<base::SingleThreadTaskRunner> ipc_task_runner)
+    : capturing_(false),
+      camera_device_delegate_(std::move(camera_device_delegate)),
+      callback_ops_(this, std::move(callback_ops_request)),
+      ipc_task_runner_(std::move(ipc_task_runner)),
+      frame_number_(0),
+      partial_result_count_(1),
+      first_frame_shutter_time_(base::TimeTicks::Now()) {}
+
+void CameraDeviceDelegate::StreamBufferManager::SetUpStreamAndBuffers(
+    VideoCaptureFormat capture_format,
+    uint32_t partial_result_count,
+    arc::mojom::Camera3StreamPtr stream) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(!stream_context_);
+
+  VLOG(2) << "Stream " << stream->id << " configured: usage=" << stream->usage
+          << " max_buffers=" << stream->max_buffers;
+
+  const size_t kMaximumAllowedBuffers = 15;
+  if (stream->max_buffers > kMaximumAllowedBuffers) {
+    camera_device_delegate_->SetErrorState(
+        FROM_HERE, std::string("Camera HAL requested ") +
+                       std::to_string(stream->max_buffers) +
+                       std::string(" buffers which exceeds the allowed maximum "
+                                   "number of buffers"));
+    return;
+  }
+
+  partial_result_count_ = partial_result_count;
+  stream_context_.reset(new StreamContext{capture_format, std::move(stream)});
+
+  // Allocate buffers.
+  size_t num_buffers = stream_context_->stream->max_buffers;
+  stream_context_->buffers.resize(num_buffers);
+  for (size_t j = 0; j < num_buffers; ++j) {
+    const VideoCaptureFormat frame_format(
+        gfx::Size(stream_context_->stream->width,
+                  stream_context_->stream->height),
+        0.0, stream_context_->capture_format.pixel_format);
+    std::unique_ptr<base::SharedMemory> buffer(new base::SharedMemory());
+    base::SharedMemoryCreateOptions options;
+    options.size = frame_format.ImageAllocationSize();
+    options.share_read_only = false;
+    bool ret = buffer->Create(options);
+    if (!ret) {
+      camera_device_delegate_->SetErrorState(
+          FROM_HERE, "Failed to create SharedMemory buffer");
+      return;
+    }
+    ret = buffer->Map(buffer->requested_size());
+    if (!ret) {
+      camera_device_delegate_->SetErrorState(
+          FROM_HERE, "Failed to map SharedMemory buffer");
+      return;
+    }
+    stream_context_->buffers[j] = std::move(buffer);
+    stream_context_->free_buffers.push(j);
+  }
+  VLOG(2) << "Allocated " << stream_context_->stream->max_buffers << " buffers";
+}
+
+void CameraDeviceDelegate::StreamBufferManager::StartCapture(
+    arc::mojom::CameraMetadataPtr settings) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(stream_context_);
+  DCHECK(stream_context_->request_settings.is_null());
+
+  capturing_ = true;
+  stream_context_->request_settings = std::move(settings);
+  // We cannot use a loop to register all the free buffers in one shot here
+  // because the camera HAL v3 API specifies that the client cannot call
+  // ProcessCaptureRequest before the previous one returns.
+  RegisterBuffer();
+}
+
+void CameraDeviceDelegate::StreamBufferManager::StopCapture() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  capturing_ = false;
+}
+
+void CameraDeviceDelegate::StreamBufferManager::RegisterBuffer() {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(stream_context_);
+
+  if (!capturing_) {
+    return;
+  }
+
+  if (stream_context_->free_buffers.empty()) {
+    return;
+  }
+
+  size_t buffer_id = stream_context_->free_buffers.front();
+  stream_context_->free_buffers.pop();
+  const base::SharedMemory* buffer = stream_context_->buffers[buffer_id].get();
+
+  VideoPixelFormat buffer_format = stream_context_->capture_format.pixel_format;
+  uint32_t drm_format = PixFormatChromiumToDrm(buffer_format);
+  if (!drm_format) {
+    camera_device_delegate_->SetErrorState(
+        FROM_HERE, std::string("Unsupported video pixel format") +
+                       VideoPixelFormatToString(buffer_format));
+    return;
+  }
+  arc::mojom::HalPixelFormat hal_pixel_format = stream_context_->stream->format;
+
+  size_t num_planes = VideoFrame::NumPlanes(buffer_format);
+  std::vector<mojo::ScopedHandle> fds(num_planes);
+  std::vector<uint32_t> strides(num_planes);
+  std::vector<uint32_t> offsets(num_planes);
+  for (size_t i = 0; i < num_planes; ++i) {
+    base::SharedMemoryHandle shm_handle = buffer->handle();
+    // Wrap the platform handle.
+    MojoHandle wrapped_handle;
+    MojoResult result = mojo::edk::CreatePlatformHandleWrapper(
+        mojo::edk::ScopedPlatformHandle(mojo::edk::PlatformHandle(
+            base::SharedMemory::DuplicateHandle(shm_handle).GetHandle())),
+        &wrapped_handle);
+    if (result != MOJO_RESULT_OK) {
+      camera_device_delegate_->SetErrorState(
+          FROM_HERE, "Failed to wrap shared memory handle");
+      return;
+    }
+    fds[i].reset(mojo::Handle(wrapped_handle));
+    strides[i] = VideoFrame::RowBytes(
+        i, buffer_format, stream_context_->capture_format.frame_size.width());
+    if (!i) {
+      offsets[i] = 0;
+    } else {
+      offsets[i] =
+          offsets[i - 1] +
+          VideoFrame::PlaneSize(buffer_format, i - 1,
+                                stream_context_->capture_format.frame_size)
+              .GetArea();
+    }
+  }
+  camera_device_delegate_->RegisterBuffer(
+      buffer_id, arc::mojom::Camera3DeviceOps::BufferType::SHM, std::move(fds),
+      drm_format, hal_pixel_format, stream_context_->stream->width,
+      stream_context_->stream->height, std::move(strides), std::move(offsets),
+      base::Bind(&CameraDeviceDelegate::StreamBufferManager::OnRegisteredBuffer,
+                 this, buffer_id));
+  VLOG(2) << "Registered buffer " << buffer_id;
+}
+
+void CameraDeviceDelegate::StreamBufferManager::OnRegisteredBuffer(
+    size_t buffer_id,
+    int32_t result) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+
+  if (!capturing_) {
+    return;
+  }
+  if (result) {
+    camera_device_delegate_->SetErrorState(
+        FROM_HERE, std::string("Failed to register buffer: ") +
+                       std::string(strerror(result)));
+    return;
+  }
+  ProcessCaptureRequest(buffer_id);
+}
+
+void CameraDeviceDelegate::StreamBufferManager::ProcessCaptureRequest(
+    size_t buffer_id) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+  DCHECK(stream_context_);
+
+  arc::mojom::Camera3StreamBufferPtr buffer =
+      arc::mojom::Camera3StreamBuffer::New();
+  buffer->stream_id = static_cast<uint64_t>(
+      arc::mojom::Camera3RequestTemplate::CAMERA3_TEMPLATE_PREVIEW);
+  buffer->buffer_id = buffer_id;
+  buffer->status = arc::mojom::Camera3BufferStatus::CAMERA3_BUFFER_STATUS_OK;
+
+  arc::mojom::Camera3CaptureRequestPtr request =
+      arc::mojom::Camera3CaptureRequest::New();
+  request->frame_number = frame_number_;
+  request->settings = stream_context_->request_settings.Clone();
+  request->output_buffers.push_back(std::move(buffer));
+
+  camera_device_delegate_->ProcessCaptureRequest(
+      std::move(request),
+      base::Bind(
+          &CameraDeviceDelegate::StreamBufferManager::OnProcessedCaptureRequest,
+          this));
+  VLOG(2) << "Requested capture for frame " << frame_number_ << " with buffer "
+          << buffer_id;
+  frame_number_++;
+  // In case |frame_number_| wraps around, we start at 1 to avoid resetting
+  // |first_frame_shutter_time_|.
+  if (!frame_number_) {
+    frame_number_++;
+  }
+}
+
+void CameraDeviceDelegate::StreamBufferManager::OnProcessedCaptureRequest(
+    int32_t result) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+
+  if (!capturing_) {
+    return;
+  }
+  if (result) {
+    camera_device_delegate_->SetErrorState(
+        FROM_HERE, std::string("Process capture request failed") +
+                       std::string(strerror(result)));
+    return;
+  }
+  RegisterBuffer();
+}
+
+void CameraDeviceDelegate::StreamBufferManager::ProcessCaptureResult(
+    arc::mojom::Camera3CaptureResultPtr result) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+
+  if (!capturing_) {
+    return;
+  }
+  uint32_t frame_number = result->frame_number;
+  // A new partial result may be created in either ProcessCaptureResult or
+  // Notify.
+  CaptureResult& partial_result = partial_results_[frame_number];
+  if (partial_results_.size() > stream_context_->stream->max_buffers) {
+    camera_device_delegate_->SetErrorState(
+        FROM_HERE,
+        "Received more capture results than the maximum number of buffers");
+    return;
+  }
+  if (result->output_buffers) {
+    if (result->output_buffers->size() != 1) {
+      camera_device_delegate_->SetErrorState(
+          FROM_HERE,
+          std::string("Incorrect number of output buffers received: ") +
+              std::to_string(result->output_buffers->size()));
+      return;
+    }
+    arc::mojom::Camera3StreamBufferPtr& stream_buffer =
+        result->output_buffers.value()[0];
+    VLOG(2) << "Received capture result for frame " << frame_number
+            << " stream_id: " << stream_buffer->stream_id;
+    // The camera HAL v3 API specifies that only one capture result can carry
+    // the result buffer for any given frame number.
+    if (!partial_result.buffer.is_null()) {
+      camera_device_delegate_->SetErrorState(
+          FROM_HERE,
+          std::string("Received multiple result buffers for frame ") +
+              std::to_string(frame_number));
+      return;
+    } else {
+      partial_result.buffer = std::move(stream_buffer);
+    }
+  }
+
+  // |result->partial_result| is set to 0 if the capture result contains only
+  // the result buffer handles and no result metadata.
+  if (result->partial_result) {
+    uint32_t result_id = result->partial_result;
+    if (result_id > partial_result_count_) {
+      camera_device_delegate_->SetErrorState(
+          FROM_HERE, std::string("Invalid partial_result id: ") +
+                         std::to_string(result_id));
+      return;
+    }
+    if (partial_result.partial_metadata_received.find(result_id) !=
+        partial_result.partial_metadata_received.end()) {
+      camera_device_delegate_->SetErrorState(
+          FROM_HERE, std::string("Received duplicated partial metadata: ") +
+                         std::to_string(result_id));
+      return;
+    }
+    partial_result.partial_metadata_received.insert(result_id);
+    MergeMetadata(&partial_result.metadata, result->result);
+  }
+
+  if (partial_result.partial_metadata_received.size() ==
+          partial_result_count_ &&
+      !partial_result.buffer.is_null()) {
+    // We can only submit the result buffer after we receive the shutter time.
+    if (partial_result.reference_time != base::TimeTicks()) {
+      SubmitCaptureResult(frame_number);
+    }
+  }
+}
+
+void CameraDeviceDelegate::StreamBufferManager::Notify(
+    arc::mojom::Camera3NotifyMsgPtr message) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+
+  if (!capturing_) {
+    return;
+  }
+  if (message->type == arc::mojom::Camera3MsgType::CAMERA3_MSG_ERROR) {
+    uint32_t frame_number = message->message->get_error()->frame_number;
+    uint64_t error_stream_id = message->message->get_error()->error_stream_id;
+    arc::mojom::Camera3ErrorMsgCode error_code =
+        message->message->get_error()->error_code;
+    HandleNotifyError(frame_number, error_stream_id, error_code);
+  } else {  // arc::mojom::Camera3MsgType::CAMERA3_MSG_SHUTTER
+    uint32_t frame_number = message->message->get_shutter()->frame_number;
+    uint64_t shutter_time = message->message->get_shutter()->timestamp;
+    // A new partial result may be created in either ProcessCaptureResult or
+    // Notify.
+    VLOG(2) << "Received shutter time for frame " << frame_number;
+    if (!shutter_time) {
+      camera_device_delegate_->SetErrorState(
+          FROM_HERE, std::string("Received invalid shutter time: ") +
+                         std::to_string(shutter_time));
+      return;
+    }
+    CaptureResult& partial_result = partial_results_[frame_number];
+    if (partial_results_.size() > stream_context_->stream->max_buffers) {
+      camera_device_delegate_->SetErrorState(
+          FROM_HERE,
+          "Received more capture results than the maximum number of buffers");
+      return;
+    }
+    // Shutter timestamp is in ns.
+    base::TimeTicks reference_time =
+        base::TimeTicks::FromInternalValue(shutter_time / 1000);
+    partial_result.reference_time = reference_time;
+    if (!frame_number) {
+      // Record the shutter time of the first frame for calculating the
+      // timestamp.
+      first_frame_shutter_time_ = reference_time;
+    }
+    partial_result.timestamp = reference_time - first_frame_shutter_time_;
+    if (partial_result.partial_metadata_received.size() ==
+            partial_result_count_ &&
+        !partial_result.buffer.is_null()) {
+      SubmitCaptureResult(frame_number);
+    }
+  }
+}
+
+void CameraDeviceDelegate::StreamBufferManager::HandleNotifyError(
+    uint32_t frame_number,
+    uint64_t error_stream_id,
+    arc::mojom::Camera3ErrorMsgCode error_code) {
+  switch (error_code) {
+    case arc::mojom::Camera3ErrorMsgCode::CAMERA3_MSG_ERROR_DEVICE:
+      // Fatal error and no more frames will be produced by the device.
+      camera_device_delegate_->SetErrorState(FROM_HERE, "Fatal device error");
+      break;
+    case arc::mojom::Camera3ErrorMsgCode::CAMERA3_MSG_ERROR_REQUEST: {
+      // An error has occurred in processing the request; the request
+      // specified by |frame_number| has been dropped by the camera device.
+      // Subsequent requests are unaffected.
+      //
+      // The HAL will call ProcessCaptureResult with the buffers' state set to
+      // STATUS_ERROR.  The content of the buffers will be dropped and the
+      // buffers will be reused in SubmitCaptureResult.
+      std::string warning_msg =
+          std::string("An error occurred while processing request for frame ") +
+          std::to_string(frame_number);
+      LOG(WARNING) << warning_msg;
+      camera_device_delegate_->LogToClient(warning_msg);
+      break;
+    }
+    case arc::mojom::Camera3ErrorMsgCode::CAMERA3_MSG_ERROR_RESULT: {
+      // An error has occurred in producing the output metadata buffer for a
+      // result; the output metadata will not be available for the frame
+      // specified by |frame_number|.  Subsequent requests are unaffected.
+      std::string warning_msg = std::string(
+                                    "An error occurred while producing result "
+                                    "metadata for frame ") +
+                                std::to_string(frame_number);
+      LOG(WARNING) << warning_msg;
+      camera_device_delegate_->LogToClient(warning_msg);
+      // The result metadata will not be complete so we don't need to wait for
+      // partial results on frame |frame_number|.
+      partial_results_[frame_number].partial_metadata_received.clear();
+      for (uint32_t i = 0; i < partial_result_count_; ++i) {
+        partial_results_[frame_number].partial_metadata_received.insert(i);
+      }
+      break;
+    }
+    case arc::mojom::Camera3ErrorMsgCode::CAMERA3_MSG_ERROR_BUFFER:
+      // An error has occurred in placing the output buffer into a stream for
+      // a request. |frame_number| specifies the request for which the buffer
+      // was dropped, and |error_stream_id| specifies the stream that dropped
+      // the buffer.
+      //
+      // The HAL will call ProcessCaptureResult with the buffer's state set to
+      // STATUS_ERROR.  The content of the buffer will be dropped and the
+      // buffer will be reused in SubmitCaptureResult.
+      camera_device_delegate_->LogToClient(
+          std::string(
+              "An error occurred while filling output buffer of stream ") +
+          std::to_string(error_stream_id) + std::string(" in frame ") +
+          std::to_string(frame_number));
+      break;
+    default:
+      // To eliminate the warning for not handling CAMERA3_MSG_NUM_ERRORS
+      break;
+  }
+}
+
+void CameraDeviceDelegate::StreamBufferManager::SubmitCaptureResult(
+    uint32_t frame_number) {
+  DCHECK(ipc_task_runner_->BelongsToCurrentThread());
+
+  if (partial_results_.begin()->first != frame_number) {
+    camera_device_delegate_->SetErrorState(
+        FROM_HERE, std::string("Received frame is out-of-order; expect ") +
+                       std::to_string(partial_results_.begin()->first) +
+                       std::string(" but got ") + std::to_string(frame_number));
+    return;
+  }
+
+  VLOG(2) << "Submit capture result of frame " << frame_number;
+  CaptureResult& partial_result = partial_results_[frame_number];
+  DCHECK_EQ(partial_result.partial_metadata_received.size(),
+            partial_result_count_);
+  DCHECK(partial_result.buffer);
+  uint32_t buffer_id = partial_result.buffer->buffer_id;
+
+  // Wait on release fence before delivering the result buffer to client.
+  if (partial_result.buffer->release_fence.is_valid()) {
+    const int kSyncWaitTimeoutMs = 1000;
+    mojo::edk::ScopedPlatformHandle fence;
+    MojoResult result = mojo::edk::PassWrappedPlatformHandle(
+        partial_result.buffer->release_fence.release().value(), &fence);
+    if (result != MOJO_RESULT_OK) {
+      camera_device_delegate_->SetErrorState(
+          FROM_HERE, "Failed to unwrap release fence fd");
+      return;
+    }
+    if (!sync_wait(fence.get().handle, kSyncWaitTimeoutMs)) {
+      camera_device_delegate_->SetErrorState(
+          FROM_HERE, "Sync wait on release fence timed out");
+      return;
+    }
+  }
+
+  // Deliver the captured data to client and then re-queue the buffer.
+  if (partial_result.buffer->status !=
+      arc::mojom::Camera3BufferStatus::CAMERA3_BUFFER_STATUS_ERROR) {
+    const base::SharedMemory* shm_buffer =
+        stream_context_->buffers[buffer_id].get();
+    camera_device_delegate_->SubmitCapturedData(
+        reinterpret_cast<uint8_t*>(shm_buffer->memory()),
+        shm_buffer->mapped_size(), stream_context_->capture_format,
+        partial_result.reference_time, partial_result.timestamp);
+  }
+  stream_context_->free_buffers.push(buffer_id);
+  partial_results_.erase(frame_number);
+  RegisterBuffer();
+}
+
+}  // namespace media