media: add video capture device for ARC++ camera HAL v3

media/capture/video/chromeos/camera_device_delegate.h

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_CAPTURE_VIDEO_CHROMEOS_CAMERA_DEVICE_DELEGATE_H_
+#define MEDIA_CAPTURE_VIDEO_CHROMEOS_CAMERA_DEVICE_DELEGATE_H_
+
+#include <map>
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "base/macros.h"
+#include "base/synchronization/waitable_event.h"
+#include "media/capture/video/chromeos/mojo/arc_camera3.mojom.h"
+#include "media/capture/video/video_capture_device.h"
+#include "media/capture/video_capture_types.h"
+#include "mojo/public/cpp/bindings/binding.h"
+
+namespace media {
+
+using namespace arc::mojom;

[chfremer, 2017/04/27 22:41:41]

As per Google C++ Style Guide, this may not be allowed.
https://google.github.io/styleguide/cppguide.html#Namespaces

[jcliang, 2017/04/28 04:47:57]

Removed.

+
+// CameraDeviceDelegate is instantiated on the capture thread where
+// AllocateAndStart of VideoCaptureDeviceArcChromeOS runs on.  All the methods
+// in CameraDeviceDelegate runs on |device_task_runner_| and hence all the
+// access to member variables is sequenced.
+class CAPTURE_EXPORT CameraDeviceDelegate final

[chfremer, 2017/04/27 22:41:41]

This class is fairly large, which makes it difficult to understanding it in its
entirety.
We should consider splitting it into separate classes for separate
responsibilities. Not sure if it is easy to split by these, but potential
responsibilities seem to be managing state, configuring/managing buffers, and
communication to the HAL service.

[jcliang, 2017/04/28 04:47:57]

I'll try if I can split the class, but it may not be easy. The methods are
deeply coupled with each other per the Android camera HAL v3 API spec. 

+    : public base::RefCountedThreadSafe<CameraDeviceDelegate>,
+      public Camera3CallbackOps {
+ public:
+  enum State {
+    // The camera device is completely stopped. This is the initial state, and
+    // is also set in OnClosed().
+    kStopped,
+
+    // The camera device is starting and waiting to be initialized.
+    //
+    // The kStarting state is set in AllocateAndStart().
+    kStarting,
+
+    // The camera device is initialized and can accept stream configuration
+    // requests.
+    //
+    // The state is transitioned to kInitialized through:
+    //
+    //   Initialize() -> OnInitialized()
+    kInitialized,
+
+    // The various capture streams are configured and the camera device is ready
+    // to process capture requests.
+    //
+    // The state is transitioned to kStreamConfigured through:
+    //
+    //   ConfigureStreams() -> OnConfiguredStreams() ->
+    //   ConstructDefaultRequestSettings() ->
+    //   OnConstructedDefaultRequestSettings()
+    kStreamConfigured,
+
+    // The camera device is capturing video streams.
+    //
+    // The kCapturing state is set in StartCapture().
+    kCapturing,
+
+    // When the camera device is in the kCapturing state, a capture loops is

[chfremer, 2017/04/27 22:41:41]

loops -> loop

[jcliang, 2017/04/28 04:47:57]

Done.

+    // constantly running:
+    //
+    // On the CameraDeviceDelegate side, we register and submit a capture
+    // request whenever a free buffer is available:
+    //
+    //   RegisterBuffer() -> OnRegisteredBuffer() ->
+    //   ProcessCaptureRequest() -> OnProcessedCaptureRequest()
+    //
+    // We get various capture metadata and error notifications from the camera
+    // HAL through the following callbacks:
+    //
+    //   ProcessCaptureResult()
+    //   Notify()
+
+    // The camera device is going through the shut down process; in order to
+    // avoid race conditions, no new Mojo messages may be sent to camera HAL in
+    // the kStopping state.
+    //
+    // The kStopping state is set in StopAndDeAllocate().
+    kStopping,
+
+    // The camera device encountered an unrecoverable error and needs to be
+    // StopAndDeAllocate()'d.
+    //
+    // The kError state is set in SetErrorState().
+    kError,
+  };
+
+  CameraDeviceDelegate(
+      VideoCaptureDeviceDescriptor device_descriptor,
+      arc::mojom::CameraMetadataPtr static_metadata,
+      mojo::InterfacePtrInfo<Camera3DeviceOps> device_ops_info,
+      const scoped_refptr<base::SingleThreadTaskRunner> device_task_runner);
+
+  // Converts the HAL pixel format |from| to Chromium pixel format.  Returns
+  // PIXEL_FORMAT_UNKNOWN if |from| is not supported.
+  static VideoPixelFormat PixFormatHalToChromium(
+      arc::mojom::HalPixelFormat from);
+
+  // Converts the Chromium pixel format |from| to DRM pixel format.  Returns 0
+  // if |from| is not supported.
+  static uint32_t PixFormatChromiumToDrm(VideoPixelFormat from);
+
+  // Delegation methods for the VideoCaptureDevice interface.
+  void AllocateAndStart(const VideoCaptureParams& params,
+                        std::unique_ptr<VideoCaptureDevice::Client> client);
+  void StopAndDeAllocate(base::WaitableEvent* closed);
+  void TakePhoto(VideoCaptureDevice::TakePhotoCallback callback);
+  void GetPhotoCapabilities(
+      VideoCaptureDevice::GetPhotoCapabilitiesCallback callback);
+  void SetPhotoOptions(mojom::PhotoSettingsPtr settings,
+                       VideoCaptureDevice::SetPhotoOptionsCallback callback);
+
+  // Sets the frame rotation angle in |rotation_|.
+  void SetRotation(int rotation);
+
+ private:
+  friend class base::RefCountedThreadSafe<CameraDeviceDelegate>;
+
+  ~CameraDeviceDelegate() = default;
+
+  // Sets the state.
+  void SetState(State state);
+
+  // Sets state to kError and call |client_->OnError| to tear down the
+  // VideoCaptureDevice.
+  void SetErrorState(const tracked_objects::Location& from_here,
+                     const std::string& reason);
+
+  // Resets the Mojo interface and bindings.
+  void ResetMojoInterface();
+
+  // Mojo connection error handler.
+  void OnMojoConnectionError();
+
+  // Callback method for the Close Mojo IPC call.  This function asynchronously

[chfremer, 2017/04/27 22:41:41]

Comment cut off?

[jcliang, 2017/04/28 04:47:57]

Done.

+  void OnClosed(base::WaitableEvent* closed, int32_t result);
+
+  // Initializes the camera HAL.  Initialize sets up the Camera3CallbackOps with
+  // the camera HAL.  OnInitialized continues to ConfigureStreams if the
+  // Initialize call succeeds.
+  void Initialize();
+  void OnInitialized(int32_t result);
+
+  // ConfigureStreams sets up stream context in |streams_| and configure the
+  // streams with the camera HAL.  OnConfiguredStreams updates |streams_| with
+  // the stream config returned, and allocates buffers as per |updated_config|
+  // indicates.  If there's no error OnConfiguredStreams notifies
+  // |client_| the capture has started by calling OnStarted, and proceeds to
+  // ConstructDefaultRequestSettings.
+  void ConfigureStreams();
+  void OnConfiguredStreams(Camera3StreamConfigurationPtr updated_config);
+
+  // ConstructDefaultRequestSettings asks the camera HAL for the default request
+  // settings of each stream in |streams_|.  OnConstructedDefaultRequestSettings
+  // sets the request settings for |stream_type| in |streams_|.  If there's no
+  // error OnConstructedDefaultRequestSettings calls StartCapture to start the
+  // video capture loop.
+  void ConstructDefaultRequestSettings(Camera3RequestTemplate stream_type);
+  void OnConstructedDefaultRequestSettings(Camera3RequestTemplate stream_type,
+                                           CameraMetadataPtr settings);
+
+  // StartCapture is the entry point to starting the video capture of
+  // |stream_type|.  The way the video capture loop works is:
+  //
+  //  (1) If there is a free buffer, RegisterBuffer registers the buffer with
+  //      the camera HAL.
+  //  (2) Once the free buffer is registered, ProcessCaptureRequest is called to
+  //      issue a capture request which will eventually fill the registered
+  //      buffer.  Goto (1) to register the remaining free buffers.
+  //  (3) The camera HAL returns shutter time of a capture request through
+  //      Notify, and the filled buffer through ProcessCaptureResult.
+  //  (4) Once all the result metadata are collected, SubmitCaptureResult is
+  //      called to deliver the filled buffer to Chrome.  After the buffer is
+  //      consumed by Chrome it is enqueued back to the free buffer queue.
+  //      Goto (1) to start another capture loop.
+  //
+  // The capture loop runs asynchronously. (1), (2), (3) and (4) may be
+  // interleaved.
+  void StartCapture(Camera3RequestTemplate stream_type);
+
+  // Registers a free buffer, if any, of |stream_type| to the camera HAL.
+  void RegisterBuffer(Camera3RequestTemplate stream_type);
+
+  // Calls ProcessCaptureRequest if the buffer indicated by |buffer_index| is
+  // successfully registered.
+  void OnRegisteredBuffer(Camera3RequestTemplate stream_type,
+                          size_t buffer_index,
+                          int32_t result);
+
+  // Creates and sends out a capture request for |stream_type|.  The capture
+  // request contains the buffer handle indicated by |buffer_index|.
+  void ProcessCaptureRequest(Camera3RequestTemplate stream_type,
+                             size_t buffer_index);
+  // Calls RegisterBuffer to attempt to register any remaining free buffers of
+  // |stream_type|.
+  void OnProcessedCaptureRequest(Camera3RequestTemplate stream_type,
+                                 int32_t result);
+
+  // Camera3CallbackOps implementations.
+
+  // ProcessCaptureResult receives the result metadata as well as the filled
+  // buffer from camera HAL.  The result metadata may be divided and delivered
+  // in several stages.  Before all the result metadata is received the partial
+  // results are kept in |partial_results_|.
+  void ProcessCaptureResult(Camera3CaptureResultPtr result) final;
+
+  // Notify receives the shutter time of capture requests and various errors
+  // from camera HAL.  The shutter time is used as the timestamp in the video
+  // frame delivered to Chrome.
+  void Notify(Camera3NotifyMsgPtr message) final;
+
+  // Submits the captured buffer of frame |frame_number_| to Chrome, then
+  // enqueues the buffer to free buffer queue for the next capture request.
+  void SubmitCaptureResult(uint32_t frame_number);
+
+  VideoCaptureDeviceDescriptor device_descriptor_;
+  std::unique_ptr<VideoCaptureDevice::Client> client_;
+
+  // Stores the static camera characteristics of the camera device.
+  arc::mojom::CameraMetadataPtr static_metadata_;
+
+  // The state the CameraDeviceDelegate currently is in.
+  State state_;
+
+  // Clockwise rotation in degrees. This value should be 0, 90, 180, or 270.
+  int rotation_;
+
+  // Mojo proxy and binding for the camera HAL device ops and callback ops API.
+  mojo::InterfacePtrInfo<Camera3DeviceOps> device_ops_info_;
+  Camera3DeviceOpsPtr device_ops_;
+  mojo::Binding<Camera3CallbackOps> callback_ops_;
+
+  // Where all the Mojo IPC calls takes place.
+  const scoped_refptr<base::SingleThreadTaskRunner> device_task_runner_;

[chfremer, 2017/04/27 22:41:41]

Would "ipc_task_runner_" be a more revealing name for this?

[jcliang, 2017/04/28 04:47:57]

Done.

+
+  // The frame number.  Increased by one for each capture request sent; reset to
+  // zero in AllocateAndStart.
+  uint32_t frame_number_;
+
+  struct StreamContext {
+    // Capture parameters provided by Chrome.
+    VideoCaptureParams params;
+    // The actual pixel format used in the capture request.
+    VideoCaptureFormat capture_format;
+    // The camera HAL stream.
+    arc::mojom::Camera3StreamPtr stream;
+    // The request settings used in the capture request of this stream.
+    arc::mojom::CameraMetadataPtr request_settings;
+    // The allocated buffers of this stream.
+    std::vector<std::unique_ptr<base::SharedMemory>> buffers;
+    // The free buffers of this stream.
+    std::queue<size_t> free_buffers;

[chfremer, 2017/04/27 22:41:41]

Are these buffers or buffer ids? Or indices into the |buffers| vector?

[jcliang, 2017/04/28 04:47:57]

It's indices into the |buffers| vector. I'll update the comment to clarify this.

+  };
+
+  StreamContext* GetStreamContext(Camera3RequestTemplate stream_type) {
+    auto it = streams_.find(stream_type);
+    if (it == streams_.end()) {
+      return nullptr;
+    }
+    return &(it->second);
+  }
+
+  // Stores the stream context of each active stream.
+  std::unordered_map<Camera3RequestTemplate, StreamContext> streams_;
+
+  // CaptureResult is used to hold the partial capture results for each frame.
+  struct CaptureResult {
+    CaptureResult()
+        : metadata(arc::mojom::CameraMetadata::New()), partial_stage(0) {}
+    // |reference_time| and |timestamp| are derived from the shutter time of
+    // this frame.  They are be passed to |client_->OnIncomingCapturedData|
+    // along with the |buffers| when the captured frame is submitted.
+    base::TimeTicks reference_time;
+    base::TimeDelta timestamp;
+    // The result metadata.  Contains various information about the captured
+    // frame.
+    arc::mojom::CameraMetadataPtr metadata;
+    // The buffer handles that hold the captured data of this frame.
+    std::unordered_map<arc::mojom::Camera3RequestTemplate,
+                       arc::mojom::Camera3StreamBufferPtr>
+        buffers;
+    // The current stage of this partial result.
+    uint32_t partial_stage;
+  };
+
+  // The number of partial stages.  |partial_result_count_| is learned by
+  // querying |static_metadata_|.  In case the result count is absent in
+  // |static_metadata_|, it defaults to one which means all the result metadata
+  // and captured buffer of a frame are returned together in one shot.
+  uint32_t partial_result_count_;
+
+  // The shutter time of the first frame.  We derive the |timestamp| of a frame
+  // using the difference between the frame's shutter time and
+  // |first_frame_shutter_time_|.
+  base::TimeTicks first_frame_shutter_time_;
+
+  // Stores the partial capture results of the current in-flight frames.
+  std::map<uint32_t, CaptureResult> partial_results_;
+
+  DISALLOW_IMPLICIT_CONSTRUCTORS(CameraDeviceDelegate);
+};
+
+}  // namespace media
+
+#endif  // MEDIA_CAPTURE_VIDEO_CHROMEOS_CAMERA_DEVICE_DELEGATE_H_