media: add video capture device for ARC++ camera HAL v3

media/capture/video/chromeos/camera_device_delegate.h

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_CAPTURE_VIDEO_CHROMEOS_CAMERA_DEVICE_DELEGATE_H_
+#define MEDIA_CAPTURE_VIDEO_CHROMEOS_CAMERA_DEVICE_DELEGATE_H_
+
+#include <map>
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "base/macros.h"
+#include "base/synchronization/waitable_event.h"
+#include "media/capture/video/chromeos/mojo/arc_camera3.mojom.h"
+#include "media/capture/video/video_capture_device.h"
+#include "media/capture/video_capture_types.h"
+#include "mojo/public/cpp/bindings/binding.h"
+
+namespace media {
+
+// CameraDeviceDelegate is instantiated on the capture thread where
+// AllocateAndStart of VideoCaptureDeviceArcChromeOS runs on.  All the methods
+// in CameraDeviceDelegate runs on |ipc_task_runner_| and hence all the

[wuchengli, 2017/05/08 04:22:03]

s/runs/run/

[jcliang, 2017/05/13 08:53:15]

Done.

+// access to member variables is sequenced.
+class CAPTURE_EXPORT CameraDeviceDelegate final
+    : public base::RefCountedThreadSafe<CameraDeviceDelegate>,
+      public arc::mojom::Camera3CallbackOps {
+ public:
+  enum State {
+    // The camera device is completely stopped. This is the initial state, and
+    // is also set in OnClosed().
+    kStopped,
+
+    // The camera device is starting and waiting to be initialized.
+    //
+    // The kStarting state is set in AllocateAndStart().
+    kStarting,
+
+    // The camera device is initialized and can accept stream configuration
+    // requests.
+    //
+    // The state is transitioned to kInitialized through:
+    //
+    //   Initialize() -> OnInitialized()
+    kInitialized,
+
+    // The various capture streams are configured and the camera device is ready
+    // to process capture requests.
+    //
+    // The state is transitioned to kStreamConfigured through:
+    //
+    //   ConfigureStreams() -> OnConfiguredStreams() ->
+    //   ConstructDefaultRequestSettings() ->
+    //   OnConstructedDefaultRequestSettings()
+    kStreamConfigured,
+
+    // The camera device is capturing video streams.
+    //
+    // The kCapturing state is set in StartCapture().
+    kCapturing,
+
+    // When the camera device is in the kCapturing state, a capture loop is
+    // constantly running:
+    //
+    // On the CameraDeviceDelegate side, we register and submit a capture
+    // request whenever a free buffer is available:
+    //
+    //   RegisterBuffer() -> OnRegisteredBuffer() ->
+    //   ProcessCaptureRequest() -> OnProcessedCaptureRequest()
+    //
+    // We get various capture metadata and error notifications from the camera
+    // HAL through the following callbacks:
+    //
+    //   ProcessCaptureResult()
+    //   Notify()
+
+    // The camera device is going through the shut down process; in order to
+    // avoid race conditions, no new Mojo messages may be sent to camera HAL in
+    // the kStopping state.
+    //
+    // The kStopping state is set in StopAndDeAllocate().
+    kStopping,
+
+    // The camera device encountered an unrecoverable error and needs to be
+    // StopAndDeAllocate()'d.
+    //
+    // The kError state is set in SetErrorState().
+    kError,
+  };
+
+  CameraDeviceDelegate(
+      VideoCaptureDeviceDescriptor device_descriptor,
+      arc::mojom::CameraMetadataPtr static_metadata,
+      mojo::InterfacePtrInfo<arc::mojom::Camera3DeviceOps> device_ops_info,
+      const scoped_refptr<base::SingleThreadTaskRunner> ipc_task_runner);
+
+  // Converts the HAL pixel format |from| to Chromium pixel format.  Returns
+  // PIXEL_FORMAT_UNKNOWN if |from| is not supported.
+  static VideoPixelFormat PixFormatHalToChromium(
+      arc::mojom::HalPixelFormat from);
+
+  // Converts the Chromium pixel format |from| to DRM pixel format.  Returns 0
+  // if |from| is not supported.
+  static uint32_t PixFormatChromiumToDrm(VideoPixelFormat from);
+
+  // Delegation methods for the VideoCaptureDevice interface.
+  void AllocateAndStart(const VideoCaptureParams& params,
+                        std::unique_ptr<VideoCaptureDevice::Client> client);
+  void StopAndDeAllocate(base::WaitableEvent* closed);
+  void TakePhoto(VideoCaptureDevice::TakePhotoCallback callback);
+  void GetPhotoCapabilities(
+      VideoCaptureDevice::GetPhotoCapabilitiesCallback callback);
+  void SetPhotoOptions(mojom::PhotoSettingsPtr settings,
+                       VideoCaptureDevice::SetPhotoOptionsCallback callback);
+
+  // Sets the frame rotation angle in |rotation_|.

[wuchengli, 2017/05/08 04:22:02]

Document this in more detail. What's this for? Is this clockwise or counter
clockwise?

[jcliang, 2017/05/13 08:53:15]

Done.

+  void SetRotation(int rotation);
+
+ private:
+  friend class base::RefCountedThreadSafe<CameraDeviceDelegate>;
+
+  ~CameraDeviceDelegate() = default;
+
+  // Sets the state.
+  void SetState(State state);
+
+  // Sets state to kError and call |client_->OnError| to tear down the
+  // VideoCaptureDevice.
+  void SetErrorState(const tracked_objects::Location& from_here,
+                     const std::string& reason);
+
+  // Resets the Mojo interface and bindings.
+  void ResetMojoInterface();
+
+  // Mojo connection error handler.
+  void OnMojoConnectionError();
+
+  // Callback method for the Close Mojo IPC call.  This method resets the Mojo
+  // connection and closes the camera device, then signals |closed_| to unblock
+  // the caller.
+  void OnClosed(int32_t result);
+
+  // Initializes the camera HAL.  Initialize sets up the Camera3CallbackOps with
+  // the camera HAL.  OnInitialized continues to ConfigureStreams if the
+  // Initialize call succeeds.
+  void Initialize();
+  void OnInitialized(int32_t result);
+
+  // ConfigureStreams sets up stream context in |streams_| and configure the
+  // streams with the camera HAL.  OnConfiguredStreams updates |streams_| with
+  // the stream config returned, and allocates buffers as per |updated_config|
+  // indicates.  If there's no error OnConfiguredStreams notifies
+  // |client_| the capture has started by calling OnStarted, and proceeds to
+  // ConstructDefaultRequestSettings.
+  void ConfigureStreams();
+  void OnConfiguredStreams(
+      arc::mojom::Camera3StreamConfigurationPtr updated_config);
+
+  // ConstructDefaultRequestSettings asks the camera HAL for the default request
+  // settings of each stream in |streams_|.  OnConstructedDefaultRequestSettings
+  // sets the request settings for |stream_type| in |streams_|.  If there's no
+  // error OnConstructedDefaultRequestSettings calls StartCapture to start the
+  // video capture loop.
+  void ConstructDefaultRequestSettings(
+      arc::mojom::Camera3RequestTemplate stream_type);
+  void OnConstructedDefaultRequestSettings(
+      arc::mojom::Camera3RequestTemplate stream_type,
+      arc::mojom::CameraMetadataPtr settings);
+
+  // StartCapture is the entry point to starting the video capture of
+  // |stream_type|.  The way the video capture loop works is:
+  //
+  //  (1) If there is a free buffer, RegisterBuffer registers the buffer with
+  //      the camera HAL.
+  //  (2) Once the free buffer is registered, ProcessCaptureRequest is called to
+  //      issue a capture request which will eventually fill the registered
+  //      buffer.  Goto (1) to register the remaining free buffers.
+  //  (3) The camera HAL returns shutter time of a capture request through
+  //      Notify, and the filled buffer through ProcessCaptureResult.
+  //  (4) Once all the result metadata are collected, SubmitCaptureResult is
+  //      called to deliver the filled buffer to Chrome.  After the buffer is
+  //      consumed by Chrome it is enqueued back to the free buffer queue.
+  //      Goto (1) to start another capture loop.
+  //
+  // The capture loop runs asynchronously. (1), (2), (3) and (4) may be

[wuchengli, 2017/05/08 04:22:03]

"The capture loop runs asynchronously." is not very clear. Rephrase this.

[jcliang, 2017/05/13 08:53:15]

I've remove this since it's not important information.

+  // interleaved.
+  void StartCapture(arc::mojom::Camera3RequestTemplate stream_type);
+
+  // Registers a free buffer, if any, of |stream_type| to the camera HAL.
+  void RegisterBuffer(arc::mojom::Camera3RequestTemplate stream_type);
+
+  // Calls ProcessCaptureRequest if the buffer indicated by |buffer_index| is
+  // successfully registered.
+  void OnRegisteredBuffer(arc::mojom::Camera3RequestTemplate stream_type,
+                          size_t buffer_index,
+                          int32_t result);
+
+  // Creates and sends out a capture request for |stream_type|.  The capture
+  // request contains the buffer handle indicated by |buffer_index|.
+  void ProcessCaptureRequest(arc::mojom::Camera3RequestTemplate stream_type,
+                             size_t buffer_index);
+  // Calls RegisterBuffer to attempt to register any remaining free buffers of
+  // |stream_type|.
+  void OnProcessedCaptureRequest(arc::mojom::Camera3RequestTemplate stream_type,
+                                 int32_t result);
+
+  // Camera3CallbackOps implementations.
+
+  // ProcessCaptureResult receives the result metadata as well as the filled
+  // buffer from camera HAL.  The result metadata may be divided and delivered
+  // in several stages.  Before all the result metadata is received the partial
+  // results are kept in |partial_results_|.
+  void ProcessCaptureResult(arc::mojom::Camera3CaptureResultPtr result) final;
+
+  // Notify receives the shutter time of capture requests and various errors
+  // from camera HAL.  The shutter time is used as the timestamp in the video
+  // frame delivered to Chrome.
+  void Notify(arc::mojom::Camera3NotifyMsgPtr message) final;
+
+  // Submits the captured buffer of frame |frame_number_| to Chrome, then
+  // enqueues the buffer to free buffer queue for the next capture request.
+  void SubmitCaptureResult(uint32_t frame_number);
+
+  VideoCaptureDeviceDescriptor device_descriptor_;
+  std::unique_ptr<VideoCaptureDevice::Client> client_;
+
+  // Stores the static camera characteristics of the camera device.
+  arc::mojom::CameraMetadataPtr static_metadata_;
+
+  // The state the CameraDeviceDelegate currently is in.
+  State state_;
+
+  base::WaitableEvent* closed_;

[wuchengli, 2017/05/08 04:22:02]

Document the client owns this.

[jcliang, 2017/05/13 08:53:15]

Done.

+
+  // Clockwise rotation in degrees. This value should be 0, 90, 180, or 270.
+  int rotation_;
+
+  // Mojo proxy and binding for the camera HAL device ops and callback ops API.
+  mojo::InterfacePtrInfo<arc::mojom::Camera3DeviceOps> device_ops_info_;
+  arc::mojom::Camera3DeviceOpsPtr device_ops_;
+  mojo::Binding<arc::mojom::Camera3CallbackOps> callback_ops_;
+
+  // Where all the Mojo IPC calls takes place.
+  const scoped_refptr<base::SingleThreadTaskRunner> ipc_task_runner_;
+
+  // The frame number.  Increased by one for each capture request sent; reset to
+  // zero in AllocateAndStart.
+  uint32_t frame_number_;

[wuchengli, 2017/05/08 04:22:02]

How does HAL handle wrap-around? If the camera is used for monitoring, camera
can run a very long time and reach int32.

[jcliang, 2017/05/13 08:53:15]

It should be okay for frame number to wrap around. The frame number should only
need to be unique among the maximum possible of concurrently in-flight frames,
which is typically < 10.

+
+  struct StreamContext {
+    // Capture parameters provided by Chrome.
+    VideoCaptureParams params;
+    // The actual pixel format used in the capture request.
+    VideoCaptureFormat capture_format;
+    // The camera HAL stream.
+    arc::mojom::Camera3StreamPtr stream;
+    // The request settings used in the capture request of this stream.
+    arc::mojom::CameraMetadataPtr request_settings;
+    // The allocated buffers of this stream.
+    std::vector<std::unique_ptr<base::SharedMemory>> buffers;
+    // The free buffers of this stream.  The queue stores indices into the
+    // |buffers| vector.
+    std::queue<size_t> free_buffers;
+  };
+
+  StreamContext* GetStreamContext(
+      arc::mojom::Camera3RequestTemplate stream_type) {
+    auto it = streams_.find(stream_type);

[wuchengli, 2017/05/08 04:22:02]

The style guides says the implementation should not be in the header file.

[jcliang, 2017/05/13 08:53:15]

Done.

+    if (it == streams_.end()) {
+      return nullptr;
+    }
+    return &(it->second);
+  }
+
+  // Stores the stream context of each active stream.
+  std::unordered_map<arc::mojom::Camera3RequestTemplate, StreamContext>

[wuchengli, 2017/05/08 04:22:03]

Let's use array or vector because we'll need to access the map 30 times per
second. The index is from 1 (or 0?) to number of types. The overhead can be
less.

[jcliang, 2017/05/13 08:53:15]

This is now a unique ptr of a StreamContext after I refactor the code to only
use preview stream.

+      streams_;
+
+  // CaptureResult is used to hold the partial capture results for each frame.
+  struct CaptureResult {
+    CaptureResult()
+        : metadata(arc::mojom::CameraMetadata::New()), partial_stage(0) {}
+    // |reference_time| and |timestamp| are derived from the shutter time of
+    // this frame.  They are be passed to |client_->OnIncomingCapturedData|
+    // along with the |buffers| when the captured frame is submitted.
+    base::TimeTicks reference_time;
+    base::TimeDelta timestamp;
+    // The result metadata.  Contains various information about the captured
+    // frame.
+    arc::mojom::CameraMetadataPtr metadata;
+    // The buffer handles that hold the captured data of this frame.
+    std::unordered_map<arc::mojom::Camera3RequestTemplate,

[wuchengli, 2017/05/08 04:22:03]

JPEG (picture callback) won't be implemented anytime soon (not in Q2 or Q3).
Let's hard code the type to preview so this file is simpler.

[jcliang, 2017/05/13 08:53:15]

Done.

+                       arc::mojom::Camera3StreamBufferPtr>
+        buffers;
+    // The current stage of this partial result.
+    uint32_t partial_stage;
+  };
+
+  // The number of partial stages.  |partial_result_count_| is learned by
+  // querying |static_metadata_|.  In case the result count is absent in
+  // |static_metadata_|, it defaults to one which means all the result metadata
+  // and captured buffer of a frame are returned together in one shot.
+  uint32_t partial_result_count_;
+
+  // The shutter time of the first frame.  We derive the |timestamp| of a frame
+  // using the difference between the frame's shutter time and
+  // |first_frame_shutter_time_|.
+  base::TimeTicks first_frame_shutter_time_;
+
+  // Stores the partial capture results of the current in-flight frames.
+  std::map<uint32_t, CaptureResult> partial_results_;
+
+  DISALLOW_IMPLICIT_CONSTRUCTORS(CameraDeviceDelegate);
+};
+
+}  // namespace media
+
+#endif  // MEDIA_CAPTURE_VIDEO_CHROMEOS_CAMERA_DEVICE_DELEGATE_H_