Binding media stream audio track to speech recognition [renderer]

content/renderer/media/speech_recognition_audio_source_provider.h

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CONTENT_RENDERER_MEDIA_SPEECH_RECOGNITION_AUDIO_SOURCE_PROVIDER_H_
+#define CONTENT_RENDERER_MEDIA_SPEECH_RECOGNITION_AUDIO_SOURCE_PROVIDER_H_
+
+#include "base/callback.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/memory/shared_memory.h"
+#include "base/sync_socket.h"
+#include "base/threading/thread_checker.h"
+#include "content/common/content_export.h"
+#include "content/public/renderer/media_stream_audio_sink.h"
+#include "content/renderer/media/media_stream_audio_source.h"
+#include "media/audio/audio_parameters.h"
+#include "media/base/audio_converter.h"
+#include "third_party/WebKit/public/platform/WebMediaStreamTrack.h"
+
+namespace media {
+class AudioBus;
+class AudioFifo;
+}
+
+namespace content {
+
+// SpeechRecognitionAudioSourceProvider works as an audio sink to the

[tommi (sloooow) - chröme, 2014/09/24 09:51:59]

Is 'source provider' a good name?  The reason I'm wondering is because usually
things are chained together like: src->track->sink, so if this is an audio sink,
it feels odd to have it be called a "source provider".  That being said, I
haven't looked at the implementation so perhaps this will make sense to me in a
bit.

[burnik, 2014/09/24 11:54:22]

Did it make sense in the impl? My reasoning that this audio data is not being
consumed here on the render, rather on the browser.

Do you think I should rename it to SpeechRecognitionAudioSink?

Shijing, your thoughts?

[no longer working on chromium, 2014/09/24 13:40:46]

From the webspeech's perspective, this class is a source provider.
From the audio track's perspective, this class is a audio sink.

I don't have strong opinion here, that says, I am fine to rename it to
SpeechRecognitionAudioSink.

+// WebRtcLocalAudioTrack. It stores the capture data into a FIFO.
+// When the FIFO has enough data for resampling, it converts it,
+// passes the buffer to the |WebSpeechRecognizer| via SharedMemory
+// and notifies it via SyncSocket followed by incrementing the |buffer_index_|.
+// |WebSpeechRecognizer| increments the shared buffer index to synchronize.
+class CONTENT_EXPORT SpeechRecognitionAudioSourceProvider
+    : NON_EXPORTED_BASE(public media::AudioConverter::InputCallback),
+      NON_EXPORTED_BASE(public MediaStreamAudioSink) {
+ public:
+
+  typedef base::Callback<void()> OnStoppedCB;
+
+  SpeechRecognitionAudioSourceProvider(const blink::WebMediaStreamTrack& track,

[tommi (sloooow) - chröme, 2014/09/24 09:51:59]

can you document these parameters? particularly how ownership is passed around
etc.

[burnik, 2014/09/24 11:54:22]

Added a TODO for next round.

+                                       const media::AudioParameters& params,
+                                       const base::SharedMemoryHandle memory,
+                                       base::SyncSocket* socket,

[tommi (sloooow) - chröme, 2014/09/24 09:51:59]

actually, if ownership is being passed here, please use scoped_ptr<> here (and
.Pass() on the caller side).

[burnik, 2014/09/24 11:54:22]

Done. I suppose I should initialize in constructor like this as well:

: socket_(socket.Pass())


Right?

+                                       OnStoppedCB on_error_cb);
+
+  virtual ~SpeechRecognitionAudioSourceProvider();
+
+  // Returns whether the provided track is supported.
+  static bool IsSupportedTrack(const blink::WebMediaStreamTrack& track);
+
+ private:
+  // content::MediaStreamAudioSink implementation.
+  virtual void OnReadyStateChanged(
+      blink::WebMediaStreamSource::ReadyState state) OVERRIDE;
+
+  virtual void OnData(const int16* audio_data, int sample_rate,
+                      int number_of_channels, int number_of_frames) OVERRIDE;
+  virtual void OnSetFormat(const media::AudioParameters& params) OVERRIDE;
+
+  // media::AudioConverter::Inputcallback implementation.
+  virtual double ProvideInput(media::AudioBus* audio_bus,
+                              base::TimeDelta buffer_delay) OVERRIDE;
+
+  // Number of frames per buffer in FIFO. When the buffer is full we convert and
+  // consume it on the |output_bus_|. Size of the buffer depends on the
+  // resampler. Example: for 44.1 to 16.0 conversion, it should be 4100 frames.
+  int fifo_buffer_size_;
+
+  // Used to DCHECK that some methods are called on the main render thread.
+  base::ThreadChecker main_render_thread_checker_;
+
+  // Used to DCHECK that some methods are called on the capture audio thread.
+  base::ThreadChecker capture_thread_checker_;
+
+  // The audio track that this source provider is connected to.
+  const blink::WebMediaStreamTrack track_;
+
+  // Shared memory used by audio buses on both browser and renderer processes.
+  base::SharedMemory shared_memory_;
+
+  // Socket for syncronization of audio bus reads/writes.
+  // Created on the renderer client and passed here. Accessed on capture thread.
+  scoped_ptr<base::SyncSocket> socket_;
+
+  // Used as a resampler to deliver appropriate format to speech recognition.
+  scoped_ptr<media::AudioConverter> audio_converter_;
+
+  // FIFO is used for queueing audio frames before we resample.
+  scoped_ptr<media::AudioFifo> fifo_;
+
+  // Audio delivered from source.
+  scoped_ptr<media::AudioBus> input_bus_;
+
+  // Audio bus shared with the browser process via |shared_memory_|.
+  scoped_ptr<media::AudioBus> output_bus_;
+
+  // Params of the source audio. Can change when |OnSetFormat()| occurs.
+  media::AudioParameters input_params_;
+
+  // Params used by speech recognition.
+  const media::AudioParameters output_params_;
+
+  // Whether the track has been stopped.
+  bool track_stopped_;
+
+  // Local counter of audio buffers for synchronization.
+  uint32 buffer_index_;
+
+  // Peer's counter of audio buffers for synchronization.
+  const uint32* peer_buffer_index_;
+
+  // Callback for the renderer client. Called when the audio track was stopped.
+  const OnStoppedCB on_stopped_cb_;
+
+  DISALLOW_COPY_AND_ASSIGN(SpeechRecognitionAudioSourceProvider);
+};
+
+}  // namespace content
+
+#endif  // CONTENT_RENDERER_MEDIA_SPEECH_RECOGNITION_AUDIO_SOURCE_PROVIDER_H_