Binding media stream audio track to speech recognition [renderer]

content/renderer/speech_recognition_audio_source_provider.h

Index: content/renderer/speech_recognition_audio_source_provider.h
diff --git a/content/renderer/speech_recognition_audio_source_provider.h b/content/renderer/speech_recognition_audio_source_provider.h
new file mode 100644
index 0000000000000000000000000000000000000000..6b1fd487c7757f78456b1cc09e2fe0e10dddf365
--- /dev/null
+++ b/content/renderer/speech_recognition_audio_source_provider.h
@@ -0,0 +1,125 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CONTENT_RENDERER_SPEECH_RECOGNITION_AUDIO_SOURCE_PROVIDER_H_
+#define CONTENT_RENDERER_SPEECH_RECOGNITION_AUDIO_SOURCE_PROVIDER_H_
+
+#include "base/callback.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/memory/shared_memory.h"
+#include "base/native_sync_socket.h"
+#include "base/threading/thread_checker.h"
+#include "base/threading/thread_restrictions.h"
+#include "content/common/content_export.h"
+#include "content/public/renderer/media_stream_audio_sink.h"
+#include "content/renderer/media/media_stream_audio_source.h"
+#include "media/audio/audio_device_thread.h"
+#include "media/base/audio_converter.h"
+#include "third_party/WebKit/public/platform/WebMediaStreamTrack.h"
+
+namespace media {
+class AudioBus;
+class AudioConverter;
+class AudioFifo;
+class AudioParameters;
+}
+
+namespace content {
+
+// SpeechRecognitionAudioSourceProvider works as a sink to the
+// WebRtcLocalAudioTrack and stores the capture data to a FIFO.
+// When the FIFO has enough buffer, it passes the buffer to
+// the speech recognizer.
+//
+class CONTENT_EXPORT SpeechRecognitionAudioSourceProvider
+    :  NON_EXPORTED_BASE(public media::AudioConverter::InputCallback),
+       NON_EXPORTED_BASE(public MediaStreamAudioSink) {
+ public:
+  SpeechRecognitionAudioSourceProvider(
+    const blink::WebMediaStreamTrack& track,
+    const media::AudioParameters& params,
+    base::SharedMemoryHandle memory,
+    base::NativeSyncSocket::Descriptor socket,
+    int memory_length);
+  virtual ~SpeechRecognitionAudioSourceProvider();
+  // determines the policy on what types of tracks are allowed

[tommi (sloooow) - chröme, 2014/08/29 11:25:31]

nit: empty line above this comment for readability. ultra nit: Comments start
with a capital letter and end with a period.

[burnik, 2014/08/29 13:26:17]

Done.

+  static bool IsAllowedAudioTrack(const blink::WebMediaStreamTrack& track);
+
+ private:
+  // MediaStreamAudioSink implementation.
+  virtual void OnData(const int16* audio_data,
+                      int sample_rate,
+                      int number_of_channels,
+                      int number_of_frames) OVERRIDE;
+  virtual void OnSetFormat(const media::AudioParameters& params) OVERRIDE;
+  virtual void OnReadyStateChanged(
+      blink::WebMediaStreamSource::ReadyState state) OVERRIDE;
+
+  // media::AudioConverter::Inputcallback implementation.
+  // This function is triggered by provideInput() on the WebAudio audio thread,
+  // so it has been under the protection of |lock_|.
+  virtual double ProvideInput(media::AudioBus* audio_bus,
+                              base::TimeDelta buffer_delay) OVERRIDE;
+
+ private:
+  // Allows for some delays on the client
+  static const int kNumberOfBuffersInFifo = 2;

[no longer working on chromium, 2014/08/29 12:23:07]

move it to the implementation.

[burnik, 2014/08/29 13:26:17]

Done.

+
+  // Number of frames per buffer in FIFO. When the buffer is full we convert and
+  // consume it on the |output_bus_|. Size of the buffer is depends on the
+  // resampler. Example: for 44.1 to 16.0 conversion, it should be 4100 frames.
+  int fifo_buffer_size_;
+
+  // Since |Receive| on |socket_| is blocking, we don't want to miss out on
+  // frames delivered to |OnData|. so we use |ReceiveWithTimeout|.
+  base::TimeDelta max_sync_delay_time_delta_;
+
+  // Used to DCHECK that some methods are called on the main render thread.
+  base::ThreadChecker main_render_thread_checker_;
+
+  // Used to DCHECK that some methods are called on the capture audio thread.
+  base::ThreadChecker capture_thread_checker_;
+
+  // The audio track that this source provider is connected to.
+  blink::WebMediaStreamTrack track_;
+
+  // Shared memory used by audio buses on both browser and renderer processes.
+  base::SharedMemory shared_memory_;
+
+  // Socket for syncronization of audio bus reads/writes.
+  base::SyncSocket socket_;
+
+  // Used as a resampler to deliver appropriate format to speech recognition.
+  scoped_ptr<media::AudioConverter> audio_converter_;
+
+  // FIFO is used for queueing audio frames before we resample.
+  scoped_ptr<media::AudioFifo> fifo_;
+
+  // Audio delivered from source.
+  scoped_ptr<media::AudioBus> input_bus_;
+
+  // Audio bus shared with the browser process via |shared_memory_|.
+  scoped_ptr<media::AudioBus> output_bus_;
+
+  // Params of the source audio.
+  media::AudioParameters input_params_;
+
+  // Params used by speech recognition.
+  media::AudioParameters output_params_;
+
+  // We attach the resampler once we have enough data in FIFO and not before.

[no longer working on chromium, 2014/08/29 12:23:07]

confusing comment, can it make it more clear on what this flag is used for?

[burnik, 2014/08/29 13:26:17]

Done.
On 2014/08/29 12:23:07, xians1 wrote:

+  bool attached_converter_;
+
+  // Whether the track has been stopped on the input.
+  bool track_stopped_;
+
+  // Making sure all buffers are consumed by the client in order.
+  uint32 buffer_index_;
+
+  DISALLOW_COPY_AND_ASSIGN(SpeechRecognitionAudioSourceProvider);
+};
+
+}  // namespace content
+
+#endif  // CONTENT_RENDERER_SPEECH_RECOGNITION_AUDIO_SOURCE_PROVIDER_H_