Binding media stream audio track to speech recognition [renderer]

content/renderer/speech_recognition_audio_source_provider.h

Index: content/renderer/speech_recognition_audio_source_provider.h
diff --git a/content/renderer/speech_recognition_audio_source_provider.h b/content/renderer/speech_recognition_audio_source_provider.h
new file mode 100644
index 0000000000000000000000000000000000000000..3f55d3e392249e7a15d51e891a626db47c7ea93d
--- /dev/null
+++ b/content/renderer/speech_recognition_audio_source_provider.h
@@ -0,0 +1,105 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CONTENT_RENDERER_SPEECH_RECOGNITION_AUDIO_SOURCE_PROVIDER_H_
+#define CONTENT_RENDERER_SPEECH_RECOGNITION_AUDIO_SOURCE_PROVIDER_H_
+
+#include "base/callback.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/memory/shared_memory.h"
+#include "base/threading/thread_checker.h"
+#include "content/common/content_export.h"
+#include "content/public/renderer/media_stream_audio_sink.h"
+#include "media/audio/audio_device_thread.h"
+#include "media/base/audio_converter.h"
+#include "third_party/WebKit/public/platform/WebMediaStreamTrack.h"
+
+namespace base {
+class TimeDelta;
+}
+
+namespace media {
+class AudioBus;
+class AudioConverter;
+class AudioFifo;
+class AudioParameters;
+}
+
+namespace content {
+
+// SpeechRecognitionAudioSourceProvider works as a sink to the
+// WebRtcLocalAudioTrack and stores the capture data to a FIFO.
+// When the FIFO has enough buffer, it passes the buffer to
+// the speech recognizer.
+//
+class CONTENT_EXPORT SpeechRecognitionAudioSourceProvider
+    :  NON_EXPORTED_BASE(public media::AudioConverter::InputCallback),
+       NON_EXPORTED_BASE(public MediaStreamAudioSink) {
+ public:
+  typedef base::Callback<void()> OnDataCB;
+  typedef base::Callback<void()> OnErrorCB;
+
+  SpeechRecognitionAudioSourceProvider(const blink::WebMediaStreamTrack& track,
+                                       const media::AudioParameters& params,
+                                       base::SharedMemoryHandle memory,
+                                       int memory_length,
+                                       const OnDataCB& on_data_callback,
+                                       const OnErrorCB& on_error_callback);
+  virtual ~SpeechRecognitionAudioSourceProvider();
+
+  // MediaStreamAudioSink implementation.

[tommi (sloooow) - chröme, 2014/08/25 14:38:46]

Do these implementations need to be a part of the public interface of
SpeechRecognitionAudioSourceProvider?
If not (i.e. if they are an implementation detail of
SpeechRecognitionAudioSourceProvider), then I'd like to make these protected or
private.

[burnik, 2014/08/29 09:18:16]

Done.

+  virtual void OnData(const int16* audio_data,
+                      int sample_rate,
+                      int number_of_channels,
+                      int number_of_frames) OVERRIDE;
+  virtual void OnSetFormat(const media::AudioParameters& params) OVERRIDE;
+  virtual void OnReadyStateChanged(
+      blink::WebMediaStreamSource::ReadyState state) OVERRIDE;
+
+  // media::AudioConverter::Inputcallback implementation.
+  // This function is triggered by provideInput() on the WebAudio audio thread,
+  // so it has been under the protection of |lock_|.
+  virtual double ProvideInput(media::AudioBus* audio_bus,
+                              base::TimeDelta buffer_delay) OVERRIDE;
+
+
+  // Called by client when consumed an AudioChunk
+  virtual void NotifyAudioBusConsumed();

[tommi (sloooow) - chröme, 2014/08/25 14:38:46]

OVERRIDE?

[burnik, 2014/08/29 09:18:16]

Removed due to sync_socket.
On 2014/08/25 14:38:46, tommi wrote:

+
+ private:
+  const int kNumberOfBuffersInFifo = 2;

[tommi (sloooow) - chröme, 2014/08/25 14:38:45]

static?

[burnik, 2014/08/29 09:18:16]

Done.

+  int fifo_buffer_size_;

[tommi (sloooow) - chröme, 2014/08/25 14:38:45]

can you add documentation about what this represents and how big the buffer can
expected to be?

+
+  // Used to DCHECK that some methods are called on the capture audio thread.
+  base::ThreadChecker capture_thread_checker_;
+
+  // The audio track that this source provider is connected to.
+  blink::WebMediaStreamTrack track_;
+
+  base::SharedMemory shared_memory_;

[henrika (OOO until Aug 14), 2014/08/25 14:46:00]

Could you add more information about what these members do?

[burnik, 2014/08/29 09:18:16]

Done.

+  scoped_ptr<media::AudioConverter> audio_converter_;
+  scoped_ptr<media::AudioFifo> fifo_;
+  scoped_ptr<media::AudioBus> input_bus_;
+  scoped_ptr<media::AudioBus> output_bus_;
+  media::AudioParameters input_params_;
+  media::AudioParameters output_params_;
+  // used for notifying client to consume audio
+  const OnDataCB on_data_callback_;
+  // used for notifying client of fifo overflow
+  const OnDataCB on_error_callback_;
+  bool attached_converter_;
+  bool track_stopped_;
+  // Used for locking the |unconsumed_audio_buffers_| since OnData is on the
+  // audio capture thread and client notifies back on the render thread
+  base::Lock lock_;
+  // |unconsumed_audio_buffers_| is used to make sure all buffers are
+  // consumed by the client in order via alternating the counter
+  int unconsumed_audio_buffers_;
+
+  DISALLOW_COPY_AND_ASSIGN(SpeechRecognitionAudioSourceProvider);
+};
+
+}  // namespace content
+
+#endif  // CONTENT_RENDERER_SPEECH_RECOGNITION_AUDIO_SOURCE_PROVIDER_H_