Binding media stream audio track to speech recognition [renderer]

content/renderer/speech_recognition_audio_source_provider.cc

Index: content/renderer/speech_recognition_audio_source_provider.cc
diff --git a/content/renderer/speech_recognition_audio_source_provider.cc b/content/renderer/speech_recognition_audio_source_provider.cc
new file mode 100644
index 0000000000000000000000000000000000000000..4da5e22a12554b966e907e03b451cde5dd7c8d9e
--- /dev/null
+++ b/content/renderer/speech_recognition_audio_source_provider.cc
@@ -0,0 +1,169 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/renderer/speech_recognition_audio_source_provider.h"
+
+#include "base/logging.h"
+#include "base/memory/shared_memory.h"
+#include "base/threading/thread_restrictions.h"

[burnik, 2014/09/16 19:10:22]

Removed.

+#include "base/time/time.h"
+#include "media/audio/audio_parameters.h"
+#include "media/base/audio_fifo.h"
+
+namespace content {
+
+SpeechRecognitionAudioSourceProvider::SpeechRecognitionAudioSourceProvider(
+    const blink::WebMediaStreamTrack& track,
+    const media::AudioParameters& params, const base::SharedMemoryHandle memory,
+    base::SyncSocket* socket, OnStoppedCB on_stopped_cb)
+    : track_(track),
+      shared_memory_(memory, false),
+      socket_(socket),
+      output_params_(params),
+      track_stopped_(false),
+      buffer_index_(0),
+      on_stopped_cb_(on_stopped_cb) {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  DCHECK(params.IsValid());

[no longer working on chromium, 2014/09/16 12:44:06]

add an DCHECK here:
DCHECK(IsAllowedAudioTrack(track));

[burnik, 2014/09/16 19:10:22]

Done. And s/IsAllowedAudioTrack/IsSupportedTrack/.

+  const size_t memory_length = media::AudioBus::CalculateMemorySize(params) +
+                               sizeof(media::AudioInputBufferParameters);
+  CHECK(shared_memory_.Map(memory_length));
+
+  uint8* ptr = static_cast<uint8*>(shared_memory_.memory());
+  media::AudioInputBuffer* buffer =
+      reinterpret_cast<media::AudioInputBuffer*>(ptr);
+  // Keep params for sync with client via |params.size| on the shared memory.

[no longer working on chromium, 2014/09/16 12:44:06]

move this comment up above uint8* ptr =
static_cast<uint8*>(shared_memory_.memory());

[burnik, 2014/09/16 19:10:22]

Done. And refactored.
On 2014/09/16 12:44:06, xians1 wrote:

+  peer_buffer_index_ = &(buffer->params.size);
+  // Client must manage his own counter and reset it.

[no longer working on chromium, 2014/09/16 12:44:06]

add an empty line before the comment.

[burnik, 2014/09/16 19:10:22]

Done.

+  DCHECK_EQ(0U, *peer_buffer_index_);
+  output_bus_ = media::AudioBus::WrapMemory(params, buffer->audio);
+  // Connect the source provider to the track as a sink.

[no longer working on chromium, 2014/09/16 12:44:06]

ditto

[burnik, 2014/09/16 19:10:23]

Done.

+  MediaStreamAudioSink::AddToAudioTrack(this, track_);
+}
+
+SpeechRecognitionAudioSourceProvider::~SpeechRecognitionAudioSourceProvider() {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  if (audio_converter_.get()) audio_converter_->RemoveInput(this);

[no longer working on chromium, 2014/09/16 12:44:06]

new line after the if (), this comment applies to the rest of the code in this
file.

[burnik, 2014/09/16 19:10:23]

Done.

+  // Notify the track before this sink goes away.
+  if (!track_stopped_) MediaStreamAudioSink::RemoveFromAudioTrack(this, track_);
+}
+
+// static
+bool SpeechRecognitionAudioSourceProvider::IsAllowedAudioTrack(
+    const blink::WebMediaStreamTrack& track) {
+  if (track.source().type() != blink::WebMediaStreamSource::TypeAudio)
+    return false;
+  MediaStreamAudioSource* native_source =

[no longer working on chromium, 2014/09/16 12:44:06]

nit, add an empty line. after the return false;

[burnik, 2014/09/16 19:10:23]

Done.

+      static_cast<MediaStreamAudioSource*>(track.source().extraData());
+  if (!native_source) return false;

[no longer working on chromium, 2014/09/16 12:44:05]

ditto

[burnik, 2014/09/16 19:10:22]

Done.

+  const StreamDeviceInfo& device_info = native_source->device_info();
+  // Purposely only support tracks from an audio device. Dissallow WebAudio.
+  return (device_info.device.type == content::MEDIA_DEVICE_AUDIO_CAPTURE);
+}
+
+void SpeechRecognitionAudioSourceProvider::OnSetFormat(
+    const media::AudioParameters& input_params) {
+  // We need detach the thread here because it will be a new capture thread
+  // calling OnSetFormat() and OnData() if the source is restarted.
+  capture_thread_checker_.DetachFromThread();
+  DCHECK(capture_thread_checker_.CalledOnValidThread());
+  DCHECK(input_params.IsValid());
+
+  input_params_ = input_params;
+  fifo_buffer_size_ = output_params_.frames_per_buffer() *
+                      input_params_.sample_rate() /
+                      output_params_.sample_rate();
+  DCHECK_GE(fifo_buffer_size_, input_params_.frames_per_buffer());
+  DCHECK_GE(fifo_buffer_size_, output_params_.frames_per_buffer());

[no longer working on chromium, 2014/09/16 12:44:06]

these two DCHECKs are not always right, for example, what if
input_params_.sample_rate() is 8K and output_params_.sample_rate() is 16K.
Some more thought is needed here.

[burnik, 2014/09/16 19:10:23]

Do we ever have mic input which is 8K? What should be the size of the FIFO then
in a general case?
On 2014/09/16 12:44:06, xians1 wrote:

[no longer working on chromium, 2014/09/17 15:55:19]

Yes, 8K as sample rate does exist on all platforms.

[burnik, 2014/09/18 19:09:21]

True, removed the second DCHECK. It doesn't really make sense anyway since the
FIFO is used for buffering the input.
On 2014/09/17 15:55:19, xians1 wrote:

+
+  // Allows for some delays on the endpoint client.
+  static const int kNumberOfBuffersInFifo = 2;
+  int frames_in_fifo = kNumberOfBuffersInFifo * fifo_buffer_size_;
+  fifo_.reset(new media::AudioFifo(input_params.channels(), frames_in_fifo));
+  input_bus_ = media::AudioBus::Create(input_params.channels(),
+                                       input_params.frames_per_buffer());
+
+  // Create the audio converter with |disable_fifo| as false so that the
+  // converter will request input_params.frames_per_buffer() each time.
+  // This will not increase the complexity as there is only one client to
+  // the converter.
+  audio_converter_.reset(
+      new media::AudioConverter(input_params, output_params_, false));
+  audio_converter_->AddInput(this);
+}
+
+void SpeechRecognitionAudioSourceProvider::OnReadyStateChanged(
+    blink::WebMediaStreamSource::ReadyState state) {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  if (track_stopped_)

[no longer working on chromium, 2014/09/16 12:44:05]

You should never get OnReadyStateChanged() more than once.

DCHECK(!track_stopped_);

[burnik, 2014/09/16 19:10:22]

Done.

+      return;
+
+  if (state == blink::WebMediaStreamSource::ReadyStateEnded) {
+    track_stopped_ = true;
+    if (!on_stopped_cb_.is_null())
+      on_stopped_cb_.Run();
+  }
+}
+
+void SpeechRecognitionAudioSourceProvider::OnData(const int16* audio_data,
+                                                  int sample_rate,
+                                                  int number_of_channels,
+                                                  int number_of_frames) {
+  DCHECK(capture_thread_checker_.CalledOnValidThread());
+  DCHECK(peer_buffer_index_);
+  DCHECK_EQ(input_bus_->frames(), number_of_frames);
+  DCHECK_EQ(input_bus_->channels(), number_of_channels);
+  if (fifo_->frames() + number_of_frames > fifo_->max_frames()) {
+    NOTREACHED() << "Audio FIFO overflow";
+    return;
+  }
+  // TODO(xians): A better way to handle the interleaved and deinterleaved
+  // format switching, see issue/317710.
+  input_bus_->FromInterleaved(audio_data, number_of_frames,
+                              sizeof(audio_data[0]));
+
+  fifo_->Push(input_bus_.get());
+  // Wait for FIFO to have at least |fifo_buffer_size_| frames ready.
+  if (fifo_->frames() < fifo_buffer_size_)
+    return;
+
+  // Make sure the previous output buffer was consumed by client before we send
+  // the next buffer. |peer_buffer_index_| is pointing to shared memory.
+  // The client must write to it (incrementing by 1) once the the buffer was
+  // consumed. This is intentional not to block this audio capturing thread.
+  if (buffer_index_ != (*peer_buffer_index_)) {
+    DLOG(WARNING) << "Buffer synchronization lag";
+    return;
+  }
+
+  audio_converter_->Convert(output_bus_.get());
+
+  // Notify client to consume buffer |buffer_index_| on |output_bus_|.
+  const size_t bytes_sent =
+      socket_->Send(&buffer_index_, sizeof(buffer_index_));
+  if (bytes_sent != sizeof(buffer_index_)) {
+    // The send usually fails if the user changes his input audio device.

[no longer working on chromium, 2014/09/16 12:44:06]

is this comment true?

[burnik, 2014/09/16 19:10:22]

As far as I've tested, yes.
On 2014/09/16 12:44:06, xians1 wrote:

[no longer working on chromium, 2014/09/17 15:55:19]

Interesting, to double check, if you changed the input device used by
getUserMedia when connecting to recognition, it will trigger this code?

[burnik, 2014/09/18 19:09:21]

If I change input device from System Sound on linux, this occurs.
On 2014/09/17 15:55:19, xians1 wrote:

+    DVLOG(1) << "Failed sending buffer index to peer";
+    // We have discarded this buffer, but could still recover on the next one.
+    // Although, if the socket was closed, this will shortly end up
+    // in |ErrorState::AUDIO_FIFO_OVERFLOW|.

[no longer working on chromium, 2014/09/16 12:44:05]

update the comment.

[burnik, 2014/09/16 19:10:22]

Done.

+    return;
+  }
+
+  // Count the sent buffer. We expect the client to do the same on his end.
+  ++buffer_index_;
+}
+
+double SpeechRecognitionAudioSourceProvider::ProvideInput(
+    media::AudioBus* audio_bus, base::TimeDelta buffer_delay) {
+  DCHECK(capture_thread_checker_.CalledOnValidThread());
+  if (fifo_->frames() >= audio_bus->frames())
+    fifo_->Consume(audio_bus, 0, audio_bus->frames());
+  else
+    audio_bus->Zero();

[no longer working on chromium, 2014/09/16 12:44:06]

can the else case be possible at all?

[burnik, 2014/09/16 19:10:23]

Yes. Shown many times while testing. This occurs when we attach the sink via
*audio_converter_->AddInput(this)*. 
This is related to the story of the now deprecated |attached_converter_| if you
look back to one of the earliest patchsets.
On 2014/09/16 12:44:06, xians1 wrote:

[no longer working on chromium, 2014/09/17 15:55:19]

Got it, thanks.

[burnik, 2014/09/18 19:09:21]

Acknowledged.

+
+  return 1.0;
+}
+
+}  // namespace content