Index: content/renderer/media/speech_recognition_audio_sink.cc |
diff --git a/content/renderer/media/speech_recognition_audio_sink.cc b/content/renderer/media/speech_recognition_audio_sink.cc |
new file mode 100644 |
index 0000000000000000000000000000000000000000..4119da07a3534cb19cfcd98820042b181e820805 |
--- /dev/null |
+++ b/content/renderer/media/speech_recognition_audio_sink.cc |
@@ -0,0 +1,184 @@ |
+// Copyright 2014 The Chromium Authors. All rights reserved. |
+// Use of this source code is governed by a BSD-style license that can be |
+// found in the LICENSE file. |
+ |
+#include "content/renderer/media/speech_recognition_audio_sink.h" |
+ |
+#include "base/logging.h" |
+#include "base/memory/shared_memory.h" |
+#include "base/time/time.h" |
+#include "media/audio/audio_parameters.h" |
+#include "media/base/audio_fifo.h" |
+ |
+namespace content { |
+ |
+SpeechRecognitionAudioSink::SpeechRecognitionAudioSink( |
+ const blink::WebMediaStreamTrack& track, |
+ const media::AudioParameters& params, |
+ const base::SharedMemoryHandle memory, |
+ scoped_ptr<base::SyncSocket> socket, |
+ const OnStoppedCB& on_stopped_cb) |
+ : track_(track), |
+ shared_memory_(memory, false), |
+ socket_(socket.Pass()), |
+ output_params_(params), |
+ track_stopped_(false), |
+ buffer_index_(0), |
+ on_stopped_cb_(on_stopped_cb) { |
+ DCHECK(socket_.get()); |
+ DCHECK(main_render_thread_checker_.CalledOnValidThread()); |
+ DCHECK(params.IsValid()); |
+ DCHECK(IsSupportedTrack(track)); |
+ const size_t memory_length = media::AudioBus::CalculateMemorySize(params) + |
+ sizeof(media::AudioInputBufferParameters); |
+ CHECK(shared_memory_.Map(memory_length)); |
+ |
+ // Buffer index for sync with client is |params.size| on the shared memory. |
henrika (OOO until Aug 14)
2014/09/29 10:38:41
Odd language here as well. Not clear to me what yo
burnik
2014/09/29 12:07:31
Done.
// Peer's buffer index is accessed via |para
|
+ uint8* ptr = static_cast<uint8*>(shared_memory_.memory()); |
+ media::AudioInputBuffer* buffer = |
+ reinterpret_cast<media::AudioInputBuffer*>(ptr); |
+ peer_buffer_index_ = &(buffer->params.size); |
+ |
+ // Client must manage his own counter and reset it. |
burnik
2014/09/29 12:07:31
s/client/peer/g
|
+ DCHECK_EQ(0U, *peer_buffer_index_); |
+ output_bus_ = media::AudioBus::WrapMemory(params, buffer->audio); |
+ |
+ // Connect this audio sink to the track |
+ MediaStreamAudioSink::AddToAudioTrack(this, track_); |
+} |
+ |
+SpeechRecognitionAudioSink::~SpeechRecognitionAudioSink() { |
+ DCHECK(main_render_thread_checker_.CalledOnValidThread()); |
+ if (audio_converter_.get()) |
+ audio_converter_->RemoveInput(this); |
+ |
+ // Notify the track before this sink goes away. |
+ if (!track_stopped_) |
+ MediaStreamAudioSink::RemoveFromAudioTrack(this, track_); |
+} |
+ |
+// static |
+bool SpeechRecognitionAudioSink::IsSupportedTrack( |
+ const blink::WebMediaStreamTrack& track) { |
+ if (track.source().type() != blink::WebMediaStreamSource::TypeAudio) |
+ return false; |
+ |
+ MediaStreamAudioSource* native_source = |
+ static_cast<MediaStreamAudioSource*>(track.source().extraData()); |
+ if (!native_source) |
+ return false; |
+ |
+ const StreamDeviceInfo& device_info = native_source->device_info(); |
+ // Purposely only support tracks from an audio device. Dissallow WebAudio. |
henrika (OOO until Aug 14)
2014/09/29 10:38:41
Dissallow WebAudio? Does it mean that it is not su
burnik
2014/09/29 12:07:31
No. Just dissallowed as documented (abuse mitigati
|
+ return (device_info.device.type == content::MEDIA_DEVICE_AUDIO_CAPTURE); |
+} |
+ |
+void SpeechRecognitionAudioSink::OnSetFormat( |
+ const media::AudioParameters& input_params) { |
+ DCHECK(input_params.IsValid()); |
+ DCHECK_LE( |
+ input_params.frames_per_buffer() * 1000 / input_params.sample_rate(), |
+ output_params_.frames_per_buffer() * 1000 / output_params_.sample_rate()); |
+ |
+ // We need detach the thread here because it will be a new capture thread |
henrika (OOO until Aug 14)
2014/09/29 10:38:41
nit, 'need to' or must detach perhaps.
burnik
2014/09/29 12:07:31
Detach the thread here because it will be a new ca
|
+ // calling OnSetFormat() and OnData() if the source is restarted. |
+ capture_thread_checker_.DetachFromThread(); |
+ |
+ input_params_ = input_params; |
+ fifo_buffer_size_ = |
+ std::ceil(output_params_.frames_per_buffer() * |
+ static_cast<double>(input_params_.sample_rate()) / |
+ output_params_.sample_rate()); |
+ DCHECK_GE(fifo_buffer_size_, input_params_.frames_per_buffer()); |
+ |
+ // Allows for some delays on the endpoint client. |
+ static const int kNumberOfBuffersInFifo = 2; |
henrika (OOO until Aug 14)
2014/09/29 10:38:41
How was 2 chosen. What would happen if it was 20 i
burnik
2014/09/29 12:07:31
The constant 2 here is a minimum integer number of
|
+ int frames_in_fifo = kNumberOfBuffersInFifo * fifo_buffer_size_; |
+ fifo_.reset(new media::AudioFifo(input_params.channels(), frames_in_fifo)); |
+ input_bus_ = media::AudioBus::Create(input_params.channels(), |
+ input_params.frames_per_buffer()); |
+ |
+ // Create the audio converter with |disable_fifo| as false so that the |
+ // converter will request input_params.frames_per_buffer() each time. |
+ // This will not increase the complexity as there is only one client to |
+ // the converter. |
+ audio_converter_.reset( |
+ new media::AudioConverter(input_params, output_params_, false)); |
+ audio_converter_->AddInput(this); |
+} |
+ |
+void SpeechRecognitionAudioSink::OnReadyStateChanged( |
+ blink::WebMediaStreamSource::ReadyState state) { |
+ DCHECK(main_render_thread_checker_.CalledOnValidThread()); |
+ DCHECK(!track_stopped_); |
+ |
+ if (state == blink::WebMediaStreamSource::ReadyStateEnded) { |
+ track_stopped_ = true; |
+ |
+ if (!on_stopped_cb_.is_null()) |
+ on_stopped_cb_.Run(); |
+ } |
+} |
+ |
+void SpeechRecognitionAudioSink::OnData(const int16* audio_data, |
+ int sample_rate, |
+ int number_of_channels, |
+ int number_of_frames) { |
+ DCHECK(capture_thread_checker_.CalledOnValidThread()); |
+ DCHECK(peer_buffer_index_); |
+ DCHECK_EQ(input_bus_->frames(), number_of_frames); |
+ DCHECK_EQ(input_bus_->channels(), number_of_channels); |
+ if (fifo_->frames() + number_of_frames > fifo_->max_frames()) { |
+ // This would indicate a serious issue with the browser process or the |
+ // SyncSocket and/or SharedMemory. We stop delivering any data to the peer. |
+ NOTREACHED() << "Audio FIFO overflow"; |
+ return; |
+ } |
+ // TODO(xians): A better way to handle the interleaved and deinterleaved |
+ // format switching, see issue/317710. |
+ input_bus_->FromInterleaved(audio_data, number_of_frames, |
+ sizeof(audio_data[0])); |
+ |
+ fifo_->Push(input_bus_.get()); |
+ // Wait for FIFO to have at least |fifo_buffer_size_| frames ready. |
+ if (fifo_->frames() < fifo_buffer_size_) |
+ return; |
+ |
+ // Make sure the previous output buffer was consumed by client before we send |
+ // the next buffer. |peer_buffer_index_| is pointing to shared memory. |
+ // The client must write to it (incrementing by 1) once the the buffer was |
+ // consumed. This is intentional not to block this audio capturing thread. |
+ if (buffer_index_ != (*peer_buffer_index_)) { |
+ DLOG(WARNING) << "Buffer synchronization lag"; |
+ return; |
+ } |
+ |
+ audio_converter_->Convert(output_bus_.get()); |
+ |
+ // Notify client to consume buffer |buffer_index_| on |output_bus_|. |
+ const size_t bytes_sent = |
+ socket_->Send(&buffer_index_, sizeof(buffer_index_)); |
+ if (bytes_sent != sizeof(buffer_index_)) { |
+ // The send ocasionally fails if the user changes his input audio device. |
+ DVLOG(1) << "Failed sending buffer index to peer"; |
+ // We have discarded this buffer, but could still recover on the next one. |
+ return; |
+ } |
+ |
+ // Count the sent buffer. We expect the client to do the same on his end. |
+ ++buffer_index_; |
+} |
+ |
+double SpeechRecognitionAudioSink::ProvideInput( |
+ media::AudioBus* audio_bus, base::TimeDelta buffer_delay) { |
+ DCHECK(capture_thread_checker_.CalledOnValidThread()); |
+ if (fifo_->frames() >= audio_bus->frames()) |
+ fifo_->Consume(audio_bus, 0, audio_bus->frames()); |
+ else |
+ audio_bus->Zero(); |
+ |
+ // Return volume greater than zero to indicate we have more data. |
+ return 1.0; |
+} |
+ |
+} // namespace content |