Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(116)

Side by Side Diff: content/renderer/media/speech_recognition_audio_sink.cc

Issue 499233003: Binding media stream audio track to speech recognition [renderer] (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Rebase on master - merge fix Created 6 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "content/renderer/media/speech_recognition_audio_sink.h"
6
7 #include "base/logging.h"
8 #include "base/memory/shared_memory.h"
9 #include "base/time/time.h"
10 #include "content/renderer/media/media_stream_audio_source.h"
11 #include "media/audio/audio_parameters.h"
12 #include "media/base/audio_fifo.h"
13
14 namespace content {
15
16 SpeechRecognitionAudioSink::SpeechRecognitionAudioSink(
17 const blink::WebMediaStreamTrack& track,
18 const media::AudioParameters& params,
19 const base::SharedMemoryHandle memory,
20 scoped_ptr<base::SyncSocket> socket,
21 const OnStoppedCB& on_stopped_cb)
22 : track_(track),
23 shared_memory_(memory, false),
24 socket_(socket.Pass()),
25 output_params_(params),
26 track_stopped_(false),
27 buffer_index_(0),
28 on_stopped_cb_(on_stopped_cb) {
29 DCHECK(socket_.get());
30 DCHECK(main_render_thread_checker_.CalledOnValidThread());
31 DCHECK(params.IsValid());
32 DCHECK(IsSupportedTrack(track));
33 const size_t memory_length = media::AudioBus::CalculateMemorySize(params) +
34 sizeof(media::AudioInputBufferParameters);
35 CHECK(shared_memory_.Map(memory_length));
36
37 media::AudioInputBuffer* buffer =
38 static_cast<media::AudioInputBuffer*>(shared_memory_.memory());
39
40 // The peer must manage their own counter and reset it to 0.
41 DCHECK_EQ(0U, buffer->params.size);
42 output_bus_ = media::AudioBus::WrapMemory(params, buffer->audio);
43
44 // Connect this audio sink to the track
45 MediaStreamAudioSink::AddToAudioTrack(this, track_);
46 }
47
48 SpeechRecognitionAudioSink::~SpeechRecognitionAudioSink() {
49 DCHECK(main_render_thread_checker_.CalledOnValidThread());
50 if (audio_converter_.get())
51 audio_converter_->RemoveInput(this);
52
53 // Notify the track before this sink goes away.
54 if (!track_stopped_)
55 MediaStreamAudioSink::RemoveFromAudioTrack(this, track_);
56 }
57
58 // static
59 bool SpeechRecognitionAudioSink::IsSupportedTrack(
60 const blink::WebMediaStreamTrack& track) {
61 if (track.source().type() != blink::WebMediaStreamSource::TypeAudio)
62 return false;
63
64 MediaStreamAudioSource* native_source =
65 static_cast<MediaStreamAudioSource*>(track.source().extraData());
66 if (!native_source)
67 return false;
68
69 const StreamDeviceInfo& device_info = native_source->device_info();
70 // Purposely only support tracks from an audio device. Dissallow WebAudio.
71 return (device_info.device.type == content::MEDIA_DEVICE_AUDIO_CAPTURE);
72 }
73
74 void SpeechRecognitionAudioSink::OnSetFormat(
75 const media::AudioParameters& input_params) {
76 DCHECK(input_params.IsValid());
77 DCHECK_LE(
78 input_params.frames_per_buffer() * 1000 / input_params.sample_rate(),
79 output_params_.frames_per_buffer() * 1000 / output_params_.sample_rate());
80
81 // Detach the thread here because it will be a new capture thread
82 // calling OnSetFormat() and OnData() if the source is restarted.
83 capture_thread_checker_.DetachFromThread();
84
85 input_params_ = input_params;
86 fifo_buffer_size_ =
87 std::ceil(output_params_.frames_per_buffer() *
88 static_cast<double>(input_params_.sample_rate()) /
89 output_params_.sample_rate());
90 DCHECK_GE(fifo_buffer_size_, input_params_.frames_per_buffer());
91
92 // Allows for some delays on the peer.
93 static const int kNumberOfBuffersInFifo = 2;
94 int frames_in_fifo = kNumberOfBuffersInFifo * fifo_buffer_size_;
95 fifo_.reset(new media::AudioFifo(input_params.channels(), frames_in_fifo));
96 input_bus_ = media::AudioBus::Create(input_params.channels(),
97 input_params.frames_per_buffer());
98
99 // Create the audio converter with |disable_fifo| as false so that the
100 // converter will request input_params.frames_per_buffer() each time.
101 // This will not increase the complexity as there is only one client to
102 // the converter.
103 audio_converter_.reset(
104 new media::AudioConverter(input_params, output_params_, false));
105 audio_converter_->AddInput(this);
106 }
107
108 void SpeechRecognitionAudioSink::OnReadyStateChanged(
109 blink::WebMediaStreamSource::ReadyState state) {
110 DCHECK(main_render_thread_checker_.CalledOnValidThread());
111 DCHECK(!track_stopped_);
112
113 if (state == blink::WebMediaStreamSource::ReadyStateEnded) {
114 track_stopped_ = true;
115
116 if (!on_stopped_cb_.is_null())
117 on_stopped_cb_.Run();
118 }
119 }
120
121 void SpeechRecognitionAudioSink::OnData(const int16* audio_data,
122 int sample_rate,
123 int number_of_channels,
124 int number_of_frames) {
125 DCHECK(capture_thread_checker_.CalledOnValidThread());
126 DCHECK_EQ(input_bus_->frames(), number_of_frames);
127 DCHECK_EQ(input_bus_->channels(), number_of_channels);
128 if (fifo_->frames() + number_of_frames > fifo_->max_frames()) {
129 // This would indicate a serious issue with the browser process or the
130 // SyncSocket and/or SharedMemory. We stop delivering any data to the peer.
131 NOTREACHED() << "Audio FIFO overflow";
132 return;
133 }
134 // TODO(xians): A better way to handle the interleaved and deinterleaved
135 // format switching, see issue/317710.
136 input_bus_->FromInterleaved(audio_data, number_of_frames,
137 sizeof(audio_data[0]));
138
139 fifo_->Push(input_bus_.get());
140 // Wait for FIFO to have at least |fifo_buffer_size_| frames ready.
141 if (fifo_->frames() < fifo_buffer_size_)
142 return;
143
144 // Make sure the previous output buffer was consumed by the peer before we
145 // send the next buffer.
146 // The peer must write to it (incrementing by 1) once the the buffer was
147 // consumed. This is intentional not to block this audio capturing thread.
148 if (buffer_index_ != GetAudioInputBuffer()->params.size) {
149 DLOG(WARNING) << "Buffer synchronization lag";
150 return;
151 }
152
153 audio_converter_->Convert(output_bus_.get());
154
155 // Notify peer to consume buffer |buffer_index_| on |output_bus_|.
156 const size_t bytes_sent =
157 socket_->Send(&buffer_index_, sizeof(buffer_index_));
158 if (bytes_sent != sizeof(buffer_index_)) {
159 // The send ocasionally fails if the user changes their input audio device.
160 DVLOG(1) << "Failed sending buffer index to peer";
161 // We have discarded this buffer, but could still recover on the next one.
162 return;
163 }
164
165 // Count the sent buffer. We expect the peer to do the same on their end.
166 ++buffer_index_;
167 }
168
169 double SpeechRecognitionAudioSink::ProvideInput(media::AudioBus* audio_bus,
170 base::TimeDelta buffer_delay) {
171 DCHECK(capture_thread_checker_.CalledOnValidThread());
172 if (fifo_->frames() >= audio_bus->frames())
173 fifo_->Consume(audio_bus, 0, audio_bus->frames());
174 else
175 audio_bus->Zero();
176
177 // Return volume greater than zero to indicate we have more data.
178 return 1.0;
179 }
180
181 media::AudioInputBuffer*
182 SpeechRecognitionAudioSink::GetAudioInputBuffer() const {
183 DCHECK(capture_thread_checker_.CalledOnValidThread());
184 DCHECK(shared_memory_.memory());
185 return static_cast<media::AudioInputBuffer*>(shared_memory_.memory());
186 }
187
188 } // namespace content
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698