OLD | NEW |
---|---|
(Empty) | |
1 // Copyright 2014 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 #include "content/renderer/speech_recognition_audio_source_provider.h" | |
6 | |
7 #include "base/logging.h" | |
8 #include "base/memory/shared_memory.h" | |
9 #include "base/threading/thread_restrictions.h" | |
burnik
2014/09/16 19:10:22
Removed.
| |
10 #include "base/time/time.h" | |
11 #include "media/audio/audio_parameters.h" | |
12 #include "media/base/audio_fifo.h" | |
13 | |
14 namespace content { | |
15 | |
16 SpeechRecognitionAudioSourceProvider::SpeechRecognitionAudioSourceProvider( | |
17 const blink::WebMediaStreamTrack& track, | |
18 const media::AudioParameters& params, const base::SharedMemoryHandle memory, | |
19 base::SyncSocket* socket, OnStoppedCB on_stopped_cb) | |
20 : track_(track), | |
21 shared_memory_(memory, false), | |
22 socket_(socket), | |
23 output_params_(params), | |
24 track_stopped_(false), | |
25 buffer_index_(0), | |
26 on_stopped_cb_(on_stopped_cb) { | |
27 DCHECK(main_render_thread_checker_.CalledOnValidThread()); | |
28 DCHECK(params.IsValid()); | |
no longer working on chromium
2014/09/16 12:44:06
add an DCHECK here:
DCHECK(IsAllowedAudioTrack(tra
burnik
2014/09/16 19:10:22
Done. And s/IsAllowedAudioTrack/IsSupportedTrack/.
| |
29 const size_t memory_length = media::AudioBus::CalculateMemorySize(params) + | |
30 sizeof(media::AudioInputBufferParameters); | |
31 CHECK(shared_memory_.Map(memory_length)); | |
32 | |
33 uint8* ptr = static_cast<uint8*>(shared_memory_.memory()); | |
34 media::AudioInputBuffer* buffer = | |
35 reinterpret_cast<media::AudioInputBuffer*>(ptr); | |
36 // Keep params for sync with client via |params.size| on the shared memory. | |
no longer working on chromium
2014/09/16 12:44:06
move this comment up above uint8* ptr = static_cas
burnik
2014/09/16 19:10:22
Done. And refactored.
On 2014/09/16 12:44:06, xian
| |
37 peer_buffer_index_ = &(buffer->params.size); | |
38 // Client must manage his own counter and reset it. | |
no longer working on chromium
2014/09/16 12:44:06
add an empty line before the comment.
burnik
2014/09/16 19:10:22
Done.
| |
39 DCHECK_EQ(0U, *peer_buffer_index_); | |
40 output_bus_ = media::AudioBus::WrapMemory(params, buffer->audio); | |
41 // Connect the source provider to the track as a sink. | |
no longer working on chromium
2014/09/16 12:44:06
ditto
burnik
2014/09/16 19:10:23
Done.
| |
42 MediaStreamAudioSink::AddToAudioTrack(this, track_); | |
43 } | |
44 | |
45 SpeechRecognitionAudioSourceProvider::~SpeechRecognitionAudioSourceProvider() { | |
46 DCHECK(main_render_thread_checker_.CalledOnValidThread()); | |
47 if (audio_converter_.get()) audio_converter_->RemoveInput(this); | |
no longer working on chromium
2014/09/16 12:44:06
new line after the if (), this comment applies to
burnik
2014/09/16 19:10:23
Done.
| |
48 // Notify the track before this sink goes away. | |
49 if (!track_stopped_) MediaStreamAudioSink::RemoveFromAudioTrack(this, track_); | |
50 } | |
51 | |
52 // static | |
53 bool SpeechRecognitionAudioSourceProvider::IsAllowedAudioTrack( | |
54 const blink::WebMediaStreamTrack& track) { | |
55 if (track.source().type() != blink::WebMediaStreamSource::TypeAudio) | |
56 return false; | |
57 MediaStreamAudioSource* native_source = | |
no longer working on chromium
2014/09/16 12:44:06
nit, add an empty line. after the return false;
burnik
2014/09/16 19:10:23
Done.
| |
58 static_cast<MediaStreamAudioSource*>(track.source().extraData()); | |
59 if (!native_source) return false; | |
no longer working on chromium
2014/09/16 12:44:05
ditto
burnik
2014/09/16 19:10:22
Done.
| |
60 const StreamDeviceInfo& device_info = native_source->device_info(); | |
61 // Purposely only support tracks from an audio device. Dissallow WebAudio. | |
62 return (device_info.device.type == content::MEDIA_DEVICE_AUDIO_CAPTURE); | |
63 } | |
64 | |
65 void SpeechRecognitionAudioSourceProvider::OnSetFormat( | |
66 const media::AudioParameters& input_params) { | |
67 // We need detach the thread here because it will be a new capture thread | |
68 // calling OnSetFormat() and OnData() if the source is restarted. | |
69 capture_thread_checker_.DetachFromThread(); | |
70 DCHECK(capture_thread_checker_.CalledOnValidThread()); | |
71 DCHECK(input_params.IsValid()); | |
72 | |
73 input_params_ = input_params; | |
74 fifo_buffer_size_ = output_params_.frames_per_buffer() * | |
75 input_params_.sample_rate() / | |
76 output_params_.sample_rate(); | |
77 DCHECK_GE(fifo_buffer_size_, input_params_.frames_per_buffer()); | |
78 DCHECK_GE(fifo_buffer_size_, output_params_.frames_per_buffer()); | |
no longer working on chromium
2014/09/16 12:44:06
these two DCHECKs are not always right, for exampl
burnik
2014/09/16 19:10:23
Do we ever have mic input which is 8K? What should
no longer working on chromium
2014/09/17 15:55:19
Yes, 8K as sample rate does exist on all platforms
burnik
2014/09/18 19:09:21
True, removed the second DCHECK. It doesn't really
| |
79 | |
80 // Allows for some delays on the endpoint client. | |
81 static const int kNumberOfBuffersInFifo = 2; | |
82 int frames_in_fifo = kNumberOfBuffersInFifo * fifo_buffer_size_; | |
83 fifo_.reset(new media::AudioFifo(input_params.channels(), frames_in_fifo)); | |
84 input_bus_ = media::AudioBus::Create(input_params.channels(), | |
85 input_params.frames_per_buffer()); | |
86 | |
87 // Create the audio converter with |disable_fifo| as false so that the | |
88 // converter will request input_params.frames_per_buffer() each time. | |
89 // This will not increase the complexity as there is only one client to | |
90 // the converter. | |
91 audio_converter_.reset( | |
92 new media::AudioConverter(input_params, output_params_, false)); | |
93 audio_converter_->AddInput(this); | |
94 } | |
95 | |
96 void SpeechRecognitionAudioSourceProvider::OnReadyStateChanged( | |
97 blink::WebMediaStreamSource::ReadyState state) { | |
98 DCHECK(main_render_thread_checker_.CalledOnValidThread()); | |
99 if (track_stopped_) | |
no longer working on chromium
2014/09/16 12:44:05
You should never get OnReadyStateChanged() more th
burnik
2014/09/16 19:10:22
Done.
| |
100 return; | |
101 | |
102 if (state == blink::WebMediaStreamSource::ReadyStateEnded) { | |
103 track_stopped_ = true; | |
104 if (!on_stopped_cb_.is_null()) | |
105 on_stopped_cb_.Run(); | |
106 } | |
107 } | |
108 | |
109 void SpeechRecognitionAudioSourceProvider::OnData(const int16* audio_data, | |
110 int sample_rate, | |
111 int number_of_channels, | |
112 int number_of_frames) { | |
113 DCHECK(capture_thread_checker_.CalledOnValidThread()); | |
114 DCHECK(peer_buffer_index_); | |
115 DCHECK_EQ(input_bus_->frames(), number_of_frames); | |
116 DCHECK_EQ(input_bus_->channels(), number_of_channels); | |
117 if (fifo_->frames() + number_of_frames > fifo_->max_frames()) { | |
118 NOTREACHED() << "Audio FIFO overflow"; | |
119 return; | |
120 } | |
121 // TODO(xians): A better way to handle the interleaved and deinterleaved | |
122 // format switching, see issue/317710. | |
123 input_bus_->FromInterleaved(audio_data, number_of_frames, | |
124 sizeof(audio_data[0])); | |
125 | |
126 fifo_->Push(input_bus_.get()); | |
127 // Wait for FIFO to have at least |fifo_buffer_size_| frames ready. | |
128 if (fifo_->frames() < fifo_buffer_size_) | |
129 return; | |
130 | |
131 // Make sure the previous output buffer was consumed by client before we send | |
132 // the next buffer. |peer_buffer_index_| is pointing to shared memory. | |
133 // The client must write to it (incrementing by 1) once the the buffer was | |
134 // consumed. This is intentional not to block this audio capturing thread. | |
135 if (buffer_index_ != (*peer_buffer_index_)) { | |
136 DLOG(WARNING) << "Buffer synchronization lag"; | |
137 return; | |
138 } | |
139 | |
140 audio_converter_->Convert(output_bus_.get()); | |
141 | |
142 // Notify client to consume buffer |buffer_index_| on |output_bus_|. | |
143 const size_t bytes_sent = | |
144 socket_->Send(&buffer_index_, sizeof(buffer_index_)); | |
145 if (bytes_sent != sizeof(buffer_index_)) { | |
146 // The send usually fails if the user changes his input audio device. | |
no longer working on chromium
2014/09/16 12:44:06
is this comment true?
burnik
2014/09/16 19:10:22
As far as I've tested, yes.
On 2014/09/16 12:44:06
no longer working on chromium
2014/09/17 15:55:19
Interesting, to double check, if you changed the i
burnik
2014/09/18 19:09:21
If I change input device from System Sound on linu
| |
147 DVLOG(1) << "Failed sending buffer index to peer"; | |
148 // We have discarded this buffer, but could still recover on the next one. | |
149 // Although, if the socket was closed, this will shortly end up | |
150 // in |ErrorState::AUDIO_FIFO_OVERFLOW|. | |
no longer working on chromium
2014/09/16 12:44:05
update the comment.
burnik
2014/09/16 19:10:22
Done.
| |
151 return; | |
152 } | |
153 | |
154 // Count the sent buffer. We expect the client to do the same on his end. | |
155 ++buffer_index_; | |
156 } | |
157 | |
158 double SpeechRecognitionAudioSourceProvider::ProvideInput( | |
159 media::AudioBus* audio_bus, base::TimeDelta buffer_delay) { | |
160 DCHECK(capture_thread_checker_.CalledOnValidThread()); | |
161 if (fifo_->frames() >= audio_bus->frames()) | |
162 fifo_->Consume(audio_bus, 0, audio_bus->frames()); | |
163 else | |
164 audio_bus->Zero(); | |
no longer working on chromium
2014/09/16 12:44:06
can the else case be possible at all?
burnik
2014/09/16 19:10:23
Yes. Shown many times while testing. This occurs w
no longer working on chromium
2014/09/17 15:55:19
Got it, thanks.
burnik
2014/09/18 19:09:21
Acknowledged.
| |
165 | |
166 return 1.0; | |
167 } | |
168 | |
169 } // namespace content | |
OLD | NEW |