OLD | NEW |
---|---|
(Empty) | |
1 // Copyright 2014 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 #include "content/renderer/speech_recognition_audio_source_provider.h" | |
6 | |
7 #include "base/logging.h" | |
8 #include "base/memory/shared_memory.h" | |
9 #include "base/threading/thread_restrictions.h" | |
10 #include "base/time/time.h" | |
11 #include "media/audio/audio_parameters.h" | |
12 #include "media/base/audio_fifo.h" | |
13 | |
14 namespace content { | |
15 | |
16 SpeechRecognitionAudioSourceProvider::SpeechRecognitionAudioSourceProvider( | |
17 const blink::WebMediaStreamTrack& track, | |
18 const media::AudioParameters& params, | |
19 base::SharedMemoryHandle memory, | |
20 base::NativeSyncSocket::Descriptor socket, | |
21 int memory_length) | |
22 : track_(track), | |
23 shared_memory_(memory, false), | |
24 socket_(base::NativeSyncSocket::Unwrap(socket)), | |
25 output_params_(params), | |
26 attached_converter_(false), | |
27 track_stopped_(false), | |
28 buffer_index_(0) { | |
29 DCHECK_EQ(memory_length, media::AudioBus::CalculateMemorySize(params)); | |
30 DCHECK(params.IsValid()); | |
31 if (!shared_memory_.Map(memory_length)) { | |
32 DLOG(ERROR) << "Could not map the shared memory"; | |
tommi (sloooow) - chröme
2014/08/29 11:25:31
Would this be a serious enough of an error to just
burnik
2014/08/29 13:26:17
Done. Although I will revisit this when I do more
| |
33 return; | |
34 } | |
35 output_bus_ = media::AudioBus::WrapMemory(params, shared_memory_.memory()); | |
36 // Connect the source provider to the track as a sink. | |
37 MediaStreamAudioSink::AddToAudioTrack(this, track_); | |
38 } | |
39 | |
40 SpeechRecognitionAudioSourceProvider::~SpeechRecognitionAudioSourceProvider() { | |
41 DCHECK(main_render_thread_checker_.CalledOnValidThread()); | |
42 if (audio_converter_.get() && attached_converter_) | |
no longer working on chromium
2014/08/29 12:23:06
you are complicating things, you should just add t
burnik
2014/08/29 13:26:16
Adding the converter on construction will trigger
burnik
2014/09/12 12:09:12
Done.
| |
43 audio_converter_->RemoveInput(this); | |
44 // If the track is still active, it is necessary to notify the track before | |
45 // the sink goes away. | |
46 if (!track_stopped_) | |
47 MediaStreamAudioSink::RemoveFromAudioTrack(this, track_); | |
48 } | |
49 | |
50 bool SpeechRecognitionAudioSourceProvider::IsAllowedAudioTrack( | |
tommi (sloooow) - chröme
2014/08/29 11:25:31
this method is static, right? If so, there should
burnik
2014/08/29 13:26:17
It is static. Done.
On 2014/08/29 11:25:31, tommi
| |
51 const blink::WebMediaStreamTrack& track ) { | |
52 DCHECK(track.source().type() == blink::WebMediaStreamSource::TypeAudio); | |
53 MediaStreamAudioSource* native_source = | |
no longer working on chromium
2014/08/29 12:23:06
you need to check if the track is local or not, if
burnik
2014/08/29 13:26:16
Here I check if it's WebAudio.
On 2014/08/29 12:23
| |
54 static_cast <MediaStreamAudioSource*>(track.source().extraData()); | |
tommi (sloooow) - chröme
2014/08/29 11:25:30
no space after static_cast
burnik
2014/08/29 13:26:17
Done.
| |
55 StreamDeviceInfo device_info = native_source->device_info(); | |
tommi (sloooow) - chröme
2014/08/29 11:25:30
no need to create a new StreamDeviceInfo instance.
burnik
2014/08/29 13:26:17
Done.
| |
56 return (device_info.device.type == content::MEDIA_DEVICE_AUDIO_CAPTURE); | |
tommi (sloooow) - chröme
2014/08/29 11:25:31
this seems to me to be a 'supported' check rather
burnik
2014/08/29 13:26:17
This is implemented as a response to a suggestion
| |
57 } | |
58 | |
59 void SpeechRecognitionAudioSourceProvider::OnSetFormat( | |
60 const media::AudioParameters& input_params) { | |
61 // We need detach the thread here because it will be a new capture thread | |
62 // calling OnSetFormat() and OnData() if the source is restarted. | |
63 capture_thread_checker_.DetachFromThread(); | |
64 DCHECK(capture_thread_checker_.CalledOnValidThread()); | |
65 DCHECK(input_params.IsValid()); | |
66 | |
67 input_params_ = input_params; | |
68 // TODO(burnik): Check if this is necessary: | |
69 // Create the audio converter with |disable_fifo| as false so that the | |
70 // converter will request input_params.frames_per_buffer() each time. | |
71 // This will not increase the complexity as there is only one client to | |
72 // the converter. | |
73 audio_converter_.reset( | |
74 new media::AudioConverter(input_params, output_params_, false)); | |
no longer working on chromium
2014/08/29 12:23:06
Call AddInput here.
burnik
2014/08/29 13:26:17
Same comment regarding the way |ProvideInput()| is
| |
75 | |
76 DCHECK_EQ(0, output_params_.frames_per_buffer() * | |
77 input_params_.sample_rate() % output_params_.sample_rate()); | |
78 fifo_buffer_size_ = output_params_.frames_per_buffer() * | |
79 input_params_.sample_rate() / output_params_.sample_rate(); | |
80 | |
81 int frames_in_fifo = kNumberOfBuffersInFifo * fifo_buffer_size_; | |
82 | |
83 max_sync_delay_time_delta_ = base::TimeDelta::FromMilliseconds( | |
84 input_params_.sample_rate() / fifo_buffer_size_); | |
85 | |
86 fifo_.reset(new media::AudioFifo(input_params.channels(), frames_in_fifo)); | |
87 input_bus_ = media::AudioBus::Create(input_params.channels(), | |
88 input_params.frames_per_buffer()); | |
89 } | |
90 | |
91 void SpeechRecognitionAudioSourceProvider::OnReadyStateChanged( | |
92 blink::WebMediaStreamSource::ReadyState state) { | |
93 DCHECK(main_render_thread_checker_.CalledOnValidThread()); | |
94 if (state == blink::WebMediaStreamSource::ReadyStateEnded) { | |
95 track_stopped_ = true; | |
96 } else { | |
97 DCHECK(!track_stopped_); | |
98 } | |
99 } | |
100 | |
101 void SpeechRecognitionAudioSourceProvider::OnData( | |
102 const int16* audio_data, | |
103 int sample_rate, | |
104 int number_of_channels, | |
105 int number_of_frames) { | |
106 DCHECK(capture_thread_checker_.CalledOnValidThread()); | |
107 DCHECK_EQ(input_bus_->frames(), number_of_frames); | |
108 DCHECK_EQ(input_bus_->channels(), number_of_channels); | |
109 DCHECK_LE(fifo_->frames() + number_of_frames, fifo_->max_frames()); | |
110 // TODO(xians): A better way to handle the interleaved and deinterleaved | |
111 // format switching, see issue/317710. | |
112 input_bus_->FromInterleaved(audio_data, number_of_frames, | |
113 sizeof(audio_data[0])); | |
114 | |
115 fifo_->Push(input_bus_.get()); | |
116 | |
117 // Wait for FIFO to have at least |fifo_buffer_size_| frames ready. | |
118 if (fifo_->frames() < fifo_buffer_size_) | |
119 return; | |
120 | |
121 // Attach converter when we first reach |fifo_buffer_size_| frames in the FIFO | |
122 if (!attached_converter_) { | |
123 audio_converter_->AddInput(this); | |
no longer working on chromium
2014/08/29 12:23:06
I guess these code is workaround to fix AudioConve
burnik
2014/08/29 13:26:17
I might try this out and see what happens.
On 2014
| |
124 attached_converter_ = true; | |
125 // we need one more buffer of |number_of_frames| before we start converting | |
126 return; | |
127 } | |
128 | |
129 // Make sure the previous output buffer was consumed by client before we send | |
130 // the next buffer. | |
131 if (buffer_index_ > 0) { | |
132 uint32 peer_buffer_index; | |
133 size_t bytes_received = socket_.ReceiveWithTimeout(&peer_buffer_index, | |
no longer working on chromium
2014/08/29 12:23:06
you have remove this ReceiveWithTimeout call, this
burnik
2014/08/29 13:26:17
Acknowledged.
| |
134 sizeof(peer_buffer_index), max_sync_delay_time_delta_); | |
135 // TODO(burnik): This should not happen. Make sure FIFO doesn't fill up. | |
136 if (bytes_received == 0) | |
tommi (sloooow) - chröme
2014/08/29 11:25:30
should there be a NOTREACHED() in the body of this
burnik
2014/08/29 13:26:16
Good point. Done.
And I'm still considering ways
| |
137 return; | |
138 DCHECK_EQ(peer_buffer_index, buffer_index_ - 1); | |
139 } | |
140 | |
141 // First call to |Convert| must have |fifo_buffer_size_| + |number_of_frames| | |
142 // waiting on the FIFO since it will trigger one extra |ProvideInput| call. | |
143 // This way it is also guaranteed to have a non-empty first output buffer | |
144 audio_converter_->Convert(output_bus_.get()); | |
145 // Notify client to consume buffer |buffer_index_| on |output_bus_|. | |
no longer working on chromium
2014/08/29 12:23:06
empty line.
| |
146 size_t bytes_sent = socket_.Send(&buffer_index_, sizeof(buffer_index_)); | |
147 // The send can fail if the user changes his input audio device | |
no longer working on chromium
2014/08/29 12:23:06
is it true?
burnik
2014/08/29 13:26:17
As far as I've tested.
On 2014/08/29 12:23:06, xia
| |
148 if (bytes_sent != sizeof(buffer_index_)) { | |
149 // TODO(burnik): See if we can stop the passing of data if this happens. | |
150 DLOG(ERROR) << "Missing a buffer"; | |
151 } | |
152 ++buffer_index_; | |
153 } | |
154 | |
155 double SpeechRecognitionAudioSourceProvider::ProvideInput( | |
156 media::AudioBus* audio_bus, base::TimeDelta buffer_delay) { | |
157 DCHECK(capture_thread_checker_.CalledOnValidThread()); | |
158 DCHECK_GE(fifo_->frames(), audio_bus->frames()); | |
159 // Consume queued input frames by passing them to |audio_converter_| | |
no longer working on chromium
2014/08/29 12:23:06
nit, empty line.
burnik
2014/08/29 13:26:17
Done.
| |
160 fifo_->Consume(audio_bus, 0, audio_bus->frames()); | |
161 return 1.0; | |
162 } | |
163 | |
164 } // namespace content | |
OLD | NEW |