OLD | NEW |
---|---|
(Empty) | |
1 // Copyright 2014 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 #ifndef CONTENT_RENDERER_SPEECH_RECOGNITION_AUDIO_SOURCE_PROVIDER_H_ | |
6 #define CONTENT_RENDERER_SPEECH_RECOGNITION_AUDIO_SOURCE_PROVIDER_H_ | |
7 | |
8 #include "base/callback.h" | |
9 #include "base/memory/scoped_ptr.h" | |
10 #include "base/memory/shared_memory.h" | |
11 #include "base/sync_socket.h" | |
12 #include "base/threading/thread_checker.h" | |
13 #include "base/threading/thread_restrictions.h" | |
14 #include "content/common/content_export.h" | |
15 #include "content/public/renderer/media_stream_audio_sink.h" | |
16 #include "content/renderer/media/media_stream_audio_source.h" | |
17 #include "media/audio/audio_device_thread.h" | |
18 #include "media/audio/audio_parameters.h" | |
19 #include "media/base/audio_converter.h" | |
20 #include "third_party/WebKit/public/platform/WebMediaStreamTrack.h" | |
21 | |
22 namespace media { | |
23 class AudioBus; | |
24 class AudioConverter; | |
25 class AudioFifo; | |
26 class AudioParameters; | |
27 } | |
28 | |
29 namespace content { | |
30 | |
31 // SpeechRecognitionAudioSourceProvider works as a sink to the | |
32 // WebRtcLocalAudioTrack and stores the capture data to a FIFO. | |
henrika (OOO until Aug 14)
2014/09/12 12:27:46
I would say, "and stores the captured data in a FI
burnik
2014/09/15 15:00:07
Comment expanded.
On 2014/09/12 12:27:46, henrika
| |
33 // When the FIFO has enough buffer, it passes the buffer to | |
34 // the speech recognizer. | |
35 // | |
36 class CONTENT_EXPORT SpeechRecognitionAudioSourceProvider | |
37 : NON_EXPORTED_BASE(public media::AudioConverter::InputCallback), | |
38 NON_EXPORTED_BASE(public MediaStreamAudioSink) { | |
39 public: | |
40 // Used for notifying the renderer client there is an issue with | |
henrika (OOO until Aug 14)
2014/09/12 12:27:46
"...if/when there is an issue"
burnik
2014/09/15 15:00:06
Removed enum.
On 2014/09/12 12:27:46, henrika wrot
| |
41 // delivering audio frames to the endpoint. | |
42 enum ErrorState { | |
43 // Indicates a notification send failed. Recoverable. | |
henrika (OOO until Aug 14)
2014/09/12 12:27:47
"Indicates that sending a notification failed"
burnik
2014/09/15 15:00:06
Ditto.
On 2014/09/12 12:27:47, henrika wrote:
| |
44 SEND_FAILED = 0, | |
45 // Indicates client hasn't consumed last buffer. Recoverable. | |
henrika (OOO until Aug 14)
2014/09/12 12:27:47
"indicates client" feels wrong; can you rewrite? W
burnik
2014/09/15 15:00:06
Ditto.
On 2014/09/12 12:27:47, henrika wrote:
| |
46 BUFFER_SYNC_LAG, | |
47 // Indiciates serious error. E.g. a disconnect on socket. Non-recoverable. | |
48 AUDIO_FIFO_OVERFLOW, | |
no longer working on chromium
2014/09/15 08:31:29
I am not sure if there is any value to most of the
burnik
2014/09/15 15:00:07
Logged instead. Only stop propagated via callback.
| |
49 // Indicates the audio track has stopped. Provider can then be destroyed. | |
henrika (OOO until Aug 14)
2014/09/12 12:27:46
"..that the audio track..."
burnik
2014/09/15 15:00:06
Ditto.
On 2014/09/12 12:27:46, henrika wrote:
| |
50 TRACK_STOPPED | |
51 }; | |
52 | |
53 typedef base::Callback<void(ErrorState)> OnErrorCB; | |
54 | |
55 SpeechRecognitionAudioSourceProvider(const blink::WebMediaStreamTrack& track, | |
56 const media::AudioParameters& params, | |
57 const base::SharedMemoryHandle memory, | |
58 base::SyncSocket* socket, | |
59 OnErrorCB on_error_cb); | |
60 | |
61 virtual ~SpeechRecognitionAudioSourceProvider(); | |
62 | |
63 // Determines the policy on what types of tracks are allowed. | |
henrika (OOO until Aug 14)
2014/09/12 12:27:47
Is this correct. How can a method which returns tr
burnik
2014/09/15 15:00:06
Good point. We determine. Implementation enforces.
| |
64 static bool IsAllowedAudioTrack(const blink::WebMediaStreamTrack& track); | |
65 | |
66 protected: | |
67 // MediaStreamAudioSink implementation. | |
henrika (OOO until Aug 14)
2014/09/12 12:27:46
No namespace here?
burnik
2014/09/15 15:00:06
It's |content| as well. But added.
On 2014/09/12 1
| |
68 virtual void OnReadyStateChanged( | |
69 blink::WebMediaStreamSource::ReadyState state) OVERRIDE; | |
70 | |
71 virtual void OnData(const int16* audio_data, int sample_rate, | |
72 int number_of_channels, int number_of_frames) OVERRIDE; | |
73 virtual void OnSetFormat(const media::AudioParameters& params) OVERRIDE; | |
74 | |
75 // media::AudioConverter::Inputcallback implementation. | |
76 // This function is triggered by provideInput() on the WebAudio audio thread, | |
77 // so it has been under the protection of |lock_|. | |
henrika (OOO until Aug 14)
2014/09/12 12:27:46
"so it has been under..." sounds odd to me. Do you
burnik
2014/09/15 15:00:06
Comment deprecated.
On 2014/09/12 12:27:46, henrik
| |
78 virtual double ProvideInput(media::AudioBus* audio_bus, | |
79 base::TimeDelta buffer_delay) OVERRIDE; | |
80 | |
81 // Notifies client there is an issue with delivering frames. | |
henrika (OOO until Aug 14)
2014/09/12 12:27:46
"Notifies client there is" does not sound correct.
burnik
2014/09/15 15:00:06
Removed from design.
On 2014/09/12 12:27:46, henri
| |
82 // TODO(burnik): Runs on capture thread. Should run on main renderer thread! | |
henrika (OOO until Aug 14)
2014/09/12 12:27:47
This TODO needs a corresponding crbug.
burnik
2014/09/15 15:00:06
TODO was for before landing. Removed from design.
| |
83 void NotifyErrorState(ErrorState error); | |
84 | |
85 private: | |
86 // Number of frames per buffer in FIFO. When the buffer is full we convert and | |
87 // consume it on the |output_bus_|. Size of the buffer is depends on the | |
henrika (OOO until Aug 14)
2014/09/12 12:27:46
"is depends"??
burnik
2014/09/15 15:00:07
Done.
| |
88 // resampler. Example: for 44.1 to 16.0 conversion, it should be 4100 frames. | |
89 int fifo_buffer_size_; | |
90 | |
91 // Used to DCHECK that some methods are called on the main render thread. | |
92 base::ThreadChecker main_render_thread_checker_; | |
93 | |
94 // Used to DCHECK that some methods are called on the capture audio thread. | |
95 base::ThreadChecker capture_thread_checker_; | |
96 | |
97 // The audio track that this source provider is connected to. | |
98 const blink::WebMediaStreamTrack track_; | |
99 | |
100 // Shared memory used by audio buses on both browser and renderer processes. | |
101 base::SharedMemory shared_memory_; | |
102 | |
103 // Socket for syncronization of audio bus reads/writes. | |
104 base::SyncSocket* socket_; | |
no longer working on chromium
2014/09/15 08:31:29
why the socket_ is raw pointer? who owns it?
burnik
2014/09/15 15:00:07
Client owns it (renderer - dispatcher). This is fo
no longer working on chromium
2014/09/16 12:44:05
I looked at the dispatcher code, it has:
audio_sou
| |
105 | |
106 // Used as a resampler to deliver appropriate format to speech recognition. | |
107 scoped_ptr<media::AudioConverter> audio_converter_; | |
108 | |
109 // FIFO is used for queueing audio frames before we resample. | |
110 scoped_ptr<media::AudioFifo> fifo_; | |
111 | |
112 // Audio delivered from source. | |
113 scoped_ptr<media::AudioBus> input_bus_; | |
114 | |
115 // Audio bus shared with the browser process via |shared_memory_|. | |
116 scoped_ptr<media::AudioBus> output_bus_; | |
117 | |
118 // Params of the source audio. Can change when |OnSetFormat| occurs. | |
119 media::AudioParameters input_params_; | |
120 | |
121 // Params used by speech recognition. | |
122 const media::AudioParameters output_params_; | |
123 | |
124 // Whether the track has been stopped on the input. | |
henrika (OOO until Aug 14)
2014/09/12 12:27:47
"stopped on the input"??
burnik
2014/09/15 15:00:06
Done.
| |
125 bool track_stopped_; | |
126 | |
127 // Local counter of audio buffers for synchronization on consumed buffers. | |
henrika (OOO until Aug 14)
2014/09/12 12:27:47
Isn't "of consumed" better?
burnik
2014/09/15 15:00:06
Looks excessive actually. Removed.
On 2014/09/12 1
| |
128 uint32 buffer_index_; | |
129 | |
130 // Peer's counter of audio buffers for synchronization on consumed buffers. | |
131 const uint32* peer_buffer_index_; | |
132 | |
133 // Callback notifying an error has occured. | |
henrika (OOO until Aug 14)
2014/09/12 12:27:47
.."notifying that an...", or "Callback which is ac
burnik
2014/09/15 15:00:06
Removed from design. Replaced by OnStoppedCB and c
| |
134 const OnErrorCB on_error_cb_; | |
135 | |
136 DISALLOW_COPY_AND_ASSIGN(SpeechRecognitionAudioSourceProvider); | |
137 }; | |
138 | |
139 } // namespace content | |
140 | |
141 #endif // CONTENT_RENDERER_SPEECH_RECOGNITION_AUDIO_SOURCE_PROVIDER_H_ | |
OLD | NEW |