OLD | NEW |
---|---|
(Empty) | |
1 // Copyright 2014 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 #ifndef CONTENT_RENDERER_SPEECH_RECOGNITION_AUDIO_SOURCE_PROVIDER_H_ | |
6 #define CONTENT_RENDERER_SPEECH_RECOGNITION_AUDIO_SOURCE_PROVIDER_H_ | |
7 | |
8 #include "base/callback.h" | |
9 #include "base/memory/scoped_ptr.h" | |
10 #include "base/memory/shared_memory.h" | |
11 #include "base/native_sync_socket.h" | |
12 #include "base/threading/thread_checker.h" | |
13 #include "base/threading/thread_restrictions.h" | |
14 #include "content/common/content_export.h" | |
15 #include "content/public/renderer/media_stream_audio_sink.h" | |
16 #include "content/renderer/media/media_stream_audio_source.h" | |
17 #include "media/audio/audio_device_thread.h" | |
18 #include "media/base/audio_converter.h" | |
19 #include "third_party/WebKit/public/platform/WebMediaStreamTrack.h" | |
20 | |
21 namespace media { | |
22 class AudioBus; | |
23 class AudioConverter; | |
24 class AudioFifo; | |
25 class AudioParameters; | |
26 } | |
27 | |
28 namespace content { | |
29 | |
30 // SpeechRecognitionAudioSourceProvider works as a sink to the | |
31 // WebRtcLocalAudioTrack and stores the capture data to a FIFO. | |
32 // When the FIFO has enough buffer, it passes the buffer to | |
33 // the speech recognizer. | |
34 // | |
35 class CONTENT_EXPORT SpeechRecognitionAudioSourceProvider | |
36 : NON_EXPORTED_BASE(public media::AudioConverter::InputCallback), | |
37 NON_EXPORTED_BASE(public MediaStreamAudioSink) { | |
38 public: | |
39 SpeechRecognitionAudioSourceProvider( | |
40 const blink::WebMediaStreamTrack& track, | |
41 const media::AudioParameters& params, | |
42 base::SharedMemoryHandle memory, | |
43 base::NativeSyncSocket::Descriptor socket, | |
44 int memory_length); | |
45 virtual ~SpeechRecognitionAudioSourceProvider(); | |
46 // determines the policy on what types of tracks are allowed | |
tommi (sloooow) - chröme
2014/08/29 11:25:31
nit: empty line above this comment for readability
burnik
2014/08/29 13:26:17
Done.
| |
47 static bool IsAllowedAudioTrack(const blink::WebMediaStreamTrack& track); | |
48 | |
49 private: | |
50 // MediaStreamAudioSink implementation. | |
51 virtual void OnData(const int16* audio_data, | |
52 int sample_rate, | |
53 int number_of_channels, | |
54 int number_of_frames) OVERRIDE; | |
55 virtual void OnSetFormat(const media::AudioParameters& params) OVERRIDE; | |
56 virtual void OnReadyStateChanged( | |
57 blink::WebMediaStreamSource::ReadyState state) OVERRIDE; | |
58 | |
59 // media::AudioConverter::Inputcallback implementation. | |
60 // This function is triggered by provideInput() on the WebAudio audio thread, | |
61 // so it has been under the protection of |lock_|. | |
62 virtual double ProvideInput(media::AudioBus* audio_bus, | |
63 base::TimeDelta buffer_delay) OVERRIDE; | |
64 | |
65 private: | |
66 // Allows for some delays on the client | |
67 static const int kNumberOfBuffersInFifo = 2; | |
no longer working on chromium
2014/08/29 12:23:07
move it to the implementation.
burnik
2014/08/29 13:26:17
Done.
| |
68 | |
69 // Number of frames per buffer in FIFO. When the buffer is full we convert and | |
70 // consume it on the |output_bus_|. Size of the buffer is depends on the | |
71 // resampler. Example: for 44.1 to 16.0 conversion, it should be 4100 frames. | |
72 int fifo_buffer_size_; | |
73 | |
74 // Since |Receive| on |socket_| is blocking, we don't want to miss out on | |
75 // frames delivered to |OnData|. so we use |ReceiveWithTimeout|. | |
76 base::TimeDelta max_sync_delay_time_delta_; | |
77 | |
78 // Used to DCHECK that some methods are called on the main render thread. | |
79 base::ThreadChecker main_render_thread_checker_; | |
80 | |
81 // Used to DCHECK that some methods are called on the capture audio thread. | |
82 base::ThreadChecker capture_thread_checker_; | |
83 | |
84 // The audio track that this source provider is connected to. | |
85 blink::WebMediaStreamTrack track_; | |
86 | |
87 // Shared memory used by audio buses on both browser and renderer processes. | |
88 base::SharedMemory shared_memory_; | |
89 | |
90 // Socket for syncronization of audio bus reads/writes. | |
91 base::SyncSocket socket_; | |
92 | |
93 // Used as a resampler to deliver appropriate format to speech recognition. | |
94 scoped_ptr<media::AudioConverter> audio_converter_; | |
95 | |
96 // FIFO is used for queueing audio frames before we resample. | |
97 scoped_ptr<media::AudioFifo> fifo_; | |
98 | |
99 // Audio delivered from source. | |
100 scoped_ptr<media::AudioBus> input_bus_; | |
101 | |
102 // Audio bus shared with the browser process via |shared_memory_|. | |
103 scoped_ptr<media::AudioBus> output_bus_; | |
104 | |
105 // Params of the source audio. | |
106 media::AudioParameters input_params_; | |
107 | |
108 // Params used by speech recognition. | |
109 media::AudioParameters output_params_; | |
110 | |
111 // We attach the resampler once we have enough data in FIFO and not before. | |
no longer working on chromium
2014/08/29 12:23:07
confusing comment, can it make it more clear on wh
burnik
2014/08/29 13:26:17
Done.
On 2014/08/29 12:23:07, xians1 wrote:
| |
112 bool attached_converter_; | |
113 | |
114 // Whether the track has been stopped on the input. | |
115 bool track_stopped_; | |
116 | |
117 // Making sure all buffers are consumed by the client in order. | |
118 uint32 buffer_index_; | |
119 | |
120 DISALLOW_COPY_AND_ASSIGN(SpeechRecognitionAudioSourceProvider); | |
121 }; | |
122 | |
123 } // namespace content | |
124 | |
125 #endif // CONTENT_RENDERER_SPEECH_RECOGNITION_AUDIO_SOURCE_PROVIDER_H_ | |
OLD | NEW |