OLD | NEW |
(Empty) | |
| 1 // Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. |
| 4 |
| 5 #ifndef CONTENT_RENDERER_MEDIA_SPEECH_RECOGNITION_AUDIO_SINK_H_ |
| 6 #define CONTENT_RENDERER_MEDIA_SPEECH_RECOGNITION_AUDIO_SINK_H_ |
| 7 |
| 8 #include "base/callback.h" |
| 9 #include "base/memory/scoped_ptr.h" |
| 10 #include "base/memory/shared_memory.h" |
| 11 #include "base/sync_socket.h" |
| 12 #include "base/threading/thread_checker.h" |
| 13 #include "content/common/content_export.h" |
| 14 #include "content/public/renderer/media_stream_audio_sink.h" |
| 15 #include "content/renderer/media/media_stream_audio_source.h" |
| 16 #include "media/audio/audio_parameters.h" |
| 17 #include "media/base/audio_converter.h" |
| 18 #include "third_party/WebKit/public/platform/WebMediaStreamTrack.h" |
| 19 |
| 20 namespace media { |
| 21 class AudioBus; |
| 22 class AudioFifo; |
| 23 } |
| 24 |
| 25 namespace content { |
| 26 |
| 27 // SpeechRecognitionAudioSink works as an audio sink to the |
| 28 // WebRtcLocalAudioTrack. It stores the capture data into a FIFO. |
| 29 // When the FIFO has enough data for resampling, it converts it, |
| 30 // passes the buffer to the WebSpeechRecognizer via SharedMemory |
| 31 // and notifies it via SyncSocket followed by incrementing the |buffer_index_|. |
| 32 // WebSpeechRecognizer increments the shared buffer index to synchronize. |
| 33 class CONTENT_EXPORT SpeechRecognitionAudioSink |
| 34 : NON_EXPORTED_BASE(public media::AudioConverter::InputCallback), |
| 35 NON_EXPORTED_BASE(public MediaStreamAudioSink) { |
| 36 public: |
| 37 typedef base::Callback<void()> OnStoppedCB; |
| 38 |
| 39 // Socket ownership is transferred to the class via constructor. |
| 40 SpeechRecognitionAudioSink(const blink::WebMediaStreamTrack& track, |
| 41 const media::AudioParameters& params, |
| 42 const base::SharedMemoryHandle memory, |
| 43 scoped_ptr<base::SyncSocket> socket, |
| 44 const OnStoppedCB& on_stopped_cb); |
| 45 |
| 46 virtual ~SpeechRecognitionAudioSink(); |
| 47 |
| 48 // Returns whether the provided track is supported. |
| 49 static bool IsSupportedTrack(const blink::WebMediaStreamTrack& track); |
| 50 |
| 51 private: |
| 52 // content::MediaStreamAudioSink implementation. |
| 53 virtual void OnReadyStateChanged( |
| 54 blink::WebMediaStreamSource::ReadyState state) OVERRIDE; |
| 55 |
| 56 virtual void OnData(const int16* audio_data, int sample_rate, |
| 57 int number_of_channels, int number_of_frames) OVERRIDE; |
| 58 virtual void OnSetFormat(const media::AudioParameters& params) OVERRIDE; |
| 59 |
| 60 // media::AudioConverter::Inputcallback implementation. |
| 61 virtual double ProvideInput(media::AudioBus* audio_bus, |
| 62 base::TimeDelta buffer_delay) OVERRIDE; |
| 63 |
| 64 // Number of frames per buffer in FIFO. When the buffer is full we convert and |
| 65 // consume it on the |output_bus_|. Size of the buffer depends on the |
| 66 // resampler. Example: for 44.1 to 16.0 conversion, it should be 4100 frames. |
| 67 int fifo_buffer_size_; |
| 68 |
| 69 // Used to DCHECK that some methods are called on the main render thread. |
| 70 base::ThreadChecker main_render_thread_checker_; |
| 71 |
| 72 // Used to DCHECK that some methods are called on the capture audio thread. |
| 73 base::ThreadChecker capture_thread_checker_; |
| 74 |
| 75 // The audio track that this audio sink is connected to. |
| 76 const blink::WebMediaStreamTrack track_; |
| 77 |
| 78 // Shared memory used by audio buses on both browser and renderer processes. |
| 79 base::SharedMemory shared_memory_; |
| 80 |
| 81 // Socket for synchronization of audio bus reads/writes. |
| 82 // Created on the renderer client and passed here. Accessed on capture thread. |
| 83 scoped_ptr<base::SyncSocket> socket_; |
| 84 |
| 85 // Used as a resampler to deliver appropriate format to speech recognition. |
| 86 scoped_ptr<media::AudioConverter> audio_converter_; |
| 87 |
| 88 // FIFO is used for queuing audio frames before we resample. |
| 89 scoped_ptr<media::AudioFifo> fifo_; |
| 90 |
| 91 // Audio delivered from source. |
| 92 scoped_ptr<media::AudioBus> input_bus_; |
| 93 |
| 94 // Audio bus shared with the browser process via |shared_memory_|. |
| 95 scoped_ptr<media::AudioBus> output_bus_; |
| 96 |
| 97 // Params of the source audio. Can change when |OnSetFormat()| occurs. |
| 98 media::AudioParameters input_params_; |
| 99 |
| 100 // Params used by speech recognition. |
| 101 const media::AudioParameters output_params_; |
| 102 |
| 103 // Whether the track has been stopped. |
| 104 bool track_stopped_; |
| 105 |
| 106 // Local counter of audio buffers for synchronization. |
| 107 uint32 buffer_index_; |
| 108 |
| 109 // Peer's counter of audio buffers for synchronization. |
| 110 const uint32* peer_buffer_index_; |
| 111 |
| 112 // Callback for the renderer client. Called when the audio track was stopped. |
| 113 const OnStoppedCB on_stopped_cb_; |
| 114 |
| 115 DISALLOW_COPY_AND_ASSIGN(SpeechRecognitionAudioSink); |
| 116 }; |
| 117 |
| 118 } // namespace content |
| 119 |
| 120 #endif // CONTENT_RENDERER_MEDIA_SPEECH_RECOGNITION_AUDIO_SINK_H_ |
OLD | NEW |