Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 // Copyright 2013 The Chromium Authors. All rights reserved. | 1 // Copyright 2013 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "media/cast/audio_receiver/audio_receiver.h" | 5 #include "media/cast/audio_receiver/audio_receiver.h" |
| 6 | 6 |
| 7 #include "base/bind.h" | 7 #include "base/bind.h" |
| 8 #include "base/logging.h" | 8 #include "base/logging.h" |
| 9 #include "base/message_loop/message_loop.h" | 9 #include "base/message_loop/message_loop.h" |
| 10 #include "base/strings/string_piece.h" | |
| 11 #include "media/cast/audio_receiver/audio_decoder.h" | 10 #include "media/cast/audio_receiver/audio_decoder.h" |
| 12 #include "media/cast/framer/framer.h" | |
| 13 #include "media/cast/rtcp/receiver_rtcp_event_subscriber.h" | |
| 14 #include "media/cast/rtcp/rtcp.h" | |
| 15 #include "media/cast/rtp_receiver/rtp_receiver.h" | |
| 16 #include "media/cast/transport/cast_transport_defines.h" | 11 #include "media/cast/transport/cast_transport_defines.h" |
| 17 | 12 |
| 18 namespace { | 13 namespace { |
| 19 | 14 |
| 20 // Max time we wait until an audio frame is due to be played out is released. | 15 // Max time we wait until an audio frame is due to be played out is released. |
| 21 static const int64 kMaxAudioFrameWaitMs = 20; | 16 static const int64 kMaxAudioFrameWaitMs = 20; |
| 22 static const int64 kMinSchedulingDelayMs = 1; | 17 static const int64 kMinSchedulingDelayMs = 1; |
| 23 | 18 |
| 24 } // namespace | 19 } // namespace |
| 25 | 20 |
| 26 namespace media { | 21 namespace media { |
| 27 namespace cast { | 22 namespace cast { |
| 28 | 23 |
| 29 DecodedAudioCallbackData::DecodedAudioCallbackData() | |
| 30 : number_of_10ms_blocks(0), desired_frequency(0), callback() {} | |
| 31 | |
| 32 DecodedAudioCallbackData::~DecodedAudioCallbackData() {} | |
| 33 | |
| 34 // Local implementation of RtpData (defined in rtp_rtcp_defines.h). | |
| 35 // Used to pass payload data into the audio receiver. | |
| 36 class LocalRtpAudioData : public RtpData { | |
| 37 public: | |
| 38 explicit LocalRtpAudioData(AudioReceiver* audio_receiver) | |
| 39 : audio_receiver_(audio_receiver) {} | |
| 40 | |
| 41 virtual void OnReceivedPayloadData(const uint8* payload_data, | |
| 42 size_t payload_size, | |
| 43 const RtpCastHeader* rtp_header) OVERRIDE { | |
| 44 audio_receiver_->IncomingParsedRtpPacket(payload_data, payload_size, | |
| 45 *rtp_header); | |
| 46 } | |
| 47 | |
| 48 private: | |
| 49 AudioReceiver* audio_receiver_; | |
| 50 }; | |
| 51 | |
| 52 // Local implementation of RtpPayloadFeedback (defined in rtp_defines.h) | |
| 53 // Used to convey cast-specific feedback from receiver to sender. | |
| 54 class LocalRtpAudioFeedback : public RtpPayloadFeedback { | |
| 55 public: | |
| 56 explicit LocalRtpAudioFeedback(AudioReceiver* audio_receiver) | |
| 57 : audio_receiver_(audio_receiver) {} | |
| 58 | |
| 59 virtual void CastFeedback(const RtcpCastMessage& cast_message) OVERRIDE { | |
| 60 audio_receiver_->CastFeedback(cast_message); | |
| 61 } | |
| 62 | |
| 63 private: | |
| 64 AudioReceiver* audio_receiver_; | |
| 65 }; | |
| 66 | |
| 67 class LocalRtpReceiverStatistics : public RtpReceiverStatistics { | |
| 68 public: | |
| 69 explicit LocalRtpReceiverStatistics(RtpReceiver* rtp_receiver) | |
| 70 : rtp_receiver_(rtp_receiver) {} | |
| 71 | |
| 72 virtual void GetStatistics(uint8* fraction_lost, | |
| 73 uint32* cumulative_lost, // 24 bits valid. | |
| 74 uint32* extended_high_sequence_number, | |
| 75 uint32* jitter) OVERRIDE { | |
| 76 rtp_receiver_->GetStatistics(fraction_lost, cumulative_lost, | |
| 77 extended_high_sequence_number, jitter); | |
| 78 } | |
| 79 | |
| 80 private: | |
| 81 RtpReceiver* rtp_receiver_; | |
| 82 }; | |
| 83 | |
| 84 AudioReceiver::AudioReceiver(scoped_refptr<CastEnvironment> cast_environment, | 24 AudioReceiver::AudioReceiver(scoped_refptr<CastEnvironment> cast_environment, |
| 85 const AudioReceiverConfig& audio_config, | 25 const AudioReceiverConfig& audio_config, |
| 86 transport::PacedPacketSender* const packet_sender) | 26 transport::PacedPacketSender* const packet_sender) |
| 87 : cast_environment_(cast_environment), | 27 : cast_environment_(cast_environment), |
| 88 event_subscriber_(kReceiverRtcpEventHistorySize, | 28 event_subscriber_(kReceiverRtcpEventHistorySize, |
| 89 ReceiverRtcpEventSubscriber::kAudioEventSubscriber), | 29 ReceiverRtcpEventSubscriber::kAudioEventSubscriber), |
| 90 codec_(audio_config.codec), | 30 codec_(audio_config.codec), |
| 91 frequency_(audio_config.frequency), | 31 frequency_(audio_config.frequency), |
| 92 audio_buffer_(), | 32 framer_(cast_environment->Clock(), |
| 93 audio_decoder_(), | 33 this, |
| 94 time_offset_(), | 34 audio_config.incoming_ssrc, |
| 35 true, | |
| 36 0), | |
| 37 rtp_receiver_(cast_environment->Clock(), &audio_config, NULL, this), | |
| 38 rtcp_(cast_environment, | |
| 39 NULL, | |
| 40 NULL, | |
| 41 packet_sender, | |
| 42 &rtp_receiver_, | |
| 43 audio_config.rtcp_mode, | |
| 44 base::TimeDelta::FromMilliseconds(audio_config.rtcp_interval), | |
| 45 audio_config.feedback_ssrc, | |
| 46 audio_config.incoming_ssrc, | |
| 47 audio_config.rtcp_c_name), | |
| 48 is_waiting_for_consecutive_frame_(false), | |
| 95 weak_factory_(this) { | 49 weak_factory_(this) { |
| 96 target_delay_delta_ = | 50 target_delay_delta_ = |
| 97 base::TimeDelta::FromMilliseconds(audio_config.rtp_max_delay_ms); | 51 base::TimeDelta::FromMilliseconds(audio_config.rtp_max_delay_ms); |
| 98 incoming_payload_callback_.reset(new LocalRtpAudioData(this)); | 52 if (!audio_config.use_external_decoder) |
| 99 incoming_payload_feedback_.reset(new LocalRtpAudioFeedback(this)); | 53 audio_decoder_.reset(new AudioDecoder(cast_environment, audio_config)); |
| 100 if (audio_config.use_external_decoder) { | |
| 101 audio_buffer_.reset(new Framer(cast_environment->Clock(), | |
| 102 incoming_payload_feedback_.get(), | |
| 103 audio_config.incoming_ssrc, true, 0)); | |
| 104 } else { | |
| 105 audio_decoder_.reset(new AudioDecoder(cast_environment, audio_config, | |
| 106 incoming_payload_feedback_.get())); | |
| 107 } | |
| 108 decryptor_.Initialize(audio_config.aes_key, audio_config.aes_iv_mask); | 54 decryptor_.Initialize(audio_config.aes_key, audio_config.aes_iv_mask); |
| 109 rtp_receiver_.reset(new RtpReceiver(cast_environment->Clock(), | 55 rtcp_.SetTargetDelay(target_delay_delta_); |
| 110 &audio_config, | |
| 111 NULL, | |
| 112 incoming_payload_callback_.get())); | |
| 113 rtp_audio_receiver_statistics_.reset( | |
| 114 new LocalRtpReceiverStatistics(rtp_receiver_.get())); | |
| 115 base::TimeDelta rtcp_interval_delta = | |
| 116 base::TimeDelta::FromMilliseconds(audio_config.rtcp_interval); | |
| 117 rtcp_.reset(new Rtcp(cast_environment, NULL, NULL, packet_sender, | |
| 118 rtp_audio_receiver_statistics_.get(), | |
| 119 audio_config.rtcp_mode, rtcp_interval_delta, | |
| 120 audio_config.feedback_ssrc, audio_config.incoming_ssrc, | |
| 121 audio_config.rtcp_c_name)); | |
| 122 // Set the target delay that will be conveyed to the sender. | |
| 123 rtcp_->SetTargetDelay(target_delay_delta_); | |
| 124 cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber_); | 56 cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber_); |
| 125 memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_)); | 57 memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_)); |
| 126 } | 58 } |
| 127 | 59 |
| 128 AudioReceiver::~AudioReceiver() { | 60 AudioReceiver::~AudioReceiver() { |
| 61 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 62 | |
| 63 // If any callbacks for encoded audio frames are queued, flush them out now. | |
| 64 while (!frame_request_queue_.empty()) { | |
| 65 frame_request_queue_.front().Run( | |
| 66 make_scoped_ptr<transport::EncodedAudioFrame>(NULL), base::TimeTicks()); | |
| 67 frame_request_queue_.pop_front(); | |
| 68 } | |
| 69 | |
| 129 cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber_); | 70 cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber_); |
| 130 } | 71 } |
| 131 | 72 |
| 132 void AudioReceiver::InitializeTimers() { | 73 void AudioReceiver::InitializeTimers() { |
| 133 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 74 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 134 ScheduleNextRtcpReport(); | 75 ScheduleNextRtcpReport(); |
| 135 ScheduleNextCastMessage(); | 76 ScheduleNextCastMessage(); |
| 136 } | 77 } |
| 137 | 78 |
| 138 void AudioReceiver::IncomingParsedRtpPacket(const uint8* payload_data, | 79 void AudioReceiver::OnReceivedPayloadData(const uint8* payload_data, |
| 139 size_t payload_size, | 80 size_t payload_size, |
| 140 const RtpCastHeader& rtp_header) { | 81 const RtpCastHeader& rtp_header) { |
| 141 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 82 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 142 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); | 83 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); |
| 143 | 84 |
| 144 frame_id_to_rtp_timestamp_[rtp_header.frame_id & 0xff] = | 85 frame_id_to_rtp_timestamp_[rtp_header.frame_id & 0xff] = |
| 145 rtp_header.webrtc.header.timestamp; | 86 rtp_header.webrtc.header.timestamp; |
| 146 cast_environment_->Logging()->InsertPacketEvent( | 87 cast_environment_->Logging()->InsertPacketEvent( |
| 147 now, kAudioPacketReceived, rtp_header.webrtc.header.timestamp, | 88 now, kAudioPacketReceived, rtp_header.webrtc.header.timestamp, |
| 148 rtp_header.frame_id, rtp_header.packet_id, rtp_header.max_packet_id, | 89 rtp_header.frame_id, rtp_header.packet_id, rtp_header.max_packet_id, |
| 149 payload_size); | 90 payload_size); |
| 150 | 91 |
| 151 // TODO(pwestin): update this as video to refresh over time. | 92 // TODO(pwestin): update this as video to refresh over time. |
| 152 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 153 if (time_first_incoming_packet_.is_null()) { | 93 if (time_first_incoming_packet_.is_null()) { |
| 154 InitializeTimers(); | 94 InitializeTimers(); |
| 155 first_incoming_rtp_timestamp_ = rtp_header.webrtc.header.timestamp; | 95 first_incoming_rtp_timestamp_ = rtp_header.webrtc.header.timestamp; |
| 156 time_first_incoming_packet_ = now; | 96 time_first_incoming_packet_ = now; |
| 157 } | 97 } |
| 158 | 98 |
| 159 if (audio_decoder_) { | |
| 160 DCHECK(!audio_buffer_) << "Invalid internal state"; | |
| 161 std::string plaintext; | |
| 162 if (decryptor_.initialized()) { | |
| 163 if (!decryptor_.Decrypt( | |
| 164 rtp_header.frame_id, | |
| 165 base::StringPiece(reinterpret_cast<const char*>(payload_data), | |
| 166 payload_size), | |
| 167 &plaintext)) | |
| 168 return; | |
| 169 } else { | |
| 170 plaintext.append(reinterpret_cast<const char*>(payload_data), | |
| 171 payload_size); | |
| 172 } | |
| 173 audio_decoder_->IncomingParsedRtpPacket( | |
| 174 reinterpret_cast<const uint8*>(plaintext.data()), plaintext.size(), | |
| 175 rtp_header); | |
| 176 if (!queued_decoded_callbacks_.empty()) { | |
| 177 DecodedAudioCallbackData decoded_data = queued_decoded_callbacks_.front(); | |
| 178 queued_decoded_callbacks_.pop_front(); | |
| 179 cast_environment_->PostTask( | |
| 180 CastEnvironment::AUDIO, FROM_HERE, | |
| 181 base::Bind(&AudioReceiver::DecodeAudioFrameThread, | |
| 182 base::Unretained(this), decoded_data.number_of_10ms_blocks, | |
| 183 decoded_data.desired_frequency, decoded_data.callback)); | |
| 184 } | |
| 185 return; | |
| 186 } | |
| 187 | |
| 188 DCHECK(audio_buffer_) << "Invalid internal state"; | |
| 189 DCHECK(!audio_decoder_) << "Invalid internal state"; | |
| 190 | |
| 191 bool duplicate = false; | 99 bool duplicate = false; |
| 192 bool complete = audio_buffer_->InsertPacket(payload_data, payload_size, | 100 const bool complete = |
| 193 rtp_header, &duplicate); | 101 framer_.InsertPacket(payload_data, payload_size, rtp_header, &duplicate); |
| 194 if (duplicate) { | 102 if (duplicate) { |
| 195 cast_environment_->Logging()->InsertPacketEvent( | 103 cast_environment_->Logging()->InsertPacketEvent( |
| 196 now, kDuplicateAudioPacketReceived, rtp_header.webrtc.header.timestamp, | 104 now, kDuplicateAudioPacketReceived, rtp_header.webrtc.header.timestamp, |
| 197 rtp_header.frame_id, rtp_header.packet_id, rtp_header.max_packet_id, | 105 rtp_header.frame_id, rtp_header.packet_id, rtp_header.max_packet_id, |
| 198 payload_size); | 106 payload_size); |
| 199 // Duplicate packets are ignored. | 107 // Duplicate packets are ignored. |
| 200 return; | 108 return; |
| 201 } | 109 } |
| 202 if (!complete) return; // Audio frame not complete; wait for more packets. | 110 if (!complete) |
| 203 if (queued_encoded_callbacks_.empty()) return; | 111 return; |
| 204 AudioFrameEncodedCallback callback = queued_encoded_callbacks_.front(); | 112 |
| 205 queued_encoded_callbacks_.pop_front(); | 113 if (!frame_request_queue_.empty()) |
|
hubbe
2014/03/28 19:50:59
This if statement seems redundant.
miu
2014/03/28 23:51:58
Done.
| |
| 206 cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, | 114 EmitAvailableEncodedFrames(); |
| 207 base::Bind(&AudioReceiver::GetEncodedAudioFrame, | |
| 208 weak_factory_.GetWeakPtr(), callback)); | |
| 209 } | 115 } |
| 210 | 116 |
| 211 void AudioReceiver::GetRawAudioFrame( | 117 void AudioReceiver::GetRawAudioFrame( |
| 212 int number_of_10ms_blocks, int desired_frequency, | |
| 213 const AudioFrameDecodedCallback& callback) { | 118 const AudioFrameDecodedCallback& callback) { |
| 214 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 119 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 215 DCHECK(audio_decoder_) << "Invalid function call in this configuration"; | 120 DCHECK(!callback.is_null()); |
| 216 // TODO(pwestin): we can skip this function by posting direct to the decoder. | 121 DCHECK(audio_decoder_.get()); |
| 217 cast_environment_->PostTask( | 122 GetEncodedAudioFrame(base::Bind(&AudioReceiver::DecodeEncodedAudioFrame, |
| 218 CastEnvironment::AUDIO, FROM_HERE, | 123 base::Unretained(this), |
|
hubbe
2014/03/28 19:50:59
weak ptr?
miu
2014/03/28 23:51:58
Not necessary. Added comments here and in the dto
| |
| 219 base::Bind(&AudioReceiver::DecodeAudioFrameThread, base::Unretained(this), | 124 callback)); |
| 220 number_of_10ms_blocks, desired_frequency, callback)); | |
| 221 } | 125 } |
| 222 | 126 |
| 223 void AudioReceiver::DecodeAudioFrameThread( | 127 void AudioReceiver::DecodeEncodedAudioFrame( |
| 224 int number_of_10ms_blocks, int desired_frequency, | 128 const AudioFrameDecodedCallback& callback, |
| 225 const AudioFrameDecodedCallback callback) { | 129 scoped_ptr<transport::EncodedAudioFrame> encoded_frame, |
| 226 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::AUDIO)); | 130 const base::TimeTicks& playout_time) { |
| 227 // TODO(mikhal): Allow the application to allocate this memory. | 131 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 228 scoped_ptr<PcmAudioFrame> audio_frame(new PcmAudioFrame()); | 132 if (!encoded_frame) { |
| 229 | 133 callback.Run(make_scoped_ptr<AudioBus>(NULL), playout_time, false); |
| 230 uint32 rtp_timestamp = 0; | |
| 231 if (!audio_decoder_->GetRawAudioFrame(number_of_10ms_blocks, | |
| 232 desired_frequency, audio_frame.get(), | |
| 233 &rtp_timestamp)) { | |
| 234 DecodedAudioCallbackData callback_data; | |
| 235 callback_data.number_of_10ms_blocks = number_of_10ms_blocks; | |
| 236 callback_data.desired_frequency = desired_frequency; | |
| 237 callback_data.callback = callback; | |
| 238 queued_decoded_callbacks_.push_back(callback_data); | |
| 239 return; | 134 return; |
| 240 } | 135 } |
| 241 | 136 const uint32 frame_id = encoded_frame->frame_id; |
| 242 cast_environment_->PostTask( | 137 const uint32 rtp_timestamp = encoded_frame->rtp_timestamp; |
| 243 CastEnvironment::MAIN, FROM_HERE, | 138 audio_decoder_->DecodeFrame(encoded_frame.Pass(), |
| 244 base::Bind(&AudioReceiver::ReturnDecodedFrameWithPlayoutDelay, | 139 base::Bind(&AudioReceiver::EmitRawAudioFrame, |
| 245 base::Unretained(this), base::Passed(&audio_frame), | 140 cast_environment_, |
| 246 rtp_timestamp, callback)); | 141 callback, |
| 142 frame_id, | |
| 143 rtp_timestamp, | |
| 144 playout_time)); | |
| 247 } | 145 } |
| 248 | 146 |
| 249 void AudioReceiver::ReturnDecodedFrameWithPlayoutDelay( | 147 // static |
| 250 scoped_ptr<PcmAudioFrame> audio_frame, uint32 rtp_timestamp, | 148 void AudioReceiver::EmitRawAudioFrame( |
| 251 const AudioFrameDecodedCallback callback) { | 149 const scoped_refptr<CastEnvironment>& cast_environment, |
| 252 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 150 const AudioFrameDecodedCallback& callback, |
| 253 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); | 151 uint32 frame_id, |
| 254 cast_environment_->Logging()->InsertFrameEvent( | 152 uint32 rtp_timestamp, |
| 255 now, kAudioFrameDecoded, rtp_timestamp, kFrameIdUnknown); | 153 const base::TimeTicks& playout_time, |
| 256 | 154 scoped_ptr<AudioBus> audio_bus, |
| 257 base::TimeTicks playout_time = GetPlayoutTime(now, rtp_timestamp); | 155 bool is_continuous) { |
| 258 | 156 DCHECK(cast_environment->CurrentlyOn(CastEnvironment::MAIN)); |
| 259 cast_environment_->Logging()->InsertFrameEventWithDelay( | 157 if (audio_bus.get()) { |
| 260 now, kAudioPlayoutDelay, rtp_timestamp, kFrameIdUnknown, | 158 const base::TimeTicks now = cast_environment->Clock()->NowTicks(); |
| 261 playout_time - now); | 159 cast_environment->Logging()->InsertFrameEvent( |
| 262 | 160 now, kAudioFrameDecoded, rtp_timestamp, frame_id); |
| 263 // Frame is ready - Send back to the caller. | 161 cast_environment->Logging()->InsertFrameEventWithDelay( |
| 264 cast_environment_->PostTask( | 162 now, kAudioPlayoutDelay, rtp_timestamp, frame_id, |
| 265 CastEnvironment::MAIN, FROM_HERE, | 163 playout_time - now); |
| 266 base::Bind(callback, base::Passed(&audio_frame), playout_time)); | |
| 267 } | |
| 268 | |
| 269 void AudioReceiver::PlayoutTimeout() { | |
| 270 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 271 DCHECK(audio_buffer_) << "Invalid function call in this configuration"; | |
| 272 if (queued_encoded_callbacks_.empty()) { | |
| 273 // Already released by incoming packet. | |
| 274 return; | |
| 275 } | 164 } |
| 276 bool next_frame = false; | 165 callback.Run(audio_bus.Pass(), playout_time, is_continuous); |
| 277 scoped_ptr<transport::EncodedAudioFrame> encoded_frame( | |
| 278 new transport::EncodedAudioFrame()); | |
| 279 | |
| 280 if (!audio_buffer_->GetEncodedAudioFrame(encoded_frame.get(), &next_frame)) { | |
| 281 // We have no audio frames. Wait for new packet(s). | |
| 282 // Since the application can post multiple AudioFrameEncodedCallback and | |
| 283 // we only check the next frame to play out we might have multiple timeout | |
| 284 // events firing after each other; however this should be a rare event. | |
| 285 VLOG(1) << "Failed to retrieved a complete frame at this point in time"; | |
| 286 return; | |
| 287 } | |
| 288 | |
| 289 if (decryptor_.initialized() && !DecryptAudioFrame(&encoded_frame)) { | |
| 290 // Logging already done. | |
| 291 return; | |
| 292 } | |
| 293 | |
| 294 if (PostEncodedAudioFrame( | |
| 295 queued_encoded_callbacks_.front(), next_frame, &encoded_frame)) { | |
| 296 // Call succeed remove callback from list. | |
| 297 queued_encoded_callbacks_.pop_front(); | |
| 298 } | |
| 299 } | 166 } |
| 300 | 167 |
| 301 void AudioReceiver::GetEncodedAudioFrame( | 168 void AudioReceiver::GetEncodedAudioFrame( |
| 302 const AudioFrameEncodedCallback& callback) { | 169 const AudioFrameEncodedCallback& callback) { |
| 303 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 170 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 304 DCHECK(audio_buffer_) << "Invalid function call in this configuration"; | 171 frame_request_queue_.push_back(callback); |
| 172 EmitAvailableEncodedFrames(); | |
| 173 } | |
| 305 | 174 |
| 306 bool next_frame = false; | 175 void AudioReceiver::EmitAvailableEncodedFrames() { |
| 307 scoped_ptr<transport::EncodedAudioFrame> encoded_frame( | 176 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 308 new transport::EncodedAudioFrame()); | |
| 309 | 177 |
| 310 if (!audio_buffer_->GetEncodedAudioFrame(encoded_frame.get(), &next_frame)) { | 178 while (!frame_request_queue_.empty()) { |
| 311 // We have no audio frames. Wait for new packet(s). | 179 // Attempt to peek at the next completed frame from the |framer_|. |
| 312 VLOG(1) << "Wait for more audio packets in frame"; | 180 // TODO(miu): We should only be peeking at the metadata, and not copying the |
| 313 queued_encoded_callbacks_.push_back(callback); | 181 // payload yet! Or, at least, peek using a StringPiece instead of a copy. |
| 314 return; | 182 scoped_ptr<transport::EncodedAudioFrame> encoded_frame( |
| 315 } | 183 new transport::EncodedAudioFrame()); |
| 316 if (decryptor_.initialized() && !DecryptAudioFrame(&encoded_frame)) { | 184 bool is_consecutively_next_frame = false; |
| 317 // Logging already done. | 185 if (!framer_.GetEncodedAudioFrame(encoded_frame.get(), |
| 318 queued_encoded_callbacks_.push_back(callback); | 186 &is_consecutively_next_frame)) { |
| 319 return; | 187 VLOG(1) << "Wait for more audio packets to produce a completed frame."; |
| 320 } | 188 return; // OnReceivedPayloadData() will invoke this method in the future. |
| 321 if (!PostEncodedAudioFrame(callback, next_frame, &encoded_frame)) { | 189 } |
| 322 // We have an audio frame; however we are missing packets and we have time | 190 |
| 323 // to wait for new packet(s). | 191 // If |framer_| has a frame ready that is out of sequence, examine the |
| 324 queued_encoded_callbacks_.push_back(callback); | 192 // playout time to determine whether it's acceptable to continue, thereby |
| 193 // skipping one or more frames. | |
| 194 const base::TimeTicks now = cast_environment_->Clock()->NowTicks(); | |
| 195 const base::TimeTicks playout_time = | |
| 196 GetPlayoutTime(now, encoded_frame->rtp_timestamp); | |
| 197 if (!is_consecutively_next_frame && playout_time > now) { | |
|
hubbe
2014/03/28 19:50:59
perhaps this should be playout_time > now - 10ms ?
miu
2014/03/28 23:51:58
Good insight. Done.
| |
| 198 VLOG(1) << "Wait for next consecutive frame, or before skipping a frame."; | |
| 199 if (!is_waiting_for_consecutive_frame_) { | |
| 200 is_waiting_for_consecutive_frame_ = true; | |
| 201 cast_environment_->PostDelayedTask( | |
| 202 CastEnvironment::MAIN, | |
| 203 FROM_HERE, | |
| 204 base::Bind(&AudioReceiver::EmitAvailableEncodedFramesAfterWaiting, | |
| 205 weak_factory_.GetWeakPtr()), | |
| 206 playout_time - now); | |
| 207 } | |
| 208 return; | |
| 209 } | |
| 210 | |
| 211 // Decrypt the payload data in the frame, if crypto is being used. | |
| 212 if (decryptor_.initialized()) { | |
| 213 std::string decrypted_audio_data; | |
| 214 if (!decryptor_.Decrypt(encoded_frame->frame_id, | |
| 215 encoded_frame->data, | |
| 216 &decrypted_audio_data)) { | |
| 217 // Decryption failed. Give up on this frame, releasing it from the | |
| 218 // jitter buffer. | |
| 219 framer_.ReleaseFrame(encoded_frame->frame_id); | |
| 220 continue; | |
| 221 } | |
| 222 encoded_frame->data.swap(decrypted_audio_data); | |
| 223 } | |
| 224 | |
| 225 // At this point, we have a decrypted EncodedAudioFrame ready to be emitted. | |
| 226 encoded_frame->codec = codec_; | |
| 227 framer_.ReleaseFrame(encoded_frame->frame_id); | |
| 228 cast_environment_->PostTask(CastEnvironment::MAIN, | |
| 229 FROM_HERE, | |
| 230 base::Bind(frame_request_queue_.front(), | |
| 231 base::Passed(&encoded_frame), | |
| 232 playout_time)); | |
| 233 frame_request_queue_.pop_front(); | |
| 325 } | 234 } |
| 326 } | 235 } |
| 327 | 236 |
| 328 bool AudioReceiver::PostEncodedAudioFrame( | 237 void AudioReceiver::EmitAvailableEncodedFramesAfterWaiting() { |
| 329 const AudioFrameEncodedCallback& callback, | |
| 330 bool next_frame, | |
| 331 scoped_ptr<transport::EncodedAudioFrame>* encoded_frame) { | |
| 332 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 238 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 333 DCHECK(audio_buffer_) << "Invalid function call in this configuration"; | 239 DCHECK(is_waiting_for_consecutive_frame_); |
| 334 DCHECK(encoded_frame) << "Invalid encoded_frame"; | 240 is_waiting_for_consecutive_frame_ = false; |
| 335 | 241 EmitAvailableEncodedFrames(); |
| 336 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); | |
| 337 base::TimeTicks playout_time = | |
| 338 GetPlayoutTime(now, (*encoded_frame)->rtp_timestamp); | |
| 339 base::TimeDelta time_until_playout = playout_time - now; | |
| 340 base::TimeDelta min_wait_delta = | |
| 341 base::TimeDelta::FromMilliseconds(kMaxAudioFrameWaitMs); | |
| 342 | |
| 343 if (!next_frame && (time_until_playout > min_wait_delta)) { | |
| 344 base::TimeDelta time_until_release = time_until_playout - min_wait_delta; | |
| 345 cast_environment_->PostDelayedTask( | |
| 346 CastEnvironment::MAIN, FROM_HERE, | |
| 347 base::Bind(&AudioReceiver::PlayoutTimeout, weak_factory_.GetWeakPtr()), | |
| 348 time_until_release); | |
| 349 VLOG(1) << "Wait until time to playout:" | |
| 350 << time_until_release.InMilliseconds(); | |
| 351 return false; | |
| 352 } | |
| 353 (*encoded_frame)->codec = codec_; | |
| 354 audio_buffer_->ReleaseFrame((*encoded_frame)->frame_id); | |
| 355 | |
| 356 cast_environment_->PostTask( | |
| 357 CastEnvironment::MAIN, FROM_HERE, | |
| 358 base::Bind(callback, base::Passed(encoded_frame), playout_time)); | |
| 359 return true; | |
| 360 } | 242 } |
| 361 | 243 |
| 362 void AudioReceiver::IncomingPacket(scoped_ptr<Packet> packet) { | 244 void AudioReceiver::IncomingPacket(scoped_ptr<Packet> packet) { |
| 363 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 245 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 364 bool rtcp_packet = Rtcp::IsRtcpPacket(&packet->front(), packet->size()); | 246 bool rtcp_packet = Rtcp::IsRtcpPacket(&packet->front(), packet->size()); |
| 365 if (!rtcp_packet) { | 247 if (!rtcp_packet) { |
| 366 rtp_receiver_->ReceivedPacket(&packet->front(), packet->size()); | 248 rtp_receiver_.ReceivedPacket(&packet->front(), packet->size()); |
| 367 } else { | 249 } else { |
| 368 rtcp_->IncomingRtcpPacket(&packet->front(), packet->size()); | 250 rtcp_.IncomingRtcpPacket(&packet->front(), packet->size()); |
| 369 } | 251 } |
| 370 } | 252 } |
| 371 | 253 |
| 372 void AudioReceiver::SetTargetDelay(base::TimeDelta target_delay) { | 254 void AudioReceiver::SetTargetDelay(base::TimeDelta target_delay) { |
| 255 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 373 target_delay_delta_ = target_delay; | 256 target_delay_delta_ = target_delay; |
| 374 rtcp_->SetTargetDelay(target_delay_delta_); | 257 rtcp_.SetTargetDelay(target_delay_delta_); |
| 375 } | 258 } |
| 376 | 259 |
| 377 void AudioReceiver::CastFeedback(const RtcpCastMessage& cast_message) { | 260 void AudioReceiver::CastFeedback(const RtcpCastMessage& cast_message) { |
| 261 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 378 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); | 262 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); |
| 379 RtpTimestamp rtp_timestamp = | 263 RtpTimestamp rtp_timestamp = |
| 380 frame_id_to_rtp_timestamp_[cast_message.ack_frame_id_ & 0xff]; | 264 frame_id_to_rtp_timestamp_[cast_message.ack_frame_id_ & 0xff]; |
| 381 cast_environment_->Logging()->InsertFrameEvent( | 265 cast_environment_->Logging()->InsertFrameEvent( |
| 382 now, kAudioAckSent, rtp_timestamp, cast_message.ack_frame_id_); | 266 now, kAudioAckSent, rtp_timestamp, cast_message.ack_frame_id_); |
| 383 | 267 |
| 384 rtcp_->SendRtcpFromRtpReceiver(&cast_message, &event_subscriber_); | 268 rtcp_.SendRtcpFromRtpReceiver(&cast_message, &event_subscriber_); |
| 385 } | 269 } |
| 386 | 270 |
| 387 base::TimeTicks AudioReceiver::GetPlayoutTime(base::TimeTicks now, | 271 base::TimeTicks AudioReceiver::GetPlayoutTime(base::TimeTicks now, |
| 388 uint32 rtp_timestamp) { | 272 uint32 rtp_timestamp) { |
| 389 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 273 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 390 // Senders time in ms when this frame was recorded. | 274 // Senders time in ms when this frame was recorded. |
| 391 // Note: the senders clock and our local clock might not be synced. | 275 // Note: the senders clock and our local clock might not be synced. |
| 392 base::TimeTicks rtp_timestamp_in_ticks; | 276 base::TimeTicks rtp_timestamp_in_ticks; |
| 393 base::TimeTicks playout_time; | 277 base::TimeTicks playout_time; |
| 394 if (time_offset_ == base::TimeDelta()) { | 278 if (time_offset_ == base::TimeDelta()) { |
| 395 if (rtcp_->RtpTimestampInSenderTime(frequency_, | 279 if (rtcp_.RtpTimestampInSenderTime(frequency_, |
| 396 first_incoming_rtp_timestamp_, | 280 first_incoming_rtp_timestamp_, |
| 397 &rtp_timestamp_in_ticks)) { | 281 &rtp_timestamp_in_ticks)) { |
| 398 time_offset_ = time_first_incoming_packet_ - rtp_timestamp_in_ticks; | 282 time_offset_ = time_first_incoming_packet_ - rtp_timestamp_in_ticks; |
| 283 // TODO(miu): As clocks drift w.r.t. each other, and other factors take | |
| 284 // effect, |time_offset_| should be updated. Otherwise, we might as well | |
| 285 // always compute the time offsets agnostic of RTCP's time data. | |
| 399 } else { | 286 } else { |
| 400 // We have not received any RTCP to sync the stream play it out as soon as | 287 // We have not received any RTCP to sync the stream play it out as soon as |
| 401 // possible. | 288 // possible. |
| 289 | |
| 290 // BUG: This means we're literally switching to a different timeline a | |
| 291 // short time after a cast receiver has been running. Re-enable | |
| 292 // End2EndTest.StartSenderBeforeReceiver once this is fixed. | |
| 293 // http://crbug.com/356942 | |
| 402 uint32 rtp_timestamp_diff = rtp_timestamp - first_incoming_rtp_timestamp_; | 294 uint32 rtp_timestamp_diff = rtp_timestamp - first_incoming_rtp_timestamp_; |
| 403 | 295 |
| 404 int frequency_khz = frequency_ / 1000; | 296 int frequency_khz = frequency_ / 1000; |
| 405 base::TimeDelta rtp_time_diff_delta = | 297 base::TimeDelta rtp_time_diff_delta = |
| 406 base::TimeDelta::FromMilliseconds(rtp_timestamp_diff / frequency_khz); | 298 base::TimeDelta::FromMilliseconds(rtp_timestamp_diff / frequency_khz); |
| 407 base::TimeDelta time_diff_delta = now - time_first_incoming_packet_; | 299 base::TimeDelta time_diff_delta = now - time_first_incoming_packet_; |
| 408 | 300 |
| 409 playout_time = now + std::max(rtp_time_diff_delta - time_diff_delta, | 301 playout_time = now + std::max(rtp_time_diff_delta - time_diff_delta, |
| 410 base::TimeDelta()); | 302 base::TimeDelta()); |
| 411 } | 303 } |
| 412 } | 304 } |
| 413 if (playout_time.is_null()) { | 305 if (playout_time.is_null()) { |
| 414 // This can fail if we have not received any RTCP packets in a long time. | 306 // This can fail if we have not received any RTCP packets in a long time. |
| 415 if (rtcp_->RtpTimestampInSenderTime(frequency_, rtp_timestamp, | 307 if (rtcp_.RtpTimestampInSenderTime(frequency_, rtp_timestamp, |
| 416 &rtp_timestamp_in_ticks)) { | 308 &rtp_timestamp_in_ticks)) { |
| 417 playout_time = | 309 playout_time = |
| 418 rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_; | 310 rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_; |
| 419 } else { | 311 } else { |
| 420 playout_time = now; | 312 playout_time = now; |
| 421 } | 313 } |
| 422 } | 314 } |
| 315 | |
| 316 // TODO(miu): This is broken since we literally switch timelines once |rtcp_| | |
| 317 // can provide us the |time_offset_|. Furthermore, this "getter" method may | |
| 318 // be called on frames received out-of-order, which means the playout times | |
| 319 // for earlier frames will be computed incorrectly. | |
| 320 #if 0 | |
| 423 // Don't allow the playout time to go backwards. | 321 // Don't allow the playout time to go backwards. |
| 424 if (last_playout_time_ > playout_time) playout_time = last_playout_time_; | 322 if (last_playout_time_ > playout_time) playout_time = last_playout_time_; |
| 425 last_playout_time_ = playout_time; | 323 last_playout_time_ = playout_time; |
| 324 #endif | |
| 325 | |
| 426 return playout_time; | 326 return playout_time; |
| 427 } | 327 } |
| 428 | 328 |
| 429 bool AudioReceiver::DecryptAudioFrame( | |
| 430 scoped_ptr<transport::EncodedAudioFrame>* audio_frame) { | |
| 431 if (!decryptor_.initialized()) | |
| 432 return false; | |
| 433 | |
| 434 std::string decrypted_audio_data; | |
| 435 if (!decryptor_.Decrypt((*audio_frame)->frame_id, | |
| 436 (*audio_frame)->data, | |
| 437 &decrypted_audio_data)) { | |
| 438 // Give up on this frame, release it from the jitter buffer. | |
| 439 audio_buffer_->ReleaseFrame((*audio_frame)->frame_id); | |
| 440 return false; | |
| 441 } | |
| 442 (*audio_frame)->data.swap(decrypted_audio_data); | |
| 443 return true; | |
| 444 } | |
| 445 | |
| 446 void AudioReceiver::ScheduleNextRtcpReport() { | 329 void AudioReceiver::ScheduleNextRtcpReport() { |
| 447 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 330 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 448 base::TimeDelta time_to_send = rtcp_->TimeToSendNextRtcpReport() - | 331 base::TimeDelta time_to_send = rtcp_.TimeToSendNextRtcpReport() - |
| 449 cast_environment_->Clock()->NowTicks(); | 332 cast_environment_->Clock()->NowTicks(); |
| 450 | 333 |
| 451 time_to_send = std::max( | 334 time_to_send = std::max( |
| 452 time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); | 335 time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); |
| 453 | 336 |
| 454 cast_environment_->PostDelayedTask( | 337 cast_environment_->PostDelayedTask( |
| 455 CastEnvironment::MAIN, FROM_HERE, | 338 CastEnvironment::MAIN, FROM_HERE, |
| 456 base::Bind(&AudioReceiver::SendNextRtcpReport, | 339 base::Bind(&AudioReceiver::SendNextRtcpReport, |
| 457 weak_factory_.GetWeakPtr()), | 340 weak_factory_.GetWeakPtr()), |
| 458 time_to_send); | 341 time_to_send); |
| 459 } | 342 } |
| 460 | 343 |
| 461 void AudioReceiver::SendNextRtcpReport() { | 344 void AudioReceiver::SendNextRtcpReport() { |
| 462 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 345 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 463 // TODO(pwestin): add logging. | 346 // TODO(pwestin): add logging. |
| 464 rtcp_->SendRtcpFromRtpReceiver(NULL, NULL); | 347 rtcp_.SendRtcpFromRtpReceiver(NULL, NULL); |
| 465 ScheduleNextRtcpReport(); | 348 ScheduleNextRtcpReport(); |
| 466 } | 349 } |
| 467 | 350 |
| 468 // Cast messages should be sent within a maximum interval. Schedule a call | 351 // Cast messages should be sent within a maximum interval. Schedule a call |
| 469 // if not triggered elsewhere, e.g. by the cast message_builder. | 352 // if not triggered elsewhere, e.g. by the cast message_builder. |
| 470 void AudioReceiver::ScheduleNextCastMessage() { | 353 void AudioReceiver::ScheduleNextCastMessage() { |
| 471 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 354 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 472 base::TimeTicks send_time; | 355 base::TimeTicks send_time; |
| 473 if (audio_buffer_) { | 356 framer_.TimeToSendNextCastMessage(&send_time); |
| 474 audio_buffer_->TimeToSendNextCastMessage(&send_time); | |
| 475 } else if (audio_decoder_) { | |
| 476 audio_decoder_->TimeToSendNextCastMessage(&send_time); | |
| 477 } else { | |
| 478 NOTREACHED(); | |
| 479 } | |
| 480 base::TimeDelta time_to_send = | 357 base::TimeDelta time_to_send = |
| 481 send_time - cast_environment_->Clock()->NowTicks(); | 358 send_time - cast_environment_->Clock()->NowTicks(); |
| 482 time_to_send = std::max( | 359 time_to_send = std::max( |
| 483 time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); | 360 time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); |
| 484 cast_environment_->PostDelayedTask( | 361 cast_environment_->PostDelayedTask( |
| 485 CastEnvironment::MAIN, FROM_HERE, | 362 CastEnvironment::MAIN, FROM_HERE, |
| 486 base::Bind(&AudioReceiver::SendNextCastMessage, | 363 base::Bind(&AudioReceiver::SendNextCastMessage, |
| 487 weak_factory_.GetWeakPtr()), | 364 weak_factory_.GetWeakPtr()), |
| 488 time_to_send); | 365 time_to_send); |
| 489 } | 366 } |
| 490 | 367 |
| 491 void AudioReceiver::SendNextCastMessage() { | 368 void AudioReceiver::SendNextCastMessage() { |
| 492 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 369 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 493 | 370 // Will only send a message if it is time. |
| 494 if (audio_buffer_) { | 371 framer_.SendCastMessage(); |
| 495 // Will only send a message if it is time. | |
| 496 audio_buffer_->SendCastMessage(); | |
| 497 } | |
| 498 if (audio_decoder_) { | |
| 499 // Will only send a message if it is time. | |
| 500 audio_decoder_->SendCastMessage(); | |
| 501 } | |
| 502 ScheduleNextCastMessage(); | 372 ScheduleNextCastMessage(); |
| 503 } | 373 } |
| 504 | 374 |
| 505 } // namespace cast | 375 } // namespace cast |
| 506 } // namespace media | 376 } // namespace media |
| OLD | NEW |