| OLD | NEW |
| 1 // Copyright 2013 The Chromium Authors. All rights reserved. | 1 // Copyright 2013 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "media/cast/audio_receiver/audio_receiver.h" | 5 #include "media/cast/audio_receiver/audio_receiver.h" |
| 6 | 6 |
| 7 #include "base/bind.h" | 7 #include "base/bind.h" |
| 8 #include "base/logging.h" | 8 #include "base/logging.h" |
| 9 #include "base/message_loop/message_loop.h" | 9 #include "base/message_loop/message_loop.h" |
| 10 #include "base/strings/string_piece.h" | |
| 11 #include "media/cast/audio_receiver/audio_decoder.h" | 10 #include "media/cast/audio_receiver/audio_decoder.h" |
| 12 #include "media/cast/framer/framer.h" | |
| 13 #include "media/cast/rtcp/receiver_rtcp_event_subscriber.h" | |
| 14 #include "media/cast/rtcp/rtcp.h" | |
| 15 #include "media/cast/rtp_receiver/rtp_receiver.h" | |
| 16 #include "media/cast/transport/cast_transport_defines.h" | 11 #include "media/cast/transport/cast_transport_defines.h" |
| 17 | 12 |
| 18 namespace { | 13 namespace { |
| 19 | 14 const int kTypicalAudioFrameDurationMs = 10; |
| 20 // Max time we wait until an audio frame is due to be played out is released. | 15 const int kMinSchedulingDelayMs = 1; |
| 21 static const int64 kMaxAudioFrameWaitMs = 20; | |
| 22 static const int64 kMinSchedulingDelayMs = 1; | |
| 23 | |
| 24 } // namespace | 16 } // namespace |
| 25 | 17 |
| 26 namespace media { | 18 namespace media { |
| 27 namespace cast { | 19 namespace cast { |
| 28 | 20 |
| 29 DecodedAudioCallbackData::DecodedAudioCallbackData() | |
| 30 : number_of_10ms_blocks(0), desired_frequency(0), callback() {} | |
| 31 | |
| 32 DecodedAudioCallbackData::~DecodedAudioCallbackData() {} | |
| 33 | |
| 34 // Local implementation of RtpPayloadFeedback (defined in rtp_defines.h) | |
| 35 // Used to convey cast-specific feedback from receiver to sender. | |
| 36 class LocalRtpAudioFeedback : public RtpPayloadFeedback { | |
| 37 public: | |
| 38 explicit LocalRtpAudioFeedback(AudioReceiver* audio_receiver) | |
| 39 : audio_receiver_(audio_receiver) {} | |
| 40 | |
| 41 virtual void CastFeedback(const RtcpCastMessage& cast_message) OVERRIDE { | |
| 42 audio_receiver_->CastFeedback(cast_message); | |
| 43 } | |
| 44 | |
| 45 private: | |
| 46 AudioReceiver* audio_receiver_; | |
| 47 }; | |
| 48 | |
| 49 AudioReceiver::AudioReceiver(scoped_refptr<CastEnvironment> cast_environment, | 21 AudioReceiver::AudioReceiver(scoped_refptr<CastEnvironment> cast_environment, |
| 50 const AudioReceiverConfig& audio_config, | 22 const AudioReceiverConfig& audio_config, |
| 51 transport::PacedPacketSender* const packet_sender) | 23 transport::PacedPacketSender* const packet_sender) |
| 52 : RtpReceiver(cast_environment->Clock(), &audio_config, NULL), | 24 : RtpReceiver(cast_environment->Clock(), &audio_config, NULL), |
| 53 cast_environment_(cast_environment), | 25 cast_environment_(cast_environment), |
| 54 event_subscriber_(kReceiverRtcpEventHistorySize, | 26 event_subscriber_(kReceiverRtcpEventHistorySize, |
| 55 ReceiverRtcpEventSubscriber::kAudioEventSubscriber), | 27 ReceiverRtcpEventSubscriber::kAudioEventSubscriber), |
| 56 codec_(audio_config.codec), | 28 codec_(audio_config.codec), |
| 57 frequency_(audio_config.frequency), | 29 frequency_(audio_config.frequency), |
| 58 audio_buffer_(), | 30 framer_(cast_environment->Clock(), |
| 59 audio_decoder_(), | 31 this, |
| 60 time_offset_(), | 32 audio_config.incoming_ssrc, |
| 33 true, |
| 34 0), |
| 35 rtcp_(cast_environment, |
| 36 NULL, |
| 37 NULL, |
| 38 packet_sender, |
| 39 GetStatistics(), |
| 40 audio_config.rtcp_mode, |
| 41 base::TimeDelta::FromMilliseconds(audio_config.rtcp_interval), |
| 42 audio_config.feedback_ssrc, |
| 43 audio_config.incoming_ssrc, |
| 44 audio_config.rtcp_c_name), |
| 45 is_waiting_for_consecutive_frame_(false), |
| 61 weak_factory_(this) { | 46 weak_factory_(this) { |
| 62 target_delay_delta_ = | 47 target_delay_delta_ = |
| 63 base::TimeDelta::FromMilliseconds(audio_config.rtp_max_delay_ms); | 48 base::TimeDelta::FromMilliseconds(audio_config.rtp_max_delay_ms); |
| 64 incoming_payload_feedback_.reset(new LocalRtpAudioFeedback(this)); | 49 if (!audio_config.use_external_decoder) |
| 65 if (audio_config.use_external_decoder) { | 50 audio_decoder_.reset(new AudioDecoder(cast_environment, audio_config)); |
| 66 audio_buffer_.reset(new Framer(cast_environment->Clock(), | |
| 67 incoming_payload_feedback_.get(), | |
| 68 audio_config.incoming_ssrc, true, 0)); | |
| 69 } else { | |
| 70 audio_decoder_.reset(new AudioDecoder(cast_environment, audio_config, | |
| 71 incoming_payload_feedback_.get())); | |
| 72 } | |
| 73 decryptor_.Initialize(audio_config.aes_key, audio_config.aes_iv_mask); | 51 decryptor_.Initialize(audio_config.aes_key, audio_config.aes_iv_mask); |
| 74 base::TimeDelta rtcp_interval_delta = | 52 rtcp_.SetTargetDelay(target_delay_delta_); |
| 75 base::TimeDelta::FromMilliseconds(audio_config.rtcp_interval); | |
| 76 rtcp_.reset(new Rtcp(cast_environment, NULL, NULL, packet_sender, | |
| 77 GetStatistics(), | |
| 78 audio_config.rtcp_mode, rtcp_interval_delta, | |
| 79 audio_config.feedback_ssrc, audio_config.incoming_ssrc, | |
| 80 audio_config.rtcp_c_name)); | |
| 81 // Set the target delay that will be conveyed to the sender. | |
| 82 rtcp_->SetTargetDelay(target_delay_delta_); | |
| 83 cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber_); | 53 cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber_); |
| 84 memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_)); | 54 memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_)); |
| 85 } | 55 } |
| 86 | 56 |
| 87 AudioReceiver::~AudioReceiver() { | 57 AudioReceiver::~AudioReceiver() { |
| 58 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 59 |
| 60 // If any callbacks for encoded audio frames are queued, flush them out now. |
| 61 // This is critical because some Closures in |frame_request_queue_| may have |
| 62 // Unretained references to |this|. |
| 63 while (!frame_request_queue_.empty()) { |
| 64 frame_request_queue_.front().Run( |
| 65 make_scoped_ptr<transport::EncodedAudioFrame>(NULL), base::TimeTicks()); |
| 66 frame_request_queue_.pop_front(); |
| 67 } |
| 68 |
| 88 cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber_); | 69 cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber_); |
| 89 } | 70 } |
| 90 | 71 |
| 91 void AudioReceiver::InitializeTimers() { | 72 void AudioReceiver::InitializeTimers() { |
| 92 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 73 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 93 ScheduleNextRtcpReport(); | 74 ScheduleNextRtcpReport(); |
| 94 ScheduleNextCastMessage(); | 75 ScheduleNextCastMessage(); |
| 95 } | 76 } |
| 96 | 77 |
| 97 void AudioReceiver::OnReceivedPayloadData(const uint8* payload_data, | 78 void AudioReceiver::OnReceivedPayloadData(const uint8* payload_data, |
| 98 size_t payload_size, | 79 size_t payload_size, |
| 99 const RtpCastHeader& rtp_header) { | 80 const RtpCastHeader& rtp_header) { |
| 100 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 81 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 101 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); | 82 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); |
| 102 | 83 |
| 103 frame_id_to_rtp_timestamp_[rtp_header.frame_id & 0xff] = | 84 frame_id_to_rtp_timestamp_[rtp_header.frame_id & 0xff] = |
| 104 rtp_header.webrtc.header.timestamp; | 85 rtp_header.webrtc.header.timestamp; |
| 105 cast_environment_->Logging()->InsertPacketEvent( | 86 cast_environment_->Logging()->InsertPacketEvent( |
| 106 now, kAudioPacketReceived, rtp_header.webrtc.header.timestamp, | 87 now, kAudioPacketReceived, rtp_header.webrtc.header.timestamp, |
| 107 rtp_header.frame_id, rtp_header.packet_id, rtp_header.max_packet_id, | 88 rtp_header.frame_id, rtp_header.packet_id, rtp_header.max_packet_id, |
| 108 payload_size); | 89 payload_size); |
| 109 | 90 |
| 110 // TODO(pwestin): update this as video to refresh over time. | 91 // TODO(pwestin): update this as video to refresh over time. |
| 111 if (time_first_incoming_packet_.is_null()) { | 92 if (time_first_incoming_packet_.is_null()) { |
| 112 InitializeTimers(); | 93 InitializeTimers(); |
| 113 first_incoming_rtp_timestamp_ = rtp_header.webrtc.header.timestamp; | 94 first_incoming_rtp_timestamp_ = rtp_header.webrtc.header.timestamp; |
| 114 time_first_incoming_packet_ = now; | 95 time_first_incoming_packet_ = now; |
| 115 } | 96 } |
| 116 | 97 |
| 117 if (audio_decoder_) { | |
| 118 DCHECK(!audio_buffer_) << "Invalid internal state"; | |
| 119 std::string plaintext; | |
| 120 if (decryptor_.initialized()) { | |
| 121 if (!decryptor_.Decrypt( | |
| 122 rtp_header.frame_id, | |
| 123 base::StringPiece(reinterpret_cast<const char*>(payload_data), | |
| 124 payload_size), | |
| 125 &plaintext)) | |
| 126 return; | |
| 127 } else { | |
| 128 plaintext.append(reinterpret_cast<const char*>(payload_data), | |
| 129 payload_size); | |
| 130 } | |
| 131 audio_decoder_->IncomingParsedRtpPacket( | |
| 132 reinterpret_cast<const uint8*>(plaintext.data()), plaintext.size(), | |
| 133 rtp_header); | |
| 134 if (!queued_decoded_callbacks_.empty()) { | |
| 135 DecodedAudioCallbackData decoded_data = queued_decoded_callbacks_.front(); | |
| 136 queued_decoded_callbacks_.pop_front(); | |
| 137 cast_environment_->PostTask( | |
| 138 CastEnvironment::AUDIO, FROM_HERE, | |
| 139 base::Bind(&AudioReceiver::DecodeAudioFrameThread, | |
| 140 base::Unretained(this), decoded_data.number_of_10ms_blocks, | |
| 141 decoded_data.desired_frequency, decoded_data.callback)); | |
| 142 } | |
| 143 return; | |
| 144 } | |
| 145 | |
| 146 DCHECK(audio_buffer_) << "Invalid internal state"; | |
| 147 DCHECK(!audio_decoder_) << "Invalid internal state"; | |
| 148 | |
| 149 bool duplicate = false; | 98 bool duplicate = false; |
| 150 bool complete = audio_buffer_->InsertPacket(payload_data, payload_size, | 99 const bool complete = |
| 151 rtp_header, &duplicate); | 100 framer_.InsertPacket(payload_data, payload_size, rtp_header, &duplicate); |
| 152 if (duplicate) { | 101 if (duplicate) { |
| 153 cast_environment_->Logging()->InsertPacketEvent( | 102 cast_environment_->Logging()->InsertPacketEvent( |
| 154 now, kDuplicateAudioPacketReceived, rtp_header.webrtc.header.timestamp, | 103 now, kDuplicateAudioPacketReceived, rtp_header.webrtc.header.timestamp, |
| 155 rtp_header.frame_id, rtp_header.packet_id, rtp_header.max_packet_id, | 104 rtp_header.frame_id, rtp_header.packet_id, rtp_header.max_packet_id, |
| 156 payload_size); | 105 payload_size); |
| 157 // Duplicate packets are ignored. | 106 // Duplicate packets are ignored. |
| 158 return; | 107 return; |
| 159 } | 108 } |
| 160 if (!complete) return; // Audio frame not complete; wait for more packets. | 109 if (!complete) |
| 161 if (queued_encoded_callbacks_.empty()) return; | 110 return; |
| 162 AudioFrameEncodedCallback callback = queued_encoded_callbacks_.front(); | 111 |
| 163 queued_encoded_callbacks_.pop_front(); | 112 EmitAvailableEncodedFrames(); |
| 164 cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, | |
| 165 base::Bind(&AudioReceiver::GetEncodedAudioFrame, | |
| 166 weak_factory_.GetWeakPtr(), callback)); | |
| 167 } | 113 } |
| 168 | 114 |
| 169 void AudioReceiver::GetRawAudioFrame( | 115 void AudioReceiver::GetRawAudioFrame( |
| 170 int number_of_10ms_blocks, int desired_frequency, | |
| 171 const AudioFrameDecodedCallback& callback) { | 116 const AudioFrameDecodedCallback& callback) { |
| 172 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 117 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 173 DCHECK(audio_decoder_) << "Invalid function call in this configuration"; | 118 DCHECK(!callback.is_null()); |
| 174 // TODO(pwestin): we can skip this function by posting direct to the decoder. | 119 DCHECK(audio_decoder_.get()); |
| 175 cast_environment_->PostTask( | 120 GetEncodedAudioFrame(base::Bind( |
| 176 CastEnvironment::AUDIO, FROM_HERE, | 121 &AudioReceiver::DecodeEncodedAudioFrame, |
| 177 base::Bind(&AudioReceiver::DecodeAudioFrameThread, base::Unretained(this), | 122 // Note: Use of Unretained is safe since this Closure is guaranteed to be |
| 178 number_of_10ms_blocks, desired_frequency, callback)); | 123 // invoked before destruction of |this|. |
| 124 base::Unretained(this), |
| 125 callback)); |
| 179 } | 126 } |
| 180 | 127 |
| 181 void AudioReceiver::DecodeAudioFrameThread( | 128 void AudioReceiver::DecodeEncodedAudioFrame( |
| 182 int number_of_10ms_blocks, int desired_frequency, | 129 const AudioFrameDecodedCallback& callback, |
| 183 const AudioFrameDecodedCallback callback) { | 130 scoped_ptr<transport::EncodedAudioFrame> encoded_frame, |
| 184 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::AUDIO)); | 131 const base::TimeTicks& playout_time) { |
| 185 // TODO(mikhal): Allow the application to allocate this memory. | 132 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 186 scoped_ptr<PcmAudioFrame> audio_frame(new PcmAudioFrame()); | 133 if (!encoded_frame) { |
| 187 | 134 callback.Run(make_scoped_ptr<AudioBus>(NULL), playout_time, false); |
| 188 uint32 rtp_timestamp = 0; | |
| 189 if (!audio_decoder_->GetRawAudioFrame(number_of_10ms_blocks, | |
| 190 desired_frequency, audio_frame.get(), | |
| 191 &rtp_timestamp)) { | |
| 192 DecodedAudioCallbackData callback_data; | |
| 193 callback_data.number_of_10ms_blocks = number_of_10ms_blocks; | |
| 194 callback_data.desired_frequency = desired_frequency; | |
| 195 callback_data.callback = callback; | |
| 196 queued_decoded_callbacks_.push_back(callback_data); | |
| 197 return; | 135 return; |
| 198 } | 136 } |
| 199 | 137 const uint32 frame_id = encoded_frame->frame_id; |
| 200 cast_environment_->PostTask( | 138 const uint32 rtp_timestamp = encoded_frame->rtp_timestamp; |
| 201 CastEnvironment::MAIN, FROM_HERE, | 139 audio_decoder_->DecodeFrame(encoded_frame.Pass(), |
| 202 base::Bind(&AudioReceiver::ReturnDecodedFrameWithPlayoutDelay, | 140 base::Bind(&AudioReceiver::EmitRawAudioFrame, |
| 203 base::Unretained(this), base::Passed(&audio_frame), | 141 cast_environment_, |
| 204 rtp_timestamp, callback)); | 142 callback, |
| 143 frame_id, |
| 144 rtp_timestamp, |
| 145 playout_time)); |
| 205 } | 146 } |
| 206 | 147 |
| 207 void AudioReceiver::ReturnDecodedFrameWithPlayoutDelay( | 148 // static |
| 208 scoped_ptr<PcmAudioFrame> audio_frame, uint32 rtp_timestamp, | 149 void AudioReceiver::EmitRawAudioFrame( |
| 209 const AudioFrameDecodedCallback callback) { | 150 const scoped_refptr<CastEnvironment>& cast_environment, |
| 210 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 151 const AudioFrameDecodedCallback& callback, |
| 211 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); | 152 uint32 frame_id, |
| 212 cast_environment_->Logging()->InsertFrameEvent( | 153 uint32 rtp_timestamp, |
| 213 now, kAudioFrameDecoded, rtp_timestamp, kFrameIdUnknown); | 154 const base::TimeTicks& playout_time, |
| 214 | 155 scoped_ptr<AudioBus> audio_bus, |
| 215 base::TimeTicks playout_time = GetPlayoutTime(now, rtp_timestamp); | 156 bool is_continuous) { |
| 216 | 157 DCHECK(cast_environment->CurrentlyOn(CastEnvironment::MAIN)); |
| 217 cast_environment_->Logging()->InsertFrameEventWithDelay( | 158 if (audio_bus.get()) { |
| 218 now, kAudioPlayoutDelay, rtp_timestamp, kFrameIdUnknown, | 159 const base::TimeTicks now = cast_environment->Clock()->NowTicks(); |
| 219 playout_time - now); | 160 cast_environment->Logging()->InsertFrameEvent( |
| 220 | 161 now, kAudioFrameDecoded, rtp_timestamp, frame_id); |
| 221 // Frame is ready - Send back to the caller. | 162 cast_environment->Logging()->InsertFrameEventWithDelay( |
| 222 cast_environment_->PostTask( | 163 now, kAudioPlayoutDelay, rtp_timestamp, frame_id, |
| 223 CastEnvironment::MAIN, FROM_HERE, | 164 playout_time - now); |
| 224 base::Bind(callback, base::Passed(&audio_frame), playout_time)); | |
| 225 } | |
| 226 | |
| 227 void AudioReceiver::PlayoutTimeout() { | |
| 228 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 229 DCHECK(audio_buffer_) << "Invalid function call in this configuration"; | |
| 230 if (queued_encoded_callbacks_.empty()) { | |
| 231 // Already released by incoming packet. | |
| 232 return; | |
| 233 } | 165 } |
| 234 bool next_frame = false; | 166 callback.Run(audio_bus.Pass(), playout_time, is_continuous); |
| 235 scoped_ptr<transport::EncodedAudioFrame> encoded_frame( | |
| 236 new transport::EncodedAudioFrame()); | |
| 237 | |
| 238 if (!audio_buffer_->GetEncodedAudioFrame(encoded_frame.get(), &next_frame)) { | |
| 239 // We have no audio frames. Wait for new packet(s). | |
| 240 // Since the application can post multiple AudioFrameEncodedCallback and | |
| 241 // we only check the next frame to play out we might have multiple timeout | |
| 242 // events firing after each other; however this should be a rare event. | |
| 243 VLOG(1) << "Failed to retrieved a complete frame at this point in time"; | |
| 244 return; | |
| 245 } | |
| 246 | |
| 247 if (decryptor_.initialized() && !DecryptAudioFrame(&encoded_frame)) { | |
| 248 // Logging already done. | |
| 249 return; | |
| 250 } | |
| 251 | |
| 252 if (PostEncodedAudioFrame( | |
| 253 queued_encoded_callbacks_.front(), next_frame, &encoded_frame)) { | |
| 254 // Call succeed remove callback from list. | |
| 255 queued_encoded_callbacks_.pop_front(); | |
| 256 } | |
| 257 } | 167 } |
| 258 | 168 |
| 259 void AudioReceiver::GetEncodedAudioFrame( | 169 void AudioReceiver::GetEncodedAudioFrame( |
| 260 const AudioFrameEncodedCallback& callback) { | 170 const AudioFrameEncodedCallback& callback) { |
| 261 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 171 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 262 DCHECK(audio_buffer_) << "Invalid function call in this configuration"; | 172 frame_request_queue_.push_back(callback); |
| 173 EmitAvailableEncodedFrames(); |
| 174 } |
| 263 | 175 |
| 264 bool next_frame = false; | 176 void AudioReceiver::EmitAvailableEncodedFrames() { |
| 265 scoped_ptr<transport::EncodedAudioFrame> encoded_frame( | 177 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 266 new transport::EncodedAudioFrame()); | |
| 267 | 178 |
| 268 if (!audio_buffer_->GetEncodedAudioFrame(encoded_frame.get(), &next_frame)) { | 179 while (!frame_request_queue_.empty()) { |
| 269 // We have no audio frames. Wait for new packet(s). | 180 // Attempt to peek at the next completed frame from the |framer_|. |
| 270 VLOG(1) << "Wait for more audio packets in frame"; | 181 // TODO(miu): We should only be peeking at the metadata, and not copying the |
| 271 queued_encoded_callbacks_.push_back(callback); | 182 // payload yet! Or, at least, peek using a StringPiece instead of a copy. |
| 272 return; | 183 scoped_ptr<transport::EncodedAudioFrame> encoded_frame( |
| 273 } | 184 new transport::EncodedAudioFrame()); |
| 274 if (decryptor_.initialized() && !DecryptAudioFrame(&encoded_frame)) { | 185 bool is_consecutively_next_frame = false; |
| 275 // Logging already done. | 186 if (!framer_.GetEncodedAudioFrame(encoded_frame.get(), |
| 276 queued_encoded_callbacks_.push_back(callback); | 187 &is_consecutively_next_frame)) { |
| 277 return; | 188 VLOG(1) << "Wait for more audio packets to produce a completed frame."; |
| 278 } | 189 return; // OnReceivedPayloadData() will invoke this method in the future. |
| 279 if (!PostEncodedAudioFrame(callback, next_frame, &encoded_frame)) { | 190 } |
| 280 // We have an audio frame; however we are missing packets and we have time | 191 |
| 281 // to wait for new packet(s). | 192 // If |framer_| has a frame ready that is out of sequence, examine the |
| 282 queued_encoded_callbacks_.push_back(callback); | 193 // playout time to determine whether it's acceptable to continue, thereby |
| 194 // skipping one or more frames. Skip if the missing frame wouldn't complete |
| 195 // playing before the start of playback of the available frame. |
| 196 const base::TimeTicks now = cast_environment_->Clock()->NowTicks(); |
| 197 const base::TimeTicks playout_time = |
| 198 GetPlayoutTime(now, encoded_frame->rtp_timestamp); |
| 199 if (!is_consecutively_next_frame) { |
| 200 const base::TimeTicks earliest_possible_end_time_of_missing_frame = |
| 201 now + base::TimeDelta::FromMilliseconds(kTypicalAudioFrameDurationMs); |
| 202 if (earliest_possible_end_time_of_missing_frame < playout_time) { |
| 203 VLOG(1) << "Wait for next consecutive frame instead of skipping."; |
| 204 if (!is_waiting_for_consecutive_frame_) { |
| 205 is_waiting_for_consecutive_frame_ = true; |
| 206 cast_environment_->PostDelayedTask( |
| 207 CastEnvironment::MAIN, |
| 208 FROM_HERE, |
| 209 base::Bind(&AudioReceiver::EmitAvailableEncodedFramesAfterWaiting, |
| 210 weak_factory_.GetWeakPtr()), |
| 211 playout_time - now); |
| 212 } |
| 213 return; |
| 214 } |
| 215 } |
| 216 |
| 217 // Decrypt the payload data in the frame, if crypto is being used. |
| 218 if (decryptor_.initialized()) { |
| 219 std::string decrypted_audio_data; |
| 220 if (!decryptor_.Decrypt(encoded_frame->frame_id, |
| 221 encoded_frame->data, |
| 222 &decrypted_audio_data)) { |
| 223 // Decryption failed. Give up on this frame, releasing it from the |
| 224 // jitter buffer. |
| 225 framer_.ReleaseFrame(encoded_frame->frame_id); |
| 226 continue; |
| 227 } |
| 228 encoded_frame->data.swap(decrypted_audio_data); |
| 229 } |
| 230 |
| 231 // At this point, we have a decrypted EncodedAudioFrame ready to be emitted. |
| 232 encoded_frame->codec = codec_; |
| 233 framer_.ReleaseFrame(encoded_frame->frame_id); |
| 234 cast_environment_->PostTask(CastEnvironment::MAIN, |
| 235 FROM_HERE, |
| 236 base::Bind(frame_request_queue_.front(), |
| 237 base::Passed(&encoded_frame), |
| 238 playout_time)); |
| 239 frame_request_queue_.pop_front(); |
| 283 } | 240 } |
| 284 } | 241 } |
| 285 | 242 |
| 286 bool AudioReceiver::PostEncodedAudioFrame( | 243 void AudioReceiver::EmitAvailableEncodedFramesAfterWaiting() { |
| 287 const AudioFrameEncodedCallback& callback, | |
| 288 bool next_frame, | |
| 289 scoped_ptr<transport::EncodedAudioFrame>* encoded_frame) { | |
| 290 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 244 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 291 DCHECK(audio_buffer_) << "Invalid function call in this configuration"; | 245 DCHECK(is_waiting_for_consecutive_frame_); |
| 292 DCHECK(encoded_frame) << "Invalid encoded_frame"; | 246 is_waiting_for_consecutive_frame_ = false; |
| 293 | 247 EmitAvailableEncodedFrames(); |
| 294 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); | |
| 295 base::TimeTicks playout_time = | |
| 296 GetPlayoutTime(now, (*encoded_frame)->rtp_timestamp); | |
| 297 base::TimeDelta time_until_playout = playout_time - now; | |
| 298 base::TimeDelta min_wait_delta = | |
| 299 base::TimeDelta::FromMilliseconds(kMaxAudioFrameWaitMs); | |
| 300 | |
| 301 if (!next_frame && (time_until_playout > min_wait_delta)) { | |
| 302 base::TimeDelta time_until_release = time_until_playout - min_wait_delta; | |
| 303 cast_environment_->PostDelayedTask( | |
| 304 CastEnvironment::MAIN, FROM_HERE, | |
| 305 base::Bind(&AudioReceiver::PlayoutTimeout, weak_factory_.GetWeakPtr()), | |
| 306 time_until_release); | |
| 307 VLOG(1) << "Wait until time to playout:" | |
| 308 << time_until_release.InMilliseconds(); | |
| 309 return false; | |
| 310 } | |
| 311 (*encoded_frame)->codec = codec_; | |
| 312 audio_buffer_->ReleaseFrame((*encoded_frame)->frame_id); | |
| 313 | |
| 314 cast_environment_->PostTask( | |
| 315 CastEnvironment::MAIN, FROM_HERE, | |
| 316 base::Bind(callback, base::Passed(encoded_frame), playout_time)); | |
| 317 return true; | |
| 318 } | 248 } |
| 319 | 249 |
| 320 void AudioReceiver::IncomingPacket(scoped_ptr<Packet> packet) { | 250 void AudioReceiver::IncomingPacket(scoped_ptr<Packet> packet) { |
| 321 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 251 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 322 bool rtcp_packet = Rtcp::IsRtcpPacket(&packet->front(), packet->size()); | 252 bool rtcp_packet = Rtcp::IsRtcpPacket(&packet->front(), packet->size()); |
| 323 if (!rtcp_packet) { | 253 if (!rtcp_packet) { |
| 324 ReceivedPacket(&packet->front(), packet->size()); | 254 ReceivedPacket(&packet->front(), packet->size()); |
| 325 } else { | 255 } else { |
| 326 rtcp_->IncomingRtcpPacket(&packet->front(), packet->size()); | 256 rtcp_.IncomingRtcpPacket(&packet->front(), packet->size()); |
| 327 } | 257 } |
| 328 } | 258 } |
| 329 | 259 |
| 330 void AudioReceiver::SetTargetDelay(base::TimeDelta target_delay) { | 260 void AudioReceiver::SetTargetDelay(base::TimeDelta target_delay) { |
| 261 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 331 target_delay_delta_ = target_delay; | 262 target_delay_delta_ = target_delay; |
| 332 rtcp_->SetTargetDelay(target_delay_delta_); | 263 rtcp_.SetTargetDelay(target_delay_delta_); |
| 333 } | 264 } |
| 334 | 265 |
| 335 void AudioReceiver::CastFeedback(const RtcpCastMessage& cast_message) { | 266 void AudioReceiver::CastFeedback(const RtcpCastMessage& cast_message) { |
| 267 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 336 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); | 268 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); |
| 337 RtpTimestamp rtp_timestamp = | 269 RtpTimestamp rtp_timestamp = |
| 338 frame_id_to_rtp_timestamp_[cast_message.ack_frame_id_ & 0xff]; | 270 frame_id_to_rtp_timestamp_[cast_message.ack_frame_id_ & 0xff]; |
| 339 cast_environment_->Logging()->InsertFrameEvent( | 271 cast_environment_->Logging()->InsertFrameEvent( |
| 340 now, kAudioAckSent, rtp_timestamp, cast_message.ack_frame_id_); | 272 now, kAudioAckSent, rtp_timestamp, cast_message.ack_frame_id_); |
| 341 | 273 |
| 342 rtcp_->SendRtcpFromRtpReceiver(&cast_message, &event_subscriber_); | 274 rtcp_.SendRtcpFromRtpReceiver(&cast_message, &event_subscriber_); |
| 343 } | 275 } |
| 344 | 276 |
| 345 base::TimeTicks AudioReceiver::GetPlayoutTime(base::TimeTicks now, | 277 base::TimeTicks AudioReceiver::GetPlayoutTime(base::TimeTicks now, |
| 346 uint32 rtp_timestamp) { | 278 uint32 rtp_timestamp) { |
| 347 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 279 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 348 // Senders time in ms when this frame was recorded. | 280 // Senders time in ms when this frame was recorded. |
| 349 // Note: the senders clock and our local clock might not be synced. | 281 // Note: the senders clock and our local clock might not be synced. |
| 350 base::TimeTicks rtp_timestamp_in_ticks; | 282 base::TimeTicks rtp_timestamp_in_ticks; |
| 351 base::TimeTicks playout_time; | 283 base::TimeTicks playout_time; |
| 352 if (time_offset_ == base::TimeDelta()) { | 284 if (time_offset_ == base::TimeDelta()) { |
| 353 if (rtcp_->RtpTimestampInSenderTime(frequency_, | 285 if (rtcp_.RtpTimestampInSenderTime(frequency_, |
| 354 first_incoming_rtp_timestamp_, | 286 first_incoming_rtp_timestamp_, |
| 355 &rtp_timestamp_in_ticks)) { | 287 &rtp_timestamp_in_ticks)) { |
| 356 time_offset_ = time_first_incoming_packet_ - rtp_timestamp_in_ticks; | 288 time_offset_ = time_first_incoming_packet_ - rtp_timestamp_in_ticks; |
| 289 // TODO(miu): As clocks drift w.r.t. each other, and other factors take |
| 290 // effect, |time_offset_| should be updated. Otherwise, we might as well |
| 291 // always compute the time offsets agnostic of RTCP's time data. |
| 357 } else { | 292 } else { |
| 358 // We have not received any RTCP to sync the stream play it out as soon as | 293 // We have not received any RTCP to sync the stream play it out as soon as |
| 359 // possible. | 294 // possible. |
| 295 |
| 296 // BUG: This means we're literally switching to a different timeline a |
| 297 // short time after a cast receiver has been running. Re-enable |
| 298 // End2EndTest.StartSenderBeforeReceiver once this is fixed. |
| 299 // http://crbug.com/356942 |
| 360 uint32 rtp_timestamp_diff = rtp_timestamp - first_incoming_rtp_timestamp_; | 300 uint32 rtp_timestamp_diff = rtp_timestamp - first_incoming_rtp_timestamp_; |
| 361 | 301 |
| 362 int frequency_khz = frequency_ / 1000; | 302 int frequency_khz = frequency_ / 1000; |
| 363 base::TimeDelta rtp_time_diff_delta = | 303 base::TimeDelta rtp_time_diff_delta = |
| 364 base::TimeDelta::FromMilliseconds(rtp_timestamp_diff / frequency_khz); | 304 base::TimeDelta::FromMilliseconds(rtp_timestamp_diff / frequency_khz); |
| 365 base::TimeDelta time_diff_delta = now - time_first_incoming_packet_; | 305 base::TimeDelta time_diff_delta = now - time_first_incoming_packet_; |
| 366 | 306 |
| 367 playout_time = now + std::max(rtp_time_diff_delta - time_diff_delta, | 307 playout_time = now + std::max(rtp_time_diff_delta - time_diff_delta, |
| 368 base::TimeDelta()); | 308 base::TimeDelta()); |
| 369 } | 309 } |
| 370 } | 310 } |
| 371 if (playout_time.is_null()) { | 311 if (playout_time.is_null()) { |
| 372 // This can fail if we have not received any RTCP packets in a long time. | 312 // This can fail if we have not received any RTCP packets in a long time. |
| 373 if (rtcp_->RtpTimestampInSenderTime(frequency_, rtp_timestamp, | 313 if (rtcp_.RtpTimestampInSenderTime(frequency_, rtp_timestamp, |
| 374 &rtp_timestamp_in_ticks)) { | 314 &rtp_timestamp_in_ticks)) { |
| 375 playout_time = | 315 playout_time = |
| 376 rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_; | 316 rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_; |
| 377 } else { | 317 } else { |
| 378 playout_time = now; | 318 playout_time = now; |
| 379 } | 319 } |
| 380 } | 320 } |
| 321 |
| 322 // TODO(miu): This is broken since we literally switch timelines once |rtcp_| |
| 323 // can provide us the |time_offset_|. Furthermore, this "getter" method may |
| 324 // be called on frames received out-of-order, which means the playout times |
| 325 // for earlier frames will be computed incorrectly. |
| 326 #if 0 |
| 381 // Don't allow the playout time to go backwards. | 327 // Don't allow the playout time to go backwards. |
| 382 if (last_playout_time_ > playout_time) playout_time = last_playout_time_; | 328 if (last_playout_time_ > playout_time) playout_time = last_playout_time_; |
| 383 last_playout_time_ = playout_time; | 329 last_playout_time_ = playout_time; |
| 330 #endif |
| 331 |
| 384 return playout_time; | 332 return playout_time; |
| 385 } | 333 } |
| 386 | 334 |
| 387 bool AudioReceiver::DecryptAudioFrame( | |
| 388 scoped_ptr<transport::EncodedAudioFrame>* audio_frame) { | |
| 389 if (!decryptor_.initialized()) | |
| 390 return false; | |
| 391 | |
| 392 std::string decrypted_audio_data; | |
| 393 if (!decryptor_.Decrypt((*audio_frame)->frame_id, | |
| 394 (*audio_frame)->data, | |
| 395 &decrypted_audio_data)) { | |
| 396 // Give up on this frame, release it from the jitter buffer. | |
| 397 audio_buffer_->ReleaseFrame((*audio_frame)->frame_id); | |
| 398 return false; | |
| 399 } | |
| 400 (*audio_frame)->data.swap(decrypted_audio_data); | |
| 401 return true; | |
| 402 } | |
| 403 | |
| 404 void AudioReceiver::ScheduleNextRtcpReport() { | 335 void AudioReceiver::ScheduleNextRtcpReport() { |
| 405 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 336 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 406 base::TimeDelta time_to_send = rtcp_->TimeToSendNextRtcpReport() - | 337 base::TimeDelta time_to_send = rtcp_.TimeToSendNextRtcpReport() - |
| 407 cast_environment_->Clock()->NowTicks(); | 338 cast_environment_->Clock()->NowTicks(); |
| 408 | 339 |
| 409 time_to_send = std::max( | 340 time_to_send = std::max( |
| 410 time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); | 341 time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); |
| 411 | 342 |
| 412 cast_environment_->PostDelayedTask( | 343 cast_environment_->PostDelayedTask( |
| 413 CastEnvironment::MAIN, FROM_HERE, | 344 CastEnvironment::MAIN, FROM_HERE, |
| 414 base::Bind(&AudioReceiver::SendNextRtcpReport, | 345 base::Bind(&AudioReceiver::SendNextRtcpReport, |
| 415 weak_factory_.GetWeakPtr()), | 346 weak_factory_.GetWeakPtr()), |
| 416 time_to_send); | 347 time_to_send); |
| 417 } | 348 } |
| 418 | 349 |
| 419 void AudioReceiver::SendNextRtcpReport() { | 350 void AudioReceiver::SendNextRtcpReport() { |
| 420 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 351 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 421 // TODO(pwestin): add logging. | 352 // TODO(pwestin): add logging. |
| 422 rtcp_->SendRtcpFromRtpReceiver(NULL, NULL); | 353 rtcp_.SendRtcpFromRtpReceiver(NULL, NULL); |
| 423 ScheduleNextRtcpReport(); | 354 ScheduleNextRtcpReport(); |
| 424 } | 355 } |
| 425 | 356 |
| 426 // Cast messages should be sent within a maximum interval. Schedule a call | 357 // Cast messages should be sent within a maximum interval. Schedule a call |
| 427 // if not triggered elsewhere, e.g. by the cast message_builder. | 358 // if not triggered elsewhere, e.g. by the cast message_builder. |
| 428 void AudioReceiver::ScheduleNextCastMessage() { | 359 void AudioReceiver::ScheduleNextCastMessage() { |
| 429 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 360 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 430 base::TimeTicks send_time; | 361 base::TimeTicks send_time; |
| 431 if (audio_buffer_) { | 362 framer_.TimeToSendNextCastMessage(&send_time); |
| 432 audio_buffer_->TimeToSendNextCastMessage(&send_time); | |
| 433 } else if (audio_decoder_) { | |
| 434 audio_decoder_->TimeToSendNextCastMessage(&send_time); | |
| 435 } else { | |
| 436 NOTREACHED(); | |
| 437 } | |
| 438 base::TimeDelta time_to_send = | 363 base::TimeDelta time_to_send = |
| 439 send_time - cast_environment_->Clock()->NowTicks(); | 364 send_time - cast_environment_->Clock()->NowTicks(); |
| 440 time_to_send = std::max( | 365 time_to_send = std::max( |
| 441 time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); | 366 time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); |
| 442 cast_environment_->PostDelayedTask( | 367 cast_environment_->PostDelayedTask( |
| 443 CastEnvironment::MAIN, FROM_HERE, | 368 CastEnvironment::MAIN, FROM_HERE, |
| 444 base::Bind(&AudioReceiver::SendNextCastMessage, | 369 base::Bind(&AudioReceiver::SendNextCastMessage, |
| 445 weak_factory_.GetWeakPtr()), | 370 weak_factory_.GetWeakPtr()), |
| 446 time_to_send); | 371 time_to_send); |
| 447 } | 372 } |
| 448 | 373 |
| 449 void AudioReceiver::SendNextCastMessage() { | 374 void AudioReceiver::SendNextCastMessage() { |
| 450 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 375 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 451 | 376 // Will only send a message if it is time. |
| 452 if (audio_buffer_) { | 377 framer_.SendCastMessage(); |
| 453 // Will only send a message if it is time. | |
| 454 audio_buffer_->SendCastMessage(); | |
| 455 } | |
| 456 if (audio_decoder_) { | |
| 457 // Will only send a message if it is time. | |
| 458 audio_decoder_->SendCastMessage(); | |
| 459 } | |
| 460 ScheduleNextCastMessage(); | 378 ScheduleNextCastMessage(); |
| 461 } | 379 } |
| 462 | 380 |
| 463 } // namespace cast | 381 } // namespace cast |
| 464 } // namespace media | 382 } // namespace media |
| OLD | NEW |