OLD | NEW |
1 // Copyright 2013 The Chromium Authors. All rights reserved. | 1 // Copyright 2013 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "media/cast/audio_sender/audio_sender.h" | 5 #include "media/cast/audio_sender/audio_sender.h" |
6 | 6 |
7 #include "base/bind.h" | 7 #include "base/bind.h" |
8 #include "base/logging.h" | 8 #include "base/logging.h" |
9 #include "base/message_loop/message_loop.h" | 9 #include "base/message_loop/message_loop.h" |
10 #include "media/cast/audio_sender/audio_encoder.h" | 10 #include "media/cast/audio_sender/audio_encoder.h" |
11 #include "media/cast/transport/cast_transport_defines.h" | 11 #include "media/cast/cast_defines.h" |
| 12 #include "media/cast/rtcp/rtcp_defines.h" |
| 13 #include "media/cast/transport/cast_transport_config.h" |
12 | 14 |
13 namespace media { | 15 namespace media { |
14 namespace cast { | 16 namespace cast { |
15 | 17 |
16 const int kNumAggressiveReportsSentAtStart = 100; | 18 const int kNumAggressiveReportsSentAtStart = 100; |
17 const int kMinSchedulingDelayMs = 1; | 19 const int kMinSchedulingDelayMs = 1; |
18 | 20 |
19 // TODO(mikhal): Reduce heap allocation when not needed. | 21 // TODO(miu): This should be specified in AudioSenderConfig, but currently it is |
| 22 // fixed to 100 FPS (i.e., 10 ms per frame), and AudioEncoder assumes this as |
| 23 // well. |
| 24 const int kAudioFrameRate = 100; |
| 25 |
20 AudioSender::AudioSender(scoped_refptr<CastEnvironment> cast_environment, | 26 AudioSender::AudioSender(scoped_refptr<CastEnvironment> cast_environment, |
21 const AudioSenderConfig& audio_config, | 27 const AudioSenderConfig& audio_config, |
22 transport::CastTransportSender* const transport_sender) | 28 transport::CastTransportSender* const transport_sender) |
23 : cast_environment_(cast_environment), | 29 : cast_environment_(cast_environment), |
| 30 target_playout_delay_(base::TimeDelta::FromMilliseconds( |
| 31 audio_config.rtp_config.max_delay_ms)), |
24 transport_sender_(transport_sender), | 32 transport_sender_(transport_sender), |
25 rtp_timestamp_helper_(audio_config.frequency), | 33 max_unacked_frames_( |
| 34 std::min(kMaxUnackedFrames, |
| 35 1 + static_cast<int>(target_playout_delay_ * |
| 36 kAudioFrameRate / |
| 37 base::TimeDelta::FromSeconds(1)))), |
| 38 configured_encoder_bitrate_(audio_config.bitrate), |
26 rtcp_(cast_environment, | 39 rtcp_(cast_environment, |
27 this, | 40 this, |
28 transport_sender_, | 41 transport_sender_, |
29 NULL, // paced sender. | 42 NULL, // paced sender. |
30 NULL, | 43 NULL, |
31 audio_config.rtcp_mode, | 44 audio_config.rtcp_mode, |
32 base::TimeDelta::FromMilliseconds(audio_config.rtcp_interval), | 45 base::TimeDelta::FromMilliseconds(audio_config.rtcp_interval), |
33 audio_config.rtp_config.ssrc, | 46 audio_config.rtp_config.ssrc, |
34 audio_config.incoming_feedback_ssrc, | 47 audio_config.incoming_feedback_ssrc, |
35 audio_config.rtcp_c_name, | 48 audio_config.rtcp_c_name, |
36 AUDIO_EVENT), | 49 AUDIO_EVENT), |
| 50 rtp_timestamp_helper_(audio_config.frequency), |
37 num_aggressive_rtcp_reports_sent_(0), | 51 num_aggressive_rtcp_reports_sent_(0), |
| 52 last_sent_frame_id_(0), |
| 53 latest_acked_frame_id_(0), |
| 54 duplicate_ack_counter_(0), |
38 cast_initialization_status_(STATUS_AUDIO_UNINITIALIZED), | 55 cast_initialization_status_(STATUS_AUDIO_UNINITIALIZED), |
39 weak_factory_(this) { | 56 weak_factory_(this) { |
40 rtcp_.SetCastReceiverEventHistorySize(kReceiverRtcpEventHistorySize); | 57 VLOG(1) << "max_unacked_frames " << max_unacked_frames_; |
| 58 DCHECK_GT(max_unacked_frames_, 0); |
| 59 |
41 if (!audio_config.use_external_encoder) { | 60 if (!audio_config.use_external_encoder) { |
42 audio_encoder_.reset( | 61 audio_encoder_.reset( |
43 new AudioEncoder(cast_environment, | 62 new AudioEncoder(cast_environment, |
44 audio_config, | 63 audio_config, |
45 base::Bind(&AudioSender::SendEncodedAudioFrame, | 64 base::Bind(&AudioSender::SendEncodedAudioFrame, |
46 weak_factory_.GetWeakPtr()))); | 65 weak_factory_.GetWeakPtr()))); |
47 cast_initialization_status_ = audio_encoder_->InitializationResult(); | 66 cast_initialization_status_ = audio_encoder_->InitializationResult(); |
48 } else { | 67 } else { |
49 NOTREACHED(); // No support for external audio encoding. | 68 NOTREACHED(); // No support for external audio encoding. |
50 cast_initialization_status_ = STATUS_AUDIO_INITIALIZED; | 69 cast_initialization_status_ = STATUS_AUDIO_UNINITIALIZED; |
51 } | 70 } |
52 | 71 |
53 media::cast::transport::CastTransportAudioConfig transport_config; | 72 media::cast::transport::CastTransportAudioConfig transport_config; |
54 transport_config.codec = audio_config.codec; | 73 transport_config.codec = audio_config.codec; |
55 transport_config.rtp.config = audio_config.rtp_config; | 74 transport_config.rtp.config = audio_config.rtp_config; |
56 transport_config.frequency = audio_config.frequency; | 75 transport_config.frequency = audio_config.frequency; |
57 transport_config.channels = audio_config.channels; | 76 transport_config.channels = audio_config.channels; |
58 transport_config.rtp.max_outstanding_frames = | 77 transport_config.rtp.max_outstanding_frames = max_unacked_frames_; |
59 audio_config.rtp_config.max_delay_ms / 100 + 1; | |
60 transport_sender_->InitializeAudio(transport_config); | 78 transport_sender_->InitializeAudio(transport_config); |
61 | 79 |
| 80 rtcp_.SetCastReceiverEventHistorySize(kReceiverRtcpEventHistorySize); |
| 81 |
62 memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_)); | 82 memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_)); |
63 } | 83 } |
64 | 84 |
65 AudioSender::~AudioSender() {} | 85 AudioSender::~AudioSender() {} |
66 | 86 |
67 void AudioSender::InsertAudio(scoped_ptr<AudioBus> audio_bus, | 87 void AudioSender::InsertAudio(scoped_ptr<AudioBus> audio_bus, |
68 const base::TimeTicks& recorded_time) { | 88 const base::TimeTicks& recorded_time) { |
69 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 89 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
70 if (cast_initialization_status_ != STATUS_AUDIO_INITIALIZED) { | 90 if (cast_initialization_status_ != STATUS_AUDIO_INITIALIZED) { |
71 NOTREACHED(); | 91 NOTREACHED(); |
72 return; | 92 return; |
73 } | 93 } |
74 DCHECK(audio_encoder_.get()) << "Invalid internal state"; | 94 DCHECK(audio_encoder_.get()) << "Invalid internal state"; |
| 95 |
| 96 if (AreTooManyFramesInFlight()) { |
| 97 VLOG(1) << "Dropping frame due to too many frames currently in-flight."; |
| 98 return; |
| 99 } |
| 100 |
75 audio_encoder_->InsertAudio(audio_bus.Pass(), recorded_time); | 101 audio_encoder_->InsertAudio(audio_bus.Pass(), recorded_time); |
76 } | 102 } |
77 | 103 |
78 void AudioSender::SendEncodedAudioFrame( | 104 void AudioSender::SendEncodedAudioFrame( |
79 scoped_ptr<transport::EncodedFrame> audio_frame) { | 105 scoped_ptr<transport::EncodedFrame> encoded_frame) { |
80 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 106 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
81 | 107 |
82 DCHECK(!audio_frame->reference_time.is_null()); | 108 const uint32 frame_id = encoded_frame->frame_id; |
83 rtp_timestamp_helper_.StoreLatestTime(audio_frame->reference_time, | 109 |
84 audio_frame->rtp_timestamp); | 110 const bool is_first_frame_to_be_sent = last_send_time_.is_null(); |
| 111 last_send_time_ = cast_environment_->Clock()->NowTicks(); |
| 112 last_sent_frame_id_ = frame_id; |
| 113 // If this is the first frame about to be sent, fake the value of |
| 114 // |latest_acked_frame_id_| to indicate the receiver starts out all caught up. |
| 115 // Also, schedule the periodic frame re-send checks. |
| 116 if (is_first_frame_to_be_sent) { |
| 117 latest_acked_frame_id_ = frame_id - 1; |
| 118 ScheduleNextResendCheck(); |
| 119 } |
| 120 |
| 121 cast_environment_->Logging()->InsertEncodedFrameEvent( |
| 122 last_send_time_, FRAME_ENCODED, AUDIO_EVENT, encoded_frame->rtp_timestamp, |
| 123 frame_id, static_cast<int>(encoded_frame->data.size()), |
| 124 encoded_frame->dependency == transport::EncodedFrame::KEY, |
| 125 configured_encoder_bitrate_); |
| 126 // Only use lowest 8 bits as key. |
| 127 frame_id_to_rtp_timestamp_[frame_id & 0xff] = encoded_frame->rtp_timestamp; |
| 128 |
| 129 DCHECK(!encoded_frame->reference_time.is_null()); |
| 130 rtp_timestamp_helper_.StoreLatestTime(encoded_frame->reference_time, |
| 131 encoded_frame->rtp_timestamp); |
85 | 132 |
86 // At the start of the session, it's important to send reports before each | 133 // At the start of the session, it's important to send reports before each |
87 // frame so that the receiver can properly compute playout times. The reason | 134 // frame so that the receiver can properly compute playout times. The reason |
88 // more than one report is sent is because transmission is not guaranteed, | 135 // more than one report is sent is because transmission is not guaranteed, |
89 // only best effort, so we send enough that one should almost certainly get | 136 // only best effort, so we send enough that one should almost certainly get |
90 // through. | 137 // through. |
91 if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) { | 138 if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) { |
92 // SendRtcpReport() will schedule future reports to be made if this is the | 139 // SendRtcpReport() will schedule future reports to be made if this is the |
93 // last "aggressive report." | 140 // last "aggressive report." |
94 ++num_aggressive_rtcp_reports_sent_; | 141 ++num_aggressive_rtcp_reports_sent_; |
95 const bool is_last_aggressive_report = | 142 const bool is_last_aggressive_report = |
96 (num_aggressive_rtcp_reports_sent_ == kNumAggressiveReportsSentAtStart); | 143 (num_aggressive_rtcp_reports_sent_ == kNumAggressiveReportsSentAtStart); |
97 VLOG_IF(1, is_last_aggressive_report) << "Sending last aggressive report."; | 144 VLOG_IF(1, is_last_aggressive_report) << "Sending last aggressive report."; |
98 SendRtcpReport(is_last_aggressive_report); | 145 SendRtcpReport(is_last_aggressive_report); |
99 } | 146 } |
100 | 147 |
101 frame_id_to_rtp_timestamp_[audio_frame->frame_id & 0xff] = | 148 transport_sender_->InsertCodedAudioFrame(*encoded_frame); |
102 audio_frame->rtp_timestamp; | |
103 transport_sender_->InsertCodedAudioFrame(*audio_frame); | |
104 } | |
105 | |
106 void AudioSender::ResendPackets( | |
107 const MissingFramesAndPacketsMap& missing_frames_and_packets) { | |
108 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
109 transport_sender_->ResendPackets(true, missing_frames_and_packets, false); | |
110 } | 149 } |
111 | 150 |
112 void AudioSender::IncomingRtcpPacket(scoped_ptr<Packet> packet) { | 151 void AudioSender::IncomingRtcpPacket(scoped_ptr<Packet> packet) { |
113 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 152 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
114 rtcp_.IncomingRtcpPacket(&packet->front(), packet->size()); | 153 rtcp_.IncomingRtcpPacket(&packet->front(), packet->size()); |
115 } | 154 } |
116 | 155 |
117 void AudioSender::ScheduleNextRtcpReport() { | 156 void AudioSender::ScheduleNextRtcpReport() { |
118 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 157 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
119 base::TimeDelta time_to_next = | 158 base::TimeDelta time_to_next = |
(...skipping 19 matching lines...) Expand all Loading... |
139 now, &now_as_rtp_timestamp)) { | 178 now, &now_as_rtp_timestamp)) { |
140 rtcp_.SendRtcpFromRtpSender(now, now_as_rtp_timestamp); | 179 rtcp_.SendRtcpFromRtpSender(now, now_as_rtp_timestamp); |
141 } else { | 180 } else { |
142 // |rtp_timestamp_helper_| should have stored a mapping by this point. | 181 // |rtp_timestamp_helper_| should have stored a mapping by this point. |
143 NOTREACHED(); | 182 NOTREACHED(); |
144 } | 183 } |
145 if (schedule_future_reports) | 184 if (schedule_future_reports) |
146 ScheduleNextRtcpReport(); | 185 ScheduleNextRtcpReport(); |
147 } | 186 } |
148 | 187 |
| 188 void AudioSender::ScheduleNextResendCheck() { |
| 189 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 190 DCHECK(!last_send_time_.is_null()); |
| 191 base::TimeDelta time_to_next = |
| 192 last_send_time_ - cast_environment_->Clock()->NowTicks() + |
| 193 target_playout_delay_; |
| 194 time_to_next = std::max( |
| 195 time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); |
| 196 cast_environment_->PostDelayedTask( |
| 197 CastEnvironment::MAIN, |
| 198 FROM_HERE, |
| 199 base::Bind(&AudioSender::ResendCheck, weak_factory_.GetWeakPtr()), |
| 200 time_to_next); |
| 201 } |
| 202 |
| 203 void AudioSender::ResendCheck() { |
| 204 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 205 DCHECK(!last_send_time_.is_null()); |
| 206 const base::TimeDelta time_since_last_send = |
| 207 cast_environment_->Clock()->NowTicks() - last_send_time_; |
| 208 if (time_since_last_send > target_playout_delay_) { |
| 209 if (latest_acked_frame_id_ == last_sent_frame_id_) { |
| 210 // Last frame acked, no point in doing anything |
| 211 } else { |
| 212 VLOG(1) << "ACK timeout; last acked frame: " << latest_acked_frame_id_; |
| 213 ResendForKickstart(); |
| 214 } |
| 215 } |
| 216 ScheduleNextResendCheck(); |
| 217 } |
| 218 |
149 void AudioSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) { | 219 void AudioSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) { |
150 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 220 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
151 | 221 |
152 if (rtcp_.is_rtt_available()) { | 222 if (rtcp_.is_rtt_available()) { |
153 // Having the RTT values implies the receiver sent back a receiver report | 223 // Having the RTT values implies the receiver sent back a receiver report |
154 // based on it having received a report from here. Therefore, ensure this | 224 // based on it having received a report from here. Therefore, ensure this |
155 // sender stops aggressively sending reports. | 225 // sender stops aggressively sending reports. |
156 if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) { | 226 if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) { |
157 VLOG(1) << "No longer a need to send reports aggressively (sent " | 227 VLOG(1) << "No longer a need to send reports aggressively (sent " |
158 << num_aggressive_rtcp_reports_sent_ << ")."; | 228 << num_aggressive_rtcp_reports_sent_ << ")."; |
159 num_aggressive_rtcp_reports_sent_ = kNumAggressiveReportsSentAtStart; | 229 num_aggressive_rtcp_reports_sent_ = kNumAggressiveReportsSentAtStart; |
160 ScheduleNextRtcpReport(); | 230 ScheduleNextRtcpReport(); |
161 } | 231 } |
162 } | 232 } |
163 | 233 |
164 if (!cast_feedback.missing_frames_and_packets_.empty()) { | 234 if (last_send_time_.is_null()) |
165 ResendPackets(cast_feedback.missing_frames_and_packets_); | 235 return; // Cannot get an ACK without having first sent a frame. |
| 236 |
| 237 if (cast_feedback.missing_frames_and_packets_.empty()) { |
| 238 // We only count duplicate ACKs when we have sent newer frames. |
| 239 if (latest_acked_frame_id_ == cast_feedback.ack_frame_id_ && |
| 240 latest_acked_frame_id_ != last_sent_frame_id_) { |
| 241 duplicate_ack_counter_++; |
| 242 } else { |
| 243 duplicate_ack_counter_ = 0; |
| 244 } |
| 245 // TODO(miu): The values "2" and "3" should be derived from configuration. |
| 246 if (duplicate_ack_counter_ >= 2 && duplicate_ack_counter_ % 3 == 2) { |
| 247 VLOG(1) << "Received duplicate ACK for frame " << latest_acked_frame_id_; |
| 248 ResendForKickstart(); |
| 249 } |
| 250 } else { |
| 251 // Only count duplicated ACKs if there is no NACK request in between. |
| 252 // This is to avoid aggresive resend. |
| 253 duplicate_ack_counter_ = 0; |
| 254 |
| 255 // A NACK is also used to cancel pending re-transmissions. |
| 256 transport_sender_->ResendPackets( |
| 257 true, cast_feedback.missing_frames_and_packets_, true); |
166 } | 258 } |
167 uint32 acked_frame_id = static_cast<uint32>(cast_feedback.ack_frame_id_); | 259 |
168 VLOG(2) << "Received audio ACK: " << acked_frame_id; | 260 const base::TimeTicks now = cast_environment_->Clock()->NowTicks(); |
169 cast_environment_->Logging()->InsertFrameEvent( | 261 |
170 cast_environment_->Clock()->NowTicks(), | 262 const RtpTimestamp rtp_timestamp = |
171 FRAME_ACK_RECEIVED, AUDIO_EVENT, | 263 frame_id_to_rtp_timestamp_[cast_feedback.ack_frame_id_ & 0xff]; |
172 frame_id_to_rtp_timestamp_[acked_frame_id & 0xff], acked_frame_id); | 264 cast_environment_->Logging()->InsertFrameEvent(now, |
| 265 FRAME_ACK_RECEIVED, |
| 266 AUDIO_EVENT, |
| 267 rtp_timestamp, |
| 268 cast_feedback.ack_frame_id_); |
| 269 |
| 270 const bool is_acked_out_of_order = |
| 271 static_cast<int32>(cast_feedback.ack_frame_id_ - |
| 272 latest_acked_frame_id_) < 0; |
| 273 VLOG(2) << "Received ACK" << (is_acked_out_of_order ? " out-of-order" : "") |
| 274 << " for frame " << cast_feedback.ack_frame_id_; |
| 275 if (!is_acked_out_of_order) { |
| 276 // Cancel resends of acked frames. |
| 277 MissingFramesAndPacketsMap missing_frames_and_packets; |
| 278 PacketIdSet missing; |
| 279 while (latest_acked_frame_id_ != cast_feedback.ack_frame_id_) { |
| 280 latest_acked_frame_id_++; |
| 281 missing_frames_and_packets[latest_acked_frame_id_] = missing; |
| 282 } |
| 283 transport_sender_->ResendPackets(true, missing_frames_and_packets, true); |
| 284 latest_acked_frame_id_ = cast_feedback.ack_frame_id_; |
| 285 } |
| 286 } |
| 287 |
| 288 bool AudioSender::AreTooManyFramesInFlight() const { |
| 289 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 290 int frames_in_flight = 0; |
| 291 if (!last_send_time_.is_null()) { |
| 292 frames_in_flight += |
| 293 static_cast<int32>(last_sent_frame_id_ - latest_acked_frame_id_); |
| 294 } |
| 295 VLOG(2) << frames_in_flight |
| 296 << " frames in flight; last sent: " << last_sent_frame_id_ |
| 297 << " latest acked: " << latest_acked_frame_id_; |
| 298 return frames_in_flight >= max_unacked_frames_; |
| 299 } |
| 300 |
| 301 void AudioSender::ResendForKickstart() { |
| 302 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 303 DCHECK(!last_send_time_.is_null()); |
| 304 VLOG(1) << "Resending last packet of frame " << last_sent_frame_id_ |
| 305 << " to kick-start."; |
| 306 // Send the first packet of the last encoded frame to kick start |
| 307 // retransmission. This gives enough information to the receiver what |
| 308 // packets and frames are missing. |
| 309 MissingFramesAndPacketsMap missing_frames_and_packets; |
| 310 PacketIdSet missing; |
| 311 missing.insert(kRtcpCastLastPacket); |
| 312 missing_frames_and_packets.insert( |
| 313 std::make_pair(last_sent_frame_id_, missing)); |
| 314 last_send_time_ = cast_environment_->Clock()->NowTicks(); |
| 315 |
| 316 // Sending this extra packet is to kick-start the session. There is |
| 317 // no need to optimize re-transmission for this case. |
| 318 transport_sender_->ResendPackets(true, missing_frames_and_packets, |
| 319 false); |
173 } | 320 } |
174 | 321 |
175 } // namespace cast | 322 } // namespace cast |
176 } // namespace media | 323 } // namespace media |
OLD | NEW |