| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2013 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 #include "media/cast/video_sender/video_sender.h" | |
| 6 | |
| 7 #include <algorithm> | |
| 8 #include <cstring> | |
| 9 | |
| 10 #include "base/bind.h" | |
| 11 #include "base/debug/trace_event.h" | |
| 12 #include "base/logging.h" | |
| 13 #include "base/message_loop/message_loop.h" | |
| 14 #include "media/cast/cast_defines.h" | |
| 15 #include "media/cast/rtcp/rtcp_defines.h" | |
| 16 #include "media/cast/transport/cast_transport_config.h" | |
| 17 #include "media/cast/video_sender/external_video_encoder.h" | |
| 18 #include "media/cast/video_sender/video_encoder_impl.h" | |
| 19 | |
| 20 namespace media { | |
| 21 namespace cast { | |
| 22 | |
| 23 const int kNumAggressiveReportsSentAtStart = 100; | |
| 24 const int kMinSchedulingDelayMs = 1; | |
| 25 | |
| 26 VideoSender::VideoSender( | |
| 27 scoped_refptr<CastEnvironment> cast_environment, | |
| 28 const VideoSenderConfig& video_config, | |
| 29 const CreateVideoEncodeAcceleratorCallback& create_vea_cb, | |
| 30 const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb, | |
| 31 transport::CastTransportSender* const transport_sender) | |
| 32 : cast_environment_(cast_environment), | |
| 33 target_playout_delay_(video_config.target_playout_delay), | |
| 34 transport_sender_(transport_sender), | |
| 35 max_unacked_frames_( | |
| 36 std::min(kMaxUnackedFrames, | |
| 37 1 + static_cast<int>(target_playout_delay_ * | |
| 38 video_config.max_frame_rate / | |
| 39 base::TimeDelta::FromSeconds(1)))), | |
| 40 rtcp_(cast_environment_, | |
| 41 this, | |
| 42 transport_sender_, | |
| 43 NULL, // paced sender. | |
| 44 NULL, | |
| 45 video_config.rtcp_mode, | |
| 46 base::TimeDelta::FromMilliseconds(video_config.rtcp_interval), | |
| 47 video_config.ssrc, | |
| 48 video_config.incoming_feedback_ssrc, | |
| 49 video_config.rtcp_c_name, | |
| 50 VIDEO_EVENT), | |
| 51 rtp_timestamp_helper_(kVideoFrequency), | |
| 52 num_aggressive_rtcp_reports_sent_(0), | |
| 53 frames_in_encoder_(0), | |
| 54 last_sent_frame_id_(0), | |
| 55 latest_acked_frame_id_(0), | |
| 56 duplicate_ack_counter_(0), | |
| 57 congestion_control_(cast_environment->Clock(), | |
| 58 video_config.max_bitrate, | |
| 59 video_config.min_bitrate, | |
| 60 max_unacked_frames_), | |
| 61 cast_initialization_status_(STATUS_VIDEO_UNINITIALIZED), | |
| 62 weak_factory_(this) { | |
| 63 VLOG(1) << "max_unacked_frames " << max_unacked_frames_; | |
| 64 DCHECK_GT(max_unacked_frames_, 0); | |
| 65 | |
| 66 if (video_config.use_external_encoder) { | |
| 67 video_encoder_.reset(new ExternalVideoEncoder(cast_environment, | |
| 68 video_config, | |
| 69 create_vea_cb, | |
| 70 create_video_encode_mem_cb)); | |
| 71 } else { | |
| 72 video_encoder_.reset(new VideoEncoderImpl( | |
| 73 cast_environment, video_config, max_unacked_frames_)); | |
| 74 } | |
| 75 cast_initialization_status_ = STATUS_VIDEO_INITIALIZED; | |
| 76 | |
| 77 media::cast::transport::CastTransportRtpConfig transport_config; | |
| 78 transport_config.ssrc = video_config.ssrc; | |
| 79 transport_config.rtp_payload_type = video_config.rtp_payload_type; | |
| 80 transport_config.stored_frames = max_unacked_frames_; | |
| 81 transport_config.aes_key = video_config.aes_key; | |
| 82 transport_config.aes_iv_mask = video_config.aes_iv_mask; | |
| 83 transport_sender_->InitializeVideo(transport_config); | |
| 84 | |
| 85 rtcp_.SetCastReceiverEventHistorySize(kReceiverRtcpEventHistorySize); | |
| 86 | |
| 87 memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_)); | |
| 88 } | |
| 89 | |
| 90 VideoSender::~VideoSender() { | |
| 91 } | |
| 92 | |
| 93 void VideoSender::InsertRawVideoFrame( | |
| 94 const scoped_refptr<media::VideoFrame>& video_frame, | |
| 95 const base::TimeTicks& capture_time) { | |
| 96 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 97 if (cast_initialization_status_ != STATUS_VIDEO_INITIALIZED) { | |
| 98 NOTREACHED(); | |
| 99 return; | |
| 100 } | |
| 101 DCHECK(video_encoder_.get()) << "Invalid state"; | |
| 102 | |
| 103 RtpTimestamp rtp_timestamp = GetVideoRtpTimestamp(capture_time); | |
| 104 cast_environment_->Logging()->InsertFrameEvent( | |
| 105 capture_time, FRAME_CAPTURE_BEGIN, VIDEO_EVENT, | |
| 106 rtp_timestamp, kFrameIdUnknown); | |
| 107 cast_environment_->Logging()->InsertFrameEvent( | |
| 108 cast_environment_->Clock()->NowTicks(), | |
| 109 FRAME_CAPTURE_END, VIDEO_EVENT, | |
| 110 rtp_timestamp, | |
| 111 kFrameIdUnknown); | |
| 112 | |
| 113 // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc | |
| 114 TRACE_EVENT_INSTANT2( | |
| 115 "cast_perf_test", "InsertRawVideoFrame", | |
| 116 TRACE_EVENT_SCOPE_THREAD, | |
| 117 "timestamp", capture_time.ToInternalValue(), | |
| 118 "rtp_timestamp", rtp_timestamp); | |
| 119 | |
| 120 if (AreTooManyFramesInFlight()) { | |
| 121 VLOG(1) << "Dropping frame due to too many frames currently in-flight."; | |
| 122 return; | |
| 123 } | |
| 124 | |
| 125 uint32 bitrate = congestion_control_.GetBitrate( | |
| 126 capture_time + target_playout_delay_, target_playout_delay_); | |
| 127 | |
| 128 video_encoder_->SetBitRate(bitrate); | |
| 129 | |
| 130 if (video_encoder_->EncodeVideoFrame( | |
| 131 video_frame, | |
| 132 capture_time, | |
| 133 base::Bind(&VideoSender::SendEncodedVideoFrame, | |
| 134 weak_factory_.GetWeakPtr(), | |
| 135 bitrate))) { | |
| 136 frames_in_encoder_++; | |
| 137 } else { | |
| 138 VLOG(1) << "Encoder rejected a frame. Skipping..."; | |
| 139 } | |
| 140 } | |
| 141 | |
| 142 void VideoSender::SendEncodedVideoFrame( | |
| 143 int requested_bitrate_before_encode, | |
| 144 scoped_ptr<transport::EncodedFrame> encoded_frame) { | |
| 145 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 146 | |
| 147 DCHECK_GT(frames_in_encoder_, 0); | |
| 148 frames_in_encoder_--; | |
| 149 | |
| 150 const uint32 frame_id = encoded_frame->frame_id; | |
| 151 | |
| 152 const bool is_first_frame_to_be_sent = last_send_time_.is_null(); | |
| 153 last_send_time_ = cast_environment_->Clock()->NowTicks(); | |
| 154 last_sent_frame_id_ = frame_id; | |
| 155 // If this is the first frame about to be sent, fake the value of | |
| 156 // |latest_acked_frame_id_| to indicate the receiver starts out all caught up. | |
| 157 // Also, schedule the periodic frame re-send checks. | |
| 158 if (is_first_frame_to_be_sent) { | |
| 159 latest_acked_frame_id_ = frame_id - 1; | |
| 160 ScheduleNextResendCheck(); | |
| 161 } | |
| 162 | |
| 163 VLOG_IF(1, encoded_frame->dependency == transport::EncodedFrame::KEY) | |
| 164 << "Send encoded key frame; frame_id: " << frame_id; | |
| 165 | |
| 166 cast_environment_->Logging()->InsertEncodedFrameEvent( | |
| 167 last_send_time_, FRAME_ENCODED, VIDEO_EVENT, encoded_frame->rtp_timestamp, | |
| 168 frame_id, static_cast<int>(encoded_frame->data.size()), | |
| 169 encoded_frame->dependency == transport::EncodedFrame::KEY, | |
| 170 requested_bitrate_before_encode); | |
| 171 // Only use lowest 8 bits as key. | |
| 172 frame_id_to_rtp_timestamp_[frame_id & 0xff] = encoded_frame->rtp_timestamp; | |
| 173 | |
| 174 // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc | |
| 175 TRACE_EVENT_INSTANT1( | |
| 176 "cast_perf_test", "VideoFrameEncoded", | |
| 177 TRACE_EVENT_SCOPE_THREAD, | |
| 178 "rtp_timestamp", encoded_frame->rtp_timestamp); | |
| 179 | |
| 180 DCHECK(!encoded_frame->reference_time.is_null()); | |
| 181 rtp_timestamp_helper_.StoreLatestTime(encoded_frame->reference_time, | |
| 182 encoded_frame->rtp_timestamp); | |
| 183 | |
| 184 // At the start of the session, it's important to send reports before each | |
| 185 // frame so that the receiver can properly compute playout times. The reason | |
| 186 // more than one report is sent is because transmission is not guaranteed, | |
| 187 // only best effort, so send enough that one should almost certainly get | |
| 188 // through. | |
| 189 if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) { | |
| 190 // SendRtcpReport() will schedule future reports to be made if this is the | |
| 191 // last "aggressive report." | |
| 192 ++num_aggressive_rtcp_reports_sent_; | |
| 193 const bool is_last_aggressive_report = | |
| 194 (num_aggressive_rtcp_reports_sent_ == kNumAggressiveReportsSentAtStart); | |
| 195 VLOG_IF(1, is_last_aggressive_report) << "Sending last aggressive report."; | |
| 196 SendRtcpReport(is_last_aggressive_report); | |
| 197 } | |
| 198 | |
| 199 congestion_control_.SendFrameToTransport( | |
| 200 frame_id, encoded_frame->data.size() * 8, last_send_time_); | |
| 201 | |
| 202 transport_sender_->InsertCodedVideoFrame(*encoded_frame); | |
| 203 } | |
| 204 | |
| 205 void VideoSender::IncomingRtcpPacket(scoped_ptr<Packet> packet) { | |
| 206 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 207 rtcp_.IncomingRtcpPacket(&packet->front(), packet->size()); | |
| 208 } | |
| 209 | |
| 210 void VideoSender::ScheduleNextRtcpReport() { | |
| 211 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 212 base::TimeDelta time_to_next = rtcp_.TimeToSendNextRtcpReport() - | |
| 213 cast_environment_->Clock()->NowTicks(); | |
| 214 | |
| 215 time_to_next = std::max( | |
| 216 time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); | |
| 217 | |
| 218 cast_environment_->PostDelayedTask( | |
| 219 CastEnvironment::MAIN, | |
| 220 FROM_HERE, | |
| 221 base::Bind(&VideoSender::SendRtcpReport, | |
| 222 weak_factory_.GetWeakPtr(), | |
| 223 true), | |
| 224 time_to_next); | |
| 225 } | |
| 226 | |
| 227 void VideoSender::SendRtcpReport(bool schedule_future_reports) { | |
| 228 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 229 const base::TimeTicks now = cast_environment_->Clock()->NowTicks(); | |
| 230 uint32 now_as_rtp_timestamp = 0; | |
| 231 if (rtp_timestamp_helper_.GetCurrentTimeAsRtpTimestamp( | |
| 232 now, &now_as_rtp_timestamp)) { | |
| 233 rtcp_.SendRtcpFromRtpSender(now, now_as_rtp_timestamp); | |
| 234 } else { | |
| 235 // |rtp_timestamp_helper_| should have stored a mapping by this point. | |
| 236 NOTREACHED(); | |
| 237 } | |
| 238 if (schedule_future_reports) | |
| 239 ScheduleNextRtcpReport(); | |
| 240 } | |
| 241 | |
| 242 void VideoSender::ScheduleNextResendCheck() { | |
| 243 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 244 DCHECK(!last_send_time_.is_null()); | |
| 245 base::TimeDelta time_to_next = | |
| 246 last_send_time_ - cast_environment_->Clock()->NowTicks() + | |
| 247 target_playout_delay_; | |
| 248 time_to_next = std::max( | |
| 249 time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); | |
| 250 cast_environment_->PostDelayedTask( | |
| 251 CastEnvironment::MAIN, | |
| 252 FROM_HERE, | |
| 253 base::Bind(&VideoSender::ResendCheck, weak_factory_.GetWeakPtr()), | |
| 254 time_to_next); | |
| 255 } | |
| 256 | |
| 257 void VideoSender::ResendCheck() { | |
| 258 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 259 DCHECK(!last_send_time_.is_null()); | |
| 260 const base::TimeDelta time_since_last_send = | |
| 261 cast_environment_->Clock()->NowTicks() - last_send_time_; | |
| 262 if (time_since_last_send > target_playout_delay_) { | |
| 263 if (latest_acked_frame_id_ == last_sent_frame_id_) { | |
| 264 // Last frame acked, no point in doing anything | |
| 265 } else { | |
| 266 VLOG(1) << "ACK timeout; last acked frame: " << latest_acked_frame_id_; | |
| 267 ResendForKickstart(); | |
| 268 } | |
| 269 } | |
| 270 ScheduleNextResendCheck(); | |
| 271 } | |
| 272 | |
| 273 void VideoSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) { | |
| 274 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 275 | |
| 276 base::TimeDelta rtt; | |
| 277 base::TimeDelta avg_rtt; | |
| 278 base::TimeDelta min_rtt; | |
| 279 base::TimeDelta max_rtt; | |
| 280 if (rtcp_.Rtt(&rtt, &avg_rtt, &min_rtt, &max_rtt)) { | |
| 281 congestion_control_.UpdateRtt(rtt); | |
| 282 | |
| 283 // Don't use a RTT lower than our average. | |
| 284 rtt = std::max(rtt, avg_rtt); | |
| 285 | |
| 286 // Having the RTT values implies the receiver sent back a receiver report | |
| 287 // based on it having received a report from here. Therefore, ensure this | |
| 288 // sender stops aggressively sending reports. | |
| 289 if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) { | |
| 290 VLOG(1) << "No longer a need to send reports aggressively (sent " | |
| 291 << num_aggressive_rtcp_reports_sent_ << ")."; | |
| 292 num_aggressive_rtcp_reports_sent_ = kNumAggressiveReportsSentAtStart; | |
| 293 ScheduleNextRtcpReport(); | |
| 294 } | |
| 295 } else { | |
| 296 // We have no measured value use default. | |
| 297 rtt = base::TimeDelta::FromMilliseconds(kStartRttMs); | |
| 298 } | |
| 299 | |
| 300 if (last_send_time_.is_null()) | |
| 301 return; // Cannot get an ACK without having first sent a frame. | |
| 302 | |
| 303 if (cast_feedback.missing_frames_and_packets_.empty()) { | |
| 304 video_encoder_->LatestFrameIdToReference(cast_feedback.ack_frame_id_); | |
| 305 | |
| 306 // We only count duplicate ACKs when we have sent newer frames. | |
| 307 if (latest_acked_frame_id_ == cast_feedback.ack_frame_id_ && | |
| 308 latest_acked_frame_id_ != last_sent_frame_id_) { | |
| 309 duplicate_ack_counter_++; | |
| 310 } else { | |
| 311 duplicate_ack_counter_ = 0; | |
| 312 } | |
| 313 // TODO(miu): The values "2" and "3" should be derived from configuration. | |
| 314 if (duplicate_ack_counter_ >= 2 && duplicate_ack_counter_ % 3 == 2) { | |
| 315 VLOG(1) << "Received duplicate ACK for frame " << latest_acked_frame_id_; | |
| 316 ResendForKickstart(); | |
| 317 } | |
| 318 } else { | |
| 319 // Only count duplicated ACKs if there is no NACK request in between. | |
| 320 // This is to avoid aggresive resend. | |
| 321 duplicate_ack_counter_ = 0; | |
| 322 | |
| 323 // A NACK is also used to cancel pending re-transmissions. | |
| 324 transport_sender_->ResendPackets( | |
| 325 false, cast_feedback.missing_frames_and_packets_, true, rtt); | |
| 326 } | |
| 327 | |
| 328 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); | |
| 329 congestion_control_.AckFrame(cast_feedback.ack_frame_id_, now); | |
| 330 | |
| 331 RtpTimestamp rtp_timestamp = | |
| 332 frame_id_to_rtp_timestamp_[cast_feedback.ack_frame_id_ & 0xff]; | |
| 333 cast_environment_->Logging()->InsertFrameEvent(now, | |
| 334 FRAME_ACK_RECEIVED, | |
| 335 VIDEO_EVENT, | |
| 336 rtp_timestamp, | |
| 337 cast_feedback.ack_frame_id_); | |
| 338 | |
| 339 const bool is_acked_out_of_order = | |
| 340 static_cast<int32>(cast_feedback.ack_frame_id_ - | |
| 341 latest_acked_frame_id_) < 0; | |
| 342 VLOG(2) << "Received ACK" << (is_acked_out_of_order ? " out-of-order" : "") | |
| 343 << " for frame " << cast_feedback.ack_frame_id_; | |
| 344 if (!is_acked_out_of_order) { | |
| 345 // Cancel resends of acked frames. | |
| 346 MissingFramesAndPacketsMap missing_frames_and_packets; | |
| 347 PacketIdSet missing; | |
| 348 while (latest_acked_frame_id_ != cast_feedback.ack_frame_id_) { | |
| 349 latest_acked_frame_id_++; | |
| 350 missing_frames_and_packets[latest_acked_frame_id_] = missing; | |
| 351 } | |
| 352 transport_sender_->ResendPackets( | |
| 353 false, missing_frames_and_packets, true, rtt); | |
| 354 latest_acked_frame_id_ = cast_feedback.ack_frame_id_; | |
| 355 } | |
| 356 } | |
| 357 | |
| 358 bool VideoSender::AreTooManyFramesInFlight() const { | |
| 359 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 360 int frames_in_flight = frames_in_encoder_; | |
| 361 if (!last_send_time_.is_null()) { | |
| 362 frames_in_flight += | |
| 363 static_cast<int32>(last_sent_frame_id_ - latest_acked_frame_id_); | |
| 364 } | |
| 365 VLOG(2) << frames_in_flight | |
| 366 << " frames in flight; last sent: " << last_sent_frame_id_ | |
| 367 << " latest acked: " << latest_acked_frame_id_ | |
| 368 << " frames in encoder: " << frames_in_encoder_; | |
| 369 return frames_in_flight >= max_unacked_frames_; | |
| 370 } | |
| 371 | |
| 372 void VideoSender::ResendForKickstart() { | |
| 373 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 374 DCHECK(!last_send_time_.is_null()); | |
| 375 VLOG(1) << "Resending last packet of frame " << last_sent_frame_id_ | |
| 376 << " to kick-start."; | |
| 377 // Send the first packet of the last encoded frame to kick start | |
| 378 // retransmission. This gives enough information to the receiver what | |
| 379 // packets and frames are missing. | |
| 380 MissingFramesAndPacketsMap missing_frames_and_packets; | |
| 381 PacketIdSet missing; | |
| 382 missing.insert(kRtcpCastLastPacket); | |
| 383 missing_frames_and_packets.insert( | |
| 384 std::make_pair(last_sent_frame_id_, missing)); | |
| 385 last_send_time_ = cast_environment_->Clock()->NowTicks(); | |
| 386 | |
| 387 base::TimeDelta rtt; | |
| 388 base::TimeDelta avg_rtt; | |
| 389 base::TimeDelta min_rtt; | |
| 390 base::TimeDelta max_rtt; | |
| 391 rtcp_.Rtt(&rtt, &avg_rtt, &min_rtt, &max_rtt); | |
| 392 | |
| 393 // Sending this extra packet is to kick-start the session. There is | |
| 394 // no need to optimize re-transmission for this case. | |
| 395 transport_sender_->ResendPackets(false, missing_frames_and_packets, | |
| 396 false, rtt); | |
| 397 } | |
| 398 | |
| 399 } // namespace cast | |
| 400 } // namespace media | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 388663003:
Cast: Reshuffle files under media/cast (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/src