OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include "webrtc/modules/rtp_rtcp/source/rtp_sender_video.h" | 11 #include "webrtc/modules/rtp_rtcp/source/rtp_sender_video.h" |
12 | 12 |
13 #include <stdlib.h> | 13 #include <stdlib.h> |
14 #include <string.h> | 14 #include <string.h> |
15 | 15 |
| 16 #include <limits> |
16 #include <memory> | 17 #include <memory> |
| 18 #include <utility> |
17 #include <vector> | 19 #include <vector> |
18 #include <utility> | |
19 | 20 |
20 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h" | 21 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h" |
21 #include "webrtc/modules/rtp_rtcp/source/byte_io.h" | 22 #include "webrtc/modules/rtp_rtcp/source/byte_io.h" |
22 #include "webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h" | 23 #include "webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h" |
23 #include "webrtc/modules/rtp_rtcp/source/rtp_format_vp8.h" | 24 #include "webrtc/modules/rtp_rtcp/source/rtp_format_vp8.h" |
24 #include "webrtc/modules/rtp_rtcp/source/rtp_format_vp9.h" | 25 #include "webrtc/modules/rtp_rtcp/source/rtp_format_vp9.h" |
25 #include "webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h" | 26 #include "webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h" |
26 #include "webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h" | 27 #include "webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h" |
27 #include "webrtc/rtc_base/checks.h" | 28 #include "webrtc/rtc_base/checks.h" |
28 #include "webrtc/rtc_base/logging.h" | 29 #include "webrtc/rtc_base/logging.h" |
29 #include "webrtc/rtc_base/ptr_util.h" | 30 #include "webrtc/rtc_base/ptr_util.h" |
30 #include "webrtc/rtc_base/trace_event.h" | 31 #include "webrtc/rtc_base/trace_event.h" |
31 | 32 |
32 namespace webrtc { | 33 namespace webrtc { |
33 | 34 |
34 namespace { | 35 namespace { |
35 constexpr size_t kRedForFecHeaderLength = 1; | 36 constexpr size_t kRedForFecHeaderLength = 1; |
| 37 constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4; |
36 | 38 |
37 void BuildRedPayload(const RtpPacketToSend& media_packet, | 39 void BuildRedPayload(const RtpPacketToSend& media_packet, |
38 RtpPacketToSend* red_packet) { | 40 RtpPacketToSend* red_packet) { |
39 uint8_t* red_payload = red_packet->AllocatePayload( | 41 uint8_t* red_payload = red_packet->AllocatePayload( |
40 kRedForFecHeaderLength + media_packet.payload_size()); | 42 kRedForFecHeaderLength + media_packet.payload_size()); |
41 RTC_DCHECK(red_payload); | 43 RTC_DCHECK(red_payload); |
42 red_payload[0] = media_packet.PayloadType(); | 44 red_payload[0] = media_packet.PayloadType(); |
43 | 45 |
44 auto media_payload = media_packet.payload(); | 46 auto media_payload = media_packet.payload(); |
45 memcpy(&red_payload[kRedForFecHeaderLength], media_payload.data(), | 47 memcpy(&red_payload[kRedForFecHeaderLength], media_payload.data(), |
46 media_payload.size()); | 48 media_payload.size()); |
47 } | 49 } |
48 } // namespace | 50 } // namespace |
49 | 51 |
50 RTPSenderVideo::RTPSenderVideo(Clock* clock, | 52 RTPSenderVideo::RTPSenderVideo(Clock* clock, |
51 RTPSender* rtp_sender, | 53 RTPSender* rtp_sender, |
52 FlexfecSender* flexfec_sender) | 54 FlexfecSender* flexfec_sender) |
53 : rtp_sender_(rtp_sender), | 55 : rtp_sender_(rtp_sender), |
54 clock_(clock), | 56 clock_(clock), |
55 video_type_(kRtpVideoGeneric), | 57 video_type_(kRtpVideoGeneric), |
56 retransmission_settings_(kRetransmitBaseLayer), | 58 retransmission_settings_(kRetransmitBaseLayer | |
| 59 kConditionallyRetransmitHigherLayers), |
57 last_rotation_(kVideoRotation_0), | 60 last_rotation_(kVideoRotation_0), |
58 red_payload_type_(-1), | 61 red_payload_type_(-1), |
59 ulpfec_payload_type_(-1), | 62 ulpfec_payload_type_(-1), |
60 flexfec_sender_(flexfec_sender), | 63 flexfec_sender_(flexfec_sender), |
61 delta_fec_params_{0, 1, kFecMaskRandom}, | 64 delta_fec_params_{0, 1, kFecMaskRandom}, |
62 key_fec_params_{0, 1, kFecMaskRandom}, | 65 key_fec_params_{0, 1, kFecMaskRandom}, |
63 fec_bitrate_(1000, RateStatistics::kBpsScale), | 66 fec_bitrate_(1000, RateStatistics::kBpsScale), |
64 video_bitrate_(1000, RateStatistics::kBpsScale) {} | 67 video_bitrate_(1000, RateStatistics::kBpsScale) {} |
65 | 68 |
66 RTPSenderVideo::~RTPSenderVideo() {} | 69 RTPSenderVideo::~RTPSenderVideo() {} |
(...skipping 218 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
285 } | 288 } |
286 | 289 |
287 bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type, | 290 bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type, |
288 FrameType frame_type, | 291 FrameType frame_type, |
289 int8_t payload_type, | 292 int8_t payload_type, |
290 uint32_t rtp_timestamp, | 293 uint32_t rtp_timestamp, |
291 int64_t capture_time_ms, | 294 int64_t capture_time_ms, |
292 const uint8_t* payload_data, | 295 const uint8_t* payload_data, |
293 size_t payload_size, | 296 size_t payload_size, |
294 const RTPFragmentationHeader* fragmentation, | 297 const RTPFragmentationHeader* fragmentation, |
295 const RTPVideoHeader* video_header) { | 298 const RTPVideoHeader* video_header, |
| 299 int64_t expected_retransmission_time_ms) { |
296 if (payload_size == 0) | 300 if (payload_size == 0) |
297 return false; | 301 return false; |
298 | 302 |
299 // Create header that will be reused in all packets. | 303 // Create header that will be reused in all packets. |
300 std::unique_ptr<RtpPacketToSend> rtp_header = rtp_sender_->AllocatePacket(); | 304 std::unique_ptr<RtpPacketToSend> rtp_header = rtp_sender_->AllocatePacket(); |
301 rtp_header->SetPayloadType(payload_type); | 305 rtp_header->SetPayloadType(payload_type); |
302 rtp_header->SetTimestamp(rtp_timestamp); | 306 rtp_header->SetTimestamp(rtp_timestamp); |
303 rtp_header->set_capture_time_ms(capture_time_ms); | 307 rtp_header->set_capture_time_ms(capture_time_ms); |
304 auto last_packet = rtc::MakeUnique<RtpPacketToSend>(*rtp_header); | 308 auto last_packet = rtc::MakeUnique<RtpPacketToSend>(*rtp_header); |
305 | 309 |
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
358 RTC_DCHECK_GT(packet_capacity, rtp_header->headers_size()); | 362 RTC_DCHECK_GT(packet_capacity, rtp_header->headers_size()); |
359 RTC_DCHECK_GT(packet_capacity, last_packet->headers_size()); | 363 RTC_DCHECK_GT(packet_capacity, last_packet->headers_size()); |
360 size_t max_data_payload_length = packet_capacity - rtp_header->headers_size(); | 364 size_t max_data_payload_length = packet_capacity - rtp_header->headers_size(); |
361 RTC_DCHECK_GE(last_packet->headers_size(), rtp_header->headers_size()); | 365 RTC_DCHECK_GE(last_packet->headers_size(), rtp_header->headers_size()); |
362 size_t last_packet_reduction_len = | 366 size_t last_packet_reduction_len = |
363 last_packet->headers_size() - rtp_header->headers_size(); | 367 last_packet->headers_size() - rtp_header->headers_size(); |
364 | 368 |
365 std::unique_ptr<RtpPacketizer> packetizer(RtpPacketizer::Create( | 369 std::unique_ptr<RtpPacketizer> packetizer(RtpPacketizer::Create( |
366 video_type, max_data_payload_length, last_packet_reduction_len, | 370 video_type, max_data_payload_length, last_packet_reduction_len, |
367 video_header ? &(video_header->codecHeader) : nullptr, frame_type)); | 371 video_header ? &(video_header->codecHeader) : nullptr, frame_type)); |
368 // Media packet storage. | 372 |
369 StorageType storage = packetizer->GetStorageType(retransmission_settings); | 373 const uint8_t temporal_id = |
| 374 video_header ? GetTemporalId(*video_header) : kNoTemporalIdx; |
| 375 StorageType storage = GetStorageType(temporal_id, retransmission_settings, |
| 376 expected_retransmission_time_ms); |
370 | 377 |
371 // TODO(changbin): we currently don't support to configure the codec to | 378 // TODO(changbin): we currently don't support to configure the codec to |
372 // output multiple partitions for VP8. Should remove below check after the | 379 // output multiple partitions for VP8. Should remove below check after the |
373 // issue is fixed. | 380 // issue is fixed. |
374 const RTPFragmentationHeader* frag = | 381 const RTPFragmentationHeader* frag = |
375 (video_type == kRtpVideoVp8) ? nullptr : fragmentation; | 382 (video_type == kRtpVideoVp8) ? nullptr : fragmentation; |
376 size_t num_packets = | 383 size_t num_packets = |
377 packetizer->SetPayloadData(payload_data, payload_size, frag); | 384 packetizer->SetPayloadData(payload_data, payload_size, frag); |
378 | 385 |
379 if (num_packets == 0) | 386 if (num_packets == 0) |
380 return false; | 387 return false; |
381 | 388 |
382 bool first_frame = first_frame_sent_(); | 389 bool first_frame = first_frame_sent_(); |
383 for (size_t i = 0; i < num_packets; ++i) { | 390 for (size_t i = 0; i < num_packets; ++i) { |
384 bool last = (i + 1) == num_packets; | 391 bool last = (i + 1) == num_packets; |
385 auto packet = last ? std::move(last_packet) | 392 auto packet = last ? std::move(last_packet) |
386 : rtc::MakeUnique<RtpPacketToSend>(*rtp_header); | 393 : rtc::MakeUnique<RtpPacketToSend>(*rtp_header); |
387 if (!packetizer->NextPacket(packet.get())) | 394 if (!packetizer->NextPacket(packet.get())) |
388 return false; | 395 return false; |
389 RTC_DCHECK_LE(packet->payload_size(), | 396 RTC_DCHECK_LE(packet->payload_size(), |
390 last ? max_data_payload_length - last_packet_reduction_len | 397 last ? max_data_payload_length - last_packet_reduction_len |
391 : max_data_payload_length); | 398 : max_data_payload_length); |
392 if (!rtp_sender_->AssignSequenceNumber(packet.get())) | 399 if (!rtp_sender_->AssignSequenceNumber(packet.get())) |
393 return false; | 400 return false; |
394 | 401 |
395 bool protect_packet = (packetizer->GetProtectionType() == kProtectedPacket); | 402 // No FEC protection for upper temporal layers, if used. |
| 403 bool protect_packet = temporal_id == 0 || temporal_id == kNoTemporalIdx; |
| 404 |
396 // Put packetization finish timestamp into extension. | 405 // Put packetization finish timestamp into extension. |
397 if (packet->HasExtension<VideoTimingExtension>()) { | 406 if (packet->HasExtension<VideoTimingExtension>()) { |
398 packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds()); | 407 packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds()); |
399 // TODO(ilnik): Due to webrtc:7859, packets with timing extensions are not | 408 // TODO(ilnik): Due to webrtc:7859, packets with timing extensions are not |
400 // protected by FEC. It reduces FEC efficiency a bit. When FEC is moved | 409 // protected by FEC. It reduces FEC efficiency a bit. When FEC is moved |
401 // below the pacer, it can be re-enabled for these packets. | 410 // below the pacer, it can be re-enabled for these packets. |
402 // NOTE: Any RTP stream processor in the network, modifying 'network' | 411 // NOTE: Any RTP stream processor in the network, modifying 'network' |
403 // timestamps in the timing frames extension have to be an end-point for | 412 // timestamps in the timing frames extension have to be an end-point for |
404 // FEC, otherwise recovered by FEC packets will be corrupted. | 413 // FEC, otherwise recovered by FEC packets will be corrupted. |
405 protect_packet = false; | 414 protect_packet = false; |
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
446 int RTPSenderVideo::SelectiveRetransmissions() const { | 455 int RTPSenderVideo::SelectiveRetransmissions() const { |
447 rtc::CritScope cs(&crit_); | 456 rtc::CritScope cs(&crit_); |
448 return retransmission_settings_; | 457 return retransmission_settings_; |
449 } | 458 } |
450 | 459 |
451 void RTPSenderVideo::SetSelectiveRetransmissions(uint8_t settings) { | 460 void RTPSenderVideo::SetSelectiveRetransmissions(uint8_t settings) { |
452 rtc::CritScope cs(&crit_); | 461 rtc::CritScope cs(&crit_); |
453 retransmission_settings_ = settings; | 462 retransmission_settings_ = settings; |
454 } | 463 } |
455 | 464 |
| 465 StorageType RTPSenderVideo::GetStorageType( |
| 466 uint8_t temporal_id, |
| 467 int32_t retransmission_settings, |
| 468 int64_t expected_retransmission_time_ms) { |
| 469 if (retransmission_settings == kRetransmitOff) |
| 470 return StorageType::kDontRetransmit; |
| 471 if (retransmission_settings == kRetransmitAllPackets) |
| 472 return StorageType::kAllowRetransmission; |
| 473 |
| 474 rtc::CritScope cs(&stats_crit_); |
| 475 // Media packet storage. |
| 476 if ((retransmission_settings & kConditionallyRetransmitHigherLayers) && |
| 477 UpdateConditionalRetransmit(temporal_id, |
| 478 expected_retransmission_time_ms)) { |
| 479 retransmission_settings |= kRetransmitHigherLayers; |
| 480 } |
| 481 |
| 482 if (temporal_id == kNoTemporalIdx) |
| 483 return kAllowRetransmission; |
| 484 |
| 485 if ((retransmission_settings & kRetransmitBaseLayer) && temporal_id == 0) |
| 486 return kAllowRetransmission; |
| 487 |
| 488 if ((retransmission_settings & kRetransmitHigherLayers) && temporal_id > 0) |
| 489 return kAllowRetransmission; |
| 490 |
| 491 return kDontRetransmit; |
| 492 } |
| 493 |
| 494 uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) { |
| 495 switch (header.codec) { |
| 496 case kRtpVideoVp8: |
| 497 return header.codecHeader.VP8.temporalIdx; |
| 498 case kRtpVideoVp9: |
| 499 return header.codecHeader.VP9.temporal_idx; |
| 500 default: |
| 501 return kNoTemporalIdx; |
| 502 } |
| 503 } |
| 504 |
| 505 bool RTPSenderVideo::UpdateConditionalRetransmit( |
| 506 uint8_t temporal_id, |
| 507 int64_t expected_retransmission_time_ms) { |
| 508 int64_t now_ms = clock_->TimeInMilliseconds(); |
| 509 // Update stats for any temporal layer. |
| 510 TemporalLayerStats* current_layer_stats = |
| 511 &frame_stats_by_temporal_layer_[temporal_id]; |
| 512 current_layer_stats->frame_rate_fp1000s.Update(1, now_ms); |
| 513 int64_t tl_frame_interval = now_ms - current_layer_stats->last_frame_time_ms; |
| 514 current_layer_stats->last_frame_time_ms = now_ms; |
| 515 |
| 516 // Conditional retransmit only applies to upper layers. |
| 517 if (temporal_id != kNoTemporalIdx && temporal_id > 0) { |
| 518 if (tl_frame_interval >= kMaxUnretransmittableFrameIntervalMs) { |
| 519 // Too long since a retransmittable frame in this layer, enable NACK |
| 520 // protection. |
| 521 return true; |
| 522 } else { |
| 523 // Estimate when the next frame of any lower layer will be sent. |
| 524 const int64_t kUndefined = std::numeric_limits<int64_t>::max(); |
| 525 int64_t expected_next_frame_time = kUndefined; |
| 526 for (int i = temporal_id - 1; i >= 0; --i) { |
| 527 TemporalLayerStats* stats = &frame_stats_by_temporal_layer_[i]; |
| 528 rtc::Optional<uint32_t> rate = stats->frame_rate_fp1000s.Rate(now_ms); |
| 529 if (rate) { |
| 530 int64_t tl_next = stats->last_frame_time_ms + 1000000 / *rate; |
| 531 if (tl_next - now_ms > -expected_retransmission_time_ms && |
| 532 tl_next < expected_next_frame_time) { |
| 533 expected_next_frame_time = tl_next; |
| 534 } |
| 535 } |
| 536 } |
| 537 |
| 538 if (expected_next_frame_time == kUndefined || |
| 539 expected_next_frame_time - now_ms > expected_retransmission_time_ms) { |
| 540 // The next frame in a lower layer is expected at a later time (or |
| 541 // unable to tell due to lack of data) than a retransmission is |
| 542 // estimated to be able to arrive, so allow this packet to be nacked. |
| 543 return true; |
| 544 } |
| 545 } |
| 546 } |
| 547 |
| 548 return false; |
| 549 } |
| 550 |
456 } // namespace webrtc | 551 } // namespace webrtc |
OLD | NEW |