| OLD | NEW |
| 1 // Copyright 2013 The Chromium Authors. All rights reserved. | 1 // Copyright 2013 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "media/cast/video_receiver/video_receiver.h" | 5 #include "media/cast/video_receiver/video_receiver.h" |
| 6 | 6 |
| 7 #include <algorithm> | 7 #include <algorithm> |
| 8 | 8 |
| 9 #include "base/bind.h" | 9 #include "base/bind.h" |
| 10 #include "base/debug/trace_event.h" | 10 #include "base/debug/trace_event.h" |
| 11 #include "base/logging.h" | 11 #include "base/logging.h" |
| 12 #include "base/message_loop/message_loop.h" | 12 #include "base/message_loop/message_loop.h" |
| 13 #include "media/cast/cast_defines.h" | 13 #include "media/base/video_frame.h" |
| 14 #include "media/cast/framer/framer.h" | 14 #include "media/cast/logging/logging_defines.h" |
| 15 #include "media/cast/rtcp/receiver_rtcp_event_subscriber.h" | 15 #include "media/cast/transport/cast_transport_defines.h" |
| 16 #include "media/cast/rtcp/rtcp_sender.h" | |
| 17 #include "media/cast/video_receiver/video_decoder.h" | 16 #include "media/cast/video_receiver/video_decoder.h" |
| 18 | 17 |
| 19 namespace { | 18 namespace { |
| 20 | 19 const int kMinSchedulingDelayMs = 1; |
| 21 static const int64 kMinSchedulingDelayMs = 1; | 20 const int kMinTimeBetweenOffsetUpdatesMs = 1000; |
| 22 static const int64 kMinTimeBetweenOffsetUpdatesMs = 1000; | 21 const int kTimeOffsetMaxCounter = 10; |
| 23 static const int kTimeOffsetMaxCounter = 10; | |
| 24 | |
| 25 } // namespace | 22 } // namespace |
| 26 | 23 |
| 27 namespace media { | 24 namespace media { |
| 28 namespace cast { | 25 namespace cast { |
| 29 | 26 |
| 30 // Local implementation of RtpPayloadFeedback (defined in rtp_defines.h) | |
| 31 // Used to convey cast-specific feedback from receiver to sender. | |
| 32 // Callback triggered by the Framer (cast message builder). | |
| 33 class LocalRtpVideoFeedback : public RtpPayloadFeedback { | |
| 34 public: | |
| 35 explicit LocalRtpVideoFeedback(VideoReceiver* video_receiver) | |
| 36 : video_receiver_(video_receiver) {} | |
| 37 | |
| 38 virtual void CastFeedback(const RtcpCastMessage& cast_message) OVERRIDE { | |
| 39 video_receiver_->CastFeedback(cast_message); | |
| 40 } | |
| 41 | |
| 42 private: | |
| 43 VideoReceiver* video_receiver_; | |
| 44 | |
| 45 DISALLOW_IMPLICIT_CONSTRUCTORS(LocalRtpVideoFeedback); | |
| 46 }; | |
| 47 | |
| 48 VideoReceiver::VideoReceiver(scoped_refptr<CastEnvironment> cast_environment, | 27 VideoReceiver::VideoReceiver(scoped_refptr<CastEnvironment> cast_environment, |
| 49 const VideoReceiverConfig& video_config, | 28 const VideoReceiverConfig& video_config, |
| 50 transport::PacedPacketSender* const packet_sender, | 29 transport::PacedPacketSender* const packet_sender) |
| 51 const SetTargetDelayCallback& target_delay_cb) | |
| 52 : RtpReceiver(cast_environment->Clock(), NULL, &video_config), | 30 : RtpReceiver(cast_environment->Clock(), NULL, &video_config), |
| 53 cast_environment_(cast_environment), | 31 cast_environment_(cast_environment), |
| 54 event_subscriber_(kReceiverRtcpEventHistorySize, | 32 event_subscriber_(kReceiverRtcpEventHistorySize, |
| 55 ReceiverRtcpEventSubscriber::kVideoEventSubscriber), | 33 ReceiverRtcpEventSubscriber::kVideoEventSubscriber), |
| 56 codec_(video_config.codec), | 34 codec_(video_config.codec), |
| 57 target_delay_delta_( | 35 target_delay_delta_( |
| 58 base::TimeDelta::FromMilliseconds(video_config.rtp_max_delay_ms)), | 36 base::TimeDelta::FromMilliseconds(video_config.rtp_max_delay_ms)), |
| 59 frame_delay_(base::TimeDelta::FromMilliseconds( | 37 expected_frame_duration_( |
| 60 1000 / video_config.max_frame_rate)), | 38 base::TimeDelta::FromSeconds(1) / video_config.max_frame_rate), |
| 61 incoming_payload_feedback_(new LocalRtpVideoFeedback(this)), | 39 framer_(cast_environment->Clock(), |
| 40 this, |
| 41 video_config.incoming_ssrc, |
| 42 video_config.decoder_faster_than_max_frame_rate, |
| 43 video_config.rtp_max_delay_ms * video_config.max_frame_rate / |
| 44 1000), |
| 45 rtcp_(cast_environment_, |
| 46 NULL, |
| 47 NULL, |
| 48 packet_sender, |
| 49 GetStatistics(), |
| 50 video_config.rtcp_mode, |
| 51 base::TimeDelta::FromMilliseconds(video_config.rtcp_interval), |
| 52 video_config.feedback_ssrc, |
| 53 video_config.incoming_ssrc, |
| 54 video_config.rtcp_c_name), |
| 62 time_offset_counter_(0), | 55 time_offset_counter_(0), |
| 63 decryptor_(), | |
| 64 time_incoming_packet_updated_(false), | 56 time_incoming_packet_updated_(false), |
| 65 incoming_rtp_timestamp_(0), | 57 incoming_rtp_timestamp_(0), |
| 66 target_delay_cb_(target_delay_cb), | 58 is_waiting_for_consecutive_frame_(false), |
| 67 weak_factory_(this) { | 59 weak_factory_(this) { |
| 68 int max_unacked_frames = | 60 DCHECK_GT(video_config.rtp_max_delay_ms, 0); |
| 69 video_config.rtp_max_delay_ms * video_config.max_frame_rate / 1000; | 61 DCHECK_GT(video_config.max_frame_rate, 0); |
| 70 DCHECK(max_unacked_frames) << "Invalid argument"; | 62 if (!video_config.use_external_decoder) { |
| 71 | 63 video_decoder_.reset(new VideoDecoder(cast_environment, video_config)); |
| 64 } |
| 72 decryptor_.Initialize(video_config.aes_key, video_config.aes_iv_mask); | 65 decryptor_.Initialize(video_config.aes_key, video_config.aes_iv_mask); |
| 73 framer_.reset(new Framer(cast_environment->Clock(), | 66 rtcp_.SetTargetDelay(target_delay_delta_); |
| 74 incoming_payload_feedback_.get(), | |
| 75 video_config.incoming_ssrc, | |
| 76 video_config.decoder_faster_than_max_frame_rate, | |
| 77 max_unacked_frames)); | |
| 78 | |
| 79 if (!video_config.use_external_decoder) { | |
| 80 video_decoder_.reset(new VideoDecoder(video_config, cast_environment)); | |
| 81 } | |
| 82 | |
| 83 rtcp_.reset( | |
| 84 new Rtcp(cast_environment_, | |
| 85 NULL, | |
| 86 NULL, | |
| 87 packet_sender, | |
| 88 GetStatistics(), | |
| 89 video_config.rtcp_mode, | |
| 90 base::TimeDelta::FromMilliseconds(video_config.rtcp_interval), | |
| 91 video_config.feedback_ssrc, | |
| 92 video_config.incoming_ssrc, | |
| 93 video_config.rtcp_c_name)); | |
| 94 // Set the target delay that will be conveyed to the sender. | |
| 95 rtcp_->SetTargetDelay(target_delay_delta_); | |
| 96 cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber_); | 67 cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber_); |
| 97 memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_)); | 68 memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_)); |
| 98 } | 69 } |
| 99 | 70 |
| 100 VideoReceiver::~VideoReceiver() { | 71 VideoReceiver::~VideoReceiver() { |
| 72 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 73 |
| 74 // If any callbacks for encoded video frames are queued, flush them out now. |
| 75 // This is critical because some Closures in |frame_request_queue_| may have |
| 76 // Unretained references to |this|. |
| 77 while (!frame_request_queue_.empty()) { |
| 78 frame_request_queue_.front().Run( |
| 79 make_scoped_ptr<transport::EncodedVideoFrame>(NULL), base::TimeTicks()); |
| 80 frame_request_queue_.pop_front(); |
| 81 } |
| 82 |
| 101 cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber_); | 83 cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber_); |
| 102 } | 84 } |
| 103 | 85 |
| 104 void VideoReceiver::InitializeTimers() { | 86 void VideoReceiver::InitializeTimers() { |
| 105 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 87 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 106 ScheduleNextRtcpReport(); | 88 ScheduleNextRtcpReport(); |
| 107 ScheduleNextCastMessage(); | 89 ScheduleNextCastMessage(); |
| 108 } | 90 } |
| 109 | 91 |
| 110 void VideoReceiver::GetRawVideoFrame( | 92 void VideoReceiver::GetRawVideoFrame( |
| 111 const VideoFrameDecodedCallback& callback) { | 93 const VideoFrameDecodedCallback& callback) { |
| 112 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 94 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 95 DCHECK(!callback.is_null()); |
| 96 DCHECK(video_decoder_.get()); |
| 113 GetEncodedVideoFrame(base::Bind( | 97 GetEncodedVideoFrame(base::Bind( |
| 114 &VideoReceiver::DecodeVideoFrame, base::Unretained(this), callback)); | 98 &VideoReceiver::DecodeEncodedVideoFrame, |
| 99 // Note: Use of Unretained is safe since this Closure is guaranteed to be |
| 100 // invoked before destruction of |this|. |
| 101 base::Unretained(this), |
| 102 callback)); |
| 115 } | 103 } |
| 116 | 104 |
| 117 // Called when we have a frame to decode. | 105 void VideoReceiver::DecodeEncodedVideoFrame( |
| 118 void VideoReceiver::DecodeVideoFrame( | |
| 119 const VideoFrameDecodedCallback& callback, | 106 const VideoFrameDecodedCallback& callback, |
| 120 scoped_ptr<transport::EncodedVideoFrame> encoded_frame, | 107 scoped_ptr<transport::EncodedVideoFrame> encoded_frame, |
| 121 const base::TimeTicks& render_time) { | 108 const base::TimeTicks& playout_time) { |
| 109 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 110 if (!encoded_frame) { |
| 111 callback.Run(make_scoped_refptr<VideoFrame>(NULL), playout_time, false); |
| 112 return; |
| 113 } |
| 114 const uint32 frame_id = encoded_frame->frame_id; |
| 115 const uint32 rtp_timestamp = encoded_frame->rtp_timestamp; |
| 116 video_decoder_->DecodeFrame(encoded_frame.Pass(), |
| 117 base::Bind(&VideoReceiver::EmitRawVideoFrame, |
| 118 cast_environment_, |
| 119 callback, |
| 120 frame_id, |
| 121 rtp_timestamp, |
| 122 playout_time)); |
| 123 } |
| 124 |
| 125 // static |
| 126 void VideoReceiver::EmitRawVideoFrame( |
| 127 const scoped_refptr<CastEnvironment>& cast_environment, |
| 128 const VideoFrameDecodedCallback& callback, |
| 129 uint32 frame_id, |
| 130 uint32 rtp_timestamp, |
| 131 const base::TimeTicks& playout_time, |
| 132 const scoped_refptr<VideoFrame>& video_frame, |
| 133 bool is_continuous) { |
| 134 DCHECK(cast_environment->CurrentlyOn(CastEnvironment::MAIN)); |
| 135 if (video_frame) { |
| 136 const base::TimeTicks now = cast_environment->Clock()->NowTicks(); |
| 137 cast_environment->Logging()->InsertFrameEvent( |
| 138 now, kVideoFrameDecoded, rtp_timestamp, frame_id); |
| 139 cast_environment->Logging()->InsertFrameEventWithDelay( |
| 140 now, kVideoRenderDelay, rtp_timestamp, frame_id, |
| 141 playout_time - now); |
| 142 // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc |
| 143 TRACE_EVENT_INSTANT1( |
| 144 "cast_perf_test", "FrameDecoded", |
| 145 TRACE_EVENT_SCOPE_THREAD, |
| 146 "rtp_timestamp", rtp_timestamp); |
| 147 } |
| 148 callback.Run(video_frame, playout_time, is_continuous); |
| 149 } |
| 150 |
| 151 void VideoReceiver::GetEncodedVideoFrame( |
| 152 const VideoFrameEncodedCallback& callback) { |
| 153 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 154 frame_request_queue_.push_back(callback); |
| 155 EmitAvailableEncodedFrames(); |
| 156 } |
| 157 |
| 158 void VideoReceiver::EmitAvailableEncodedFrames() { |
| 122 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 159 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 123 | 160 |
| 124 // Hand the ownership of the encoded frame to the decode thread. | 161 while (!frame_request_queue_.empty()) { |
| 125 cast_environment_->PostTask(CastEnvironment::VIDEO, | 162 // Attempt to peek at the next completed frame from the |framer_|. |
| 126 FROM_HERE, | 163 // TODO(miu): We should only be peeking at the metadata, and not copying the |
| 127 base::Bind(&VideoReceiver::DecodeVideoFrameThread, | 164 // payload yet! Or, at least, peek using a StringPiece instead of a copy. |
| 128 base::Unretained(this), | 165 scoped_ptr<transport::EncodedVideoFrame> encoded_frame( |
| 129 base::Passed(&encoded_frame), | 166 new transport::EncodedVideoFrame()); |
| 130 render_time, | 167 bool is_consecutively_next_frame = false; |
| 131 callback)); | 168 if (!framer_.GetEncodedVideoFrame(encoded_frame.get(), |
| 132 } | 169 &is_consecutively_next_frame)) { |
| 170 VLOG(1) << "Wait for more video packets to produce a completed frame."; |
| 171 return; // OnReceivedPayloadData() will invoke this method in the future. |
| 172 } |
| 133 | 173 |
| 134 // Utility function to run the decoder on a designated decoding thread. | 174 // If |framer_| has a frame ready that is out of sequence, examine the |
| 135 void VideoReceiver::DecodeVideoFrameThread( | 175 // playout time to determine whether it's acceptable to continue, thereby |
| 136 scoped_ptr<transport::EncodedVideoFrame> encoded_frame, | 176 // skipping one or more frames. Skip if the missing frame wouldn't complete |
| 137 const base::TimeTicks render_time, | 177 // playing before the start of playback of the available frame. |
| 138 const VideoFrameDecodedCallback& frame_decoded_callback) { | 178 const base::TimeTicks now = cast_environment_->Clock()->NowTicks(); |
| 139 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::VIDEO)); | 179 const base::TimeTicks playout_time = |
| 140 DCHECK(video_decoder_); | 180 GetPlayoutTime(now, encoded_frame->rtp_timestamp); |
| 181 if (!is_consecutively_next_frame) { |
| 182 // TODO(miu): Also account for expected decode time here? |
| 183 const base::TimeTicks earliest_possible_end_time_of_missing_frame = |
| 184 now + expected_frame_duration_; |
| 185 if (earliest_possible_end_time_of_missing_frame < playout_time) { |
| 186 VLOG(1) << "Wait for next consecutive frame instead of skipping."; |
| 187 if (!is_waiting_for_consecutive_frame_) { |
| 188 is_waiting_for_consecutive_frame_ = true; |
| 189 cast_environment_->PostDelayedTask( |
| 190 CastEnvironment::MAIN, |
| 191 FROM_HERE, |
| 192 base::Bind(&VideoReceiver::EmitAvailableEncodedFramesAfterWaiting, |
| 193 weak_factory_.GetWeakPtr()), |
| 194 playout_time - now); |
| 195 } |
| 196 return; |
| 197 } |
| 198 } |
| 141 | 199 |
| 142 if (!(video_decoder_->DecodeVideoFrame( | 200 // Decrypt the payload data in the frame, if crypto is being used. |
| 143 encoded_frame.get(), render_time, frame_decoded_callback))) { | 201 if (decryptor_.initialized()) { |
| 144 // This will happen if we decide to decode but not show a frame. | 202 std::string decrypted_video_data; |
| 203 if (!decryptor_.Decrypt(encoded_frame->frame_id, |
| 204 encoded_frame->data, |
| 205 &decrypted_video_data)) { |
| 206 // Decryption failed. Give up on this frame, releasing it from the |
| 207 // jitter buffer. |
| 208 framer_.ReleaseFrame(encoded_frame->frame_id); |
| 209 continue; |
| 210 } |
| 211 encoded_frame->data.swap(decrypted_video_data); |
| 212 } |
| 213 |
| 214 // At this point, we have a decrypted EncodedVideoFrame ready to be emitted. |
| 215 encoded_frame->codec = codec_; |
| 216 framer_.ReleaseFrame(encoded_frame->frame_id); |
| 217 // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc |
| 218 TRACE_EVENT_INSTANT2( |
| 219 "cast_perf_test", "PullEncodedVideoFrame", |
| 220 TRACE_EVENT_SCOPE_THREAD, |
| 221 "rtp_timestamp", encoded_frame->rtp_timestamp, |
| 222 // TODO(miu): Need to find an alternative to using ToInternalValue(): |
| 223 "render_time", playout_time.ToInternalValue()); |
| 145 cast_environment_->PostTask(CastEnvironment::MAIN, | 224 cast_environment_->PostTask(CastEnvironment::MAIN, |
| 146 FROM_HERE, | 225 FROM_HERE, |
| 147 base::Bind(&VideoReceiver::GetRawVideoFrame, | 226 base::Bind(frame_request_queue_.front(), |
| 148 base::Unretained(this), | 227 base::Passed(&encoded_frame), |
| 149 frame_decoded_callback)); | 228 playout_time)); |
| 229 frame_request_queue_.pop_front(); |
| 150 } | 230 } |
| 151 } | 231 } |
| 152 | 232 |
| 153 bool VideoReceiver::DecryptVideoFrame( | 233 void VideoReceiver::EmitAvailableEncodedFramesAfterWaiting() { |
| 154 scoped_ptr<transport::EncodedVideoFrame>* video_frame) { | |
| 155 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 234 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 156 | 235 DCHECK(is_waiting_for_consecutive_frame_); |
| 157 if (!decryptor_.initialized()) | 236 is_waiting_for_consecutive_frame_ = false; |
| 158 return false; | 237 EmitAvailableEncodedFrames(); |
| 159 | |
| 160 std::string decrypted_video_data; | |
| 161 if (!decryptor_.Decrypt((*video_frame)->frame_id, | |
| 162 (*video_frame)->data, | |
| 163 &decrypted_video_data)) { | |
| 164 // Give up on this frame, release it from jitter buffer. | |
| 165 framer_->ReleaseFrame((*video_frame)->frame_id); | |
| 166 return false; | |
| 167 } | |
| 168 (*video_frame)->data.swap(decrypted_video_data); | |
| 169 return true; | |
| 170 } | 238 } |
| 171 | 239 |
| 172 // Called from the main cast thread. | 240 base::TimeTicks VideoReceiver::GetPlayoutTime(base::TimeTicks now, |
| 173 void VideoReceiver::GetEncodedVideoFrame( | 241 uint32 rtp_timestamp) { |
| 174 const VideoFrameEncodedCallback& callback) { | 242 // TODO(miu): This and AudioReceiver::GetPlayoutTime() need to be reconciled! |
| 175 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 176 scoped_ptr<transport::EncodedVideoFrame> encoded_frame( | |
| 177 new transport::EncodedVideoFrame()); | |
| 178 bool next_frame = false; | |
| 179 | 243 |
| 180 if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &next_frame)) { | |
| 181 // We have no video frames. Wait for new packet(s). | |
| 182 queued_encoded_callbacks_.push_back(callback); | |
| 183 return; | |
| 184 } | |
| 185 | |
| 186 if (decryptor_.initialized() && !DecryptVideoFrame(&encoded_frame)) { | |
| 187 // Logging already done. | |
| 188 queued_encoded_callbacks_.push_back(callback); | |
| 189 return; | |
| 190 } | |
| 191 | |
| 192 base::TimeTicks render_time; | |
| 193 if (PullEncodedVideoFrame(next_frame, &encoded_frame, &render_time)) { | |
| 194 cast_environment_->PostTask( | |
| 195 CastEnvironment::MAIN, | |
| 196 FROM_HERE, | |
| 197 base::Bind(callback, base::Passed(&encoded_frame), render_time)); | |
| 198 } else { | |
| 199 // We have a video frame; however we are missing packets and we have time | |
| 200 // to wait for new packet(s). | |
| 201 queued_encoded_callbacks_.push_back(callback); | |
| 202 } | |
| 203 } | |
| 204 | |
| 205 // Should we pull the encoded video frame from the framer? decided by if this is | |
| 206 // the next frame or we are running out of time and have to pull the following | |
| 207 // frame. | |
| 208 // If the frame is too old to be rendered we set the don't show flag in the | |
| 209 // video bitstream where possible. | |
| 210 bool VideoReceiver::PullEncodedVideoFrame( | |
| 211 bool next_frame, | |
| 212 scoped_ptr<transport::EncodedVideoFrame>* encoded_frame, | |
| 213 base::TimeTicks* render_time) { | |
| 214 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 215 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); | |
| 216 *render_time = GetRenderTime(now, (*encoded_frame)->rtp_timestamp); | |
| 217 | |
| 218 // TODO(mikhal): Store actual render time and not diff. | |
| 219 cast_environment_->Logging()->InsertFrameEventWithDelay( | |
| 220 now, | |
| 221 kVideoRenderDelay, | |
| 222 (*encoded_frame)->rtp_timestamp, | |
| 223 (*encoded_frame)->frame_id, | |
| 224 now - *render_time); | |
| 225 | |
| 226 // Minimum time before a frame is due to be rendered before we pull it for | |
| 227 // decode. | |
| 228 base::TimeDelta min_wait_delta = frame_delay_; | |
| 229 base::TimeDelta time_until_render = *render_time - now; | |
| 230 if (!next_frame && (time_until_render > min_wait_delta)) { | |
| 231 // Example: | |
| 232 // We have decoded frame 1 and we have received the complete frame 3, but | |
| 233 // not frame 2. If we still have time before frame 3 should be rendered we | |
| 234 // will wait for 2 to arrive, however if 2 never show up this timer will hit | |
| 235 // and we will pull out frame 3 for decoding and rendering. | |
| 236 base::TimeDelta time_until_release = time_until_render - min_wait_delta; | |
| 237 cast_environment_->PostDelayedTask( | |
| 238 CastEnvironment::MAIN, | |
| 239 FROM_HERE, | |
| 240 base::Bind(&VideoReceiver::PlayoutTimeout, weak_factory_.GetWeakPtr()), | |
| 241 time_until_release); | |
| 242 VLOG(1) << "Wait before releasing frame " | |
| 243 << static_cast<int>((*encoded_frame)->frame_id) << " time " | |
| 244 << time_until_release.InMilliseconds(); | |
| 245 return false; | |
| 246 } | |
| 247 | |
| 248 base::TimeDelta dont_show_timeout_delta = | |
| 249 base::TimeDelta::FromMilliseconds(-kDontShowTimeoutMs); | |
| 250 if (codec_ == transport::kVp8 && | |
| 251 time_until_render < dont_show_timeout_delta) { | |
| 252 (*encoded_frame)->data[0] &= 0xef; | |
| 253 VLOG(1) << "Don't show frame " | |
| 254 << static_cast<int>((*encoded_frame)->frame_id) | |
| 255 << " time_until_render:" << time_until_render.InMilliseconds(); | |
| 256 } else { | |
| 257 VLOG(2) << "Show frame " << static_cast<int>((*encoded_frame)->frame_id) | |
| 258 << " time_until_render:" << time_until_render.InMilliseconds(); | |
| 259 } | |
| 260 // We have a copy of the frame, release this one. | |
| 261 framer_->ReleaseFrame((*encoded_frame)->frame_id); | |
| 262 (*encoded_frame)->codec = codec_; | |
| 263 | |
| 264 // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc | |
| 265 TRACE_EVENT_INSTANT2( | |
| 266 "cast_perf_test", "PullEncodedVideoFrame", | |
| 267 TRACE_EVENT_SCOPE_THREAD, | |
| 268 "rtp_timestamp", (*encoded_frame)->rtp_timestamp, | |
| 269 "render_time", render_time->ToInternalValue()); | |
| 270 | |
| 271 return true; | |
| 272 } | |
| 273 | |
| 274 void VideoReceiver::PlayoutTimeout() { | |
| 275 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | |
| 276 if (queued_encoded_callbacks_.empty()) | |
| 277 return; | |
| 278 | |
| 279 bool next_frame = false; | |
| 280 scoped_ptr<transport::EncodedVideoFrame> encoded_frame( | |
| 281 new transport::EncodedVideoFrame()); | |
| 282 | |
| 283 if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &next_frame)) { | |
| 284 // We have no video frames. Wait for new packet(s). | |
| 285 // Since the application can post multiple VideoFrameEncodedCallback and | |
| 286 // we only check the next frame to play out we might have multiple timeout | |
| 287 // events firing after each other; however this should be a rare event. | |
| 288 VLOG(1) << "Failed to retrieved a complete frame at this point in time"; | |
| 289 return; | |
| 290 } | |
| 291 VLOG(2) << "PlayoutTimeout retrieved frame " | |
| 292 << static_cast<int>(encoded_frame->frame_id); | |
| 293 | |
| 294 if (decryptor_.initialized() && !DecryptVideoFrame(&encoded_frame)) { | |
| 295 // Logging already done. | |
| 296 return; | |
| 297 } | |
| 298 | |
| 299 base::TimeTicks render_time; | |
| 300 if (PullEncodedVideoFrame(next_frame, &encoded_frame, &render_time)) { | |
| 301 if (!queued_encoded_callbacks_.empty()) { | |
| 302 VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front(); | |
| 303 queued_encoded_callbacks_.pop_front(); | |
| 304 cast_environment_->PostTask( | |
| 305 CastEnvironment::MAIN, | |
| 306 FROM_HERE, | |
| 307 base::Bind(callback, base::Passed(&encoded_frame), render_time)); | |
| 308 } | |
| 309 } | |
| 310 // Else we have a video frame; however we are missing packets and we have time | |
| 311 // to wait for new packet(s). | |
| 312 } | |
| 313 | |
| 314 base::TimeTicks VideoReceiver::GetRenderTime(base::TimeTicks now, | |
| 315 uint32 rtp_timestamp) { | |
| 316 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 244 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 317 // Senders time in ms when this frame was captured. | 245 // Senders time in ms when this frame was captured. |
| 318 // Note: the senders clock and our local clock might not be synced. | 246 // Note: the senders clock and our local clock might not be synced. |
| 319 base::TimeTicks rtp_timestamp_in_ticks; | 247 base::TimeTicks rtp_timestamp_in_ticks; |
| 320 | 248 |
| 321 // Compute the time offset_in_ticks based on the incoming_rtp_timestamp_. | 249 // Compute the time offset_in_ticks based on the incoming_rtp_timestamp_. |
| 322 if (time_offset_counter_ == 0) { | 250 if (time_offset_counter_ == 0) { |
| 323 // Check for received RTCP to sync the stream play it out asap. | 251 // Check for received RTCP to sync the stream play it out asap. |
| 324 if (rtcp_->RtpTimestampInSenderTime(kVideoFrequency, | 252 if (rtcp_.RtpTimestampInSenderTime(kVideoFrequency, |
| 325 incoming_rtp_timestamp_, | 253 incoming_rtp_timestamp_, |
| 326 &rtp_timestamp_in_ticks)) { | 254 &rtp_timestamp_in_ticks)) { |
| 327 | |
| 328 ++time_offset_counter_; | 255 ++time_offset_counter_; |
| 329 } | 256 } |
| 330 return now; | |
| 331 } else if (time_incoming_packet_updated_) { | 257 } else if (time_incoming_packet_updated_) { |
| 332 if (rtcp_->RtpTimestampInSenderTime(kVideoFrequency, | 258 if (rtcp_.RtpTimestampInSenderTime(kVideoFrequency, |
| 333 incoming_rtp_timestamp_, | 259 incoming_rtp_timestamp_, |
| 334 &rtp_timestamp_in_ticks)) { | 260 &rtp_timestamp_in_ticks)) { |
| 335 // Time to update the time_offset. | 261 // Time to update the time_offset. |
| 336 base::TimeDelta time_offset = | 262 base::TimeDelta time_offset = |
| 337 time_incoming_packet_ - rtp_timestamp_in_ticks; | 263 time_incoming_packet_ - rtp_timestamp_in_ticks; |
| 338 // Taking the minimum of the first kTimeOffsetMaxCounter values. We are | 264 // Taking the minimum of the first kTimeOffsetMaxCounter values. We are |
| 339 // assuming that we are looking for the minimum offset, which will occur | 265 // assuming that we are looking for the minimum offset, which will occur |
| 340 // when network conditions are the best. This should occur at least once | 266 // when network conditions are the best. This should occur at least once |
| 341 // within the first kTimeOffsetMaxCounter samples. Any drift should be | 267 // within the first kTimeOffsetMaxCounter samples. Any drift should be |
| 342 // very slow, and negligible for this use case. | 268 // very slow, and negligible for this use case. |
| 343 if (time_offset_counter_ == 1) | 269 if (time_offset_counter_ == 1) |
| 344 time_offset_ = time_offset; | 270 time_offset_ = time_offset; |
| 345 else if (time_offset_counter_ < kTimeOffsetMaxCounter) { | 271 else if (time_offset_counter_ < kTimeOffsetMaxCounter) { |
| 346 time_offset_ = std::min(time_offset_, time_offset); | 272 time_offset_ = std::min(time_offset_, time_offset); |
| 347 } | 273 } |
| 348 ++time_offset_counter_; | 274 if (time_offset_counter_ < kTimeOffsetMaxCounter) |
| 275 ++time_offset_counter_; |
| 349 } | 276 } |
| 350 } | 277 } |
| 351 // Reset |time_incoming_packet_updated_| to enable a future measurement. | 278 // Reset |time_incoming_packet_updated_| to enable a future measurement. |
| 352 time_incoming_packet_updated_ = false; | 279 time_incoming_packet_updated_ = false; |
| 353 // Compute the actual rtp_timestamp_in_ticks based on the current timestamp. | 280 // Compute the actual rtp_timestamp_in_ticks based on the current timestamp. |
| 354 if (!rtcp_->RtpTimestampInSenderTime( | 281 if (!rtcp_.RtpTimestampInSenderTime( |
| 355 kVideoFrequency, rtp_timestamp, &rtp_timestamp_in_ticks)) { | 282 kVideoFrequency, rtp_timestamp, &rtp_timestamp_in_ticks)) { |
| 356 // This can fail if we have not received any RTCP packets in a long time. | 283 // This can fail if we have not received any RTCP packets in a long time. |
| 357 return now; | 284 // BUG: These calculations are a placeholder, and to be revisited in a |
| 285 // soon-upcoming change. http://crbug.com/356942 |
| 286 const int frequency_khz = kVideoFrequency / 1000; |
| 287 const base::TimeDelta delta_based_on_rtp_timestamps = |
| 288 base::TimeDelta::FromMilliseconds( |
| 289 static_cast<int32>(rtp_timestamp - incoming_rtp_timestamp_) / |
| 290 frequency_khz); |
| 291 return time_incoming_packet_ + delta_based_on_rtp_timestamps; |
| 358 } | 292 } |
| 359 | 293 |
| 360 base::TimeTicks render_time = | 294 base::TimeTicks render_time = |
| 361 rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_; | 295 rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_; |
| 296 // TODO(miu): This is broken since this "getter" method may be called on |
| 297 // frames received out-of-order, which means the playout times for earlier |
| 298 // frames will be computed incorrectly. |
| 299 #if 0 |
| 362 if (last_render_time_ > render_time) | 300 if (last_render_time_ > render_time) |
| 363 render_time = last_render_time_; | 301 render_time = last_render_time_; |
| 364 last_render_time_ = render_time; | 302 last_render_time_ = render_time; |
| 303 #endif |
| 304 |
| 365 return render_time; | 305 return render_time; |
| 366 } | 306 } |
| 367 | 307 |
| 368 void VideoReceiver::IncomingPacket(scoped_ptr<Packet> packet) { | 308 void VideoReceiver::IncomingPacket(scoped_ptr<Packet> packet) { |
| 369 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 309 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 370 if (Rtcp::IsRtcpPacket(&packet->front(), packet->size())) { | 310 if (Rtcp::IsRtcpPacket(&packet->front(), packet->size())) { |
| 371 rtcp_->IncomingRtcpPacket(&packet->front(), packet->size()); | 311 rtcp_.IncomingRtcpPacket(&packet->front(), packet->size()); |
| 372 } else { | 312 } else { |
| 373 ReceivedPacket(&packet->front(), packet->size()); | 313 ReceivedPacket(&packet->front(), packet->size()); |
| 374 } | 314 } |
| 375 } | 315 } |
| 376 | 316 |
| 377 void VideoReceiver::OnReceivedPayloadData(const uint8* payload_data, | 317 void VideoReceiver::OnReceivedPayloadData(const uint8* payload_data, |
| 378 size_t payload_size, | 318 size_t payload_size, |
| 379 const RtpCastHeader& rtp_header) { | 319 const RtpCastHeader& rtp_header) { |
| 380 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 320 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 381 | 321 |
| (...skipping 18 matching lines...) Expand all Loading... |
| 400 cast_environment_->Logging()->InsertPacketEvent( | 340 cast_environment_->Logging()->InsertPacketEvent( |
| 401 now, | 341 now, |
| 402 kVideoPacketReceived, | 342 kVideoPacketReceived, |
| 403 rtp_header.webrtc.header.timestamp, | 343 rtp_header.webrtc.header.timestamp, |
| 404 rtp_header.frame_id, | 344 rtp_header.frame_id, |
| 405 rtp_header.packet_id, | 345 rtp_header.packet_id, |
| 406 rtp_header.max_packet_id, | 346 rtp_header.max_packet_id, |
| 407 payload_size); | 347 payload_size); |
| 408 | 348 |
| 409 bool duplicate = false; | 349 bool duplicate = false; |
| 410 bool complete = | 350 const bool complete = |
| 411 framer_->InsertPacket(payload_data, payload_size, rtp_header, &duplicate); | 351 framer_.InsertPacket(payload_data, payload_size, rtp_header, &duplicate); |
| 412 | |
| 413 if (duplicate) { | 352 if (duplicate) { |
| 414 cast_environment_->Logging()->InsertPacketEvent( | 353 cast_environment_->Logging()->InsertPacketEvent( |
| 415 now, | 354 now, |
| 416 kDuplicateVideoPacketReceived, | 355 kDuplicateVideoPacketReceived, |
| 417 rtp_header.webrtc.header.timestamp, | 356 rtp_header.webrtc.header.timestamp, |
| 418 rtp_header.frame_id, | 357 rtp_header.frame_id, |
| 419 rtp_header.packet_id, | 358 rtp_header.packet_id, |
| 420 rtp_header.max_packet_id, | 359 rtp_header.max_packet_id, |
| 421 payload_size); | 360 payload_size); |
| 422 // Duplicate packets are ignored. | 361 // Duplicate packets are ignored. |
| 423 return; | 362 return; |
| 424 } | 363 } |
| 425 if (!complete) | 364 if (!complete) |
| 426 return; // Video frame not complete; wait for more packets. | 365 return; // Video frame not complete; wait for more packets. |
| 427 if (queued_encoded_callbacks_.empty()) | |
| 428 return; // No pending callback. | |
| 429 | 366 |
| 430 VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front(); | 367 EmitAvailableEncodedFrames(); |
| 431 queued_encoded_callbacks_.pop_front(); | |
| 432 cast_environment_->PostTask(CastEnvironment::MAIN, | |
| 433 FROM_HERE, | |
| 434 base::Bind(&VideoReceiver::GetEncodedVideoFrame, | |
| 435 weak_factory_.GetWeakPtr(), | |
| 436 callback)); | |
| 437 } | 368 } |
| 438 | 369 |
| 439 // Send a cast feedback message. Actual message created in the framer (cast | 370 // Send a cast feedback message. Actual message created in the framer (cast |
| 440 // message builder). | 371 // message builder). |
| 441 void VideoReceiver::CastFeedback(const RtcpCastMessage& cast_message) { | 372 void VideoReceiver::CastFeedback(const RtcpCastMessage& cast_message) { |
| 442 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 373 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 443 | 374 |
| 444 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); | 375 base::TimeTicks now = cast_environment_->Clock()->NowTicks(); |
| 445 RtpTimestamp rtp_timestamp = | 376 RtpTimestamp rtp_timestamp = |
| 446 frame_id_to_rtp_timestamp_[cast_message.ack_frame_id_ & 0xff]; | 377 frame_id_to_rtp_timestamp_[cast_message.ack_frame_id_ & 0xff]; |
| 447 cast_environment_->Logging()->InsertFrameEvent( | 378 cast_environment_->Logging()->InsertFrameEvent( |
| 448 now, kVideoAckSent, rtp_timestamp, cast_message.ack_frame_id_); | 379 now, kVideoAckSent, rtp_timestamp, cast_message.ack_frame_id_); |
| 449 | 380 |
| 450 rtcp_->SendRtcpFromRtpReceiver(&cast_message, &event_subscriber_); | 381 rtcp_.SendRtcpFromRtpReceiver(&cast_message, &event_subscriber_); |
| 451 } | 382 } |
| 452 | 383 |
| 453 // Cast messages should be sent within a maximum interval. Schedule a call | 384 // Cast messages should be sent within a maximum interval. Schedule a call |
| 454 // if not triggered elsewhere, e.g. by the cast message_builder. | 385 // if not triggered elsewhere, e.g. by the cast message_builder. |
| 455 void VideoReceiver::ScheduleNextCastMessage() { | 386 void VideoReceiver::ScheduleNextCastMessage() { |
| 456 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 387 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 457 base::TimeTicks send_time; | 388 base::TimeTicks send_time; |
| 458 framer_->TimeToSendNextCastMessage(&send_time); | 389 framer_.TimeToSendNextCastMessage(&send_time); |
| 459 | |
| 460 base::TimeDelta time_to_send = | 390 base::TimeDelta time_to_send = |
| 461 send_time - cast_environment_->Clock()->NowTicks(); | 391 send_time - cast_environment_->Clock()->NowTicks(); |
| 462 time_to_send = std::max( | 392 time_to_send = std::max( |
| 463 time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); | 393 time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); |
| 464 cast_environment_->PostDelayedTask( | 394 cast_environment_->PostDelayedTask( |
| 465 CastEnvironment::MAIN, | 395 CastEnvironment::MAIN, |
| 466 FROM_HERE, | 396 FROM_HERE, |
| 467 base::Bind(&VideoReceiver::SendNextCastMessage, | 397 base::Bind(&VideoReceiver::SendNextCastMessage, |
| 468 weak_factory_.GetWeakPtr()), | 398 weak_factory_.GetWeakPtr()), |
| 469 time_to_send); | 399 time_to_send); |
| 470 } | 400 } |
| 471 | 401 |
| 472 void VideoReceiver::SendNextCastMessage() { | 402 void VideoReceiver::SendNextCastMessage() { |
| 473 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 403 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 474 framer_->SendCastMessage(); // Will only send a message if it is time. | 404 framer_.SendCastMessage(); // Will only send a message if it is time. |
| 475 ScheduleNextCastMessage(); | 405 ScheduleNextCastMessage(); |
| 476 } | 406 } |
| 477 | 407 |
| 478 // Schedule the next RTCP report to be sent back to the sender. | |
| 479 void VideoReceiver::ScheduleNextRtcpReport() { | 408 void VideoReceiver::ScheduleNextRtcpReport() { |
| 480 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 409 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 481 base::TimeDelta time_to_next = rtcp_->TimeToSendNextRtcpReport() - | 410 base::TimeDelta time_to_next = rtcp_.TimeToSendNextRtcpReport() - |
| 482 cast_environment_->Clock()->NowTicks(); | 411 cast_environment_->Clock()->NowTicks(); |
| 483 | 412 |
| 484 time_to_next = std::max( | 413 time_to_next = std::max( |
| 485 time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); | 414 time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); |
| 486 | 415 |
| 487 cast_environment_->PostDelayedTask( | 416 cast_environment_->PostDelayedTask( |
| 488 CastEnvironment::MAIN, | 417 CastEnvironment::MAIN, |
| 489 FROM_HERE, | 418 FROM_HERE, |
| 490 base::Bind(&VideoReceiver::SendNextRtcpReport, | 419 base::Bind(&VideoReceiver::SendNextRtcpReport, |
| 491 weak_factory_.GetWeakPtr()), | 420 weak_factory_.GetWeakPtr()), |
| 492 time_to_next); | 421 time_to_next); |
| 493 } | 422 } |
| 494 | 423 |
| 495 void VideoReceiver::SendNextRtcpReport() { | 424 void VideoReceiver::SendNextRtcpReport() { |
| 496 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); | 425 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); |
| 497 rtcp_->SendRtcpFromRtpReceiver(NULL, NULL); | 426 rtcp_.SendRtcpFromRtpReceiver(NULL, NULL); |
| 498 ScheduleNextRtcpReport(); | 427 ScheduleNextRtcpReport(); |
| 499 } | 428 } |
| 500 | 429 |
| 501 void VideoReceiver::UpdateTargetDelay() { | |
| 502 NOTIMPLEMENTED(); | |
| 503 rtcp_->SetTargetDelay(target_delay_delta_); | |
| 504 target_delay_cb_.Run(target_delay_delta_); | |
| 505 } | |
| 506 | |
| 507 } // namespace cast | 430 } // namespace cast |
| 508 } // namespace media | 431 } // namespace media |
| OLD | NEW |