OLD | NEW |
| (Empty) |
1 // Copyright 2016 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 #include "media/remoting/remote_renderer_impl.h" | |
6 | |
7 #include <algorithm> | |
8 #include <limits> | |
9 #include <utility> | |
10 | |
11 #include "base/bind.h" | |
12 #include "base/bind_helpers.h" | |
13 #include "base/callback_helpers.h" | |
14 #include "base/memory/ptr_util.h" | |
15 #include "base/message_loop/message_loop.h" | |
16 #include "base/numerics/safe_math.h" | |
17 #include "base/threading/thread_task_runner_handle.h" | |
18 #include "base/time/time.h" | |
19 #include "media/base/bind_to_current_loop.h" | |
20 #include "media/base/demuxer_stream_provider.h" | |
21 #include "media/remoting/remote_demuxer_stream_adapter.h" | |
22 #include "media/remoting/remoting_renderer_controller.h" | |
23 #include "media/remoting/rpc/proto_enum_utils.h" | |
24 #include "media/remoting/rpc/proto_utils.h" | |
25 | |
26 namespace { | |
27 | |
28 // The moving time window to track the media time and statistics updates. | |
29 constexpr base::TimeDelta kTrackingWindow = base::TimeDelta::FromSeconds(5); | |
30 | |
31 // The allowed delay for the remoting playback. When continuously exceeds this | |
32 // limit for |kPlaybackDelayCountThreshold| times, the user experience is likely | |
33 // poor and the controller is notified. | |
34 constexpr base::TimeDelta kMediaPlaybackDelayThreshold = | |
35 base::TimeDelta::FromMilliseconds(750); | |
36 constexpr int kPlaybackDelayCountThreshold = 3; | |
37 | |
38 // The allowed percentage of the number of video frames dropped vs. the number | |
39 // of the video frames decoded. When exceeds this limit, the user experience is | |
40 // likely poor and the controller is notified. | |
41 constexpr int kMaxNumVideoFramesDroppedPercentage = 3; | |
42 | |
43 // The time period to allow receiver get stable after playback rate change or | |
44 // Flush(). | |
45 constexpr base::TimeDelta kStabilizationPeriod = | |
46 base::TimeDelta::FromSeconds(2); | |
47 | |
48 // The amount of time between polling the DemuxerStreamAdapters to measure their | |
49 // data flow rates for metrics. | |
50 constexpr base::TimeDelta kDataFlowPollPeriod = | |
51 base::TimeDelta::FromSeconds(10); | |
52 | |
53 } // namespace | |
54 | |
55 namespace media { | |
56 | |
57 RemoteRendererImpl::RemoteRendererImpl( | |
58 scoped_refptr<base::SingleThreadTaskRunner> media_task_runner, | |
59 const base::WeakPtr<RemotingRendererController>& | |
60 remoting_renderer_controller, | |
61 VideoRendererSink* video_renderer_sink) | |
62 : state_(STATE_UNINITIALIZED), | |
63 main_task_runner_(base::ThreadTaskRunnerHandle::Get()), | |
64 media_task_runner_(std::move(media_task_runner)), | |
65 demuxer_stream_provider_(nullptr), | |
66 client_(nullptr), | |
67 remoting_renderer_controller_(remoting_renderer_controller), | |
68 rpc_broker_(remoting_renderer_controller_->GetRpcBroker()), | |
69 rpc_handle_(rpc_broker_->GetUniqueHandle()), | |
70 remote_renderer_handle_(remoting::kInvalidHandle), | |
71 video_renderer_sink_(video_renderer_sink), | |
72 weak_factory_(this) { | |
73 VLOG(2) << __func__; | |
74 // The constructor is running on the main thread. | |
75 DCHECK(remoting_renderer_controller_); | |
76 remoting_renderer_controller_->SetShowInterstitialCallback( | |
77 base::Bind(&RemoteRendererImpl::RequestUpdateInterstitialOnMainThread, | |
78 media_task_runner_, weak_factory_.GetWeakPtr())); | |
79 | |
80 const remoting::RpcBroker::ReceiveMessageCallback receive_callback = | |
81 base::Bind(&RemoteRendererImpl::OnMessageReceivedOnMainThread, | |
82 media_task_runner_, weak_factory_.GetWeakPtr()); | |
83 rpc_broker_->RegisterMessageReceiverCallback(rpc_handle_, receive_callback); | |
84 } | |
85 | |
86 RemoteRendererImpl::~RemoteRendererImpl() { | |
87 VLOG(2) << __func__; | |
88 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
89 | |
90 UpdateInterstitial(interstitial_background_, canvas_size_, | |
91 RemotingInterstitialType::BETWEEN_SESSIONS); | |
92 | |
93 // Post task on main thread to unset the interstial callback. | |
94 main_task_runner_->PostTask( | |
95 FROM_HERE, | |
96 base::Bind(&RemotingRendererController::SetShowInterstitialCallback, | |
97 remoting_renderer_controller_, | |
98 RemotingRendererController::ShowInterstitialCallback())); | |
99 | |
100 // Post task on main thread to unregister message receiver. | |
101 main_task_runner_->PostTask( | |
102 FROM_HERE, | |
103 base::Bind(&remoting::RpcBroker::UnregisterMessageReceiverCallback, | |
104 rpc_broker_, rpc_handle_)); | |
105 } | |
106 | |
107 void RemoteRendererImpl::Initialize( | |
108 DemuxerStreamProvider* demuxer_stream_provider, | |
109 media::RendererClient* client, | |
110 const PipelineStatusCB& init_cb) { | |
111 VLOG(2) << __func__; | |
112 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
113 DCHECK(demuxer_stream_provider); | |
114 DCHECK(client); | |
115 | |
116 if (state_ != STATE_UNINITIALIZED) { | |
117 media_task_runner_->PostTask( | |
118 FROM_HERE, base::Bind(init_cb, PIPELINE_ERROR_INVALID_STATE)); | |
119 return; | |
120 } | |
121 | |
122 demuxer_stream_provider_ = demuxer_stream_provider; | |
123 client_ = client; | |
124 init_workflow_done_callback_ = init_cb; | |
125 | |
126 state_ = STATE_CREATE_PIPE; | |
127 // Create audio mojo data pipe handles if audio is available. | |
128 ::media::DemuxerStream* audio_demuxer_stream = | |
129 demuxer_stream_provider_->GetStream(::media::DemuxerStream::AUDIO); | |
130 std::unique_ptr<mojo::DataPipe> audio_data_pipe; | |
131 if (audio_demuxer_stream) { | |
132 audio_data_pipe = base::WrapUnique(remoting::CreateDataPipe()); | |
133 } | |
134 | |
135 // Create video mojo data pipe handles if video is available. | |
136 ::media::DemuxerStream* video_demuxer_stream = | |
137 demuxer_stream_provider_->GetStream(::media::DemuxerStream::VIDEO); | |
138 std::unique_ptr<mojo::DataPipe> video_data_pipe; | |
139 if (video_demuxer_stream) { | |
140 video_data_pipe = base::WrapUnique(remoting::CreateDataPipe()); | |
141 } | |
142 | |
143 // Establish remoting data pipe connection using main thread. | |
144 const RemotingSourceImpl::DataPipeStartCallback data_pipe_callback = | |
145 base::Bind(&RemoteRendererImpl::OnDataPipeCreatedOnMainThread, | |
146 media_task_runner_, weak_factory_.GetWeakPtr(), rpc_broker_); | |
147 main_task_runner_->PostTask( | |
148 FROM_HERE, | |
149 base::Bind(&RemotingRendererController::StartDataPipe, | |
150 remoting_renderer_controller_, base::Passed(&audio_data_pipe), | |
151 base::Passed(&video_data_pipe), data_pipe_callback)); | |
152 } | |
153 | |
154 void RemoteRendererImpl::SetCdm(CdmContext* cdm_context, | |
155 const CdmAttachedCB& cdm_attached_cb) { | |
156 VLOG(2) << __func__ << " cdm_id:" << cdm_context->GetCdmId(); | |
157 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
158 | |
159 // TODO(erickung): add implementation once Remote CDM implementation is done. | |
160 // Right now it returns callback immediately. | |
161 if (!cdm_attached_cb.is_null()) { | |
162 cdm_attached_cb.Run(false); | |
163 } | |
164 } | |
165 | |
166 void RemoteRendererImpl::Flush(const base::Closure& flush_cb) { | |
167 VLOG(2) << __func__; | |
168 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
169 DCHECK(flush_cb_.is_null()); | |
170 | |
171 if (state_ != STATE_PLAYING) { | |
172 DCHECK_EQ(state_, STATE_ERROR); | |
173 // In the error state, this renderer will be shut down shortly. To prevent | |
174 // breaking the pipeline impl, just run the done callback (interface | |
175 // requirement). | |
176 media_task_runner_->PostTask(FROM_HERE, flush_cb); | |
177 return; | |
178 } | |
179 | |
180 state_ = STATE_FLUSHING; | |
181 base::Optional<uint32_t> flush_audio_count; | |
182 if (audio_demuxer_stream_adapter_) | |
183 flush_audio_count = audio_demuxer_stream_adapter_->SignalFlush(true); | |
184 base::Optional<uint32_t> flush_video_count; | |
185 if (video_demuxer_stream_adapter_) | |
186 flush_video_count = video_demuxer_stream_adapter_->SignalFlush(true); | |
187 // Makes sure flush count is valid if stream is available or both audio and | |
188 // video agrees on the same flushing state. | |
189 if ((audio_demuxer_stream_adapter_ && !flush_audio_count.has_value()) || | |
190 (video_demuxer_stream_adapter_ && !flush_video_count.has_value()) || | |
191 (audio_demuxer_stream_adapter_ && video_demuxer_stream_adapter_ && | |
192 flush_audio_count.has_value() != flush_video_count.has_value())) { | |
193 VLOG(1) << "Ignoring flush request while under flushing operation"; | |
194 return; | |
195 } | |
196 | |
197 flush_cb_ = flush_cb; | |
198 | |
199 // Issues RPC_R_FLUSHUNTIL RPC message. | |
200 std::unique_ptr<remoting::pb::RpcMessage> rpc(new remoting::pb::RpcMessage()); | |
201 rpc->set_handle(remote_renderer_handle_); | |
202 rpc->set_proc(remoting::pb::RpcMessage::RPC_R_FLUSHUNTIL); | |
203 remoting::pb::RendererFlushUntil* message = | |
204 rpc->mutable_renderer_flushuntil_rpc(); | |
205 if (flush_audio_count.has_value()) | |
206 message->set_audio_count(*flush_audio_count); | |
207 if (flush_video_count.has_value()) | |
208 message->set_video_count(*flush_video_count); | |
209 message->set_callback_handle(rpc_handle_); | |
210 VLOG(2) << __func__ << ": Sending RPC_R_FLUSHUNTIL to " << rpc->handle() | |
211 << " with audio_count=" << message->audio_count() | |
212 << ", video_count=" << message->video_count() | |
213 << ", callback_handle=" << message->callback_handle(); | |
214 SendRpcToRemote(std::move(rpc)); | |
215 } | |
216 | |
217 void RemoteRendererImpl::StartPlayingFrom(base::TimeDelta time) { | |
218 VLOG(2) << __func__ << ": " << time.InMicroseconds(); | |
219 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
220 | |
221 if (state_ != STATE_PLAYING) { | |
222 DCHECK_EQ(state_, STATE_ERROR); | |
223 return; | |
224 } | |
225 | |
226 // Issues RPC_R_STARTPLAYINGFROM RPC message. | |
227 std::unique_ptr<remoting::pb::RpcMessage> rpc(new remoting::pb::RpcMessage()); | |
228 rpc->set_handle(remote_renderer_handle_); | |
229 rpc->set_proc(remoting::pb::RpcMessage::RPC_R_STARTPLAYINGFROM); | |
230 rpc->set_integer64_value(time.InMicroseconds()); | |
231 VLOG(2) << __func__ << ": Sending RPC_R_STARTPLAYINGFROM to " << rpc->handle() | |
232 << " with time_usec=" << rpc->integer64_value(); | |
233 SendRpcToRemote(std::move(rpc)); | |
234 | |
235 { | |
236 base::AutoLock auto_lock(time_lock_); | |
237 current_media_time_ = time; | |
238 } | |
239 ResetMeasurements(); | |
240 } | |
241 | |
242 void RemoteRendererImpl::SetPlaybackRate(double playback_rate) { | |
243 VLOG(2) << __func__ << ": " << playback_rate; | |
244 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
245 | |
246 if (state_ != STATE_FLUSHING && state_ != STATE_PLAYING) { | |
247 DCHECK_EQ(state_, STATE_ERROR); | |
248 return; | |
249 } | |
250 | |
251 // Issues RPC_R_SETPLAYBACKRATE RPC message. | |
252 std::unique_ptr<remoting::pb::RpcMessage> rpc(new remoting::pb::RpcMessage()); | |
253 rpc->set_handle(remote_renderer_handle_); | |
254 rpc->set_proc(remoting::pb::RpcMessage::RPC_R_SETPLAYBACKRATE); | |
255 rpc->set_double_value(playback_rate); | |
256 VLOG(2) << __func__ << ": Sending RPC_R_SETPLAYBACKRATE to " << rpc->handle() | |
257 << " with rate=" << rpc->double_value(); | |
258 SendRpcToRemote(std::move(rpc)); | |
259 playback_rate_ = playback_rate; | |
260 ResetMeasurements(); | |
261 } | |
262 | |
263 void RemoteRendererImpl::SetVolume(float volume) { | |
264 VLOG(2) << __func__ << ": " << volume; | |
265 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
266 | |
267 if (state_ != STATE_FLUSHING && state_ != STATE_PLAYING) { | |
268 DCHECK_EQ(state_, STATE_ERROR); | |
269 return; | |
270 } | |
271 | |
272 // Issues RPC_R_SETVOLUME RPC message. | |
273 std::unique_ptr<remoting::pb::RpcMessage> rpc(new remoting::pb::RpcMessage()); | |
274 rpc->set_handle(remote_renderer_handle_); | |
275 rpc->set_proc(remoting::pb::RpcMessage::RPC_R_SETVOLUME); | |
276 rpc->set_double_value(volume); | |
277 VLOG(2) << __func__ << ": Sending RPC_R_SETVOLUME to " << rpc->handle() | |
278 << " with volume=" << rpc->double_value(); | |
279 SendRpcToRemote(std::move(rpc)); | |
280 } | |
281 | |
282 base::TimeDelta RemoteRendererImpl::GetMediaTime() { | |
283 // No BelongsToCurrentThread() checking because this can be called from other | |
284 // threads. | |
285 // TODO(erickung): Interpolate current media time using local system time. | |
286 // Current receiver is to update |current_media_time_| every 250ms. But it | |
287 // needs to lower the update frequency in order to reduce network usage. Hence | |
288 // the interpolation is needed after receiver implementation is changed. | |
289 base::AutoLock auto_lock(time_lock_); | |
290 return current_media_time_; | |
291 } | |
292 | |
293 // static | |
294 void RemoteRendererImpl::OnDataPipeCreatedOnMainThread( | |
295 scoped_refptr<base::SingleThreadTaskRunner> media_task_runner, | |
296 base::WeakPtr<RemoteRendererImpl> self, | |
297 base::WeakPtr<remoting::RpcBroker> rpc_broker, | |
298 mojom::RemotingDataStreamSenderPtrInfo audio, | |
299 mojom::RemotingDataStreamSenderPtrInfo video, | |
300 mojo::ScopedDataPipeProducerHandle audio_handle, | |
301 mojo::ScopedDataPipeProducerHandle video_handle) { | |
302 media_task_runner->PostTask( | |
303 FROM_HERE, | |
304 base::Bind( | |
305 &RemoteRendererImpl::OnDataPipeCreated, self, base::Passed(&audio), | |
306 base::Passed(&video), base::Passed(&audio_handle), | |
307 base::Passed(&video_handle), | |
308 rpc_broker ? rpc_broker->GetUniqueHandle() : remoting::kInvalidHandle, | |
309 rpc_broker ? rpc_broker->GetUniqueHandle() | |
310 : remoting::kInvalidHandle)); | |
311 } | |
312 | |
313 void RemoteRendererImpl::OnDataPipeCreated( | |
314 mojom::RemotingDataStreamSenderPtrInfo audio, | |
315 mojom::RemotingDataStreamSenderPtrInfo video, | |
316 mojo::ScopedDataPipeProducerHandle audio_handle, | |
317 mojo::ScopedDataPipeProducerHandle video_handle, | |
318 int audio_rpc_handle, | |
319 int video_rpc_handle) { | |
320 VLOG(2) << __func__; | |
321 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
322 DCHECK(!init_workflow_done_callback_.is_null()); | |
323 | |
324 if (state_ == STATE_ERROR) | |
325 return; // Abort because something went wrong in the meantime. | |
326 DCHECK_EQ(state_, STATE_CREATE_PIPE); | |
327 | |
328 // Create audio demuxer stream adapter if audio is available. | |
329 ::media::DemuxerStream* audio_demuxer_stream = | |
330 demuxer_stream_provider_->GetStream(::media::DemuxerStream::AUDIO); | |
331 if (audio_demuxer_stream && audio.is_valid() && audio_handle.is_valid() && | |
332 audio_rpc_handle != remoting::kInvalidHandle) { | |
333 VLOG(2) << "Initialize audio"; | |
334 audio_demuxer_stream_adapter_.reset( | |
335 new remoting::RemoteDemuxerStreamAdapter( | |
336 main_task_runner_, media_task_runner_, "audio", | |
337 audio_demuxer_stream, rpc_broker_, audio_rpc_handle, | |
338 std::move(audio), std::move(audio_handle), | |
339 base::Bind(&RemoteRendererImpl::OnFatalError, | |
340 base::Unretained(this)))); | |
341 } | |
342 | |
343 // Create video demuxer stream adapter if video is available. | |
344 ::media::DemuxerStream* video_demuxer_stream = | |
345 demuxer_stream_provider_->GetStream(::media::DemuxerStream::VIDEO); | |
346 if (video_demuxer_stream && video.is_valid() && video_handle.is_valid() && | |
347 video_rpc_handle != remoting::kInvalidHandle) { | |
348 VLOG(2) << "Initialize video"; | |
349 video_demuxer_stream_adapter_.reset( | |
350 new remoting::RemoteDemuxerStreamAdapter( | |
351 main_task_runner_, media_task_runner_, "video", | |
352 video_demuxer_stream, rpc_broker_, video_rpc_handle, | |
353 std::move(video), std::move(video_handle), | |
354 base::Bind(&RemoteRendererImpl::OnFatalError, | |
355 base::Unretained(this)))); | |
356 } | |
357 | |
358 // Checks if data pipe is created successfully. | |
359 if (!audio_demuxer_stream_adapter_ && !video_demuxer_stream_adapter_) { | |
360 OnFatalError(remoting::DATA_PIPE_CREATE_ERROR); | |
361 return; | |
362 } | |
363 | |
364 state_ = STATE_ACQUIRING; | |
365 // Issues RPC_ACQUIRE_RENDERER RPC message. | |
366 std::unique_ptr<remoting::pb::RpcMessage> rpc(new remoting::pb::RpcMessage()); | |
367 rpc->set_handle(remoting::kReceiverHandle); | |
368 rpc->set_proc(remoting::pb::RpcMessage::RPC_ACQUIRE_RENDERER); | |
369 rpc->set_integer_value(rpc_handle_); | |
370 VLOG(2) << __func__ << ": Sending RPC_ACQUIRE_RENDERER to " << rpc->handle() | |
371 << " with rpc_handle=" << rpc->integer_value(); | |
372 SendRpcToRemote(std::move(rpc)); | |
373 } | |
374 | |
375 // static | |
376 void RemoteRendererImpl::OnMessageReceivedOnMainThread( | |
377 scoped_refptr<base::SingleThreadTaskRunner> media_task_runner, | |
378 base::WeakPtr<RemoteRendererImpl> self, | |
379 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
380 media_task_runner->PostTask( | |
381 FROM_HERE, base::Bind(&RemoteRendererImpl::OnReceivedRpc, self, | |
382 base::Passed(&message))); | |
383 } | |
384 | |
385 void RemoteRendererImpl::OnReceivedRpc( | |
386 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
387 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
388 DCHECK(message); | |
389 switch (message->proc()) { | |
390 case remoting::pb::RpcMessage::RPC_ACQUIRE_RENDERER_DONE: | |
391 AcquireRendererDone(std::move(message)); | |
392 break; | |
393 case remoting::pb::RpcMessage::RPC_R_INITIALIZE_CALLBACK: | |
394 InitializeCallback(std::move(message)); | |
395 break; | |
396 case remoting::pb::RpcMessage::RPC_R_FLUSHUNTIL_CALLBACK: | |
397 FlushUntilCallback(); | |
398 break; | |
399 case remoting::pb::RpcMessage::RPC_R_SETCDM_CALLBACK: | |
400 SetCdmCallback(std::move(message)); | |
401 break; | |
402 case remoting::pb::RpcMessage::RPC_RC_ONTIMEUPDATE: | |
403 OnTimeUpdate(std::move(message)); | |
404 break; | |
405 case remoting::pb::RpcMessage::RPC_RC_ONBUFFERINGSTATECHANGE: | |
406 OnBufferingStateChange(std::move(message)); | |
407 break; | |
408 case remoting::pb::RpcMessage::RPC_RC_ONENDED: | |
409 VLOG(2) << __func__ << ": Received RPC_RC_ONENDED."; | |
410 client_->OnEnded(); | |
411 break; | |
412 case remoting::pb::RpcMessage::RPC_RC_ONERROR: | |
413 VLOG(2) << __func__ << ": Received RPC_RC_ONERROR."; | |
414 OnFatalError(remoting::RECEIVER_PIPELINE_ERROR); | |
415 break; | |
416 case remoting::pb::RpcMessage::RPC_RC_ONVIDEONATURALSIZECHANGE: | |
417 OnVideoNaturalSizeChange(std::move(message)); | |
418 break; | |
419 case remoting::pb::RpcMessage::RPC_RC_ONVIDEOOPACITYCHANGE: | |
420 OnVideoOpacityChange(std::move(message)); | |
421 break; | |
422 case remoting::pb::RpcMessage::RPC_RC_ONSTATISTICSUPDATE: | |
423 OnStatisticsUpdate(std::move(message)); | |
424 break; | |
425 case remoting::pb::RpcMessage::RPC_RC_ONWAITINGFORDECRYPTIONKEY: | |
426 VLOG(2) << __func__ << ": Received RPC_RC_ONWAITINGFORDECRYPTIONKEY."; | |
427 client_->OnWaitingForDecryptionKey(); | |
428 break; | |
429 case remoting::pb::RpcMessage::RPC_RC_ONDURATIONCHANGE: | |
430 OnDurationChange(std::move(message)); | |
431 break; | |
432 | |
433 default: | |
434 LOG(ERROR) << "Unknown rpc: " << message->proc(); | |
435 } | |
436 } | |
437 | |
438 void RemoteRendererImpl::SendRpcToRemote( | |
439 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
440 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
441 DCHECK(main_task_runner_); | |
442 main_task_runner_->PostTask( | |
443 FROM_HERE, base::Bind(&remoting::RpcBroker::SendMessageToRemote, | |
444 rpc_broker_, base::Passed(&message))); | |
445 } | |
446 | |
447 void RemoteRendererImpl::AcquireRendererDone( | |
448 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
449 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
450 DCHECK(message); | |
451 | |
452 remote_renderer_handle_ = message->integer_value(); | |
453 VLOG(2) << __func__ | |
454 << ": Received RPC_ACQUIRE_RENDERER_DONE with remote_renderer_handle=" | |
455 << remote_renderer_handle_; | |
456 | |
457 if (state_ != STATE_ACQUIRING || init_workflow_done_callback_.is_null()) { | |
458 LOG(WARNING) << "Unexpected acquire renderer done RPC."; | |
459 OnFatalError(remoting::PEERS_OUT_OF_SYNC); | |
460 return; | |
461 } | |
462 state_ = STATE_INITIALIZING; | |
463 | |
464 // Issues RPC_R_INITIALIZE RPC message to initialize renderer. | |
465 std::unique_ptr<remoting::pb::RpcMessage> rpc(new remoting::pb::RpcMessage()); | |
466 rpc->set_handle(remote_renderer_handle_); | |
467 rpc->set_proc(remoting::pb::RpcMessage::RPC_R_INITIALIZE); | |
468 remoting::pb::RendererInitialize* init = | |
469 rpc->mutable_renderer_initialize_rpc(); | |
470 init->set_client_handle(rpc_handle_); | |
471 init->set_audio_demuxer_handle( | |
472 audio_demuxer_stream_adapter_ | |
473 ? audio_demuxer_stream_adapter_->rpc_handle() | |
474 : remoting::kInvalidHandle); | |
475 init->set_video_demuxer_handle( | |
476 video_demuxer_stream_adapter_ | |
477 ? video_demuxer_stream_adapter_->rpc_handle() | |
478 : remoting::kInvalidHandle); | |
479 init->set_callback_handle(rpc_handle_); | |
480 VLOG(2) << __func__ << ": Sending RPC_R_INITIALIZE to " << rpc->handle() | |
481 << " with client_handle=" << init->client_handle() | |
482 << ", audio_demuxer_handle=" << init->audio_demuxer_handle() | |
483 << ", video_demuxer_handle=" << init->video_demuxer_handle() | |
484 << ", callback_handle=" << init->callback_handle(); | |
485 SendRpcToRemote(std::move(rpc)); | |
486 } | |
487 | |
488 void RemoteRendererImpl::InitializeCallback( | |
489 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
490 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
491 DCHECK(message); | |
492 | |
493 const bool success = message->boolean_value(); | |
494 VLOG(2) << __func__ | |
495 << ": Received RPC_R_INITIALIZE_CALLBACK with success=" << success; | |
496 | |
497 if (state_ != STATE_INITIALIZING || init_workflow_done_callback_.is_null()) { | |
498 LOG(WARNING) << "Unexpected initialize callback RPC."; | |
499 OnFatalError(remoting::PEERS_OUT_OF_SYNC); | |
500 return; | |
501 } | |
502 | |
503 if (!success) { | |
504 OnFatalError(remoting::RECEIVER_INITIALIZE_FAILED); | |
505 return; | |
506 } | |
507 | |
508 metrics_recorder_.OnRendererInitialized(); | |
509 | |
510 state_ = STATE_PLAYING; | |
511 base::ResetAndReturn(&init_workflow_done_callback_).Run(::media::PIPELINE_OK); | |
512 } | |
513 | |
514 void RemoteRendererImpl::FlushUntilCallback() { | |
515 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
516 VLOG(2) << __func__ << ": Received RPC_R_FLUSHUNTIL_CALLBACK"; | |
517 | |
518 if (state_ != STATE_FLUSHING || flush_cb_.is_null()) { | |
519 LOG(WARNING) << "Unexpected flushuntil callback RPC."; | |
520 OnFatalError(remoting::PEERS_OUT_OF_SYNC); | |
521 return; | |
522 } | |
523 | |
524 state_ = STATE_PLAYING; | |
525 if (audio_demuxer_stream_adapter_) | |
526 audio_demuxer_stream_adapter_->SignalFlush(false); | |
527 if (video_demuxer_stream_adapter_) | |
528 video_demuxer_stream_adapter_->SignalFlush(false); | |
529 base::ResetAndReturn(&flush_cb_).Run(); | |
530 ResetMeasurements(); | |
531 } | |
532 | |
533 void RemoteRendererImpl::SetCdmCallback( | |
534 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
535 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
536 DCHECK(message); | |
537 VLOG(2) << __func__ << ": Received RPC_R_SETCDM_CALLBACK with cdm_id=" | |
538 << message->renderer_set_cdm_rpc().cdm_id() << ", callback_handle=" | |
539 << message->renderer_set_cdm_rpc().callback_handle(); | |
540 // TODO(erickung): add implementation once Remote CDM implementation is done. | |
541 NOTIMPLEMENTED(); | |
542 } | |
543 | |
544 void RemoteRendererImpl::OnTimeUpdate( | |
545 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
546 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
547 DCHECK(message); | |
548 // Shutdown remoting session if receiving malformed RPC message. | |
549 if (!message->has_rendererclient_ontimeupdate_rpc()) { | |
550 VLOG(1) << __func__ << " missing required RPC message"; | |
551 OnFatalError(remoting::RPC_INVALID); | |
552 return; | |
553 } | |
554 const int64_t time_usec = | |
555 message->rendererclient_ontimeupdate_rpc().time_usec(); | |
556 const int64_t max_time_usec = | |
557 message->rendererclient_ontimeupdate_rpc().max_time_usec(); | |
558 VLOG(2) << __func__ | |
559 << ": Received RPC_RC_ONTIMEUPDATE with time_usec=" << time_usec | |
560 << ", max_time_usec=" << max_time_usec; | |
561 // Ignores invalid time, such as negative value, or time larger than max value | |
562 // (usually the time stamp that all streams are pushed into AV pipeline). | |
563 if (time_usec < 0 || max_time_usec < 0 || time_usec > max_time_usec) | |
564 return; | |
565 | |
566 { | |
567 // Updates current time information. | |
568 base::AutoLock auto_lock(time_lock_); | |
569 current_media_time_ = base::TimeDelta::FromMicroseconds(time_usec); | |
570 current_max_time_ = base::TimeDelta::FromMicroseconds(max_time_usec); | |
571 } | |
572 | |
573 metrics_recorder_.OnEvidenceOfPlayoutAtReceiver(); | |
574 OnMediaTimeUpdated(); | |
575 } | |
576 | |
577 void RemoteRendererImpl::OnBufferingStateChange( | |
578 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
579 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
580 DCHECK(message); | |
581 if (!message->has_rendererclient_onbufferingstatechange_rpc()) { | |
582 VLOG(1) << __func__ << " missing required RPC message"; | |
583 OnFatalError(remoting::RPC_INVALID); | |
584 return; | |
585 } | |
586 VLOG(2) << __func__ << ": Received RPC_RC_ONBUFFERINGSTATECHANGE with state=" | |
587 << message->rendererclient_onbufferingstatechange_rpc().state(); | |
588 base::Optional<::media::BufferingState> state = | |
589 remoting::ToMediaBufferingState( | |
590 message->rendererclient_onbufferingstatechange_rpc().state()); | |
591 if (!state.has_value()) | |
592 return; | |
593 client_->OnBufferingStateChange(state.value()); | |
594 } | |
595 | |
596 void RemoteRendererImpl::OnVideoNaturalSizeChange( | |
597 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
598 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
599 DCHECK(message); | |
600 // Shutdown remoting session if receiving malformed RPC message. | |
601 if (!message->has_rendererclient_onvideonatualsizechange_rpc()) { | |
602 VLOG(1) << __func__ << " missing required RPC message"; | |
603 OnFatalError(remoting::RPC_INVALID); | |
604 return; | |
605 } | |
606 const auto& size_change = | |
607 message->rendererclient_onvideonatualsizechange_rpc(); | |
608 VLOG(2) << __func__ << ": Received RPC_RC_ONVIDEONATURALSIZECHANGE with size=" | |
609 << size_change.width() << 'x' << size_change.height(); | |
610 if (size_change.width() <= 0 || size_change.height() <= 0) | |
611 return; | |
612 client_->OnVideoNaturalSizeChange( | |
613 gfx::Size(size_change.width(), size_change.height())); | |
614 } | |
615 | |
616 void RemoteRendererImpl::OnVideoOpacityChange( | |
617 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
618 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
619 DCHECK(message); | |
620 const bool opaque = message->boolean_value(); | |
621 VLOG(2) << __func__ | |
622 << ": Received RPC_RC_ONVIDEOOPACITYCHANGE with opaque=" << opaque; | |
623 client_->OnVideoOpacityChange(opaque); | |
624 } | |
625 | |
626 void RemoteRendererImpl::OnStatisticsUpdate( | |
627 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
628 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
629 DCHECK(message); | |
630 // Shutdown remoting session if receiving malformed RPC message. | |
631 if (!message->has_rendererclient_onstatisticsupdate_rpc()) { | |
632 VLOG(1) << __func__ << " missing required RPC message"; | |
633 OnFatalError(remoting::RPC_INVALID); | |
634 return; | |
635 } | |
636 const auto& rpc_message = message->rendererclient_onstatisticsupdate_rpc(); | |
637 ::media::PipelineStatistics stats; | |
638 // Note: Each |stats| value is a delta, not the aggregate amount. | |
639 stats.audio_bytes_decoded = rpc_message.audio_bytes_decoded(); | |
640 stats.video_bytes_decoded = rpc_message.video_bytes_decoded(); | |
641 stats.video_frames_decoded = rpc_message.video_frames_decoded(); | |
642 stats.video_frames_dropped = rpc_message.video_frames_dropped(); | |
643 stats.audio_memory_usage = rpc_message.audio_memory_usage(); | |
644 stats.video_memory_usage = rpc_message.video_memory_usage(); | |
645 VLOG(2) << __func__ | |
646 << ": Received RPC_RC_ONSTATISTICSUPDATE with audio_bytes_decoded=" | |
647 << stats.audio_bytes_decoded | |
648 << ", video_bytes_decoded=" << stats.video_bytes_decoded | |
649 << ", video_frames_decoded=" << stats.video_frames_decoded | |
650 << ", video_frames_dropped=" << stats.video_frames_dropped | |
651 << ", audio_memory_usage=" << stats.audio_memory_usage | |
652 << ", video_memory_usage=" << stats.video_memory_usage; | |
653 | |
654 if (stats.audio_bytes_decoded > 0 || stats.video_frames_decoded > 0 || | |
655 stats.video_frames_dropped > 0) { | |
656 metrics_recorder_.OnEvidenceOfPlayoutAtReceiver(); | |
657 } | |
658 UpdateVideoStatsQueue(stats.video_frames_decoded, stats.video_frames_dropped); | |
659 client_->OnStatisticsUpdate(stats); | |
660 } | |
661 | |
662 void RemoteRendererImpl::OnDurationChange( | |
663 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
664 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
665 DCHECK(message); | |
666 VLOG(2) << __func__ << ": Received RPC_RC_ONDURATIONCHANGE with usec=" | |
667 << message->integer64_value(); | |
668 if (message->integer64_value() < 0) | |
669 return; | |
670 client_->OnDurationChange( | |
671 base::TimeDelta::FromMicroseconds(message->integer64_value())); | |
672 } | |
673 | |
674 void RemoteRendererImpl::OnFatalError(remoting::StopTrigger stop_trigger) { | |
675 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
676 DCHECK_NE(remoting::UNKNOWN_STOP_TRIGGER, stop_trigger); | |
677 | |
678 VLOG(2) << __func__ << " with StopTrigger " << stop_trigger; | |
679 | |
680 // If this is the first error, notify the controller. It is expected the | |
681 // controller will shut down this renderer shortly. | |
682 if (state_ != STATE_ERROR) { | |
683 state_ = STATE_ERROR; | |
684 main_task_runner_->PostTask( | |
685 FROM_HERE, base::Bind(&RemotingRendererController::OnRendererFatalError, | |
686 remoting_renderer_controller_, stop_trigger)); | |
687 } | |
688 | |
689 data_flow_poll_timer_.Stop(); | |
690 | |
691 if (!init_workflow_done_callback_.is_null()) { | |
692 base::ResetAndReturn(&init_workflow_done_callback_) | |
693 .Run(PIPELINE_ERROR_INITIALIZATION_FAILED); | |
694 return; | |
695 } | |
696 | |
697 if (!flush_cb_.is_null()) | |
698 base::ResetAndReturn(&flush_cb_).Run(); | |
699 } | |
700 | |
701 // static | |
702 void RemoteRendererImpl::RequestUpdateInterstitialOnMainThread( | |
703 scoped_refptr<base::SingleThreadTaskRunner> media_task_runner, | |
704 base::WeakPtr<RemoteRendererImpl> remote_renderer_impl, | |
705 const base::Optional<SkBitmap>& background_image, | |
706 const gfx::Size& canvas_size, | |
707 RemotingInterstitialType interstitial_type) { | |
708 media_task_runner->PostTask( | |
709 FROM_HERE, | |
710 base::Bind(&RemoteRendererImpl::UpdateInterstitial, remote_renderer_impl, | |
711 background_image, canvas_size, interstitial_type)); | |
712 } | |
713 | |
714 void RemoteRendererImpl::UpdateInterstitial( | |
715 const base::Optional<SkBitmap>& background_image, | |
716 const gfx::Size& canvas_size, | |
717 RemotingInterstitialType interstitial_type) { | |
718 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
719 if (background_image.has_value()) | |
720 interstitial_background_ = background_image.value(); | |
721 canvas_size_ = canvas_size; | |
722 PaintRemotingInterstitial(interstitial_background_, canvas_size_, | |
723 interstitial_type, video_renderer_sink_); | |
724 } | |
725 | |
726 void RemoteRendererImpl::OnMediaTimeUpdated() { | |
727 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
728 if (!flush_cb_.is_null()) | |
729 return; // Don't manage and check the queue when Flush() is on-going. | |
730 | |
731 base::TimeTicks current_time = base::TimeTicks::Now(); | |
732 if (current_time < ignore_updates_until_time_) | |
733 return; // Not stable yet. | |
734 | |
735 media_time_queue_.push_back( | |
736 std::make_pair(current_time, current_media_time_)); | |
737 base::TimeDelta window_duration = | |
738 current_time - media_time_queue_.front().first; | |
739 if (window_duration < kTrackingWindow) | |
740 return; // Not enough data to make a reliable decision. | |
741 | |
742 base::TimeDelta media_duration = | |
743 media_time_queue_.back().second - media_time_queue_.front().second; | |
744 base::TimeDelta update_duration = | |
745 (media_time_queue_.back().first - media_time_queue_.front().first) * | |
746 playback_rate_; | |
747 if ((media_duration - update_duration).magnitude() >= | |
748 kMediaPlaybackDelayThreshold) { | |
749 VLOG(1) << "Irregular playback detected: Media playback delayed." | |
750 << " media_duration = " << media_duration | |
751 << " update_duration = " << update_duration; | |
752 ++times_playback_delayed_; | |
753 if (times_playback_delayed_ == kPlaybackDelayCountThreshold) | |
754 OnFatalError(remoting::PACING_TOO_SLOWLY); | |
755 } else { | |
756 times_playback_delayed_ = 0; | |
757 } | |
758 | |
759 // Prune |media_time_queue_|. | |
760 while (media_time_queue_.back().first - media_time_queue_.front().first >= | |
761 kTrackingWindow) | |
762 media_time_queue_.pop_front(); | |
763 } | |
764 | |
765 void RemoteRendererImpl::UpdateVideoStatsQueue(int video_frames_decoded, | |
766 int video_frames_dropped) { | |
767 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
768 if (!flush_cb_.is_null()) | |
769 return; // Don't manage and check the queue when Flush() is on-going. | |
770 | |
771 if (!stats_updated_) { | |
772 if (video_frames_decoded) | |
773 stats_updated_ = true; | |
774 // Ignore the first stats since it may include the information during | |
775 // unstable period. | |
776 return; | |
777 } | |
778 | |
779 base::TimeTicks current_time = base::TimeTicks::Now(); | |
780 if (current_time < ignore_updates_until_time_) | |
781 return; // Not stable yet. | |
782 | |
783 video_stats_queue_.push_back(std::make_tuple( | |
784 current_time, video_frames_decoded, video_frames_dropped)); | |
785 sum_video_frames_decoded_ += video_frames_decoded; | |
786 sum_video_frames_dropped_ += video_frames_dropped; | |
787 base::TimeDelta window_duration = | |
788 current_time - std::get<0>(video_stats_queue_.front()); | |
789 if (window_duration < kTrackingWindow) | |
790 return; // Not enough data to make a reliable decision. | |
791 | |
792 if (sum_video_frames_decoded_ && | |
793 sum_video_frames_dropped_ * 100 > | |
794 sum_video_frames_decoded_ * kMaxNumVideoFramesDroppedPercentage) { | |
795 VLOG(1) << "Irregular playback detected: Too many video frames dropped." | |
796 << " video_frames_decoded= " << sum_video_frames_decoded_ | |
797 << " video_frames_dropped= " << sum_video_frames_dropped_; | |
798 OnFatalError(remoting::FRAME_DROP_RATE_HIGH); | |
799 } | |
800 // Prune |video_stats_queue_|. | |
801 while (std::get<0>(video_stats_queue_.back()) - | |
802 std::get<0>(video_stats_queue_.front()) >= | |
803 kTrackingWindow) { | |
804 sum_video_frames_decoded_ -= std::get<1>(video_stats_queue_.front()); | |
805 sum_video_frames_dropped_ -= std::get<2>(video_stats_queue_.front()); | |
806 video_stats_queue_.pop_front(); | |
807 } | |
808 } | |
809 | |
810 void RemoteRendererImpl::ResetMeasurements() { | |
811 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
812 media_time_queue_.clear(); | |
813 video_stats_queue_.clear(); | |
814 sum_video_frames_dropped_ = 0; | |
815 sum_video_frames_decoded_ = 0; | |
816 stats_updated_ = false; | |
817 ignore_updates_until_time_ = base::TimeTicks::Now() + kStabilizationPeriod; | |
818 | |
819 if (state_ != STATE_ERROR && | |
820 (audio_demuxer_stream_adapter_ || video_demuxer_stream_adapter_)) { | |
821 data_flow_poll_timer_.Start(FROM_HERE, kDataFlowPollPeriod, this, | |
822 &RemoteRendererImpl::MeasureAndRecordDataRates); | |
823 } | |
824 } | |
825 | |
826 void RemoteRendererImpl::MeasureAndRecordDataRates() { | |
827 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
828 | |
829 // Whenever media is first started or flushed/seeked, there is a "burst | |
830 // bufferring" period as the remote device rapidly fills its buffer before | |
831 // resuming playback. Since the goal here is to measure the sustained content | |
832 // bitrates, ignore the byte counts the first time since the last | |
833 // ResetMeasurements() call. | |
834 const base::TimeTicks current_time = base::TimeTicks::Now(); | |
835 if (current_time < ignore_updates_until_time_ + kDataFlowPollPeriod) { | |
836 if (audio_demuxer_stream_adapter_) | |
837 audio_demuxer_stream_adapter_->GetBytesWrittenAndReset(); | |
838 if (video_demuxer_stream_adapter_) | |
839 video_demuxer_stream_adapter_->GetBytesWrittenAndReset(); | |
840 return; | |
841 } | |
842 | |
843 const int kBytesPerKilobit = 1024 / 8; | |
844 if (audio_demuxer_stream_adapter_) { | |
845 const double kilobits_per_second = | |
846 (audio_demuxer_stream_adapter_->GetBytesWrittenAndReset() / | |
847 kDataFlowPollPeriod.InSecondsF()) / | |
848 kBytesPerKilobit; | |
849 DCHECK_GE(kilobits_per_second, 0); | |
850 const base::CheckedNumeric<int> checked_kbps = kilobits_per_second; | |
851 metrics_recorder_.OnAudioRateEstimate( | |
852 checked_kbps.ValueOrDefault(std::numeric_limits<int>::max())); | |
853 } | |
854 if (video_demuxer_stream_adapter_) { | |
855 const double kilobits_per_second = | |
856 (video_demuxer_stream_adapter_->GetBytesWrittenAndReset() / | |
857 kDataFlowPollPeriod.InSecondsF()) / | |
858 kBytesPerKilobit; | |
859 DCHECK_GE(kilobits_per_second, 0); | |
860 const base::CheckedNumeric<int> checked_kbps = kilobits_per_second; | |
861 metrics_recorder_.OnVideoRateEstimate( | |
862 checked_kbps.ValueOrDefault(std::numeric_limits<int>::max())); | |
863 } | |
864 } | |
865 | |
866 } // namespace media | |
OLD | NEW |