OLD | NEW |
| (Empty) |
1 // Copyright 2016 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 #include "media/remoting/remote_renderer_impl.h" | |
6 | |
7 #include <algorithm> | |
8 #include <limits> | |
9 #include <utility> | |
10 | |
11 #include "base/bind.h" | |
12 #include "base/bind_helpers.h" | |
13 #include "base/callback_helpers.h" | |
14 #include "base/memory/ptr_util.h" | |
15 #include "base/message_loop/message_loop.h" | |
16 #include "base/numerics/safe_math.h" | |
17 #include "base/threading/thread_task_runner_handle.h" | |
18 #include "base/time/time.h" | |
19 #include "media/base/bind_to_current_loop.h" | |
20 #include "media/base/demuxer_stream_provider.h" | |
21 #include "media/remoting/remote_demuxer_stream_adapter.h" | |
22 #include "media/remoting/remoting_renderer_controller.h" | |
23 #include "media/remoting/rpc/proto_enum_utils.h" | |
24 #include "media/remoting/rpc/proto_utils.h" | |
25 | |
26 namespace { | |
27 | |
28 // The moving time window to track the media time and statistics updates. | |
29 constexpr base::TimeDelta kTrackingWindow = base::TimeDelta::FromSeconds(3); | |
30 | |
31 // The allowed delay for the remoting playback. When exceeds this limit, the | |
32 // user experience is likely poor and the controller is notified. | |
33 constexpr base::TimeDelta kMediaPlaybackDelayThreshold = | |
34 base::TimeDelta::FromMilliseconds(450); | |
35 | |
36 // The allowed percentage of the number of video frames dropped vs. the number | |
37 // of the video frames decoded. When exceeds this limit, the user experience is | |
38 // likely poor and the controller is notified. | |
39 constexpr int kMaxNumVideoFramesDroppedPercentage = 3; | |
40 | |
41 // The time period to allow receiver get stable after playback rate change or | |
42 // Flush(). | |
43 constexpr base::TimeDelta kStabilizationPeriod = | |
44 base::TimeDelta::FromSeconds(2); | |
45 | |
46 // The amount of time between polling the DemuxerStreamAdapters to measure their | |
47 // data flow rates for metrics. | |
48 constexpr base::TimeDelta kDataFlowPollPeriod = | |
49 base::TimeDelta::FromSeconds(10); | |
50 | |
51 } // namespace | |
52 | |
53 namespace media { | |
54 | |
55 RemoteRendererImpl::RemoteRendererImpl( | |
56 scoped_refptr<base::SingleThreadTaskRunner> media_task_runner, | |
57 const base::WeakPtr<RemotingRendererController>& | |
58 remoting_renderer_controller, | |
59 VideoRendererSink* video_renderer_sink) | |
60 : state_(STATE_UNINITIALIZED), | |
61 main_task_runner_(base::ThreadTaskRunnerHandle::Get()), | |
62 media_task_runner_(std::move(media_task_runner)), | |
63 demuxer_stream_provider_(nullptr), | |
64 client_(nullptr), | |
65 remoting_renderer_controller_(remoting_renderer_controller), | |
66 rpc_broker_(remoting_renderer_controller_->GetRpcBroker()), | |
67 rpc_handle_(rpc_broker_->GetUniqueHandle()), | |
68 remote_renderer_handle_(remoting::kInvalidHandle), | |
69 video_renderer_sink_(video_renderer_sink), | |
70 weak_factory_(this) { | |
71 VLOG(2) << __func__; | |
72 // The constructor is running on the main thread. | |
73 DCHECK(remoting_renderer_controller_); | |
74 remoting_renderer_controller_->SetShowInterstitialCallback( | |
75 base::Bind(&RemoteRendererImpl::RequestUpdateInterstitialOnMainThread, | |
76 media_task_runner_, weak_factory_.GetWeakPtr())); | |
77 | |
78 const remoting::RpcBroker::ReceiveMessageCallback receive_callback = | |
79 base::Bind(&RemoteRendererImpl::OnMessageReceivedOnMainThread, | |
80 media_task_runner_, weak_factory_.GetWeakPtr()); | |
81 rpc_broker_->RegisterMessageReceiverCallback(rpc_handle_, receive_callback); | |
82 } | |
83 | |
84 RemoteRendererImpl::~RemoteRendererImpl() { | |
85 VLOG(2) << __func__; | |
86 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
87 | |
88 UpdateInterstitial(interstitial_background_, canvas_size_, | |
89 RemotingInterstitialType::BETWEEN_SESSIONS); | |
90 | |
91 // Post task on main thread to unset the interstial callback. | |
92 main_task_runner_->PostTask( | |
93 FROM_HERE, | |
94 base::Bind(&RemotingRendererController::SetShowInterstitialCallback, | |
95 remoting_renderer_controller_, | |
96 RemotingRendererController::ShowInterstitialCallback())); | |
97 | |
98 // Post task on main thread to unregister message receiver. | |
99 main_task_runner_->PostTask( | |
100 FROM_HERE, | |
101 base::Bind(&remoting::RpcBroker::UnregisterMessageReceiverCallback, | |
102 rpc_broker_, rpc_handle_)); | |
103 } | |
104 | |
105 void RemoteRendererImpl::Initialize( | |
106 DemuxerStreamProvider* demuxer_stream_provider, | |
107 media::RendererClient* client, | |
108 const PipelineStatusCB& init_cb) { | |
109 VLOG(2) << __func__; | |
110 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
111 DCHECK(demuxer_stream_provider); | |
112 DCHECK(client); | |
113 | |
114 if (state_ != STATE_UNINITIALIZED) { | |
115 media_task_runner_->PostTask( | |
116 FROM_HERE, base::Bind(init_cb, PIPELINE_ERROR_INVALID_STATE)); | |
117 return; | |
118 } | |
119 | |
120 demuxer_stream_provider_ = demuxer_stream_provider; | |
121 client_ = client; | |
122 init_workflow_done_callback_ = init_cb; | |
123 | |
124 state_ = STATE_CREATE_PIPE; | |
125 // Create audio mojo data pipe handles if audio is available. | |
126 ::media::DemuxerStream* audio_demuxer_stream = | |
127 demuxer_stream_provider_->GetStream(::media::DemuxerStream::AUDIO); | |
128 std::unique_ptr<mojo::DataPipe> audio_data_pipe; | |
129 if (audio_demuxer_stream) { | |
130 audio_data_pipe = base::WrapUnique(remoting::CreateDataPipe()); | |
131 } | |
132 | |
133 // Create video mojo data pipe handles if video is available. | |
134 ::media::DemuxerStream* video_demuxer_stream = | |
135 demuxer_stream_provider_->GetStream(::media::DemuxerStream::VIDEO); | |
136 std::unique_ptr<mojo::DataPipe> video_data_pipe; | |
137 if (video_demuxer_stream) { | |
138 video_data_pipe = base::WrapUnique(remoting::CreateDataPipe()); | |
139 } | |
140 | |
141 // Establish remoting data pipe connection using main thread. | |
142 const RemotingSourceImpl::DataPipeStartCallback data_pipe_callback = | |
143 base::Bind(&RemoteRendererImpl::OnDataPipeCreatedOnMainThread, | |
144 media_task_runner_, weak_factory_.GetWeakPtr(), rpc_broker_); | |
145 main_task_runner_->PostTask( | |
146 FROM_HERE, | |
147 base::Bind(&RemotingRendererController::StartDataPipe, | |
148 remoting_renderer_controller_, base::Passed(&audio_data_pipe), | |
149 base::Passed(&video_data_pipe), data_pipe_callback)); | |
150 } | |
151 | |
152 void RemoteRendererImpl::SetCdm(CdmContext* cdm_context, | |
153 const CdmAttachedCB& cdm_attached_cb) { | |
154 VLOG(2) << __func__ << " cdm_id:" << cdm_context->GetCdmId(); | |
155 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
156 | |
157 // TODO(erickung): add implementation once Remote CDM implementation is done. | |
158 // Right now it returns callback immediately. | |
159 if (!cdm_attached_cb.is_null()) { | |
160 cdm_attached_cb.Run(false); | |
161 } | |
162 } | |
163 | |
164 void RemoteRendererImpl::Flush(const base::Closure& flush_cb) { | |
165 VLOG(2) << __func__; | |
166 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
167 DCHECK(flush_cb_.is_null()); | |
168 | |
169 if (state_ != STATE_PLAYING) { | |
170 DCHECK_EQ(state_, STATE_ERROR); | |
171 // In the error state, this renderer will be shut down shortly. To prevent | |
172 // breaking the pipeline impl, just run the done callback (interface | |
173 // requirement). | |
174 media_task_runner_->PostTask(FROM_HERE, flush_cb); | |
175 return; | |
176 } | |
177 | |
178 state_ = STATE_FLUSHING; | |
179 base::Optional<uint32_t> flush_audio_count; | |
180 if (audio_demuxer_stream_adapter_) | |
181 flush_audio_count = audio_demuxer_stream_adapter_->SignalFlush(true); | |
182 base::Optional<uint32_t> flush_video_count; | |
183 if (video_demuxer_stream_adapter_) | |
184 flush_video_count = video_demuxer_stream_adapter_->SignalFlush(true); | |
185 // Makes sure flush count is valid if stream is available or both audio and | |
186 // video agrees on the same flushing state. | |
187 if ((audio_demuxer_stream_adapter_ && !flush_audio_count.has_value()) || | |
188 (video_demuxer_stream_adapter_ && !flush_video_count.has_value()) || | |
189 (audio_demuxer_stream_adapter_ && video_demuxer_stream_adapter_ && | |
190 flush_audio_count.has_value() != flush_video_count.has_value())) { | |
191 VLOG(1) << "Ignoring flush request while under flushing operation"; | |
192 return; | |
193 } | |
194 | |
195 flush_cb_ = flush_cb; | |
196 | |
197 // Issues RPC_R_FLUSHUNTIL RPC message. | |
198 std::unique_ptr<remoting::pb::RpcMessage> rpc(new remoting::pb::RpcMessage()); | |
199 rpc->set_handle(remote_renderer_handle_); | |
200 rpc->set_proc(remoting::pb::RpcMessage::RPC_R_FLUSHUNTIL); | |
201 remoting::pb::RendererFlushUntil* message = | |
202 rpc->mutable_renderer_flushuntil_rpc(); | |
203 if (flush_audio_count.has_value()) | |
204 message->set_audio_count(*flush_audio_count); | |
205 if (flush_video_count.has_value()) | |
206 message->set_video_count(*flush_video_count); | |
207 message->set_callback_handle(rpc_handle_); | |
208 VLOG(2) << __func__ << ": Sending RPC_R_FLUSHUNTIL to " << rpc->handle() | |
209 << " with audio_count=" << message->audio_count() | |
210 << ", video_count=" << message->video_count() | |
211 << ", callback_handle=" << message->callback_handle(); | |
212 SendRpcToRemote(std::move(rpc)); | |
213 } | |
214 | |
215 void RemoteRendererImpl::StartPlayingFrom(base::TimeDelta time) { | |
216 VLOG(2) << __func__ << ": " << time.InMicroseconds(); | |
217 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
218 | |
219 if (state_ != STATE_PLAYING) { | |
220 DCHECK_EQ(state_, STATE_ERROR); | |
221 return; | |
222 } | |
223 | |
224 // Issues RPC_R_STARTPLAYINGFROM RPC message. | |
225 std::unique_ptr<remoting::pb::RpcMessage> rpc(new remoting::pb::RpcMessage()); | |
226 rpc->set_handle(remote_renderer_handle_); | |
227 rpc->set_proc(remoting::pb::RpcMessage::RPC_R_STARTPLAYINGFROM); | |
228 rpc->set_integer64_value(time.InMicroseconds()); | |
229 VLOG(2) << __func__ << ": Sending RPC_R_STARTPLAYINGFROM to " << rpc->handle() | |
230 << " with time_usec=" << rpc->integer64_value(); | |
231 SendRpcToRemote(std::move(rpc)); | |
232 | |
233 { | |
234 base::AutoLock auto_lock(time_lock_); | |
235 current_media_time_ = time; | |
236 } | |
237 ResetMeasurements(); | |
238 } | |
239 | |
240 void RemoteRendererImpl::SetPlaybackRate(double playback_rate) { | |
241 VLOG(2) << __func__ << ": " << playback_rate; | |
242 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
243 | |
244 if (state_ != STATE_FLUSHING && state_ != STATE_PLAYING) { | |
245 DCHECK_EQ(state_, STATE_ERROR); | |
246 return; | |
247 } | |
248 | |
249 // Issues RPC_R_SETPLAYBACKRATE RPC message. | |
250 std::unique_ptr<remoting::pb::RpcMessage> rpc(new remoting::pb::RpcMessage()); | |
251 rpc->set_handle(remote_renderer_handle_); | |
252 rpc->set_proc(remoting::pb::RpcMessage::RPC_R_SETPLAYBACKRATE); | |
253 rpc->set_double_value(playback_rate); | |
254 VLOG(2) << __func__ << ": Sending RPC_R_SETPLAYBACKRATE to " << rpc->handle() | |
255 << " with rate=" << rpc->double_value(); | |
256 SendRpcToRemote(std::move(rpc)); | |
257 playback_rate_ = playback_rate; | |
258 ResetMeasurements(); | |
259 } | |
260 | |
261 void RemoteRendererImpl::SetVolume(float volume) { | |
262 VLOG(2) << __func__ << ": " << volume; | |
263 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
264 | |
265 if (state_ != STATE_FLUSHING && state_ != STATE_PLAYING) { | |
266 DCHECK_EQ(state_, STATE_ERROR); | |
267 return; | |
268 } | |
269 | |
270 // Issues RPC_R_SETVOLUME RPC message. | |
271 std::unique_ptr<remoting::pb::RpcMessage> rpc(new remoting::pb::RpcMessage()); | |
272 rpc->set_handle(remote_renderer_handle_); | |
273 rpc->set_proc(remoting::pb::RpcMessage::RPC_R_SETVOLUME); | |
274 rpc->set_double_value(volume); | |
275 VLOG(2) << __func__ << ": Sending RPC_R_SETVOLUME to " << rpc->handle() | |
276 << " with volume=" << rpc->double_value(); | |
277 SendRpcToRemote(std::move(rpc)); | |
278 } | |
279 | |
280 base::TimeDelta RemoteRendererImpl::GetMediaTime() { | |
281 // No BelongsToCurrentThread() checking because this can be called from other | |
282 // threads. | |
283 // TODO(erickung): Interpolate current media time using local system time. | |
284 // Current receiver is to update |current_media_time_| every 250ms. But it | |
285 // needs to lower the update frequency in order to reduce network usage. Hence | |
286 // the interpolation is needed after receiver implementation is changed. | |
287 base::AutoLock auto_lock(time_lock_); | |
288 return current_media_time_; | |
289 } | |
290 | |
291 // static | |
292 void RemoteRendererImpl::OnDataPipeCreatedOnMainThread( | |
293 scoped_refptr<base::SingleThreadTaskRunner> media_task_runner, | |
294 base::WeakPtr<RemoteRendererImpl> self, | |
295 base::WeakPtr<remoting::RpcBroker> rpc_broker, | |
296 mojom::RemotingDataStreamSenderPtrInfo audio, | |
297 mojom::RemotingDataStreamSenderPtrInfo video, | |
298 mojo::ScopedDataPipeProducerHandle audio_handle, | |
299 mojo::ScopedDataPipeProducerHandle video_handle) { | |
300 media_task_runner->PostTask( | |
301 FROM_HERE, | |
302 base::Bind( | |
303 &RemoteRendererImpl::OnDataPipeCreated, self, base::Passed(&audio), | |
304 base::Passed(&video), base::Passed(&audio_handle), | |
305 base::Passed(&video_handle), | |
306 rpc_broker ? rpc_broker->GetUniqueHandle() : remoting::kInvalidHandle, | |
307 rpc_broker ? rpc_broker->GetUniqueHandle() | |
308 : remoting::kInvalidHandle)); | |
309 } | |
310 | |
311 void RemoteRendererImpl::OnDataPipeCreated( | |
312 mojom::RemotingDataStreamSenderPtrInfo audio, | |
313 mojom::RemotingDataStreamSenderPtrInfo video, | |
314 mojo::ScopedDataPipeProducerHandle audio_handle, | |
315 mojo::ScopedDataPipeProducerHandle video_handle, | |
316 int audio_rpc_handle, | |
317 int video_rpc_handle) { | |
318 VLOG(2) << __func__; | |
319 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
320 DCHECK(!init_workflow_done_callback_.is_null()); | |
321 | |
322 if (state_ == STATE_ERROR) | |
323 return; // Abort because something went wrong in the meantime. | |
324 DCHECK_EQ(state_, STATE_CREATE_PIPE); | |
325 | |
326 // Create audio demuxer stream adapter if audio is available. | |
327 ::media::DemuxerStream* audio_demuxer_stream = | |
328 demuxer_stream_provider_->GetStream(::media::DemuxerStream::AUDIO); | |
329 if (audio_demuxer_stream && audio.is_valid() && audio_handle.is_valid() && | |
330 audio_rpc_handle != remoting::kInvalidHandle) { | |
331 VLOG(2) << "Initialize audio"; | |
332 audio_demuxer_stream_adapter_.reset( | |
333 new remoting::RemoteDemuxerStreamAdapter( | |
334 main_task_runner_, media_task_runner_, "audio", | |
335 audio_demuxer_stream, rpc_broker_, audio_rpc_handle, | |
336 std::move(audio), std::move(audio_handle), | |
337 base::Bind(&RemoteRendererImpl::OnFatalError, | |
338 base::Unretained(this)))); | |
339 } | |
340 | |
341 // Create video demuxer stream adapter if video is available. | |
342 ::media::DemuxerStream* video_demuxer_stream = | |
343 demuxer_stream_provider_->GetStream(::media::DemuxerStream::VIDEO); | |
344 if (video_demuxer_stream && video.is_valid() && video_handle.is_valid() && | |
345 video_rpc_handle != remoting::kInvalidHandle) { | |
346 VLOG(2) << "Initialize video"; | |
347 video_demuxer_stream_adapter_.reset( | |
348 new remoting::RemoteDemuxerStreamAdapter( | |
349 main_task_runner_, media_task_runner_, "video", | |
350 video_demuxer_stream, rpc_broker_, video_rpc_handle, | |
351 std::move(video), std::move(video_handle), | |
352 base::Bind(&RemoteRendererImpl::OnFatalError, | |
353 base::Unretained(this)))); | |
354 } | |
355 | |
356 // Checks if data pipe is created successfully. | |
357 if (!audio_demuxer_stream_adapter_ && !video_demuxer_stream_adapter_) { | |
358 OnFatalError(remoting::DATA_PIPE_CREATE_ERROR); | |
359 return; | |
360 } | |
361 | |
362 state_ = STATE_ACQUIRING; | |
363 // Issues RPC_ACQUIRE_RENDERER RPC message. | |
364 std::unique_ptr<remoting::pb::RpcMessage> rpc(new remoting::pb::RpcMessage()); | |
365 rpc->set_handle(remoting::kReceiverHandle); | |
366 rpc->set_proc(remoting::pb::RpcMessage::RPC_ACQUIRE_RENDERER); | |
367 rpc->set_integer_value(rpc_handle_); | |
368 VLOG(2) << __func__ << ": Sending RPC_ACQUIRE_RENDERER to " << rpc->handle() | |
369 << " with rpc_handle=" << rpc->integer_value(); | |
370 SendRpcToRemote(std::move(rpc)); | |
371 } | |
372 | |
373 // static | |
374 void RemoteRendererImpl::OnMessageReceivedOnMainThread( | |
375 scoped_refptr<base::SingleThreadTaskRunner> media_task_runner, | |
376 base::WeakPtr<RemoteRendererImpl> self, | |
377 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
378 media_task_runner->PostTask( | |
379 FROM_HERE, base::Bind(&RemoteRendererImpl::OnReceivedRpc, self, | |
380 base::Passed(&message))); | |
381 } | |
382 | |
383 void RemoteRendererImpl::OnReceivedRpc( | |
384 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
385 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
386 DCHECK(message); | |
387 switch (message->proc()) { | |
388 case remoting::pb::RpcMessage::RPC_ACQUIRE_RENDERER_DONE: | |
389 AcquireRendererDone(std::move(message)); | |
390 break; | |
391 case remoting::pb::RpcMessage::RPC_R_INITIALIZE_CALLBACK: | |
392 InitializeCallback(std::move(message)); | |
393 break; | |
394 case remoting::pb::RpcMessage::RPC_R_FLUSHUNTIL_CALLBACK: | |
395 FlushUntilCallback(); | |
396 break; | |
397 case remoting::pb::RpcMessage::RPC_R_SETCDM_CALLBACK: | |
398 SetCdmCallback(std::move(message)); | |
399 break; | |
400 case remoting::pb::RpcMessage::RPC_RC_ONTIMEUPDATE: | |
401 OnTimeUpdate(std::move(message)); | |
402 break; | |
403 case remoting::pb::RpcMessage::RPC_RC_ONBUFFERINGSTATECHANGE: | |
404 OnBufferingStateChange(std::move(message)); | |
405 break; | |
406 case remoting::pb::RpcMessage::RPC_RC_ONENDED: | |
407 VLOG(2) << __func__ << ": Received RPC_RC_ONENDED."; | |
408 client_->OnEnded(); | |
409 break; | |
410 case remoting::pb::RpcMessage::RPC_RC_ONERROR: | |
411 VLOG(2) << __func__ << ": Received RPC_RC_ONERROR."; | |
412 OnFatalError(remoting::RECEIVER_PIPELINE_ERROR); | |
413 break; | |
414 case remoting::pb::RpcMessage::RPC_RC_ONVIDEONATURALSIZECHANGE: | |
415 OnVideoNaturalSizeChange(std::move(message)); | |
416 break; | |
417 case remoting::pb::RpcMessage::RPC_RC_ONVIDEOOPACITYCHANGE: | |
418 OnVideoOpacityChange(std::move(message)); | |
419 break; | |
420 case remoting::pb::RpcMessage::RPC_RC_ONSTATISTICSUPDATE: | |
421 OnStatisticsUpdate(std::move(message)); | |
422 break; | |
423 case remoting::pb::RpcMessage::RPC_RC_ONWAITINGFORDECRYPTIONKEY: | |
424 VLOG(2) << __func__ << ": Received RPC_RC_ONWAITINGFORDECRYPTIONKEY."; | |
425 client_->OnWaitingForDecryptionKey(); | |
426 break; | |
427 case remoting::pb::RpcMessage::RPC_RC_ONDURATIONCHANGE: | |
428 OnDurationChange(std::move(message)); | |
429 break; | |
430 | |
431 default: | |
432 LOG(ERROR) << "Unknown rpc: " << message->proc(); | |
433 } | |
434 } | |
435 | |
436 void RemoteRendererImpl::SendRpcToRemote( | |
437 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
438 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
439 DCHECK(main_task_runner_); | |
440 main_task_runner_->PostTask( | |
441 FROM_HERE, base::Bind(&remoting::RpcBroker::SendMessageToRemote, | |
442 rpc_broker_, base::Passed(&message))); | |
443 } | |
444 | |
445 void RemoteRendererImpl::AcquireRendererDone( | |
446 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
447 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
448 DCHECK(message); | |
449 | |
450 remote_renderer_handle_ = message->integer_value(); | |
451 VLOG(2) << __func__ | |
452 << ": Received RPC_ACQUIRE_RENDERER_DONE with remote_renderer_handle=" | |
453 << remote_renderer_handle_; | |
454 | |
455 if (state_ != STATE_ACQUIRING || init_workflow_done_callback_.is_null()) { | |
456 LOG(WARNING) << "Unexpected acquire renderer done RPC."; | |
457 OnFatalError(remoting::PEERS_OUT_OF_SYNC); | |
458 return; | |
459 } | |
460 state_ = STATE_INITIALIZING; | |
461 | |
462 // Issues RPC_R_INITIALIZE RPC message to initialize renderer. | |
463 std::unique_ptr<remoting::pb::RpcMessage> rpc(new remoting::pb::RpcMessage()); | |
464 rpc->set_handle(remote_renderer_handle_); | |
465 rpc->set_proc(remoting::pb::RpcMessage::RPC_R_INITIALIZE); | |
466 remoting::pb::RendererInitialize* init = | |
467 rpc->mutable_renderer_initialize_rpc(); | |
468 init->set_client_handle(rpc_handle_); | |
469 init->set_audio_demuxer_handle( | |
470 audio_demuxer_stream_adapter_ | |
471 ? audio_demuxer_stream_adapter_->rpc_handle() | |
472 : remoting::kInvalidHandle); | |
473 init->set_video_demuxer_handle( | |
474 video_demuxer_stream_adapter_ | |
475 ? video_demuxer_stream_adapter_->rpc_handle() | |
476 : remoting::kInvalidHandle); | |
477 init->set_callback_handle(rpc_handle_); | |
478 VLOG(2) << __func__ << ": Sending RPC_R_INITIALIZE to " << rpc->handle() | |
479 << " with client_handle=" << init->client_handle() | |
480 << ", audio_demuxer_handle=" << init->audio_demuxer_handle() | |
481 << ", video_demuxer_handle=" << init->video_demuxer_handle() | |
482 << ", callback_handle=" << init->callback_handle(); | |
483 SendRpcToRemote(std::move(rpc)); | |
484 } | |
485 | |
486 void RemoteRendererImpl::InitializeCallback( | |
487 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
488 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
489 DCHECK(message); | |
490 | |
491 const bool success = message->boolean_value(); | |
492 VLOG(2) << __func__ | |
493 << ": Received RPC_R_INITIALIZE_CALLBACK with success=" << success; | |
494 | |
495 if (state_ != STATE_INITIALIZING || init_workflow_done_callback_.is_null()) { | |
496 LOG(WARNING) << "Unexpected initialize callback RPC."; | |
497 OnFatalError(remoting::PEERS_OUT_OF_SYNC); | |
498 return; | |
499 } | |
500 | |
501 if (!success) { | |
502 OnFatalError(remoting::RECEIVER_INITIALIZE_FAILED); | |
503 return; | |
504 } | |
505 | |
506 metrics_recorder_.OnRendererInitialized(); | |
507 | |
508 state_ = STATE_PLAYING; | |
509 base::ResetAndReturn(&init_workflow_done_callback_).Run(::media::PIPELINE_OK); | |
510 } | |
511 | |
512 void RemoteRendererImpl::FlushUntilCallback() { | |
513 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
514 VLOG(2) << __func__ << ": Received RPC_R_FLUSHUNTIL_CALLBACK"; | |
515 | |
516 if (state_ != STATE_FLUSHING || flush_cb_.is_null()) { | |
517 LOG(WARNING) << "Unexpected flushuntil callback RPC."; | |
518 OnFatalError(remoting::PEERS_OUT_OF_SYNC); | |
519 return; | |
520 } | |
521 | |
522 state_ = STATE_PLAYING; | |
523 if (audio_demuxer_stream_adapter_) | |
524 audio_demuxer_stream_adapter_->SignalFlush(false); | |
525 if (video_demuxer_stream_adapter_) | |
526 video_demuxer_stream_adapter_->SignalFlush(false); | |
527 base::ResetAndReturn(&flush_cb_).Run(); | |
528 ResetMeasurements(); | |
529 } | |
530 | |
531 void RemoteRendererImpl::SetCdmCallback( | |
532 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
533 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
534 DCHECK(message); | |
535 VLOG(2) << __func__ << ": Received RPC_R_SETCDM_CALLBACK with cdm_id=" | |
536 << message->renderer_set_cdm_rpc().cdm_id() << ", callback_handle=" | |
537 << message->renderer_set_cdm_rpc().callback_handle(); | |
538 // TODO(erickung): add implementation once Remote CDM implementation is done. | |
539 NOTIMPLEMENTED(); | |
540 } | |
541 | |
542 void RemoteRendererImpl::OnTimeUpdate( | |
543 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
544 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
545 DCHECK(message); | |
546 // Shutdown remoting session if receiving malformed RPC message. | |
547 if (!message->has_rendererclient_ontimeupdate_rpc()) { | |
548 VLOG(1) << __func__ << " missing required RPC message"; | |
549 OnFatalError(remoting::RPC_INVALID); | |
550 return; | |
551 } | |
552 const int64_t time_usec = | |
553 message->rendererclient_ontimeupdate_rpc().time_usec(); | |
554 const int64_t max_time_usec = | |
555 message->rendererclient_ontimeupdate_rpc().max_time_usec(); | |
556 VLOG(2) << __func__ | |
557 << ": Received RPC_RC_ONTIMEUPDATE with time_usec=" << time_usec | |
558 << ", max_time_usec=" << max_time_usec; | |
559 // Ignores invalid time, such as negative value, or time larger than max value | |
560 // (usually the time stamp that all streams are pushed into AV pipeline). | |
561 if (time_usec < 0 || max_time_usec < 0 || time_usec > max_time_usec) | |
562 return; | |
563 | |
564 { | |
565 // Updates current time information. | |
566 base::AutoLock auto_lock(time_lock_); | |
567 current_media_time_ = base::TimeDelta::FromMicroseconds(time_usec); | |
568 current_max_time_ = base::TimeDelta::FromMicroseconds(max_time_usec); | |
569 } | |
570 | |
571 metrics_recorder_.OnEvidenceOfPlayoutAtReceiver(); | |
572 OnMediaTimeUpdated(); | |
573 } | |
574 | |
575 void RemoteRendererImpl::OnBufferingStateChange( | |
576 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
577 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
578 DCHECK(message); | |
579 if (!message->has_rendererclient_onbufferingstatechange_rpc()) { | |
580 VLOG(1) << __func__ << " missing required RPC message"; | |
581 OnFatalError(remoting::RPC_INVALID); | |
582 return; | |
583 } | |
584 VLOG(2) << __func__ << ": Received RPC_RC_ONBUFFERINGSTATECHANGE with state=" | |
585 << message->rendererclient_onbufferingstatechange_rpc().state(); | |
586 base::Optional<::media::BufferingState> state = | |
587 remoting::ToMediaBufferingState( | |
588 message->rendererclient_onbufferingstatechange_rpc().state()); | |
589 if (!state.has_value()) | |
590 return; | |
591 client_->OnBufferingStateChange(state.value()); | |
592 } | |
593 | |
594 void RemoteRendererImpl::OnVideoNaturalSizeChange( | |
595 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
596 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
597 DCHECK(message); | |
598 // Shutdown remoting session if receiving malformed RPC message. | |
599 if (!message->has_rendererclient_onvideonatualsizechange_rpc()) { | |
600 VLOG(1) << __func__ << " missing required RPC message"; | |
601 OnFatalError(remoting::RPC_INVALID); | |
602 return; | |
603 } | |
604 const auto& size_change = | |
605 message->rendererclient_onvideonatualsizechange_rpc(); | |
606 VLOG(2) << __func__ << ": Received RPC_RC_ONVIDEONATURALSIZECHANGE with size=" | |
607 << size_change.width() << 'x' << size_change.height(); | |
608 if (size_change.width() <= 0 || size_change.height() <= 0) | |
609 return; | |
610 client_->OnVideoNaturalSizeChange( | |
611 gfx::Size(size_change.width(), size_change.height())); | |
612 } | |
613 | |
614 void RemoteRendererImpl::OnVideoOpacityChange( | |
615 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
616 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
617 DCHECK(message); | |
618 const bool opaque = message->boolean_value(); | |
619 VLOG(2) << __func__ | |
620 << ": Received RPC_RC_ONVIDEOOPACITYCHANGE with opaque=" << opaque; | |
621 client_->OnVideoOpacityChange(opaque); | |
622 } | |
623 | |
624 void RemoteRendererImpl::OnStatisticsUpdate( | |
625 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
626 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
627 DCHECK(message); | |
628 // Shutdown remoting session if receiving malformed RPC message. | |
629 if (!message->has_rendererclient_onstatisticsupdate_rpc()) { | |
630 VLOG(1) << __func__ << " missing required RPC message"; | |
631 OnFatalError(remoting::RPC_INVALID); | |
632 return; | |
633 } | |
634 const auto& rpc_message = message->rendererclient_onstatisticsupdate_rpc(); | |
635 ::media::PipelineStatistics stats; | |
636 // Note: Each |stats| value is a delta, not the aggregate amount. | |
637 stats.audio_bytes_decoded = rpc_message.audio_bytes_decoded(); | |
638 stats.video_bytes_decoded = rpc_message.video_bytes_decoded(); | |
639 stats.video_frames_decoded = rpc_message.video_frames_decoded(); | |
640 stats.video_frames_dropped = rpc_message.video_frames_dropped(); | |
641 stats.audio_memory_usage = rpc_message.audio_memory_usage(); | |
642 stats.video_memory_usage = rpc_message.video_memory_usage(); | |
643 VLOG(2) << __func__ | |
644 << ": Received RPC_RC_ONSTATISTICSUPDATE with audio_bytes_decoded=" | |
645 << stats.audio_bytes_decoded | |
646 << ", video_bytes_decoded=" << stats.video_bytes_decoded | |
647 << ", video_frames_decoded=" << stats.video_frames_decoded | |
648 << ", video_frames_dropped=" << stats.video_frames_dropped | |
649 << ", audio_memory_usage=" << stats.audio_memory_usage | |
650 << ", video_memory_usage=" << stats.video_memory_usage; | |
651 | |
652 if (stats.audio_bytes_decoded > 0 || stats.video_frames_decoded > 0 || | |
653 stats.video_frames_dropped > 0) { | |
654 metrics_recorder_.OnEvidenceOfPlayoutAtReceiver(); | |
655 } | |
656 UpdateVideoStatsQueue(stats.video_frames_decoded, stats.video_frames_dropped); | |
657 client_->OnStatisticsUpdate(stats); | |
658 } | |
659 | |
660 void RemoteRendererImpl::OnDurationChange( | |
661 std::unique_ptr<remoting::pb::RpcMessage> message) { | |
662 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
663 DCHECK(message); | |
664 VLOG(2) << __func__ << ": Received RPC_RC_ONDURATIONCHANGE with usec=" | |
665 << message->integer64_value(); | |
666 if (message->integer64_value() < 0) | |
667 return; | |
668 client_->OnDurationChange( | |
669 base::TimeDelta::FromMicroseconds(message->integer64_value())); | |
670 } | |
671 | |
672 void RemoteRendererImpl::OnFatalError(remoting::StopTrigger stop_trigger) { | |
673 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
674 DCHECK_NE(remoting::UNKNOWN_STOP_TRIGGER, stop_trigger); | |
675 | |
676 VLOG(2) << __func__ << " with StopTrigger " << stop_trigger; | |
677 | |
678 // If this is the first error, notify the controller. It is expected the | |
679 // controller will shut down this renderer shortly. | |
680 if (state_ != STATE_ERROR) { | |
681 state_ = STATE_ERROR; | |
682 main_task_runner_->PostTask( | |
683 FROM_HERE, base::Bind(&RemotingRendererController::OnRendererFatalError, | |
684 remoting_renderer_controller_, stop_trigger)); | |
685 } | |
686 | |
687 data_flow_poll_timer_.Stop(); | |
688 | |
689 if (!init_workflow_done_callback_.is_null()) { | |
690 base::ResetAndReturn(&init_workflow_done_callback_) | |
691 .Run(PIPELINE_ERROR_INITIALIZATION_FAILED); | |
692 return; | |
693 } | |
694 | |
695 if (!flush_cb_.is_null()) | |
696 base::ResetAndReturn(&flush_cb_).Run(); | |
697 } | |
698 | |
699 // static | |
700 void RemoteRendererImpl::RequestUpdateInterstitialOnMainThread( | |
701 scoped_refptr<base::SingleThreadTaskRunner> media_task_runner, | |
702 base::WeakPtr<RemoteRendererImpl> remote_renderer_impl, | |
703 const base::Optional<SkBitmap>& background_image, | |
704 const gfx::Size& canvas_size, | |
705 RemotingInterstitialType interstitial_type) { | |
706 media_task_runner->PostTask( | |
707 FROM_HERE, | |
708 base::Bind(&RemoteRendererImpl::UpdateInterstitial, remote_renderer_impl, | |
709 background_image, canvas_size, interstitial_type)); | |
710 } | |
711 | |
712 void RemoteRendererImpl::UpdateInterstitial( | |
713 const base::Optional<SkBitmap>& background_image, | |
714 const gfx::Size& canvas_size, | |
715 RemotingInterstitialType interstitial_type) { | |
716 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
717 if (background_image.has_value()) | |
718 interstitial_background_ = background_image.value(); | |
719 canvas_size_ = canvas_size; | |
720 PaintRemotingInterstitial(interstitial_background_, canvas_size_, | |
721 interstitial_type, video_renderer_sink_); | |
722 } | |
723 | |
724 void RemoteRendererImpl::OnMediaTimeUpdated() { | |
725 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
726 if (!flush_cb_.is_null()) | |
727 return; // Don't manage and check the queue when Flush() is on-going. | |
728 | |
729 base::TimeTicks current_time = base::TimeTicks::Now(); | |
730 if (current_time < ignore_updates_until_time_) | |
731 return; // Not stable yet. | |
732 | |
733 media_time_queue_.push_back( | |
734 std::make_pair(current_time, current_media_time_)); | |
735 base::TimeDelta window_duration = | |
736 current_time - media_time_queue_.front().first; | |
737 if (window_duration < kTrackingWindow) | |
738 return; // Not enough data to make a reliable decision. | |
739 | |
740 base::TimeDelta media_duration = | |
741 media_time_queue_.back().second - media_time_queue_.front().second; | |
742 base::TimeDelta update_duration = | |
743 (media_time_queue_.back().first - media_time_queue_.front().first) * | |
744 playback_rate_; | |
745 if ((media_duration - update_duration).magnitude() >= | |
746 kMediaPlaybackDelayThreshold) { | |
747 VLOG(1) << "Irregular playback detected: Media playback delayed." | |
748 << " media_duration = " << media_duration | |
749 << " update_duration = " << update_duration; | |
750 OnFatalError(remoting::PACING_TOO_SLOWLY); | |
751 } | |
752 // Prune |media_time_queue_|. | |
753 while (media_time_queue_.back().first - media_time_queue_.front().first >= | |
754 kTrackingWindow) | |
755 media_time_queue_.pop_front(); | |
756 } | |
757 | |
758 void RemoteRendererImpl::UpdateVideoStatsQueue(int video_frames_decoded, | |
759 int video_frames_dropped) { | |
760 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
761 if (!flush_cb_.is_null()) | |
762 return; // Don't manage and check the queue when Flush() is on-going. | |
763 | |
764 if (!stats_updated_) { | |
765 if (video_frames_decoded) | |
766 stats_updated_ = true; | |
767 // Ignore the first stats since it may include the information during | |
768 // unstable period. | |
769 return; | |
770 } | |
771 | |
772 base::TimeTicks current_time = base::TimeTicks::Now(); | |
773 if (current_time < ignore_updates_until_time_) | |
774 return; // Not stable yet. | |
775 | |
776 video_stats_queue_.push_back(std::make_tuple( | |
777 current_time, video_frames_decoded, video_frames_dropped)); | |
778 sum_video_frames_decoded_ += video_frames_decoded; | |
779 sum_video_frames_dropped_ += video_frames_dropped; | |
780 base::TimeDelta window_duration = | |
781 current_time - std::get<0>(video_stats_queue_.front()); | |
782 if (window_duration < kTrackingWindow) | |
783 return; // Not enough data to make a reliable decision. | |
784 | |
785 if (sum_video_frames_decoded_ && | |
786 sum_video_frames_dropped_ * 100 > | |
787 sum_video_frames_decoded_ * kMaxNumVideoFramesDroppedPercentage) { | |
788 VLOG(1) << "Irregular playback detected: Too many video frames dropped." | |
789 << " video_frames_decoded= " << sum_video_frames_decoded_ | |
790 << " video_frames_dropped= " << sum_video_frames_dropped_; | |
791 OnFatalError(remoting::FRAME_DROP_RATE_HIGH); | |
792 } | |
793 // Prune |video_stats_queue_|. | |
794 while (std::get<0>(video_stats_queue_.back()) - | |
795 std::get<0>(video_stats_queue_.front()) >= | |
796 kTrackingWindow) { | |
797 sum_video_frames_decoded_ -= std::get<1>(video_stats_queue_.front()); | |
798 sum_video_frames_dropped_ -= std::get<2>(video_stats_queue_.front()); | |
799 video_stats_queue_.pop_front(); | |
800 } | |
801 } | |
802 | |
803 void RemoteRendererImpl::ResetMeasurements() { | |
804 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
805 media_time_queue_.clear(); | |
806 video_stats_queue_.clear(); | |
807 sum_video_frames_dropped_ = 0; | |
808 sum_video_frames_decoded_ = 0; | |
809 stats_updated_ = false; | |
810 ignore_updates_until_time_ = base::TimeTicks::Now() + kStabilizationPeriod; | |
811 | |
812 if (state_ != STATE_ERROR && | |
813 (audio_demuxer_stream_adapter_ || video_demuxer_stream_adapter_)) { | |
814 data_flow_poll_timer_.Start(FROM_HERE, kDataFlowPollPeriod, this, | |
815 &RemoteRendererImpl::MeasureAndRecordDataRates); | |
816 } | |
817 } | |
818 | |
819 void RemoteRendererImpl::MeasureAndRecordDataRates() { | |
820 DCHECK(media_task_runner_->BelongsToCurrentThread()); | |
821 | |
822 // Whenever media is first started or flushed/seeked, there is a "burst | |
823 // bufferring" period as the remote device rapidly fills its buffer before | |
824 // resuming playback. Since the goal here is to measure the sustained content | |
825 // bitrates, ignore the byte counts the first time since the last | |
826 // ResetMeasurements() call. | |
827 const base::TimeTicks current_time = base::TimeTicks::Now(); | |
828 if (current_time < ignore_updates_until_time_ + kDataFlowPollPeriod) { | |
829 if (audio_demuxer_stream_adapter_) | |
830 audio_demuxer_stream_adapter_->GetBytesWrittenAndReset(); | |
831 if (video_demuxer_stream_adapter_) | |
832 video_demuxer_stream_adapter_->GetBytesWrittenAndReset(); | |
833 return; | |
834 } | |
835 | |
836 const int kBytesPerKilobit = 1024 / 8; | |
837 if (audio_demuxer_stream_adapter_) { | |
838 const double kilobits_per_second = | |
839 (audio_demuxer_stream_adapter_->GetBytesWrittenAndReset() / | |
840 kDataFlowPollPeriod.InSecondsF()) / | |
841 kBytesPerKilobit; | |
842 DCHECK_GE(kilobits_per_second, 0); | |
843 const base::CheckedNumeric<int> checked_kbps = kilobits_per_second; | |
844 metrics_recorder_.OnAudioRateEstimate( | |
845 checked_kbps.ValueOrDefault(std::numeric_limits<int>::max())); | |
846 } | |
847 if (video_demuxer_stream_adapter_) { | |
848 const double kilobits_per_second = | |
849 (video_demuxer_stream_adapter_->GetBytesWrittenAndReset() / | |
850 kDataFlowPollPeriod.InSecondsF()) / | |
851 kBytesPerKilobit; | |
852 DCHECK_GE(kilobits_per_second, 0); | |
853 const base::CheckedNumeric<int> checked_kbps = kilobits_per_second; | |
854 metrics_recorder_.OnVideoRateEstimate( | |
855 checked_kbps.ValueOrDefault(std::numeric_limits<int>::max())); | |
856 } | |
857 } | |
858 | |
859 } // namespace media | |
OLD | NEW |