Render inband text tracks in the media pipeline

media/base/pipeline.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/base/pipeline.h"
 
 #include <algorithm>
 
 #include "base/bind.h"
 #include "base/callback.h"
 #include "base/callback_helpers.h"
 #include "base/compiler_specific.h"
 #include "base/message_loop/message_loop.h"
 #include "base/metrics/histogram.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_util.h"
 #include "base/synchronization/condition_variable.h"
 #include "media/base/audio_decoder.h"
 #include "media/base/audio_renderer.h"
 #include "media/base/clock.h"
 #include "media/base/filter_collection.h"
 #include "media/base/media_log.h"
+#include "media/base/text_renderer.h"
 #include "media/base/video_decoder.h"
 #include "media/base/video_decoder_config.h"
 #include "media/base/video_renderer.h"
 
 using base::TimeDelta;
 
 namespace media {
 
 Pipeline::Pipeline(const scoped_refptr<base::MessageLoopProxy>& message_loop,
                    MediaLog* media_log)
     : message_loop_(message_loop),
       media_log_(media_log),
       running_(false),
       did_loading_progress_(false),
       total_bytes_(0),
       natural_size_(0, 0),
       volume_(1.0f),
       playback_rate_(0.0f),
       clock_(new Clock(&default_tick_clock_)),
       waiting_for_clock_update_(false),
       status_(PIPELINE_OK),
       has_audio_(false),
       has_video_(false),
       state_(kCreated),
       audio_ended_(false),
       video_ended_(false),
+      text_ended_(false),
       audio_disabled_(false),
       demuxer_(NULL),
       creation_time_(default_tick_clock_.NowTicks()) {
   media_log_->AddEvent(media_log_->CreatePipelineStateChangedEvent(kCreated));
   media_log_->AddEvent(
       media_log_->CreateEvent(MediaLogEvent::PIPELINE_CREATED));
 }
 
 Pipeline::~Pipeline() {
   DCHECK(thread_checker_.CalledOnValidThread())
@@ skipping 226 matching lines @@
       break;
   }
   NOTREACHED() << "State has no transition: " << state_;
   return state_;
 }
 
 void Pipeline::OnDemuxerError(PipelineStatus error) {
   SetError(error);
 }
 
+void Pipeline::AddTextStream(DemuxerStream* text_stream,
+                             const TextTrackConfig& config) {
+  message_loop_->PostTask(FROM_HERE, base::Bind(
+    &Pipeline::AddTextStreamTask, base::Unretained(this),
+    text_stream, config));
+}
+
+void Pipeline::RemoveTextStream(DemuxerStream* text_stream) {
+#if 0

[acolwell GONE FROM CHROMIUM, 2013/10/24 18:57:51]

Remove #if

[Matthew Heaney (Chromium), 2013/10/25 03:05:38]

Done.

+  DCHECK(message_loop_->BelongsToCurrentThread());
+  if (text_renderer_)
+    text_renderer_->RemoveTextStream(text_stream);
+#else
+  message_loop_->PostTask(FROM_HERE, base::Bind(
+    &Pipeline::RemoveTextStreamTask, base::Unretained(this),
+    text_stream));
+#endif
+}
+
 void Pipeline::SetError(PipelineStatus error) {
   DCHECK(IsRunning());
   DCHECK_NE(PIPELINE_OK, error);
   VLOG(1) << "Media pipeline error: " << error;
 
   message_loop_->PostTask(FROM_HERE, base::Bind(
       &Pipeline::ErrorChangedTask, base::Unretained(this), error));
 
   media_log_->AddEvent(media_log_->CreatePipelineErrorEvent(error));
 }
@@ skipping 224 matching lines @@
 
   // Pause.
   if (audio_renderer_) {
     bound_fns.Push(base::Bind(
         &AudioRenderer::Pause, base::Unretained(audio_renderer_.get())));
   }
   if (video_renderer_) {
     bound_fns.Push(base::Bind(
         &VideoRenderer::Pause, base::Unretained(video_renderer_.get())));
   }
+  if (text_renderer_) {
+    bound_fns.Push(base::Bind(
+        &TextRenderer::Pause, base::Unretained(text_renderer_.get())));
+  }
 
   // Flush.
   if (audio_renderer_) {
     bound_fns.Push(base::Bind(
         &AudioRenderer::Flush, base::Unretained(audio_renderer_.get())));
   }
   if (video_renderer_) {
     bound_fns.Push(base::Bind(
         &VideoRenderer::Flush, base::Unretained(video_renderer_.get())));
   }
+  if (text_renderer_) {
+    bound_fns.Push(base::Bind(
+        &TextRenderer::Flush, base::Unretained(text_renderer_.get())));
+  }
 
   // Seek demuxer.
   bound_fns.Push(base::Bind(
       &Demuxer::Seek, base::Unretained(demuxer_), seek_timestamp));
 
   // Preroll renderers.
   if (audio_renderer_) {
     bound_fns.Push(base::Bind(
         &AudioRenderer::Preroll, base::Unretained(audio_renderer_.get()),
         seek_timestamp));
@@ skipping 19 matching lines @@
   if (audio_renderer_) {
     bound_fns.Push(base::Bind(
         &AudioRenderer::Play, base::Unretained(audio_renderer_.get())));
   }
 
   if (video_renderer_) {
     bound_fns.Push(base::Bind(
         &VideoRenderer::Play, base::Unretained(video_renderer_.get())));
   }
 
+  if (text_renderer_) {
+    bound_fns.Push(base::Bind(
+        &TextRenderer::Play, base::Unretained(text_renderer_.get())));
+  }
+
   pending_callbacks_ = SerialRunner::Run(bound_fns, done_cb);
 }
 
 void Pipeline::DoStop(const PipelineStatusCB& done_cb) {
   DCHECK(message_loop_->BelongsToCurrentThread());
   DCHECK(!pending_callbacks_.get());
   SerialRunner::Queue bound_fns;
 
   if (demuxer_) {
     bound_fns.Push(base::Bind(
         &Demuxer::Stop, base::Unretained(demuxer_)));
   }
 
   if (audio_renderer_) {
     bound_fns.Push(base::Bind(
         &AudioRenderer::Stop, base::Unretained(audio_renderer_.get())));
   }
 
   if (video_renderer_) {
     bound_fns.Push(base::Bind(
         &VideoRenderer::Stop, base::Unretained(video_renderer_.get())));
   }
 
+  if (text_renderer_) {
+    bound_fns.Push(base::Bind(
+        &TextRenderer::Stop, base::Unretained(text_renderer_.get())));
+  }
+
   pending_callbacks_ = SerialRunner::Run(bound_fns, done_cb);
 }
 
 void Pipeline::OnStopCompleted(PipelineStatus status) {
   DCHECK(message_loop_->BelongsToCurrentThread());
   DCHECK_EQ(state_, kStopping);
   {
     base::AutoLock l(lock_);
     running_ = false;
   }
 
   SetState(kStopped);
   pending_callbacks_.reset();
   filter_collection_.reset();
   audio_renderer_.reset();
   video_renderer_.reset();
+  text_renderer_.reset();
   demuxer_ = NULL;
 
   // If we stop during initialization/seeking we want to run |seek_cb_|
   // followed by |stop_cb_| so we don't leave outstanding callbacks around.
   if (!seek_cb_.is_null()) {
     base::ResetAndReturn(&seek_cb_).Run(status_);
     error_cb_.Reset();
   }
   if (!stop_cb_.is_null()) {
     error_cb_.Reset();
@@ skipping 40 matching lines @@
   media_log_->AddEvent(media_log_->CreateEvent(MediaLogEvent::AUDIO_ENDED));
 }
 
 void Pipeline::OnVideoRendererEnded() {
   // Force post to process ended messages after current execution frame.
   message_loop_->PostTask(FROM_HERE, base::Bind(
       &Pipeline::DoVideoRendererEnded, base::Unretained(this)));
   media_log_->AddEvent(media_log_->CreateEvent(MediaLogEvent::VIDEO_ENDED));
 }
 
+void Pipeline::OnTextRendererEnded() {
+  // Force post to process ended messages after current execution frame.
+  message_loop_->PostTask(FROM_HERE, base::Bind(
+      &Pipeline::DoTextRendererEnded, base::Unretained(this)));
+  media_log_->AddEvent(media_log_->CreateEvent(MediaLogEvent::TEXT_ENDED));
+}
+
 // Called from any thread.
 void Pipeline::OnUpdateStatistics(const PipelineStatistics& stats) {
   base::AutoLock auto_lock(lock_);
   statistics_.audio_bytes_decoded += stats.audio_bytes_decoded;
   statistics_.video_bytes_decoded += stats.video_bytes_decoded;
   statistics_.video_frames_decoded += stats.video_frames_decoded;
   statistics_.video_frames_dropped += stats.video_frames_dropped;
 }
 
 void Pipeline::StartTask(scoped_ptr<FilterCollection> filter_collection,
                          const base::Closure& ended_cb,
                          const PipelineStatusCB& error_cb,
                          const PipelineStatusCB& seek_cb,
                          const BufferingStateCB& buffering_state_cb,
                          const base::Closure& duration_change_cb) {
   DCHECK(message_loop_->BelongsToCurrentThread());
   CHECK_EQ(kCreated, state_)
       << "Media pipeline cannot be started more than once";
 
   filter_collection_ = filter_collection.Pass();
   ended_cb_ = ended_cb;
   error_cb_ = error_cb;
   seek_cb_ = seek_cb;
   buffering_state_cb_ = buffering_state_cb;
   duration_change_cb_ = duration_change_cb;
 
+  text_renderer_ = filter_collection_->GetTextRenderer();
+
+  if (text_renderer_) {
+    text_renderer_->Initialize(
+        base::Bind(&Pipeline::OnTextRendererEnded, base::Unretained(this)));
+  }
+
   StateTransitionTask(PIPELINE_OK);
 }
 
 void Pipeline::StopTask(const base::Closure& stop_cb) {
   DCHECK(message_loop_->BelongsToCurrentThread());
   DCHECK(stop_cb_.is_null());
 
   if (state_ == kStopped) {
     stop_cb.Run();
     return;
@@ skipping 69 matching lines @@
     return;
   }
 
   DCHECK(seek_cb_.is_null());
 
   SetState(kSeeking);
   base::TimeDelta seek_timestamp = std::max(time, demuxer_->GetStartTime());
   seek_cb_ = seek_cb;
   audio_ended_ = false;
   video_ended_ = false;
+  text_ended_ = false;
 
   // Kick off seeking!
   {
     base::AutoLock auto_lock(lock_);
     if (clock_->IsPlaying())
       clock_->Pause();
     clock_->SetTime(seek_timestamp, seek_timestamp);
   }
   DoSeek(seek_timestamp, base::Bind(
       &Pipeline::OnStateTransition, base::Unretained(this)));
@@ skipping 23 matching lines @@
 
   if (state_ != kStarted)
     return;
 
   DCHECK(!video_ended_);
   video_ended_ = true;
 
   RunEndedCallbackIfNeeded();
 }
 
+void Pipeline::DoTextRendererEnded() {
+  DCHECK(message_loop_->BelongsToCurrentThread());
+
+  if (state_ != kStarted)
+    return;
+
+  DCHECK(!text_ended_);
+  text_ended_ = true;
+
+  RunEndedCallbackIfNeeded();
+}
+
 void Pipeline::RunEndedCallbackIfNeeded() {
   DCHECK(message_loop_->BelongsToCurrentThread());
 
   if (audio_renderer_ && !audio_ended_ && !audio_disabled_)
     return;
 
   if (video_renderer_ && !video_ended_)
     return;
 
+  if (text_renderer_ && text_renderer_->HasTracks() && !text_ended_)
+    return;
+
   {
     base::AutoLock auto_lock(lock_);
     clock_->EndOfStream();
   }
 
   DCHECK_EQ(status_, PIPELINE_OK);
   ended_cb_.Run();
 }
 
 void Pipeline::AudioDisabledTask() {
   DCHECK(message_loop_->BelongsToCurrentThread());
 
   base::AutoLock auto_lock(lock_);
   has_audio_ = false;
   audio_disabled_ = true;
 
   // Notify our demuxer that we're no longer rendering audio.
   demuxer_->OnAudioRendererDisabled();
 
   // Start clock since there is no more audio to trigger clock updates.
   clock_->SetMaxTime(clock_->Duration());
   StartClockIfWaitingForTimeUpdate_Locked();
 }
 
+void Pipeline::AddTextStreamTask(DemuxerStream* text_stream,
+                                 const TextTrackConfig& config) {
+  DCHECK(message_loop_->BelongsToCurrentThread());
+  text_renderer_->AddTextStream(text_stream, config);
+}
+
+void Pipeline::RemoveTextStreamTask(DemuxerStream* text_stream) {
+  DCHECK(message_loop_->BelongsToCurrentThread());
+  if (text_renderer_)

[acolwell GONE FROM CHROMIUM, 2013/10/24 18:57:51]

Why is this needed? It shouldn't be possible for this to get called after the
pipeline has reached the kStopped state. The demuxers should not call
Add/RemoveTextStream after they have called the callback passed to Stop().

[Matthew Heaney (Chromium), 2013/10/25 03:05:38]

Done.

+    text_renderer_->RemoveTextStream(text_stream);
+}
+
 void Pipeline::InitializeDemuxer(const PipelineStatusCB& done_cb) {
   DCHECK(message_loop_->BelongsToCurrentThread());
 
   demuxer_ = filter_collection_->GetDemuxer();
   demuxer_->Initialize(this, done_cb);

[acolwell GONE FROM CHROMIUM, 2013/10/24 18:57:51]

WDYT about passing the enable_text_tracks bool here instead of through the
demuxer constructors? You could use !text_renderer_.is_null() here and that
would simplify the code in WebMediaPlayerImpl and prevent a situation where the
pipeline and demuxers disagree about whether text tracks are enabled.

[acolwell GONE FROM CHROMIUM, 2013/10/29 21:14:19]

Please address this comment.

[Matthew Heaney (Chromium), 2013/11/05 04:52:40]

Done.

[Matthew Heaney (Chromium), 2013/11/05 04:52:40]

Done.

 }
 
 void Pipeline::InitializeAudioRenderer(const PipelineStatusCB& done_cb) {
   DCHECK(message_loop_->BelongsToCurrentThread());
 
   audio_renderer_ = filter_collection_->GetAudioRenderer();
   audio_renderer_->Initialize(
       demuxer_->GetStream(DemuxerStream::AUDIO),
       done_cb,
       base::Bind(&Pipeline::OnUpdateStatistics, base::Unretained(this)),
@@ skipping 46 matching lines @@
 void Pipeline::StartClockIfWaitingForTimeUpdate_Locked() {
   lock_.AssertAcquired();
   if (!waiting_for_clock_update_)
     return;
 
   waiting_for_clock_update_ = false;
   clock_->Play();
 }
 
 }  // namespace media