Render inband text tracks in the media pipeline

content/renderer/media/webmediaplayer_impl.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/webmediaplayer_impl.h"
 
 #include <algorithm>
 #include <limits>
 #include <string>
 #include <vector>
 
 #include "base/bind.h"
 #include "base/callback.h"
 #include "base/command_line.h"
 #include "base/debug/crash_logging.h"
 #include "base/message_loop/message_loop_proxy.h"
 #include "base/metrics/histogram.h"
+#include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/synchronization/waitable_event.h"
 #include "cc/layers/video_layer.h"
 #include "content/public/common/content_switches.h"
 #include "content/renderer/media/buffered_data_source.h"
 #include "content/renderer/media/crypto/key_systems.h"
 #include "content/renderer/media/texttrack_impl.h"
 #include "content/renderer/media/webaudiosourceprovider_impl.h"
 #include "content/renderer/media/webinbandtexttrack_impl.h"
 #include "content/renderer/media/webmediaplayer_delegate.h"
 #include "content/renderer/media/webmediaplayer_params.h"
 #include "content/renderer/media/webmediaplayer_util.h"
 #include "content/renderer/media/webmediasource_impl.h"
 #include "content/renderer/pepper/pepper_webplugin_impl.h"
 #include "gpu/GLES2/gl2extchromium.h"
 #include "media/audio/null_audio_sink.h"
 #include "media/base/bind_to_loop.h"
 #include "media/base/filter_collection.h"
 #include "media/base/limits.h"
 #include "media/base/media_log.h"
 #include "media/base/media_switches.h"
 #include "media/base/pipeline.h"
+#include "media/base/text_cue.h"
+#include "media/base/text_renderer.h"
 #include "media/base/video_frame.h"
 #include "media/filters/audio_renderer_impl.h"
 #include "media/filters/chunk_demuxer.h"
 #include "media/filters/ffmpeg_audio_decoder.h"
 #include "media/filters/ffmpeg_demuxer.h"
 #include "media/filters/ffmpeg_video_decoder.h"
 #include "media/filters/gpu_video_accelerator_factories.h"
 #include "media/filters/gpu_video_decoder.h"
 #include "media/filters/opus_audio_decoder.h"
+#include "media/filters/text_decoder_impl.h"
 #include "media/filters/video_renderer_base.h"
 #include "media/filters/vpx_video_decoder.h"
 #include "third_party/WebKit/public/platform/WebMediaSource.h"
 #include "third_party/WebKit/public/platform/WebRect.h"
 #include "third_party/WebKit/public/platform/WebSize.h"
 #include "third_party/WebKit/public/platform/WebString.h"
 #include "third_party/WebKit/public/platform/WebURL.h"
 #include "third_party/WebKit/public/web/WebRuntimeFeatures.h"
 #include "third_party/WebKit/public/web/WebView.h"
 #include "v8/include/v8.h"
@@ skipping 894 matching lines @@
   const WebKit::WebString weblanguage = WebKit::WebString::fromUTF8(language);
 
   WebInbandTextTrackImpl* const text_track =
     new WebInbandTextTrackImpl(webkind, weblabel, weblanguage,
                                text_track_index_++);
 
   return scoped_ptr<media::TextTrack>(new TextTrackImpl(GetClient(),
                                                         text_track));
 }
 
+void WebMediaPlayerImpl::AddTextStream(media::DemuxerStream* text_stream,
+                                       media::TextKind kind,
+                                       const std::string& label,
+                                       const std::string& language) {
+  DCHECK(main_loop_->BelongsToCurrentThread());
+  scoped_ptr<media::TextTrack> text_track = OnTextTrack(kind, label, language);
+  text_track_map_[text_stream] = text_track.release();

[acolwell GONE FROM CHROMIUM, 2013/09/25 23:23:10]

I don't think you need the map here. You can hide the thread hopping business in
TextTrackImpl and let the renderer own the TextTrack instance.

I think if you use the following signature instead you'd avoid the need for the
map here and avoid map lookups on each cue. It would also allow the TextRenderer
to control the text track lifetime instead of having to coordinate with this map
here.
typedef base::Callback<void(scoped_ptr<media::TextTrack>)>
AddTextTrack(media::TextKind kind,
             const std::string& label,
             const std::string& language,
             const AddTextTrackDoneCB& done_cb)

[Matthew Heaney (Chromium), 2013/09/29 03:31:23]

Done.

+}
+
 void WebMediaPlayerImpl::OnKeyError(const std::string& session_id,
                                     media::MediaKeys::KeyError error_code,
                                     int system_code) {
   DCHECK(main_loop_->BelongsToCurrentThread());
 
   EmeUMAHistogramEnumeration(current_key_system_, "KeyError",
                              error_code, media::MediaKeys::kMaxKeyError);
 
   GetClient()->keyError(
       current_key_system_,
@@ skipping 48 matching lines @@
 }
 
 void WebMediaPlayerImpl::StartPipeline() {
   const CommandLine* cmd_line = CommandLine::ForCurrentProcess();
   bool increase_preroll_on_underflow = true;
 
   // Keep track if this is a MSE or non-MSE playback.
   UMA_HISTOGRAM_BOOLEAN("Media.MSE.Playback",
                         (load_type_ == LoadTypeMediaSource));
 
+  const bool enable_inband_text_tracks =
+      cmd_line->HasSwitch(switches::kEnableInbandTextTracks);
+
   // Figure out which demuxer to use.
   if (load_type_ != LoadTypeMediaSource) {
     DCHECK(!chunk_demuxer_);
     DCHECK(data_source_);
 
     demuxer_.reset(new media::FFmpegDemuxer(
         media_loop_, data_source_.get(),
         BIND_TO_RENDER_LOOP_1(&WebMediaPlayerImpl::OnNeedKey, ""),
+        enable_inband_text_tracks,
         media_log_));
   } else {
     DCHECK(!chunk_demuxer_);
     DCHECK(!data_source_);
 
+    // TODO(matthewjheaney): Once we have resolved the threading issues,
+    // change the chunk demuxer to user the same mechanism as for ffmpeg.
     media::AddTextTrackCB add_text_track_cb;
 
-    if (cmd_line->HasSwitch(switches::kEnableInbandTextTracks)) {
+    if (enable_inband_text_tracks) {
       add_text_track_cb =
           base::Bind(&WebMediaPlayerImpl::OnTextTrack, base::Unretained(this));
     }
 
     chunk_demuxer_ = new media::ChunkDemuxer(
         BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnDemuxerOpened),
         BIND_TO_RENDER_LOOP_1(&WebMediaPlayerImpl::OnNeedKey, ""),
         add_text_track_cb,
         base::Bind(&LogMediaSourceError, media_log_));
     demuxer_.reset(chunk_demuxer_);
@@ skipping 57 matching lines @@
   scoped_ptr<media::VideoRenderer> video_renderer(
       new media::VideoRendererBase(
           media_loop_,
           video_decoders.Pass(),
           set_decryptor_ready_cb,
           base::Bind(&WebMediaPlayerImpl::FrameReady, base::Unretained(this)),
           BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::SetOpaque),
           true));
   filter_collection->SetVideoRenderer(video_renderer.Pass());
 
+  if (load_type_ != LoadTypeMediaSource && enable_inband_text_tracks) {
+    scoped_ptr<media::TextDecoder> text_decoder(
+        new media::TextDecoderImpl(media_loop_));
+
+    scoped_ptr<media::TextRenderer> text_renderer(
+        new media::TextRenderer(
+            media_loop_,
+            text_decoder.Pass(),
+            BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::AddTextStream),
+            BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::CueReady)));
+
+    filter_collection->SetTextRenderer(text_renderer.Pass());
+  }
+
   // ... and we're ready to go!
   starting_ = true;
   pipeline_->Start(
       filter_collection.Pass(),
       BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnPipelineEnded),
       BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnPipelineError),
       BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnPipelineSeek),
       BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnPipelineBufferingState),
       BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnDurationChange));
 }
@@ skipping 36 matching lines @@
     gpu_factories_ = NULL;
   }
 
   // Make sure to kill the pipeline so there's no more media threads running.
   // Note: stopping the pipeline might block for a long time.
   base::WaitableEvent waiter(false, false);
   pipeline_->Stop(base::Bind(
       &base::WaitableEvent::Signal, base::Unretained(&waiter)));
   waiter.Wait();
 
+  STLDeleteValues(&text_track_map_);
+
   // Let V8 know we are not using extra resources anymore.
   if (incremented_externally_allocated_memory_) {
     v8::V8::AdjustAmountOfExternalAllocatedMemory(-kPlayerExtraMemory);
     incremented_externally_allocated_memory_ = false;
   }
 
   // Release any final references now that everything has stopped.
   pipeline_.reset();
   demuxer_.reset();
   data_source_.reset();
@@ skipping 46 matching lines @@
   current_frame_ = frame;
 
   if (pending_repaint_)
     return;
 
   pending_repaint_ = true;
   main_loop_->PostTask(FROM_HERE, base::Bind(
       &WebMediaPlayerImpl::Repaint, AsWeakPtr()));
 }
 
+void WebMediaPlayerImpl::CueReady(

[acolwell GONE FROM CHROMIUM, 2013/09/25 23:23:10]

You shouldn't need this. You can hide the thread hopping in TextTrackImpl and
the TextRenderer should just be able to interact with TextTrack objects.

[Matthew Heaney (Chromium), 2013/09/29 03:31:23]

Done.

+    media::DemuxerStream* text_stream,
+    const scoped_refptr<media::TextCue>& text_cue) {
+  DCHECK(main_loop_->BelongsToCurrentThread());
+  DCHECK_NE(text_stream, static_cast<media::DemuxerStream*>(NULL));
+  DCHECK(text_cue != NULL && !text_cue->text().empty());
+
+  TextTrackMap::iterator it = text_track_map_.find(text_stream);
+  DCHECK(it != text_track_map_.end());
+
+  media::TextTrack* text_track = it->second;
+  DCHECK_NE(text_track, static_cast<media::TextTrack*>(NULL));
+
+  base::TimeDelta start = text_cue->timestamp();
+  base::TimeDelta end = start + text_cue->duration();
+
+  text_track->addWebVTTCue(start, end,
+                           text_cue->id(),
+                           text_cue->text(),
+                           text_cue->settings());
+}
+
 }  // namespace content