Render inband text tracks in the media pipeline

content/renderer/media/webmediaplayer_impl.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/webmediaplayer_impl.h"
 
 #include <algorithm>
 #include <limits>
 #include <string>
 #include <vector>
 
 #include "base/bind.h"
 #include "base/callback.h"
 #include "base/command_line.h"
 #include "base/debug/crash_logging.h"
 #include "base/message_loop/message_loop_proxy.h"
 #include "base/metrics/histogram.h"
+#include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/synchronization/waitable_event.h"
 #include "cc/layers/video_layer.h"
 #include "content/public/common/content_switches.h"
 #include "content/renderer/media/buffered_data_source.h"
 #include "content/renderer/media/crypto/key_systems.h"
 #include "content/renderer/media/texttrack_impl.h"
 #include "content/renderer/media/webaudiosourceprovider_impl.h"
 #include "content/renderer/media/webinbandtexttrack_impl.h"
 #include "content/renderer/media/webmediaplayer_delegate.h"
 #include "content/renderer/media/webmediaplayer_params.h"
 #include "content/renderer/media/webmediaplayer_util.h"
 #include "content/renderer/media/webmediasource_impl.h"
 #include "content/renderer/pepper/pepper_webplugin_impl.h"
 #include "gpu/GLES2/gl2extchromium.h"
 #include "media/audio/null_audio_sink.h"
 #include "media/base/bind_to_loop.h"
 #include "media/base/filter_collection.h"
 #include "media/base/limits.h"
 #include "media/base/media_log.h"
 #include "media/base/media_switches.h"
 #include "media/base/pipeline.h"
+#include "media/base/text_buffer.h"
 #include "media/base/video_frame.h"
 #include "media/filters/audio_renderer_impl.h"
 #include "media/filters/chunk_demuxer.h"
 #include "media/filters/ffmpeg_audio_decoder.h"
 #include "media/filters/ffmpeg_demuxer.h"
+#include "media/filters/ffmpeg_text_decoder.h"
 #include "media/filters/ffmpeg_video_decoder.h"
 #include "media/filters/gpu_video_accelerator_factories.h"
 #include "media/filters/gpu_video_decoder.h"
 #include "media/filters/opus_audio_decoder.h"
+#include "media/filters/text_renderer_impl.h"
 #include "media/filters/video_renderer_base.h"
 #include "media/filters/vpx_video_decoder.h"
 #include "third_party/WebKit/public/platform/WebMediaSource.h"
 #include "third_party/WebKit/public/platform/WebRect.h"
 #include "third_party/WebKit/public/platform/WebSize.h"
 #include "third_party/WebKit/public/platform/WebString.h"
 #include "third_party/WebKit/public/platform/WebURL.h"
 #include "third_party/WebKit/public/web/WebRuntimeFeatures.h"
 #include "third_party/WebKit/public/web/WebView.h"
 #include "v8/include/v8.h"
@@ skipping 894 matching lines @@
   const WebKit::WebString weblanguage = WebKit::WebString::fromUTF8(language);
 
   WebInbandTextTrackImpl* const text_track =
     new WebInbandTextTrackImpl(webkind, weblabel, weblanguage,
                                text_track_index_++);
 
   return scoped_ptr<media::TextTrack>(new TextTrackImpl(GetClient(),
                                                         text_track));
 }
 
+void WebMediaPlayerImpl::OnFFmpegTextTrack(media::TextKind kind,
+                                           const std::string& label,
+                                           const std::string& language,
+                                           int index) {
+  DCHECK(main_loop_->BelongsToCurrentThread());
+  if (ffmpeg_text_track_map_.find(index) == ffmpeg_text_track_map_.end()) {

[acolwell GONE FROM CHROMIUM, 2013/09/12 00:15:15]

nit: It seems like we should DCHECK instead. It would be a bug if the index was
already being used right?

[Matthew Heaney (Chromium), 2013/09/13 19:51:54]

Done.

+    scoped_ptr<media::TextTrack> text_track =
+        OnTextTrack(kind, label, language);
+    ffmpeg_text_track_map_.insert(std::make_pair(index, text_track.release()));
+  }
+}
+
 void WebMediaPlayerImpl::OnKeyError(const std::string& session_id,
                                     media::MediaKeys::KeyError error_code,
                                     int system_code) {
   DCHECK(main_loop_->BelongsToCurrentThread());
 
   EmeUMAHistogramEnumeration(current_key_system_, "KeyError",
                              error_code, media::MediaKeys::kMaxKeyError);
 
   GetClient()->keyError(
       current_key_system_,
@@ skipping 53 matching lines @@
 
   // Keep track if this is a MSE or non-MSE playback.
   UMA_HISTOGRAM_BOOLEAN("Media.MSE.Playback",
                         (load_type_ == LoadTypeMediaSource));
 
   // Figure out which demuxer to use.
   if (load_type_ != LoadTypeMediaSource) {
     DCHECK(!chunk_demuxer_);
     DCHECK(data_source_);
 
+    media::FFmpegAddTextTrackCB add_text_track_cb;
+
+    if (cmd_line->HasSwitch(switches::kEnableInbandTextTracks)) {
+      add_text_track_cb =
+          BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnFFmpegTextTrack);
+    }
+
     demuxer_.reset(new media::FFmpegDemuxer(
         media_loop_, data_source_.get(),
         BIND_TO_RENDER_LOOP_1(&WebMediaPlayerImpl::OnNeedKey, ""),
+        add_text_track_cb,

[acolwell GONE FROM CHROMIUM, 2013/09/12 00:15:15]

This doesn't seem right to me. It seems like Demuxers should notify the pipeline
to add/remove tracks through the DemuxerHost interface. This should in turn
cause streams to be added/removed from the TextRenderer. It seems like the
TextRenderer should be one to receive this particular callback.

[Matthew Heaney (Chromium), 2013/09/13 19:51:54]

OK, just to make sure I understand.  The FFmpegDemuxer should hold a DemuxerHost
pointer.  When it detects a text stream, it should call AddStream on the
DemuxerHost pointer, instead of calling the add_text_track_cb.

Is my understanding correct?

[acolwell GONE FROM CHROMIUM, 2013/09/13 20:57:30]

Yes. The demuxers already hold a reference to the DemuxerHost. It is passed in
via the Initialize() method.

[Matthew Heaney (Chromium), 2013/09/20 23:53:54]

The ffmpeg demuxer now forwards the AddTextStream request to the DemuxerHost,
and the Pipeline in turn forwards it to the TextRenderer.

         media_log_));
   } else {
     DCHECK(!chunk_demuxer_);
     DCHECK(!data_source_);
 
     media::AddTextTrackCB add_text_track_cb;
 
     if (cmd_line->HasSwitch(switches::kEnableInbandTextTracks)) {
       add_text_track_cb =
           base::Bind(&WebMediaPlayerImpl::OnTextTrack, base::Unretained(this));
@@ skipping 65 matching lines @@
   scoped_ptr<media::VideoRenderer> video_renderer(
       new media::VideoRendererBase(
           media_loop_,
           video_decoders.Pass(),
           set_decryptor_ready_cb,
           base::Bind(&WebMediaPlayerImpl::FrameReady, base::Unretained(this)),
           BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::SetOpaque),
           true));
   filter_collection->SetVideoRenderer(video_renderer.Pass());
 
+  if (load_type_ != LoadTypeMediaSource &&

[acolwell GONE FROM CHROMIUM, 2013/09/12 00:15:15]

This should be load_type_ independent. ChunkDemuxer and FFmpegDemuxer should
both use the text renderer mechanism.

[Matthew Heaney (Chromium), 2013/09/13 19:51:54]

Agreed.  I figure I can make changes to DemuxerHost first, and then change the
ChunkDemuxer.

+      cmd_line->HasSwitch(switches::kEnableInbandTextTracks)) {
+    scoped_ptr<media::TextDecoder> text_decoder(
+        new media::FFmpegTextDecoder(media_loop_));
+
+    scoped_ptr<media::TextRenderer> text_renderer(
+        new media::TextRendererImpl(
+            media_loop_,
+            text_decoder.Pass(),
+            BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::CueReady)));
+    filter_collection->SetTextRenderer(text_renderer.Pass());
+  }
+
   // ... and we're ready to go!
   starting_ = true;
   pipeline_->Start(
       filter_collection.Pass(),
       BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnPipelineEnded),
       BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnPipelineError),
       BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnPipelineSeek),
       BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnPipelineBufferingState),
       BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnDurationChange));
 }
@@ skipping 46 matching lines @@
   // Let V8 know we are not using extra resources anymore.
   if (incremented_externally_allocated_memory_) {
     v8::V8::AdjustAmountOfExternalAllocatedMemory(-kPlayerExtraMemory);
     incremented_externally_allocated_memory_ = false;
   }
 
   // Release any final references now that everything has stopped.
   pipeline_.reset();
   demuxer_.reset();
   data_source_.reset();
+
+  STLDeleteValues(&ffmpeg_text_track_map_);
 }
 
 WebKit::WebMediaPlayerClient* WebMediaPlayerImpl::GetClient() {
   DCHECK(main_loop_->BelongsToCurrentThread());
   DCHECK(client_);
   return client_;
 }
 
 WebKit::WebAudioSourceProvider* WebMediaPlayerImpl::audioSourceProvider() {
   return audio_source_provider_.get();
@@ skipping 36 matching lines @@
   current_frame_ = frame;
 
   if (pending_repaint_)
     return;
 
   pending_repaint_ = true;
   main_loop_->PostTask(FROM_HERE, base::Bind(
       &WebMediaPlayerImpl::Repaint, AsWeakPtr()));
 }
 
+void WebMediaPlayerImpl::CueReady(
+    int index,
+    const scoped_refptr<media::TextBuffer>& buf) {
+  DCHECK(main_loop_->BelongsToCurrentThread());
+
+  if (buf == NULL || buf->text().empty())

[acolwell GONE FROM CHROMIUM, 2013/09/12 00:15:15]

nit: These should be DCHECKS.

[Matthew Heaney (Chromium), 2013/09/13 19:51:54]

Done.

+    return;
+
+  TextTrackMap::iterator it = ffmpeg_text_track_map_.find(index);

[acolwell GONE FROM CHROMIUM, 2013/09/12 00:15:15]

The TextRenderer seems like a better home for this map.

[Matthew Heaney (Chromium), 2013/09/13 19:51:54]

Agreed.  I think this will go away once I make the changes to the DemuxerHost.

[Matthew Heaney (Chromium), 2013/09/20 23:53:54]

Actually, I had to leave this here, so that it could be run on the main thread. 
This is required by Blink.

+  if (it == ffmpeg_text_track_map_.end())

[acolwell GONE FROM CHROMIUM, 2013/09/12 00:15:15]

nit: Shouldn't this be a DCHECK? It seems like this would be a bug if we got a
cue for a track we didn't have.

[Matthew Heaney (Chromium), 2013/09/13 19:51:54]

Done.

+    return;
+
+  media::TextTrack* track = it->second;
+  base::TimeDelta start = buf->timestamp();
+  base::TimeDelta end = start + buf->duration();
+
+  track->addWebVTTCue(start, end,
+                      buf->id(), buf->text(), buf->settings());
+}
+
 }  // namespace content