Render inband text tracks in the media pipeline

content/renderer/media/webmediaplayer_impl.h

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Delegate calls from WebCore::MediaPlayerPrivate to Chrome's video player.
 // It contains Pipeline which is the actual media player pipeline, it glues
 // the media player pipeline, data source, audio renderer and renderer.
 // Pipeline would creates multiple threads and access some public methods
 // of this class, so we need to be extra careful about concurrent access of
 // methods and members.
 //
 // Other issues:
 // During tear down of the whole browser or a tab, the DOM tree may not be
 // destructed nicely, and there will be some dangling media threads trying to
 // the main thread, so we need this class to listen to destruction event of the
 // main thread and cleanup the media threads when the even is received. Also
 // at destruction of this class we will need to unhook it from destruction event
 // list of the main thread.
 
 #ifndef CONTENT_RENDERER_MEDIA_WEBMEDIAPLAYER_IMPL_H_
 #define CONTENT_RENDERER_MEDIA_WEBMEDIAPLAYER_IMPL_H_
 
+#include <map>
 #include <string>
 #include <vector>
 
 #include "base/basictypes.h"
 #include "base/memory/ref_counted.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/memory/weak_ptr.h"
 #include "base/threading/thread.h"
 #include "cc/layers/video_frame_provider.h"
 #include "content/renderer/media/crypto/proxy_decryptor.h"
@@ skipping 18 matching lines @@
 
 namespace base {
 class MessageLoopProxy;
 }
 
 namespace media {
 class ChunkDemuxer;
 class FFmpegDemuxer;
 class GpuVideoAcceleratorFactories;
 class MediaLog;
+class TextBuffer;
 }
 
 namespace webkit {
 class WebLayerImpl;
 }
 
 namespace content {
 class BufferedDataSource;
 class WebAudioSourceProviderImpl;
 class WebMediaPlayerDelegate;
@@ skipping 122 matching lines @@
                   int system_code);
   void OnKeyMessage(const std::string& session_id,
                     const std::vector<uint8>& message,
                     const std::string& default_url);
   void OnNeedKey(const std::string& type,
                  const std::string& session_id,
                  const std::vector<uint8>& init_data);
   scoped_ptr<media::TextTrack> OnTextTrack(media::TextKind kind,
                                            const std::string& label,
                                            const std::string& language);
+  void OnFFmpegTextTrack(media::TextKind kind,

[acolwell GONE FROM CHROMIUM, 2013/09/12 00:15:15]

A separate method should not be needed for FFmpegDemuxer vs ChunkDemuxer. This
code should not be aware of the difference.

[Matthew Heaney (Chromium), 2013/09/13 19:51:54]

Agreed.  I'm thinking I should make the DemuxerHost changes first, and then when
that's working, change the ChunkDemuxer to use that too.

[Matthew Heaney (Chromium), 2013/09/20 23:53:54]

I haven't touched the ChunkDemuxer yet.  I just wanted to make sure I had the
threading model correct first.  Assuming it's correct, I can either fix the
ChunkDemuxer in a separate patch for this CL, or defer the change to a future
CL.

+                         const std::string& label,
+                         const std::string& language,
+                         int index);
   void SetOpaque(bool);
 
  private:
   // Called after |defer_load_cb_| has decided to allow the load. If
   // |defer_load_cb_| is null this is called immediately.
   void DoLoad(LoadType load_type,
               const WebKit::WebURL& url,
               CORSMode cors_mode);
 
   // Called after asynchronous initialization of a data source completed.
@@ skipping 38 matching lines @@
   // Gets the duration value reported by the pipeline.
   double GetPipelineDuration() const;
 
   // Notifies WebKit of the duration change.
   void OnDurationChange();
 
   // Called by VideoRendererBase on its internal thread with the new frame to be
   // painted.
   void FrameReady(const scoped_refptr<media::VideoFrame>& frame);
 
+  // Called by TextRendererImpl when a decoded cue has been delivered on the
+  // text stream having the given |index|, and hence can be added to the
+  // cue list for this text track.
+  void CueReady(int index, const scoped_refptr<media::TextBuffer>& cue);
+
   WebKit::WebFrame* frame_;
 
   // TODO(hclam): get rid of these members and read from the pipeline directly.
   WebKit::WebMediaPlayer::NetworkState network_state_;
   WebKit::WebMediaPlayer::ReadyState ready_state_;
 
   // Keep a list of buffered time ranges.
   WebKit::WebTimeRanges buffered_;
 
   // Message loops for posting tasks on Chrome's main thread. Also used
@@ skipping 89 matching lines @@
   // playback.
   scoped_ptr<webkit::WebLayerImpl> video_weblayer_;
 
   // A pointer back to the compositor to inform it about state changes. This is
   // not NULL while the compositor is actively using this webmediaplayer.
   cc::VideoFrameProvider::Client* video_frame_provider_client_;
 
   // Text track objects get a unique index value when they're created.
   int text_track_index_;
 
+  // Text track objects for the FFmpeg-based pipeline.
+  typedef std::map<int, media::TextTrack*> TextTrackMap;
+  TextTrackMap ffmpeg_text_track_map_;

[acolwell GONE FROM CHROMIUM, 2013/09/12 00:15:15]

The ChunkDemuxer didn't need this map here. Why does the FFmpeg code need it?

[Matthew Heaney (Chromium), 2013/09/13 19:51:54]

I was having some trouble getting the callback mechanism working (I'm still
learning the threading model), and this was the solution that managed to get
working.  No surprise that it's not optimal.  I think this will go away as a
side effect of the DemuxerHost changes.

[Matthew Heaney (Chromium), 2013/09/20 23:53:54]

As it turns out, creating a TextTrack(Impl) object must be done on the main
thread (there's a check in WebMediaPlayerImpl::client()).  Text cues must also
be dispatched to the Blink on the main thread (there's a check deep inside
blink).  So I made the decision to just keep all knowledge of TextTrack(Impl)
objects in WebMediaPlayerImpl, so I left this map here.  (Though in this patch,
the key type did change from int to DemuxerStream.)

+
   DISALLOW_COPY_AND_ASSIGN(WebMediaPlayerImpl);
 };
 
 }  // namespace content
 
 #endif  // CONTENT_RENDERER_MEDIA_WEBMEDIAPLAYER_IMPL_H_