Render inband text tracks in the media pipeline

media/filters/ffmpeg_demuxer.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/filters/ffmpeg_demuxer.h"
 
 #include <algorithm>
 #include <string>
 
 #include "base/base64.h"
 #include "base/bind.h"
 #include "base/callback.h"
 #include "base/callback_helpers.h"
-#include "base/command_line.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/message_loop/message_loop.h"
 #include "base/metrics/sparse_histogram.h"
-#include "base/stl_util.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/sys_byteorder.h"
 #include "base/task_runner_util.h"
 #include "base/time/time.h"
 #include "media/base/audio_decoder_config.h"
 #include "media/base/bind_to_loop.h"
 #include "media/base/decoder_buffer.h"
 #include "media/base/decrypt_config.h"
 #include "media/base/limits.h"
 #include "media/base/media_log.h"
-#include "media/base/media_switches.h"
 #include "media/base/video_decoder_config.h"
 #include "media/ffmpeg/ffmpeg_common.h"
 #include "media/filters/ffmpeg_glue.h"
 #include "media/filters/ffmpeg_h264_to_annex_b_bitstream_converter.h"
+#include "media/filters/webvtt_util.h"
 #include "media/webm/webm_crypto_helpers.h"
 
 namespace media {
 
 //
 // FFmpegDemuxerStream
 //
 FFmpegDemuxerStream::FFmpegDemuxerStream(
     FFmpegDemuxer* demuxer,
     AVStream* stream)
@@ skipping 13 matching lines @@
     case AVMEDIA_TYPE_AUDIO:
       type_ = AUDIO;
       AVStreamToAudioDecoderConfig(stream, &audio_config_, true);
       is_encrypted = audio_config_.is_encrypted();
       break;
     case AVMEDIA_TYPE_VIDEO:
       type_ = VIDEO;
       AVStreamToVideoDecoderConfig(stream, &video_config_, true);
       is_encrypted = video_config_.is_encrypted();
       break;
+    case AVMEDIA_TYPE_SUBTITLE:
+      type_ = TEXT;
+      break;
     default:
       NOTREACHED();
       break;
   }
 
   // Calculate the duration.
   duration_ = ConvertStreamTimestamp(stream->time_base, stream->duration);
 
   if (stream_->codec->codec_id == AV_CODEC_ID_H264) {
     bitstream_converter_.reset(
@@ skipping 30 matching lines @@
   // Convert the packet if there is a bitstream filter.
   if (packet->data && bitstream_converter_enabled_ &&
       !bitstream_converter_->ConvertPacket(packet.get())) {
     LOG(ERROR) << "Format conversion failed.";
   }
 
   // Get side data if any. For now, the only type of side_data is VP8 Alpha. We
   // keep this generic so that other side_data types in the future can be
   // handled the same way as well.
   av_packet_split_side_data(packet.get());
-  int side_data_size = 0;
-  uint8* side_data = av_packet_get_side_data(
-      packet.get(),
-      AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL,
-      &side_data_size);
+  scoped_refptr<DecoderBuffer> buffer;
 
-  // If a packet is returned by FFmpeg's av_parser_parse2() the packet will
-  // reference inner memory of FFmpeg.  As such we should transfer the packet
-  // into memory we control.
-  scoped_refptr<DecoderBuffer> buffer;
-  if (side_data_size > 0) {
+  if (type() == DemuxerStream::TEXT) {
+    int id_size = 0;
+    uint8* id_data = av_packet_get_side_data(
+        packet.get(),
+        AV_PKT_DATA_WEBVTT_IDENTIFIER,
+        &id_size);
+
+    int settings_size = 0;
+    uint8* settings_data = av_packet_get_side_data(
+        packet.get(),
+        AV_PKT_DATA_WEBVTT_SETTINGS,
+        &settings_size);
+
+    std::vector<uint8> side_data;
+    MakeSideData(id_data, id_data + id_size,
+                 settings_data, settings_data + settings_size,
+                 &side_data);
+
     buffer = DecoderBuffer::CopyFrom(packet.get()->data, packet.get()->size,
-                                     side_data, side_data_size);
+                                     side_data.data(), side_data.size());
   } else {
-    buffer = DecoderBuffer::CopyFrom(packet.get()->data, packet.get()->size);
-  }
+    int side_data_size = 0;
+    uint8* side_data = av_packet_get_side_data(
+        packet.get(),
+        AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL,
+        &side_data_size);
 
-  int skip_samples_size = 0;
-  uint8* skip_samples = av_packet_get_side_data(packet.get(),
-                                                AV_PKT_DATA_SKIP_SAMPLES,
-                                                &skip_samples_size);
-  const int kSkipSamplesValidSize = 10;
-  const int kSkipSamplesOffset = 4;
-  if (skip_samples_size >= kSkipSamplesValidSize) {
-    int discard_padding_samples = base::ByteSwapToLE32(
-        *(reinterpret_cast<const uint32*>(skip_samples +
-                                          kSkipSamplesOffset)));
-    // TODO(vigneshv): Change decoder buffer to use number of samples so that
-    // this conversion can be avoided.
-    buffer->set_discard_padding(base::TimeDelta::FromMicroseconds(
-        discard_padding_samples * 1000000.0 /
-        audio_decoder_config().samples_per_second()));
+    // If a packet is returned by FFmpeg's av_parser_parse2() the packet will
+    // reference inner memory of FFmpeg.  As such we should transfer the packet
+    // into memory we control.
+    if (side_data_size > 0) {
+      buffer = DecoderBuffer::CopyFrom(packet.get()->data, packet.get()->size,
+                                       side_data, side_data_size);
+    } else {
+      buffer = DecoderBuffer::CopyFrom(packet.get()->data, packet.get()->size);
+    }
+
+    int skip_samples_size = 0;
+    uint8* skip_samples = av_packet_get_side_data(packet.get(),
+                                                  AV_PKT_DATA_SKIP_SAMPLES,
+                                                  &skip_samples_size);
+    const int kSkipSamplesValidSize = 10;
+    const int kSkipSamplesOffset = 4;
+    if (skip_samples_size >= kSkipSamplesValidSize) {
+      int discard_padding_samples = base::ByteSwapToLE32(
+          *(reinterpret_cast<const uint32*>(skip_samples +
+                                            kSkipSamplesOffset)));
+      // TODO(vigneshv): Change decoder buffer to use number of samples so that
+      // this conversion can be avoided.
+      buffer->set_discard_padding(base::TimeDelta::FromMicroseconds(
+          discard_padding_samples * 1000000.0 /
+          audio_decoder_config().samples_per_second()));
+    }
   }
 
   if ((type() == DemuxerStream::AUDIO && audio_config_.is_encrypted()) ||
       (type() == DemuxerStream::VIDEO && video_config_.is_encrypted())) {
     scoped_ptr<DecryptConfig> config(WebMCreateDecryptConfig(
-        packet->data,  packet->size,
+        packet->data, packet->size,
         reinterpret_cast<const uint8*>(encryption_key_id_.data()),
         encryption_key_id_.size()));
     if (!config)
       LOG(ERROR) << "Creation of DecryptConfig failed.";
     buffer->set_decrypt_config(config.Pass());
   }
 
   buffer->set_timestamp(ConvertStreamTimestamp(
       stream_->time_base, packet->pts));
   buffer->set_duration(ConvertStreamTimestamp(
@@ skipping 117 matching lines @@
   // TODO(scherkus): Remove early return and reenable time-based capacity
   // after our data sources support canceling/concurrent reads, see
   // http://crbug.com/165762 for details.
   return !read_cb_.is_null();
 
   // Try to have one second's worth of encoded data per stream.
   const base::TimeDelta kCapacity = base::TimeDelta::FromSeconds(1);
   return buffer_queue_.IsEmpty() || buffer_queue_.Duration() < kCapacity;
 }
 
+TextKind FFmpegDemuxerStream::GetTextKind() const {
+  DCHECK_EQ(type_, DemuxerStream::TEXT);
+
+  if (stream_->disposition & AV_DISPOSITION_CAPTIONS)
+    return kTextCaptions;
+
+  if (stream_->disposition & AV_DISPOSITION_DESCRIPTIONS)
+    return kTextDescriptions;
+
+  if (stream_->disposition & AV_DISPOSITION_METADATA)
+    return kTextMetadata;
+
+  return kTextSubtitles;
+}
+
+std::string FFmpegDemuxerStream::GetMetadata(const char* key) const {
+  const AVDictionaryEntry* entry =
+      av_dict_get(stream_->metadata, key, NULL, 0);
+  return (entry == NULL || entry->value == NULL) ? "" : entry->value;
+}
+
 // static
 base::TimeDelta FFmpegDemuxerStream::ConvertStreamTimestamp(
     const AVRational& time_base, int64 timestamp) {
   if (timestamp == static_cast<int64>(AV_NOPTS_VALUE))
     return kNoTimestamp();
 
   return ConvertFromTimeBase(time_base, timestamp);
 }
 
 //
 // FFmpegDemuxer
 //
 FFmpegDemuxer::FFmpegDemuxer(
     const scoped_refptr<base::MessageLoopProxy>& message_loop,
     DataSource* data_source,
     const NeedKeyCB& need_key_cb,
     const scoped_refptr<MediaLog>& media_log)
     : host_(NULL),
       message_loop_(message_loop),
       weak_factory_(this),
       blocking_thread_("FFmpegDemuxer"),
       pending_read_(false),
       pending_seek_(false),
       data_source_(data_source),
       media_log_(media_log),
       bitrate_(0),
       start_time_(kNoTimestamp()),
       audio_disabled_(false),
+      text_enabled_(false),
       duration_known_(false),
       url_protocol_(data_source, BindToLoop(message_loop_, base::Bind(
           &FFmpegDemuxer::OnDataSourceError, base::Unretained(this)))),
       need_key_cb_(need_key_cb) {
   DCHECK(message_loop_.get());
   DCHECK(data_source_);
 }
 
 FFmpegDemuxer::~FFmpegDemuxer() {}
 
@@ skipping 37 matching lines @@
   audio_disabled_ = true;
   StreamVector::iterator iter;
   for (iter = streams_.begin(); iter != streams_.end(); ++iter) {
     if (*iter && (*iter)->type() == DemuxerStream::AUDIO) {
       (*iter)->Stop();
     }
   }
 }
 
 void FFmpegDemuxer::Initialize(DemuxerHost* host,
-                               const PipelineStatusCB& status_cb) {
+                               const PipelineStatusCB& status_cb,
+                               bool enable_text_tracks) {
   DCHECK(message_loop_->BelongsToCurrentThread());
   host_ = host;
   weak_this_ = weak_factory_.GetWeakPtr();
+  text_enabled_ = enable_text_tracks;
 
   // TODO(scherkus): DataSource should have a host by this point,
   // see http://crbug.com/122071
   data_source_->set_host(host);
 
   glue_.reset(new FFmpegGlue(&url_protocol_));
   AVFormatContext* format_context = glue_->format_context();
 
   // Disable ID3v1 tag reading to avoid costly seeks to end of file for data we
   // don't use.  FFmpeg will only read ID3v1 tags if no other metadata is
@@ skipping 23 matching lines @@
     }
   }
   return NULL;
 }
 
 base::TimeDelta FFmpegDemuxer::GetStartTime() const {
   DCHECK(message_loop_->BelongsToCurrentThread());
   return start_time_;
 }
 
+void FFmpegDemuxer::AddTextStreams() {
+  DCHECK(message_loop_->BelongsToCurrentThread());
+
+  for (StreamVector::size_type idx = 0; idx < streams_.size(); ++idx) {
+    FFmpegDemuxerStream* stream = streams_[idx];
+    if (stream == NULL || stream->type() != DemuxerStream::TEXT)
+      continue;
+
+    TextKind kind = stream->GetTextKind();
+    std::string title = stream->GetMetadata("title");
+    std::string language = stream->GetMetadata("language");
+
+    host_->AddTextStream(stream, TextTrackConfig(kind, title, language));
+  }
+}
+
 // Helper for calculating the bitrate of the media based on information stored
 // in |format_context| or failing that the size and duration of the media.
 //
 // Returns 0 if a bitrate could not be determined.
 static int CalculateBitrate(
     AVFormatContext* format_context,
     const base::TimeDelta& duration,
     int64 filesize_in_bytes) {
   // If there is a bitrate set on the container, use it.
   if (format_context->bit_rate > 0)
@@ skipping 98 matching lines @@
       // Log the codec detected, whether it is supported or not.
       UMA_HISTOGRAM_SPARSE_SLOWLY("Media.DetectedVideoCodec",
                                   codec_context->codec_id);
       // Ensure the codec is supported. IsValidConfig() also checks that the
       // frame size and visible size are valid.
       AVStreamToVideoDecoderConfig(stream, &video_config, false);
 
       if (!video_config.IsValidConfig())
         continue;
       video_stream = stream;
+    } else if (codec_type == AVMEDIA_TYPE_SUBTITLE) {
+      if (codec_context->codec_id != AV_CODEC_ID_WEBVTT || !text_enabled_) {
+        continue;
+      }
     } else {
       continue;
     }
 
     streams_[i] = new FFmpegDemuxerStream(this, stream);
     max_duration = std::max(max_duration, streams_[i]->duration());
 
     if (stream->first_dts != static_cast<int64_t>(AV_NOPTS_VALUE)) {
       const base::TimeDelta first_dts = ConvertFromTimeBase(
           stream->time_base, stream->first_dts);
       if (start_time_ == kNoTimestamp() || first_dts < start_time_)
         start_time_ = first_dts;
     }
   }
 
   if (!audio_stream && !video_stream) {
     status_cb.Run(DEMUXER_ERROR_NO_SUPPORTED_STREAMS);
     return;
   }
 
+  if (text_enabled_)
+    AddTextStreams();
+
   if (format_context->duration != static_cast<int64_t>(AV_NOPTS_VALUE)) {
     // If there is a duration value in the container use that to find the
     // maximum between it and the duration from A/V streams.
     const AVRational av_time_base = {1, AV_TIME_BASE};
     max_duration =
         std::max(max_duration,
                  ConvertFromTimeBase(av_time_base, format_context->duration));
   } else {
     // The duration is unknown, in which case this is likely a live stream.
     max_duration = kInfiniteDuration();
@@ skipping 274 matching lines @@
   }
   for (size_t i = 0; i < buffered.size(); ++i)
     host_->AddBufferedTimeRange(buffered.start(i), buffered.end(i));
 }
 
 void FFmpegDemuxer::OnDataSourceError() {
   host_->OnDemuxerError(PIPELINE_ERROR_READ);
 }
 
 }  // namespace media