Encrypted Media: Enforcing the CDM to decode audio into S16 integers.

webkit/media/crypto/ppapi/ffmpeg_cdm_audio_decoder.cc

Index: webkit/media/crypto/ppapi/ffmpeg_cdm_audio_decoder.cc
diff --git a/webkit/media/crypto/ppapi/ffmpeg_cdm_audio_decoder.cc b/webkit/media/crypto/ppapi/ffmpeg_cdm_audio_decoder.cc
index 8f76256dd915388f5a64d3bb91f17e84a77f05f6..e23d4c2c71b06dfbc1d60ea83e3f458525127417 100644
--- a/webkit/media/crypto/ppapi/ffmpeg_cdm_audio_decoder.cc
+++ b/webkit/media/crypto/ppapi/ffmpeg_cdm_audio_decoder.cc
@@ -7,7 +7,10 @@
 #include <algorithm>
 
 #include "base/logging.h"
+#include "media/base/audio_bus.h"
+#include "media/base/audio_timestamp_helper.h"
 #include "media/base/buffers.h"
+#include "media/base/data_buffer.h"
 #include "media/base/limits.h"
 #include "webkit/media/crypto/ppapi/cdm/content_decryption_module.h"
 
@@ -44,6 +47,8 @@ static void CdmAudioDecoderConfigToAVCodecContext(
   codec_context->codec_type = AVMEDIA_TYPE_AUDIO;
   codec_context->codec_id = CdmAudioCodecToCodecID(config.codec);
 
+  LOG(ERROR) << "ClearKey CDM bpc: " << config.bits_per_channel;

[DaleCurtis, 2013/01/10 01:24:35]

Remove?

[xhwang, 2013/01/10 18:17:23]

Done.

+
   switch (config.bits_per_channel) {
     case 8:
       codec_context->sample_fmt = AV_SAMPLE_FMT_U8;
@@ -85,8 +90,6 @@ FFmpegCdmAudioDecoder::FFmpegCdmAudioDecoder(cdm::Allocator* allocator)
       bits_per_channel_(0),
       samples_per_second_(0),
       bytes_per_frame_(0),
-      output_timestamp_base_(media::kNoTimestamp()),
-      total_frames_decoded_(0),
       last_input_timestamp_(media::kNoTimestamp()),
       output_bytes_to_drop_(0) {
 }
@@ -112,22 +115,44 @@ bool FFmpegCdmAudioDecoder::Initialize(const cdm::AudioDecoderConfig& config) {
   codec_context_ = avcodec_alloc_context3(NULL);
   CdmAudioDecoderConfigToAVCodecContext(config, codec_context_);
 
+  // MP3 decodes to S16P which we don't support, tell it to use S16 instead.
+  if (codec_context_->sample_fmt == AV_SAMPLE_FMT_S16P)
+    codec_context_->request_sample_fmt = AV_SAMPLE_FMT_S16;
+
   AVCodec* codec = avcodec_find_decoder(codec_context_->codec_id);
-  if (!codec) {
-    LOG(ERROR) << "Initialize(): avcodec_find_decoder failed.";
+  if (!codec || avcodec_open2(codec_context_, codec, NULL) < 0) {
+    DLOG(ERROR) << "Could not initialize audio decoder: "
+                << codec_context_->codec_id;
     return false;
   }
 
-  int status;
-  if ((status = avcodec_open2(codec_context_, codec, NULL)) < 0) {
-    LOG(ERROR) << "Initialize(): avcodec_open2 failed: " << status;
+  // Ensure avcodec_open2() respected our format request.
+  if (codec_context_->sample_fmt == AV_SAMPLE_FMT_S16P) {
+    DLOG(ERROR) << "Unable to configure a supported sample format: "
+                << codec_context_->sample_fmt;
     return false;
   }
 
+  // Some codecs will only output float data, so we need to convert to integer
+  // before returning the decoded buffer.
+  if (codec_context_->sample_fmt == AV_SAMPLE_FMT_FLTP ||
+      codec_context_->sample_fmt == AV_SAMPLE_FMT_FLT) {
+    // Preallocate the AudioBus for float conversions.  We can treat interleaved
+    // float data as a single planar channel since our output is expected in an
+    // interleaved format anyways.
+    int channels = codec_context_->channels;
+    if (codec_context_->sample_fmt == AV_SAMPLE_FMT_FLT)
+      channels = 1;
+    converter_bus_ = media::AudioBus::CreateWrapper(channels);
+  }
+
+  // Success!
   av_frame_ = avcodec_alloc_frame();
   bits_per_channel_ = config.bits_per_channel;
   samples_per_second_ = config.samples_per_second;
   bytes_per_frame_ = codec_context_->channels * bits_per_channel_ / 8;
+  output_timestamp_helper_.reset(new media::AudioTimestampHelper(
+      bytes_per_frame_, config.samples_per_second));
   serialized_audio_frames_.reserve(bytes_per_frame_ * samples_per_second_);
   is_initialized_ = true;
 
@@ -138,13 +163,13 @@ void FFmpegCdmAudioDecoder::Deinitialize() {
   DVLOG(1) << "Deinitialize()";
   ReleaseFFmpegResources();
   is_initialized_ = false;
-  ResetAudioTimingData();
+  ResetTimestampState();
 }
 
 void FFmpegCdmAudioDecoder::Reset() {
   DVLOG(1) << "Reset()";
   avcodec_flush_buffers(codec_context_);
-  ResetAudioTimingData();
+  ResetTimestampState();
 }
 
 // static
@@ -168,10 +193,11 @@ cdm::Status FFmpegCdmAudioDecoder::DecodeBuffer(
   const bool is_end_of_stream = compressed_buffer_size == 0;
   base::TimeDelta timestamp =
       base::TimeDelta::FromMicroseconds(input_timestamp);
+
+  bool is_vorbis = codec_context_->codec_id == CODEC_ID_VORBIS;
   if (!is_end_of_stream) {
     if (last_input_timestamp_ == media::kNoTimestamp()) {
-      if (codec_context_->codec_id == CODEC_ID_VORBIS &&
-          timestamp < base::TimeDelta()) {
+      if (is_vorbis && timestamp < base::TimeDelta()) {
         // Dropping frames for negative timestamps as outlined in section A.2
         // in the Vorbis spec. http://xiph.org/vorbis/doc/Vorbis_I_spec.html
         int frames_to_drop = floor(
@@ -230,17 +256,19 @@ cdm::Status FFmpegCdmAudioDecoder::DecodeBuffer(
     packet.size -= result;
     packet.data += result;
 
-    if (output_timestamp_base_ == media::kNoTimestamp() && !is_end_of_stream) {
+    if (output_timestamp_helper_->base_timestamp() == media::kNoTimestamp() &&
+        !is_end_of_stream) {
       DCHECK(timestamp != media::kNoTimestamp());
       if (output_bytes_to_drop_ > 0) {
+        // Currently Vorbis is the only codec that causes us to drop samples.
         // If we have to drop samples it always means the timeline starts at 0.
-        output_timestamp_base_ = base::TimeDelta();
+        DCHECK_EQ(codec_context_->codec_id, CODEC_ID_VORBIS);
+        output_timestamp_helper_->SetBaseTimestamp(base::TimeDelta());
       } else {
-        output_timestamp_base_ = timestamp;
+        output_timestamp_helper_->SetBaseTimestamp(timestamp);
       }
     }
 
-    const uint8_t* decoded_audio_data = NULL;
     int decoded_audio_size = 0;
     if (frame_decoded) {
       int output_sample_rate = av_frame_->sample_rate;
@@ -250,35 +278,76 @@ cdm::Status FFmpegCdmAudioDecoder::DecodeBuffer(
         return cdm::kDecodeError;
       }
 
-      decoded_audio_data = av_frame_->data[0];
-      decoded_audio_size =
-          av_samples_get_buffer_size(NULL,
-                                     codec_context_->channels,
-                                     av_frame_->nb_samples,
-                                     codec_context_->sample_fmt,
-                                     1);
+      decoded_audio_size = av_samples_get_buffer_size(
+          NULL, codec_context_->channels, av_frame_->nb_samples,
+          codec_context_->sample_fmt, 1);
+      // If we're decoding into float, adjust audio size.
+      if (converter_bus_ && bits_per_channel_ / 8 != sizeof(float)) {
+        DCHECK(codec_context_->sample_fmt == AV_SAMPLE_FMT_FLT ||
+               codec_context_->sample_fmt == AV_SAMPLE_FMT_FLTP);
+        decoded_audio_size *=
+            static_cast<float>(bits_per_channel_ / 8) / sizeof(float);
+      }
     }
 
+    int start_sample = 0;
     if (decoded_audio_size > 0 && output_bytes_to_drop_ > 0) {
+      DCHECK_EQ(decoded_audio_size % bytes_per_frame_, 0)
+          << "Decoder didn't output full frames";
+
       int dropped_size = std::min(decoded_audio_size, output_bytes_to_drop_);
-      decoded_audio_data += dropped_size;
+      start_sample = dropped_size / bytes_per_frame_;
       decoded_audio_size -= dropped_size;
       output_bytes_to_drop_ -= dropped_size;
     }
 
+    scoped_refptr<media::DataBuffer> output;
     if (decoded_audio_size > 0) {
       DCHECK_EQ(decoded_audio_size % bytes_per_frame_, 0)
           << "Decoder didn't output full frames";
 
-      base::TimeDelta output_timestamp = GetNextOutputTimestamp();
-      total_frames_decoded_ += decoded_audio_size / bytes_per_frame_;
+      // Convert float data using an AudioBus.
+      if (converter_bus_) {
+        // Setup the AudioBus as a wrapper of the AVFrame data and then use
+        // AudioBus::ToInterleaved() to convert the data as necessary.
+        int skip_frames = start_sample;
+        int total_frames = av_frame_->nb_samples - start_sample;
+        if (codec_context_->sample_fmt == AV_SAMPLE_FMT_FLT) {
+          DCHECK_EQ(converter_bus_->channels(), 1);
+          total_frames *= codec_context_->channels;
+          skip_frames *= codec_context_->channels;
+        }
+        converter_bus_->set_frames(total_frames);
+        DCHECK_EQ(decoded_audio_size,
+                  converter_bus_->frames() * bytes_per_frame_);
+
+        for (int i = 0; i < converter_bus_->channels(); ++i) {
+          converter_bus_->SetChannelData(i, reinterpret_cast<float*>(
+              av_frame_->extended_data[i]) + skip_frames);
+        }
+
+        output = new media::DataBuffer(decoded_audio_size);
+        output->SetDataSize(decoded_audio_size);
+        converter_bus_->ToInterleaved(
+            converter_bus_->frames(), bits_per_channel_ / 8,
+            output->GetWritableData());
+      } else {
+        output = new media::DataBuffer(
+            av_frame_->extended_data[0] + start_sample * bytes_per_frame_,
+            decoded_audio_size);
+      }
+
+      base::TimeDelta output_timestamp =
+          output_timestamp_helper_->GetTimestamp();
+      output_timestamp_helper_->AddBytes(decoded_audio_size);
 
       // Serialize the audio samples into |serialized_audio_frames_|.
       SerializeInt64(output_timestamp.InMicroseconds());
-      SerializeInt64(decoded_audio_size);
-      serialized_audio_frames_.insert(serialized_audio_frames_.end(),
-                                      decoded_audio_data,
-                                      decoded_audio_data + decoded_audio_size);
+      SerializeInt64(output->GetDataSize());
+      serialized_audio_frames_.insert(
+          serialized_audio_frames_.end(),
+          output->GetData(),
+          output->GetData() + output->GetDataSize());
     }
   } while (packet.size > 0);
 
@@ -301,9 +370,8 @@ cdm::Status FFmpegCdmAudioDecoder::DecodeBuffer(
   return cdm::kNeedMoreData;
 }
 
-void FFmpegCdmAudioDecoder::ResetAudioTimingData() {
-  output_timestamp_base_ = media::kNoTimestamp();
-  total_frames_decoded_ = 0;
+void FFmpegCdmAudioDecoder::ResetTimestampState() {
+  output_timestamp_helper_->SetBaseTimestamp(media::kNoTimestamp());
   last_input_timestamp_ = media::kNoTimestamp();
   output_bytes_to_drop_ = 0;
 }
@@ -323,15 +391,6 @@ void FFmpegCdmAudioDecoder::ReleaseFFmpegResources() {
   }
 }
 
-base::TimeDelta FFmpegCdmAudioDecoder::GetNextOutputTimestamp() const {
-  DCHECK(output_timestamp_base_ != media::kNoTimestamp());
-  const double total_frames_decoded = total_frames_decoded_;
-  const double decoded_us = (total_frames_decoded / samples_per_second_) *
-      base::Time::kMicrosecondsPerSecond;
-  return output_timestamp_base_ +
-      base::TimeDelta::FromMicroseconds(decoded_us);
-}
-
 void FFmpegCdmAudioDecoder::SerializeInt64(int64 value) {
   int previous_size = serialized_audio_frames_.size();
   serialized_audio_frames_.resize(previous_size + sizeof(value));