Encrypted Media: Enforcing the CDM to decode audio into S16 integers.

webkit/media/crypto/ppapi/ffmpeg_cdm_audio_decoder.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "webkit/media/crypto/ppapi/ffmpeg_cdm_audio_decoder.h"
 
 #include <algorithm>
 
 #include "base/logging.h"
+#include "media/base/audio_bus.h"
+#include "media/base/audio_timestamp_helper.h"
 #include "media/base/buffers.h"
+#include "media/base/data_buffer.h"
 #include "media/base/limits.h"
 #include "webkit/media/crypto/ppapi/cdm/content_decryption_module.h"
 
 // Include FFmpeg header files.
 extern "C" {
 // Temporarily disable possible loss of data warning.
 MSVC_PUSH_DISABLE_WARNING(4244);
 #include <libavcodec/avcodec.h>
 MSVC_POP_WARNING();
 }  // extern "C"
@@ skipping 16 matching lines @@
       return CODEC_ID_NONE;
   }
 }
 
 static void CdmAudioDecoderConfigToAVCodecContext(
     const cdm::AudioDecoderConfig& config,
     AVCodecContext* codec_context) {
   codec_context->codec_type = AVMEDIA_TYPE_AUDIO;
   codec_context->codec_id = CdmAudioCodecToCodecID(config.codec);
 
+  LOG(ERROR) << "ClearKey CDM bpc: " << config.bits_per_channel;

[DaleCurtis, 2013/01/10 01:24:35]

Remove?

[xhwang, 2013/01/10 18:17:23]

Done.

+
   switch (config.bits_per_channel) {
     case 8:
       codec_context->sample_fmt = AV_SAMPLE_FMT_U8;
       break;
     case 16:
       codec_context->sample_fmt = AV_SAMPLE_FMT_S16;
       break;
     case 32:
       codec_context->sample_fmt = AV_SAMPLE_FMT_S32;
       break;
@@ skipping 21 matching lines @@
 }
 
 FFmpegCdmAudioDecoder::FFmpegCdmAudioDecoder(cdm::Allocator* allocator)
     : is_initialized_(false),
       allocator_(allocator),
       codec_context_(NULL),
       av_frame_(NULL),
       bits_per_channel_(0),
       samples_per_second_(0),
       bytes_per_frame_(0),
-      output_timestamp_base_(media::kNoTimestamp()),
-      total_frames_decoded_(0),
       last_input_timestamp_(media::kNoTimestamp()),
       output_bytes_to_drop_(0) {
 }
 
 FFmpegCdmAudioDecoder::~FFmpegCdmAudioDecoder() {
   ReleaseFFmpegResources();
 }
 
 bool FFmpegCdmAudioDecoder::Initialize(const cdm::AudioDecoderConfig& config) {
   DVLOG(1) << "Initialize()";
 
   if (!IsValidConfig(config)) {
     LOG(ERROR) << "Initialize(): invalid audio decoder configuration.";
     return false;
   }
 
   if (is_initialized_) {
     LOG(ERROR) << "Initialize(): Already initialized.";
     return false;
   }
 
   // Initialize AVCodecContext structure.
   codec_context_ = avcodec_alloc_context3(NULL);
   CdmAudioDecoderConfigToAVCodecContext(config, codec_context_);
 
+  // MP3 decodes to S16P which we don't support, tell it to use S16 instead.
+  if (codec_context_->sample_fmt == AV_SAMPLE_FMT_S16P)
+    codec_context_->request_sample_fmt = AV_SAMPLE_FMT_S16;
+
   AVCodec* codec = avcodec_find_decoder(codec_context_->codec_id);
-  if (!codec) {
-    LOG(ERROR) << "Initialize(): avcodec_find_decoder failed.";
+  if (!codec || avcodec_open2(codec_context_, codec, NULL) < 0) {
+    DLOG(ERROR) << "Could not initialize audio decoder: "
+                << codec_context_->codec_id;
     return false;
   }
 
-  int status;
-  if ((status = avcodec_open2(codec_context_, codec, NULL)) < 0) {
-    LOG(ERROR) << "Initialize(): avcodec_open2 failed: " << status;
+  // Ensure avcodec_open2() respected our format request.
+  if (codec_context_->sample_fmt == AV_SAMPLE_FMT_S16P) {
+    DLOG(ERROR) << "Unable to configure a supported sample format: "
+                << codec_context_->sample_fmt;
     return false;
   }
 
+  // Some codecs will only output float data, so we need to convert to integer
+  // before returning the decoded buffer.
+  if (codec_context_->sample_fmt == AV_SAMPLE_FMT_FLTP ||
+      codec_context_->sample_fmt == AV_SAMPLE_FMT_FLT) {
+    // Preallocate the AudioBus for float conversions.  We can treat interleaved
+    // float data as a single planar channel since our output is expected in an
+    // interleaved format anyways.
+    int channels = codec_context_->channels;
+    if (codec_context_->sample_fmt == AV_SAMPLE_FMT_FLT)
+      channels = 1;
+    converter_bus_ = media::AudioBus::CreateWrapper(channels);
+  }
+
+  // Success!
   av_frame_ = avcodec_alloc_frame();
   bits_per_channel_ = config.bits_per_channel;
   samples_per_second_ = config.samples_per_second;
   bytes_per_frame_ = codec_context_->channels * bits_per_channel_ / 8;
+  output_timestamp_helper_.reset(new media::AudioTimestampHelper(
+      bytes_per_frame_, config.samples_per_second));
   serialized_audio_frames_.reserve(bytes_per_frame_ * samples_per_second_);
   is_initialized_ = true;
 
   return true;
 }
 
 void FFmpegCdmAudioDecoder::Deinitialize() {
   DVLOG(1) << "Deinitialize()";
   ReleaseFFmpegResources();
   is_initialized_ = false;
-  ResetAudioTimingData();
+  ResetTimestampState();
 }
 
 void FFmpegCdmAudioDecoder::Reset() {
   DVLOG(1) << "Reset()";
   avcodec_flush_buffers(codec_context_);
-  ResetAudioTimingData();
+  ResetTimestampState();
 }
 
 // static
 bool FFmpegCdmAudioDecoder::IsValidConfig(
     const cdm::AudioDecoderConfig& config) {
   return config.codec != cdm::AudioDecoderConfig::kUnknownAudioCodec &&
          config.channel_count > 0 &&
          config.channel_count <= kMaxChannels &&
          config.bits_per_channel > 0 &&
          config.bits_per_channel <= media::limits::kMaxBitsPerSample &&
          config.samples_per_second > 0 &&
          config.samples_per_second <= media::limits::kMaxSampleRate;
 }
 
 cdm::Status FFmpegCdmAudioDecoder::DecodeBuffer(
     const uint8_t* compressed_buffer,
     int32_t compressed_buffer_size,
     int64_t input_timestamp,
     cdm::AudioFrames* decoded_frames) {
   DVLOG(1) << "DecodeBuffer()";
   const bool is_end_of_stream = compressed_buffer_size == 0;
   base::TimeDelta timestamp =
       base::TimeDelta::FromMicroseconds(input_timestamp);
+
+  bool is_vorbis = codec_context_->codec_id == CODEC_ID_VORBIS;
   if (!is_end_of_stream) {
     if (last_input_timestamp_ == media::kNoTimestamp()) {
-      if (codec_context_->codec_id == CODEC_ID_VORBIS &&
-          timestamp < base::TimeDelta()) {
+      if (is_vorbis && timestamp < base::TimeDelta()) {
         // Dropping frames for negative timestamps as outlined in section A.2
         // in the Vorbis spec. http://xiph.org/vorbis/doc/Vorbis_I_spec.html
         int frames_to_drop = floor(
             0.5 + -timestamp.InSecondsF() * samples_per_second_);
         output_bytes_to_drop_ = bytes_per_frame_ * frames_to_drop;
       } else {
         last_input_timestamp_ = timestamp;
       }
     } else if (timestamp != media::kNoTimestamp()) {
       if (timestamp < last_input_timestamp_) {
@@ skipping 38 matching lines @@
           << compressed_buffer_size << " bytes";
 
       return cdm::kDecodeError;
     }
 
     // Update packet size and data pointer in case we need to call the decoder
     // with the remaining bytes from this packet.
     packet.size -= result;
     packet.data += result;
 
-    if (output_timestamp_base_ == media::kNoTimestamp() && !is_end_of_stream) {
+    if (output_timestamp_helper_->base_timestamp() == media::kNoTimestamp() &&
+        !is_end_of_stream) {
       DCHECK(timestamp != media::kNoTimestamp());
       if (output_bytes_to_drop_ > 0) {
+        // Currently Vorbis is the only codec that causes us to drop samples.
         // If we have to drop samples it always means the timeline starts at 0.
-        output_timestamp_base_ = base::TimeDelta();
+        DCHECK_EQ(codec_context_->codec_id, CODEC_ID_VORBIS);
+        output_timestamp_helper_->SetBaseTimestamp(base::TimeDelta());
       } else {
-        output_timestamp_base_ = timestamp;
+        output_timestamp_helper_->SetBaseTimestamp(timestamp);
       }
     }
 
-    const uint8_t* decoded_audio_data = NULL;
     int decoded_audio_size = 0;
     if (frame_decoded) {
       int output_sample_rate = av_frame_->sample_rate;
       if (output_sample_rate != samples_per_second_) {
         DLOG(ERROR) << "Output sample rate (" << output_sample_rate
                     << ") doesn't match expected rate " << samples_per_second_;
         return cdm::kDecodeError;
       }
 
-      decoded_audio_data = av_frame_->data[0];
-      decoded_audio_size =
-          av_samples_get_buffer_size(NULL,
-                                     codec_context_->channels,
-                                     av_frame_->nb_samples,
-                                     codec_context_->sample_fmt,
-                                     1);
+      decoded_audio_size = av_samples_get_buffer_size(
+          NULL, codec_context_->channels, av_frame_->nb_samples,
+          codec_context_->sample_fmt, 1);
+      // If we're decoding into float, adjust audio size.
+      if (converter_bus_ && bits_per_channel_ / 8 != sizeof(float)) {
+        DCHECK(codec_context_->sample_fmt == AV_SAMPLE_FMT_FLT ||
+               codec_context_->sample_fmt == AV_SAMPLE_FMT_FLTP);
+        decoded_audio_size *=
+            static_cast<float>(bits_per_channel_ / 8) / sizeof(float);
+      }
     }
 
+    int start_sample = 0;
     if (decoded_audio_size > 0 && output_bytes_to_drop_ > 0) {
+      DCHECK_EQ(decoded_audio_size % bytes_per_frame_, 0)
+          << "Decoder didn't output full frames";
+
       int dropped_size = std::min(decoded_audio_size, output_bytes_to_drop_);
-      decoded_audio_data += dropped_size;
+      start_sample = dropped_size / bytes_per_frame_;
       decoded_audio_size -= dropped_size;
       output_bytes_to_drop_ -= dropped_size;
     }
 
+    scoped_refptr<media::DataBuffer> output;
     if (decoded_audio_size > 0) {
       DCHECK_EQ(decoded_audio_size % bytes_per_frame_, 0)
           << "Decoder didn't output full frames";
 
-      base::TimeDelta output_timestamp = GetNextOutputTimestamp();
-      total_frames_decoded_ += decoded_audio_size / bytes_per_frame_;
+      // Convert float data using an AudioBus.
+      if (converter_bus_) {
+        // Setup the AudioBus as a wrapper of the AVFrame data and then use
+        // AudioBus::ToInterleaved() to convert the data as necessary.
+        int skip_frames = start_sample;
+        int total_frames = av_frame_->nb_samples - start_sample;
+        if (codec_context_->sample_fmt == AV_SAMPLE_FMT_FLT) {
+          DCHECK_EQ(converter_bus_->channels(), 1);
+          total_frames *= codec_context_->channels;
+          skip_frames *= codec_context_->channels;
+        }
+        converter_bus_->set_frames(total_frames);
+        DCHECK_EQ(decoded_audio_size,
+                  converter_bus_->frames() * bytes_per_frame_);
+
+        for (int i = 0; i < converter_bus_->channels(); ++i) {
+          converter_bus_->SetChannelData(i, reinterpret_cast<float*>(
+              av_frame_->extended_data[i]) + skip_frames);
+        }
+
+        output = new media::DataBuffer(decoded_audio_size);
+        output->SetDataSize(decoded_audio_size);
+        converter_bus_->ToInterleaved(
+            converter_bus_->frames(), bits_per_channel_ / 8,
+            output->GetWritableData());
+      } else {
+        output = new media::DataBuffer(
+            av_frame_->extended_data[0] + start_sample * bytes_per_frame_,
+            decoded_audio_size);
+      }
+
+      base::TimeDelta output_timestamp =
+          output_timestamp_helper_->GetTimestamp();
+      output_timestamp_helper_->AddBytes(decoded_audio_size);
 
       // Serialize the audio samples into |serialized_audio_frames_|.
       SerializeInt64(output_timestamp.InMicroseconds());
-      SerializeInt64(decoded_audio_size);
-      serialized_audio_frames_.insert(serialized_audio_frames_.end(),
-                                      decoded_audio_data,
-                                      decoded_audio_data + decoded_audio_size);
+      SerializeInt64(output->GetDataSize());
+      serialized_audio_frames_.insert(
+          serialized_audio_frames_.end(),
+          output->GetData(),
+          output->GetData() + output->GetDataSize());
     }
   } while (packet.size > 0);
 
   if (!serialized_audio_frames_.empty()) {
     decoded_frames->SetFrameBuffer(
         allocator_->Allocate(serialized_audio_frames_.size()));
     if (!decoded_frames->FrameBuffer()) {
       LOG(ERROR) << "DecodeBuffer() cdm::Allocator::Allocate failed.";
       return cdm::kDecodeError;
     }
     memcpy(decoded_frames->FrameBuffer()->Data(),
            &serialized_audio_frames_[0],
            serialized_audio_frames_.size());
     decoded_frames->FrameBuffer()->SetSize(serialized_audio_frames_.size());
     serialized_audio_frames_.clear();
 
     return cdm::kSuccess;
   }
 
   return cdm::kNeedMoreData;
 }
 
-void FFmpegCdmAudioDecoder::ResetAudioTimingData() {
-  output_timestamp_base_ = media::kNoTimestamp();
-  total_frames_decoded_ = 0;
+void FFmpegCdmAudioDecoder::ResetTimestampState() {
+  output_timestamp_helper_->SetBaseTimestamp(media::kNoTimestamp());
   last_input_timestamp_ = media::kNoTimestamp();
   output_bytes_to_drop_ = 0;
 }
 
 void FFmpegCdmAudioDecoder::ReleaseFFmpegResources() {
   DVLOG(1) << "ReleaseFFmpegResources()";
 
   if (codec_context_) {
     av_free(codec_context_->extradata);
     avcodec_close(codec_context_);
     av_free(codec_context_);
     codec_context_ = NULL;
   }
   if (av_frame_) {
     av_free(av_frame_);
     av_frame_ = NULL;
   }
 }
 
-base::TimeDelta FFmpegCdmAudioDecoder::GetNextOutputTimestamp() const {
-  DCHECK(output_timestamp_base_ != media::kNoTimestamp());
-  const double total_frames_decoded = total_frames_decoded_;
-  const double decoded_us = (total_frames_decoded / samples_per_second_) *
-      base::Time::kMicrosecondsPerSecond;
-  return output_timestamp_base_ +
-      base::TimeDelta::FromMicroseconds(decoded_us);
-}
-
 void FFmpegCdmAudioDecoder::SerializeInt64(int64 value) {
   int previous_size = serialized_audio_frames_.size();
   serialized_audio_frames_.resize(previous_size + sizeof(value));
   memcpy(&serialized_audio_frames_[0] + previous_size, &value, sizeof(value));
 }
 
 }  // namespace webkit_media