[cast] AAC encoder for OS X and iOS based on AudioConverter and AudioFile

media/cast/sender/audio_encoder.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/cast/sender/audio_encoder.h"
 
 #include <algorithm>
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/location.h"
 #include "base/stl_util.h"
 #include "base/sys_byteorder.h"
 #include "base/time/time.h"
 #include "media/base/audio_bus.h"
 #include "media/cast/cast_defines.h"
 #include "media/cast/cast_environment.h"
+
+#if !defined(OS_IOS)
 #include "third_party/opus/src/include/opus.h"
+#endif
+
+#if defined(OS_MACOSX)
+#include <AudioToolbox/AudioToolbox.h>
+#endif
 
 namespace media {
 namespace cast {
 
 
 // Base class that handles the common problem of feeding one or more AudioBus'
 // data into a buffer and then, once the buffer is full, encoding the signal and
 // emitting an EncodedFrame via the FrameEncodedCallback.
 //
 // Subclasses complete the implementation by handling the actual encoding
@@ skipping 163 matching lines @@
   // the RTP timestamps.
   base::TimeTicks frame_capture_time_;
 
   // Set to non-zero to indicate the next output frame skipped over audio
   // samples in order to recover from an input underrun.
   int samples_dropped_from_buffer_;
 
   DISALLOW_COPY_AND_ASSIGN(ImplBase);
 };
 
+#if !defined(OS_IOS)
 class AudioEncoder::OpusImpl : public AudioEncoder::ImplBase {
  public:
   OpusImpl(const scoped_refptr<CastEnvironment>& cast_environment,
            int num_channels,
            int sampling_rate,
            int bitrate,
            const FrameEncodedCallback& callback)
       : ImplBase(cast_environment,
                  CODEC_AUDIO_OPUS,
                  num_channels,
@@ skipping 71 matching lines @@
   // This is the recommended value, according to documentation in
   // third_party/opus/src/include/opus.h, so that the Opus encoder does not
   // degrade the audio due to memory constraints.
   //
   // Note: Whereas other RTP implementations do not, the cast library is
   // perfectly capable of transporting larger than MTU-sized audio frames.
   static const int kOpusMaxPayloadSize = 4000;
 
   DISALLOW_COPY_AND_ASSIGN(OpusImpl);
 };
+#endif
+
+#if defined(OS_MACOSX)
+class AudioEncoder::AacAudioConverterImpl : public AudioEncoder::ImplBase {
+ public:
+  AacAudioConverterImpl(const scoped_refptr<CastEnvironment>& cast_environment,
+                        int num_channels,
+                        int sampling_rate,
+                        int bitrate,
+                        const FrameEncodedCallback& callback)
+      : ImplBase(cast_environment,
+                 CODEC_AUDIO_AAC,
+                 num_channels,
+                 sampling_rate,
+                 1024, /* AudioConverter produces 1024 samples blocks */

[miu, 2014/09/29 19:54:47]

s/AudioConverter/AacImpl/

+                 callback),
+        input_buffer_(AudioBus::Create(num_channels, 1024)),
+        output_buffer_(nullptr),
+        converter_(nullptr),
+        file_(nullptr),
+        num_encoded_packets_(0),
+        max_packet_size_(0),
+        can_resume_(true) {
+    if (ImplBase::cast_initialization_status_ != STATUS_AUDIO_UNINITIALIZED) {
+      return;
+    }
+    if (!Initialize(sampling_rate, bitrate)) {
+      ImplBase::cast_initialization_status_ =
+          STATUS_INVALID_AUDIO_CONFIGURATION;
+      return;
+    }
+    ImplBase::cast_initialization_status_ = STATUS_AUDIO_INITIALIZED;
+  }
+
+ private:
+  // The |AudioConverterFillComplexBuffer| input callback function. Returns the
+  // data that has been copied into the encoder's |buffer_| by the
+  // |TransferSamplesIntoBuffer| method.

[miu, 2014/09/29 19:54:47]

Just a sanity-check: This callback is called synchronously, right?  Meaning, the
call to AudioConverterFillComplexBuffer() won't call this at some future point
(on the same or different thread)?

Either way, it is worth mentioning this in the comment here.

[jfroy, 2014/10/14 01:06:34]

Yep. I'll add the comment.

+  static OSStatus ConverterFillDataCallback(

[miu, 2014/09/29 19:54:47]

style nit: static methods need to be declared/defined after instance methods.

+      AudioConverterRef in_converter,
+      UInt32* io_num_packets,
+      AudioBufferList* io_data,
+      AudioStreamPacketDescription** out_packet_desc,
+      void* in_encoder) {
+    DCHECK(io_data);
+    DCHECK(in_encoder);
+    auto encoder = reinterpret_cast<const AacAudioConverterImpl*>(in_encoder);
+
+    // Provide the entire content of the input buffer.
+    const int num_frames = encoder->input_buffer_->frames();
+    *io_num_packets = num_frames;
+
+    DCHECK_EQ(io_data->mNumberBuffers,
+              static_cast<unsigned>(encoder->input_buffer_->channels()));
+    for (int i_buf = 0, end = io_data->mNumberBuffers; i_buf < end; ++i_buf) {
+      io_data->mBuffers[i_buf].mNumberChannels = 1;
+      io_data->mBuffers[i_buf].mDataByteSize = sizeof(float) * num_frames;
+      io_data->mBuffers[i_buf].mData = encoder->input_buffer_->channel(i_buf);
+    }
+    return noErr;
+  }
+
+  // The AudioFile write callback function. Appends the data to the encoder's
+  // current output buffer.
+  static OSStatus FileWriteCallback(void* in_encoder,
+                                    SInt64 in_position,
+                                    UInt32 in_size,
+                                    const void* in_buffer,
+                                    UInt32* out_size) {
+    DCHECK(in_encoder);
+    auto encoder = reinterpret_cast<const AacAudioConverterImpl*>(in_encoder);
+    DCHECK(in_buffer);
+    auto buffer = reinterpret_cast<const std::string::value_type*>(in_buffer);
+
+    std::string* const output_buffer = encoder->output_buffer_;
+    DCHECK(output_buffer);
+
+    output_buffer->append(buffer, in_size);
+    *out_size = in_size;
+    return noErr;
+  }
+
+  virtual ~AacAudioConverterImpl() { Teardown(); }
+
+  // Destroys the existing audio converter and file, if any.
+  void Teardown() {
+    if (converter_) {
+      AudioConverterDispose(converter_);
+      converter_ = nullptr;
+    }
+    if (file_) {
+      AudioFileClose(file_);
+      file_ = nullptr;
+    }
+  }
+
+  // Initializes the audio converter and file. Calls Teardown to destroy any
+  // existing state. This is so that Initialize() may be called to setup another
+  // converter after a non-resumable interruption.
+  bool Initialize(int sampling_rate, int bitrate) {
+    // Teardown previous audio converter and file.
+    Teardown();
+
+    // Input data comes from AudioBus objects, which carry deinterleaved packed
+    // native-endian float samples.
+    AudioStreamBasicDescription in_asbd;
+    in_asbd.mSampleRate = sampling_rate;
+    in_asbd.mFormatID = kAudioFormatLinearPCM;
+    in_asbd.mFormatFlags =
+        kAudioFormatFlagsNativeFloatPacked | kAudioFormatFlagIsNonInterleaved;
+    in_asbd.mChannelsPerFrame = num_channels_;
+    in_asbd.mBitsPerChannel = sizeof(float) * 8;
+    in_asbd.mFramesPerPacket = 1;
+    in_asbd.mBytesPerPacket = in_asbd.mBytesPerFrame = sizeof(float);

[miu, 2014/09/29 19:54:47]

Somehow, this doesn't seem right.  Only 4 bytes per packet?

I can understand 4 bytes per frame, if the concept used by AudioToolbox is that
one sample is the same thing as a frame.  However, more often one audio frame is
defined as a batch of samples (e.g., 480 for a 10ms frame at 48 kHz).

[jfroy, 2014/10/14 01:06:34]

In Core Audio, a frame of audio is one sample across all channels at a
particular timestamp. For uncompressed audio, one packet contains one frame. For
de-interleaved audio, the per-frame fields are actually per-channel (or per
buffer), *except* mChannelsPerFrame which still represents the overall channel
count (i.e. the buffer count).

+    in_asbd.mReserved = 0;
+
+    // Request AAC-LC encoding, with no downmixing or downsampling.
+    AudioStreamBasicDescription out_asbd;
+    memset(&out_asbd, 0, sizeof(AudioStreamBasicDescription));
+    out_asbd.mSampleRate = sampling_rate;
+    out_asbd.mFormatID = kAudioFormatMPEG4AAC;
+    out_asbd.mChannelsPerFrame = num_channels_;
+    UInt32 prop_size = sizeof(out_asbd);
+    if (AudioFormatGetProperty(kAudioFormatProperty_FormatInfo,
+                               0,
+                               nullptr,
+                               &prop_size,
+                               &out_asbd) != noErr) {
+      return false;
+    }
+
+    if (AudioConverterNew(&in_asbd, &out_asbd, &converter_) != noErr) {
+      return false;
+    }
+
+    // The converter is allowed to modify (really fill-in) the output format.
+    // After creation, we can query the final output format.
+    prop_size = sizeof(out_asbd);
+    if (AudioConverterGetProperty(converter_,
+                                  kAudioConverterCurrentOutputStreamDescription,
+                                  &prop_size,
+                                  &out_asbd) != noErr) {
+      return false;
+    }
+
+    // If bitrate is <= 0, allow the encoder to pick a suitable value.
+    // Otherwise, set the bitrate (which can fail if the value is not suitable
+    // or compatible with the output sampling rate or channels).
+    if (bitrate > 0) {
+      prop_size = sizeof(int);
+      if (AudioConverterSetProperty(
+              converter_, kAudioConverterEncodeBitRate, prop_size, &bitrate) !=
+          noErr) {
+        return false;
+      }
+    }
+
+    // See the comment next to |can_resume_| for details on resumption. Some
+    // converters can return kAudioConverterErr_PropertyNotSupported, in which
+    // case resumption is implicitly supported.
+    uint32_t can_resume;
+    prop_size = sizeof(can_resume);
+    OSStatus oserr = AudioConverterGetProperty(
+        converter_,
+        kAudioConverterPropertyCanResumeFromInterruption,
+        &prop_size,
+        &can_resume);
+    if (oserr == noErr) {
+      const_cast<bool&>(can_resume_) = can_resume != 0;
+    }
+
+    // mBytesPerPacket will be 0 for variable packet size configurations, in
+    // which case we must query the maximum possible packet size.
+    const_cast<uint32_t&>(max_packet_size_) = out_asbd.mBytesPerPacket;
+    if (max_packet_size_ == 0) {
+      prop_size = sizeof(max_packet_size_);
+      if (AudioConverterGetProperty(
+              converter_,
+              kAudioConverterPropertyMaximumOutputPacketSize,
+              &prop_size,
+              const_cast<uint32_t*>(&max_packet_size_)) != noErr) {
+        return false;
+      }
+    }
+
+    // This is the only location where we modify the packet buffer.
+    const_cast<scoped_ptr<uint8[]>&>(packet_buffer_)
+        .reset(new uint8[max_packet_size_]);
+
+    // The "magic cookie" is an encoder state vector required for decoding and
+    // packing. We query it from the converter and set it on the file below.
+    UInt32 cookie_size;
+    if (AudioConverterGetPropertyInfo(converter_,
+                                      kAudioConverterCompressionMagicCookie,
+                                      &cookie_size,
+                                      nullptr) != noErr) {
+      return false;
+    }
+    scoped_ptr<int8[]> cookie_data(new int8[cookie_size]);

[miu, 2014/09/29 19:54:47]

Sanity-check: Did you mean uint8 here, or is int8 the type provided by their
API?

[jfroy, 2014/10/14 01:06:34]

Their API use void*, which doesn't play well in C++. I've traditionally used
char/int8 in these situations. I don't interpret the data, I just hand it over
to the next API (which also takes void*).

+    if (AudioConverterGetProperty(converter_,
+                                  kAudioConverterCompressionMagicCookie,
+                                  &cookie_size,
+                                  cookie_data.get()) != noErr) {
+      return false;
+    }
+
+    if (AudioFileInitializeWithCallbacks(this,
+                                         nullptr,
+                                         &FileWriteCallback,
+                                         nullptr,
+                                         nullptr,
+                                         kAudioFileAAC_ADTSType,
+                                         &out_asbd,
+                                         0,
+                                         &file_) != noErr) {
+      return false;
+    }
+
+    if (AudioFileSetProperty(file_,
+                             kAudioFilePropertyMagicCookieData,
+                             cookie_size,
+                             cookie_data.get()) != noErr) {
+      return false;
+    }
+
+    return true;
+  }
+
+  virtual void TransferSamplesIntoBuffer(const AudioBus* audio_bus,
+                                         int source_offset,
+                                         int buffer_fill_offset,
+                                         int num_samples) override {
+    audio_bus->CopyPartialFramesTo(
+        source_offset, num_samples, buffer_fill_offset, input_buffer_.get());
+  }
+
+  virtual bool EncodeFromFilledBuffer(std::string* out) override {
+    AudioBufferList out_abl;

[miu, 2014/09/29 19:54:47]

nit: Can out_abl be initialized once in Initialize(), and then re-used over and
over in this method without having to re-assign to its members?  Then, you don't
need the |max_packet_size_| member since it would only be used inside
Initialize().

[jfroy, 2014/10/14 01:06:34]

Good idea.

+    out_abl.mNumberBuffers = 1;
+    out_abl.mBuffers[0].mNumberChannels = num_channels_;
+    out_abl.mBuffers[0].mDataByteSize = max_packet_size_;
+    out_abl.mBuffers[0].mData = packet_buffer_.get();
+
+    UInt32 io_num_packets = 1;
+    AudioStreamPacketDescription packet_description;
+    if (AudioConverterFillComplexBuffer(converter_,
+                                        &ConverterFillDataCallback,
+                                        this,
+                                        &io_num_packets,
+                                        &out_abl,
+                                        &packet_description) != noErr ||
+        io_num_packets == 0) {
+      return false;
+    }
+
+    // Reserve enough space to write the packet and a basic ADTS header.
+    out->reserve(packet_description.mDataByteSize + 7);

[miu, 2014/09/29 19:54:47]

Where does 7 come from?  Can you specify this as sizeof(SomeStruct)?  Otherwise,
please define an integer constant for it.

[jfroy, 2014/10/14 01:06:34]

It's the ADTS header size. I've switched to a constant for this.

+
+    output_buffer_ = out;
+    OSStatus oserr = AudioFileWritePackets(file_,
+                                           false,
+                                           out_abl.mBuffers[0].mDataByteSize,
+                                           &packet_description,
+                                           num_encoded_packets_,
+                                           &io_num_packets,
+                                           out_abl.mBuffers[0].mData);
+    output_buffer_ = nullptr;
+    if (oserr != noErr || io_num_packets == 0) {
+      return false;
+    }
+    num_encoded_packets_ += io_num_packets;
+    return true;
+  }
+
+  // A buffer that holds one encoded packet worth of audio frames. Initialized
+  // during construction.
+  const scoped_ptr<AudioBus> input_buffer_;
+
+  // A buffer that holds one encoded packet. Initialized in |Initialize| once
+  // the maximum packet size is known.
+  const scoped_ptr<uint8[]> packet_buffer_;
+
+  // A temporary pointer to the current output buffer. Only non-null when
+  // writing a packet for use by the AudioFile write callback.
+  std::string* output_buffer_;
+
+  // The AudioConverter is responsible for encoding LPCM frames to AAC packets.
+  AudioConverterRef converter_;
+
+  // The AudioFile is responsible for wrapping AAC packets into ADTS packets.
+  AudioFileID file_;
+
+  // The number of encoded packets emitted so far by the encoder.
+  uint64_t num_encoded_packets_;
+
+  // The maximum packet size that the encoder can emit.
+  const uint32_t max_packet_size_;
+
+  // On iOS, AudioConverter can be interrupted by other services (such as an
+  // audio alert or phone call). Depending on the underlying hardware and
+  // configuration, the converter may have to be thrown away and re-initialized
+  // after such an interruption. This flag tracks if we can resume or not.
+  const bool can_resume_;

[miu, 2014/09/29 19:54:47]

Is this used anywhere?

[jfroy, 2014/10/14 01:06:34]

Not yet but I'll be filing a bug to add support for encoder re-initialization
after an interruption. This will be needed on certain hardware configurations.

+
+  DISALLOW_COPY_AND_ASSIGN(AacAudioConverterImpl);
+};
+#endif  // defined(OS_MACOSX)
 
 class AudioEncoder::Pcm16Impl : public AudioEncoder::ImplBase {
  public:
   Pcm16Impl(const scoped_refptr<CastEnvironment>& cast_environment,
             int num_channels,
             int sampling_rate,
             const FrameEncodedCallback& callback)
       : ImplBase(cast_environment,
                  CODEC_AUDIO_PCM16,
                  num_channels,
@@ skipping 42 matching lines @@
     int num_channels,
     int sampling_rate,
     int bitrate,
     Codec codec,
     const FrameEncodedCallback& frame_encoded_callback)
     : cast_environment_(cast_environment) {
   // Note: It doesn't matter which thread constructs AudioEncoder, just so long
   // as all calls to InsertAudio() are by the same thread.
   insert_thread_checker_.DetachFromThread();
   switch (codec) {
+#if !defined(OS_IOS)
     case CODEC_AUDIO_OPUS:
       impl_ = new OpusImpl(cast_environment,
                            num_channels,
                            sampling_rate,
                            bitrate,
                            frame_encoded_callback);
       break;
+#endif
+#if defined(OS_MACOSX)
+    case CODEC_AUDIO_AAC:
+      impl_ = new AacAudioConverterImpl(cast_environment,
+                                        num_channels,
+                                        sampling_rate,
+                                        bitrate,
+                                        frame_encoded_callback);
+      break;
+#endif  // defined(OS_MACOSX)
     case CODEC_AUDIO_PCM16:
       impl_ = new Pcm16Impl(cast_environment,
                             num_channels,
                             sampling_rate,
                             frame_encoded_callback);
       break;
     default:
       NOTREACHED() << "Unsupported or unspecified codec for audio encoder";
       break;
   }
@@ skipping 29 matching lines @@
   cast_environment_->PostTask(CastEnvironment::AUDIO,
                               FROM_HERE,
                               base::Bind(&AudioEncoder::ImplBase::EncodeAudio,
                                          impl_,
                                          base::Passed(&audio_bus),
                                          recorded_time));
 }
 
 }  // namespace cast
 }  // namespace media