Used native deinterleaved and float point format for the input streams.

media/audio/mac/audio_low_latency_input_mac.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/audio/mac/audio_low_latency_input_mac.h"
 
 #include <CoreServices/CoreServices.h>
 
 #include "base/basictypes.h"
 #include "base/logging.h"
 #include "base/mac/mac_logging.h"
 #include "media/audio/mac/audio_manager_mac.h"
+#include "media/base/audio_block_fifo.h"
 #include "media/base/audio_bus.h"
 #include "media/base/data_buffer.h"
 
 namespace media {
 
 // Number of blocks of buffers used in the |fifo_|.
 const int kNumberOfBlocksBufferInFifo = 2;
 
 static std::ostream& operator<<(std::ostream& os,
                                 const AudioStreamBasicDescription& format) {
   os << "sample rate       : " << format.mSampleRate << std::endl
      << "format ID         : " << format.mFormatID << std::endl
      << "format flags      : " << format.mFormatFlags << std::endl
      << "bytes per packet  : " << format.mBytesPerPacket << std::endl
      << "frames per packet : " << format.mFramesPerPacket << std::endl
      << "bytes per frame   : " << format.mBytesPerFrame << std::endl
      << "channels per frame: " << format.mChannelsPerFrame << std::endl
      << "bits per channel  : " << format.mBitsPerChannel;
   return os;
 }
 
+static void WrapBufferList(AudioBufferList* buffer_list,
+                           AudioBus* bus,
+                           int frames) {
+  DCHECK(buffer_list);
+  DCHECK(bus);
+  const int channels = bus->channels();
+  const int buffer_list_channels = buffer_list->mNumberBuffers;
+  CHECK_EQ(channels, buffer_list_channels);
+
+  // Copy pointers from AudioBufferList.
+  for (int i = 0; i < channels; ++i) {

[DaleCurtis, 2014/08/25 17:42:23]

{} not necessary.

[no longer working on chromium, 2014/08/26 09:39:52]

Done.

+    bus->SetChannelData(i, static_cast<float*>(buffer_list->mBuffers[i].mData));
+  }
+
+  // Finally set the actual length.
+  bus->set_frames(frames);
+}
+
 // See "Technical Note TN2091 - Device input using the HAL Output Audio Unit"
 // http://developer.apple.com/library/mac/#technotes/tn2091/_index.html
 // for more details and background regarding this implementation.
 
 AUAudioInputStream::AUAudioInputStream(AudioManagerMac* manager,
                                        const AudioParameters& input_params,
                                        AudioDeviceID audio_device_id)
     : manager_(manager),
       number_of_frames_(input_params.frames_per_buffer()),
       sink_(NULL),
       audio_unit_(0),
       input_device_id_(audio_device_id),
+      audio_buffer_list_(NULL),
       started_(false),
       hardware_latency_frames_(0),
       number_of_channels_in_frame_(0),
-      fifo_(input_params.channels(),
-            number_of_frames_,
-            kNumberOfBlocksBufferInFifo) {
+      audio_wrapper_(AudioBus::CreateWrapper(input_params.channels())) {
   DCHECK(manager_);
 
   // Set up the desired (output) format specified by the client.
   format_.mSampleRate = input_params.sample_rate();
   format_.mFormatID = kAudioFormatLinearPCM;
-  format_.mFormatFlags = kLinearPCMFormatFlagIsPacked |
-                         kLinearPCMFormatFlagIsSignedInteger;
-  format_.mBitsPerChannel = input_params.bits_per_sample();
+  format_.mFormatFlags =

[DaleCurtis, 2014/08/25 17:42:23]

Hmm, I thought you tried this before creating the AudioBlockFifo and found it
didn't work for input?  What changed since then?

[no longer working on chromium, 2014/08/26 09:39:52]

It is the |format_| + |audio_buffer_list_| failed the previous attempt.
Particularly, previously audio_buffer_list_ is a variable, it only allocates a
sizeof(UInt32) + sizeof(AudioBuffer) memory which does not work for stereo
recording, which badly confused me.
In this CL, I change it to be a pointer so that we can dynamically allocate
enough memory, then it works.

We need to set the correct |format_| and also allocate enough memory to
|audio_buffer_list_|, otherwise AudioUnitRender() will fail or crash.

[DaleCurtis, 2014/08/26 18:51:54]

Ah I see, that's great that it works.  We should probably unify this code into a
auhal_common.cc class or something so it can be shared between the input and
output drivers.  I'm okay if you do that in a separate CL though.

[no longer working on chromium, 2014/08/27 12:22:10]

Ok, lets do it in a separate refactoring CL then.

+      kAudioFormatFlagsNativeFloatPacked | kLinearPCMFormatFlagIsNonInterleaved;
+  size_t bytes_per_sample = sizeof(Float32);
+  format_.mBitsPerChannel = bytes_per_sample * 8;
   format_.mChannelsPerFrame = input_params.channels();
-  format_.mFramesPerPacket = 1;  // uncompressed audio
-  format_.mBytesPerPacket = (format_.mBitsPerChannel *
-                             input_params.channels()) / 8;
-  format_.mBytesPerFrame = format_.mBytesPerPacket;
+  format_.mFramesPerPacket = 1;
+  format_.mBytesPerFrame = bytes_per_sample;
+  format_.mBytesPerPacket = format_.mBytesPerFrame * format_.mFramesPerPacket;

[DaleCurtis, 2014/08/25 17:42:23]

This is not the same setting as output?

[no longer working on chromium, 2014/08/26 09:39:52]

Practically it is the same, since format_.mFramesPerPacket == 1, so
mBytesPerPacket is sizeof(Float32).

The output code is making use of the fact that format_.mFramesPerPacket == 1

[DaleCurtis, 2014/08/26 18:51:54]

Acknowledged.

   format_.mReserved = 0;
 
   DVLOG(1) << "Desired ouput format: " << format_;
 
-  // Derive size (in bytes) of the buffers that we will render to.
-  UInt32 data_byte_size = number_of_frames_ * format_.mBytesPerFrame;
-  DVLOG(1) << "Size of data buffer in bytes : " << data_byte_size;
+  // Allocate AudioBufferList based on the number of channels.
+  audio_buffer_list_ = static_cast<AudioBufferList*>(

[DaleCurtis, 2014/08/25 17:42:23]

You allocate an audio_buffer_list_, but I don't see it actually being used...
What am I missing?  In output, the buffer list is provided by the callback.

[no longer working on chromium, 2014/08/26 09:39:52]

It is used in AudioUnitRender(), which is the method to get audio data from the
device, we need to provide our own AudioBufferList* memory to the method.
AUAudioInputStream::InputProc() contains the details.

[DaleCurtis, 2014/08/26 18:51:54]

Ahh, searching for audio_buffer_list_ showed no use, but I see it uses the class
accessor for AudioUnitRender().

In any case, please make this a scoped_ptr instead of a raw allocation.

[no longer working on chromium, 2014/08/27 12:22:09]

Thanks for pointing out, I forgot scoped_ptr could specify deleter, done with
using scoped_ptr

+      malloc(sizeof(UInt32) + input_params.channels() * sizeof(AudioBuffer)));
+  audio_buffer_list_->mNumberBuffers = input_params.channels();
 
   // Allocate AudioBuffers to be used as storage for the received audio.
   // The AudioBufferList structure works as a placeholder for the
   // AudioBuffer structure, which holds a pointer to the actual data buffer.
-  audio_data_buffer_.reset(new uint8[data_byte_size]);
-  audio_buffer_list_.mNumberBuffers = 1;
-
-  AudioBuffer* audio_buffer = audio_buffer_list_.mBuffers;
-  audio_buffer->mNumberChannels = input_params.channels();
-  audio_buffer->mDataByteSize = data_byte_size;
-  audio_buffer->mData = audio_data_buffer_.get();
+  UInt32 data_byte_size = number_of_frames_ * format_.mBytesPerFrame;
+  audio_data_buffer_.reset(
+      new uint8[data_byte_size * audio_buffer_list_->mNumberBuffers]);
+  AudioBuffer* audio_buffer = audio_buffer_list_->mBuffers;
+  for (UInt32 i = 0; i < audio_buffer_list_->mNumberBuffers; ++i) {
+    audio_buffer[i].mNumberChannels = 1;
+    audio_buffer[i].mDataByteSize = data_byte_size;
+    audio_buffer[i].mData = &audio_data_buffer_[i * data_byte_size];
+  }
 }
 
-AUAudioInputStream::~AUAudioInputStream() {}
+AUAudioInputStream::~AUAudioInputStream() {
+  audio_data_buffer_.reset();

[DaleCurtis, 2014/08/26 18:51:54]

No need for any of this after converting to scoped_ptr?

[no longer working on chromium, 2014/08/27 12:22:09]

Done.

+  free(audio_buffer_list_);
+  audio_buffer_list_ = NULL;
+}
 
 // Obtain and open the AUHAL AudioOutputUnit for recording.
 bool AUAudioInputStream::Open() {
   // Verify that we are not already opened.
   if (audio_unit_)
     return false;
 
   // Verify that we have a valid device.
   if (input_device_id_ == kAudioObjectUnknown) {
     NOTREACHED() << "Device ID is unknown";
@@ skipping 62 matching lines @@
                                 kAudioOutputUnitProperty_CurrentDevice,
                                 kAudioUnitScope_Global,
                                 0,
                                 &input_device_id_,
                                 sizeof(input_device_id_));
   if (result) {
     HandleError(result);
     return false;
   }
 
-  // Register the input procedure for the AUHAL.
-  // This procedure will be called when the AUHAL has received new data
-  // from the input device.
-  AURenderCallbackStruct callback;
-  callback.inputProc = InputProc;
-  callback.inputProcRefCon = this;
-  result = AudioUnitSetProperty(audio_unit_,
-                                kAudioOutputUnitProperty_SetInputCallback,
-                                kAudioUnitScope_Global,
-                                0,
-                                &callback,
-                                sizeof(callback));
-  if (result) {
-    HandleError(result);
-    return false;
-  }
-
   // Set up the the desired (output) format.
   // For obtaining input from a device, the device format is always expressed
   // on the output scope of the AUHAL's Element 1.
   result = AudioUnitSetProperty(audio_unit_,
                                 kAudioUnitProperty_StreamFormat,
                                 kAudioUnitScope_Output,
                                 1,
                                 &format_,
                                 sizeof(format_));
   if (result) {
@@ skipping 27 matching lines @@
                                   kAudioUnitScope_Output,
                                   1,
                                   &buffer_size,
                                   sizeof(buffer_size));
     if (result != noErr) {
       HandleError(result);
       return false;
     }
   }
 
+  // Register the input procedure for the AUHAL.
+  // This procedure will be called when the AUHAL has received new data
+  // from the input device.
+  AURenderCallbackStruct callback;
+  callback.inputProc = InputProc;
+  callback.inputProcRefCon = this;
+  result = AudioUnitSetProperty(audio_unit_,
+                                kAudioOutputUnitProperty_SetInputCallback,
+                                kAudioUnitScope_Global,
+                                0,
+                                &callback,
+                                sizeof(callback));
+  if (result) {
+    HandleError(result);
+    return false;
+  }
+
   // Finally, initialize the audio unit and ensure that it is ready to render.
   // Allocates memory according to the maximum number of audio frames
   // it can produce in response to a single render call.
   result = AudioUnitInitialize(audio_unit_);
   if (result) {
     HandleError(result);
     return false;
   }
 
   // The hardware latency is fixed and will not change during the call.
@@ skipping 93 matching lines @@
   DCHECK_LE(volume, 1.0);
 
   // Verify that we have a valid device.
   if (input_device_id_ == kAudioObjectUnknown) {
     NOTREACHED() << "Device ID is unknown";
     return;
   }
 
   Float32 volume_float32 = static_cast<Float32>(volume);
   AudioObjectPropertyAddress property_address = {
-    kAudioDevicePropertyVolumeScalar,
-    kAudioDevicePropertyScopeInput,
-    kAudioObjectPropertyElementMaster
+      kAudioDevicePropertyVolumeScalar,
+      kAudioDevicePropertyScopeInput,
+      kAudioObjectPropertyElementMaster
   };
 
   // Try to set the volume for master volume channel.
   if (IsVolumeSettableOnChannel(kAudioObjectPropertyElementMaster)) {
     OSStatus result = AudioObjectSetPropertyData(input_device_id_,
                                                  &property_address,
                                                  0,
                                                  NULL,
                                                  sizeof(volume_float32),
                                                  &volume_float32);
@@ skipping 25 matching lines @@
   // Update the AGC volume level based on the last setting above. Note that,
   // the volume-level resolution is not infinite and it is therefore not
   // possible to assume that the volume provided as input parameter can be
   // used directly. Instead, a new query to the audio hardware is required.
   // This method does nothing if AGC is disabled.
   UpdateAgcVolume();
 }
 
 double AUAudioInputStream::GetVolume() {
   // Verify that we have a valid device.
-  if (input_device_id_ == kAudioObjectUnknown){
+  if (input_device_id_ == kAudioObjectUnknown) {
     NOTREACHED() << "Device ID is unknown";
     return 0.0;
   }
 
   AudioObjectPropertyAddress property_address = {
-    kAudioDevicePropertyVolumeScalar,
-    kAudioDevicePropertyScopeInput,
-    kAudioObjectPropertyElementMaster
+      kAudioDevicePropertyVolumeScalar,
+      kAudioDevicePropertyScopeInput,
+      kAudioObjectPropertyElementMaster
   };
 
   if (AudioObjectHasProperty(input_device_id_, &property_address)) {
     // The device supports master volume control, get the volume from the
     // master channel.
     Float32 volume_float32 = 0.0;
     UInt32 size = sizeof(volume_float32);
-    OSStatus result = AudioObjectGetPropertyData(input_device_id_,
-                                                 &property_address,
-                                                 0,
-                                                 NULL,
-                                                 &size,
-                                                 &volume_float32);
+    OSStatus result = AudioObjectGetPropertyData(
+        input_device_id_, &property_address, 0, NULL, &size, &volume_float32);
     if (result == noErr)
       return static_cast<double>(volume_float32);
   } else {
     // There is no master volume control, try to get the average volume of
     // all the channels.
     Float32 volume_float32 = 0.0;
     int successful_channels = 0;
     for (int i = 1; i <= number_of_channels_in_frame_; ++i) {
       property_address.mElement = static_cast<UInt32>(i);
       if (AudioObjectHasProperty(input_device_id_, &property_address)) {
@@ skipping 40 matching lines @@
   OSStatus result = AudioUnitRender(audio_input->audio_unit(),
                                     flags,
                                     time_stamp,
                                     bus_number,
                                     number_of_frames,
                                     audio_input->audio_buffer_list());
   if (result)
     return result;
 
   // Deliver recorded data to the consumer as a callback.
-  return audio_input->Provide(number_of_frames,
-                              audio_input->audio_buffer_list(),
-                              time_stamp);
+  return audio_input->Provide(
+      number_of_frames, audio_input->audio_buffer_list(), time_stamp);
 }
 
 OSStatus AUAudioInputStream::Provide(UInt32 number_of_frames,
                                      AudioBufferList* io_data,
                                      const AudioTimeStamp* time_stamp) {
   // Update the capture latency.
   double capture_latency_frames = GetCaptureLatency(time_stamp);
 
   // The AGC volume level is updated once every second on a separate thread.
   // Note that, |volume| is also updated each time SetVolume() is called
   // through IPC by the render-side AGC.
   double normalized_volume = 0.0;
   GetAgcVolume(&normalized_volume);
 
   AudioBuffer& buffer = io_data->mBuffers[0];
   uint8* audio_data = reinterpret_cast<uint8*>(buffer.mData);
-  uint32 capture_delay_bytes = static_cast<uint32>
-      ((capture_latency_frames + 0.5) * format_.mBytesPerFrame);
+  uint32 capture_delay_bytes = static_cast<uint32>(
+      (capture_latency_frames + 0.5) * format_.mBytesPerFrame);
   DCHECK(audio_data);
   if (!audio_data)
     return kAudioUnitErr_InvalidElement;
 
-  // Copy captured (and interleaved) data into FIFO.
-  fifo_.Push(audio_data, number_of_frames, format_.mBitsPerChannel / 8);
+  // Wrap the output AudioBufferList to |audio_wrapper_|.
+  WrapBufferList(io_data, audio_wrapper_.get(), number_of_frames);
 
+  // If the stream parameters change for any reason, we need to insert a FIFO
+  // since the OnMoreData() pipeline can't handle frame size changes.
+  if (number_of_frames != number_of_frames_) {
+    // Create a FIFO on the fly to handle any discrepancies in callback rates.
+    if (!fifo_) {
+      fifo_.reset(new AudioBlockFifo(audio_wrapper_->channels(),
+                                     number_of_frames_,
+                                     kNumberOfBlocksBufferInFifo));
+    }
+  }
+
+  // When FIFO does not kick in, data will be directly passed to the callback.
+  if (!fifo_) {
+    CHECK_EQ(audio_wrapper_->frames(), static_cast<int>(number_of_frames_));
+    sink_->OnData(
+        this, audio_wrapper_.get(), capture_delay_bytes, normalized_volume);
+    return noErr;
+  }
+
+  // Compensate the audio delay caused by the FIFO.
+  capture_delay_bytes += fifo_->GetAvailableFrames() * format_.mBytesPerFrame;
+
+  fifo_->Push(audio_wrapper_.get());
   // Consume and deliver the data when the FIFO has a block of available data.
-  while (fifo_.available_blocks()) {
-    const AudioBus* audio_bus = fifo_.Consume();
+  while (fifo_->available_blocks()) {
+    const AudioBus* audio_bus = fifo_->Consume();
     DCHECK_EQ(audio_bus->frames(), static_cast<int>(number_of_frames_));
-
-    // Compensate the audio delay caused by the FIFO.
-    capture_delay_bytes += fifo_.GetAvailableFrames() * format_.mBytesPerFrame;
     sink_->OnData(this, audio_bus, capture_delay_bytes, normalized_volume);
   }
 
   return noErr;
 }
 
 int AUAudioInputStream::HardwareSampleRate() {
   // Determine the default input device's sample-rate.
   AudioDeviceID device_id = kAudioObjectUnknown;
   UInt32 info_size = sizeof(device_id);
 
   AudioObjectPropertyAddress default_input_device_address = {
-    kAudioHardwarePropertyDefaultInputDevice,
-    kAudioObjectPropertyScopeGlobal,
-    kAudioObjectPropertyElementMaster
+      kAudioHardwarePropertyDefaultInputDevice,
+      kAudioObjectPropertyScopeGlobal,
+      kAudioObjectPropertyElementMaster
   };
   OSStatus result = AudioObjectGetPropertyData(kAudioObjectSystemObject,
                                                &default_input_device_address,
                                                0,
                                                0,
                                                &info_size,
                                                &device_id);
   if (result != noErr)
     return 0.0;
 
   Float64 nominal_sample_rate;
   info_size = sizeof(nominal_sample_rate);
 
   AudioObjectPropertyAddress nominal_sample_rate_address = {
-    kAudioDevicePropertyNominalSampleRate,
-    kAudioObjectPropertyScopeGlobal,
-    kAudioObjectPropertyElementMaster
-  };
+      kAudioDevicePropertyNominalSampleRate, kAudioObjectPropertyScopeGlobal,
+      kAudioObjectPropertyElementMaster};
   result = AudioObjectGetPropertyData(device_id,
                                       &nominal_sample_rate_address,
                                       0,
                                       0,
                                       &info_size,
                                       &nominal_sample_rate);
   if (result != noErr)
     return 0.0;
 
   return static_cast<int>(nominal_sample_rate);
@@ skipping 12 matching lines @@
                                          kAudioUnitProperty_Latency,
                                          kAudioUnitScope_Global,
                                          0,
                                          &audio_unit_latency_sec,
                                          &size);
   OSSTATUS_DLOG_IF(WARNING, result != noErr, result)
       << "Could not get audio unit latency";
 
   // Get input audio device latency.
   AudioObjectPropertyAddress property_address = {
-    kAudioDevicePropertyLatency,
-    kAudioDevicePropertyScopeInput,
-    kAudioObjectPropertyElementMaster
+      kAudioDevicePropertyLatency,
+      kAudioDevicePropertyScopeInput,
+      kAudioObjectPropertyElementMaster
   };
   UInt32 device_latency_frames = 0;
   size = sizeof(device_latency_frames);
   result = AudioObjectGetPropertyData(input_device_id_,
                                       &property_address,
                                       0,
                                       NULL,
                                       &size,
                                       &device_latency_frames);
   DLOG_IF(WARNING, result != noErr) << "Could not get audio device latency.";
 
-  return static_cast<double>((audio_unit_latency_sec *
-      format_.mSampleRate) + device_latency_frames);
+  return static_cast<double>((audio_unit_latency_sec * format_.mSampleRate) +
+                             device_latency_frames);
 }
 
 double AUAudioInputStream::GetCaptureLatency(
     const AudioTimeStamp* input_time_stamp) {
   // Get the delay between between the actual recording instant and the time
   // when the data packet is provided as a callback.
-  UInt64 capture_time_ns = AudioConvertHostTimeToNanos(
-      input_time_stamp->mHostTime);
+  UInt64 capture_time_ns =
+      AudioConvertHostTimeToNanos(input_time_stamp->mHostTime);
   UInt64 now_ns = AudioConvertHostTimeToNanos(AudioGetCurrentHostTime());
-  double delay_frames = static_cast<double>
-      (1e-9 * (now_ns - capture_time_ns) * format_.mSampleRate);
+  double delay_frames = static_cast<double>(1e-9 * (now_ns - capture_time_ns) *
+                                            format_.mSampleRate);
 
   // Total latency is composed by the dynamic latency and the fixed
   // hardware latency.
   return (delay_frames + hardware_latency_frames_);
 }
 
 int AUAudioInputStream::GetNumberOfChannelsFromStream() {
   // Get the stream format, to be able to read the number of channels.
   AudioObjectPropertyAddress property_address = {
-    kAudioDevicePropertyStreamFormat,
-    kAudioDevicePropertyScopeInput,
-    kAudioObjectPropertyElementMaster
+      kAudioDevicePropertyStreamFormat,
+      kAudioDevicePropertyScopeInput,
+      kAudioObjectPropertyElementMaster
   };
   AudioStreamBasicDescription stream_format;
   UInt32 size = sizeof(stream_format);
-  OSStatus result = AudioObjectGetPropertyData(input_device_id_,
-                                               &property_address,
-                                               0,
-                                               NULL,
-                                               &size,
-                                               &stream_format);
+  OSStatus result = AudioObjectGetPropertyData(
+      input_device_id_, &property_address, 0, NULL, &size, &stream_format);
   if (result != noErr) {
     DLOG(WARNING) << "Could not get stream format";
     return 0;
   }
 
   return static_cast<int>(stream_format.mChannelsPerFrame);
 }
 
 void AUAudioInputStream::HandleError(OSStatus err) {
-  NOTREACHED() << "error " << GetMacOSStatusErrorString(err)
-               << " (" << err << ")";
+  NOTREACHED() << "error " << GetMacOSStatusErrorString(err) << " (" << err
+               << ")";
   if (sink_)
     sink_->OnError(this);
 }
 
 bool AUAudioInputStream::IsVolumeSettableOnChannel(int channel) {
   Boolean is_settable = false;
   AudioObjectPropertyAddress property_address = {
-    kAudioDevicePropertyVolumeScalar,
-    kAudioDevicePropertyScopeInput,
-    static_cast<UInt32>(channel)
+      kAudioDevicePropertyVolumeScalar,
+      kAudioDevicePropertyScopeInput,
+      static_cast<UInt32>(channel)
   };
-  OSStatus result = AudioObjectIsPropertySettable(input_device_id_,
-                                                  &property_address,
-                                                  &is_settable);
+  OSStatus result = AudioObjectIsPropertySettable(
+      input_device_id_, &property_address, &is_settable);
   return (result == noErr) ? is_settable : false;
 }
 
 }  // namespace media