Add basic resample support to WASAPIAudioInputStream.

media/audio/win/audio_low_latency_input_win.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/audio/win/audio_low_latency_input_win.h"
 
+#include <math.h>

[DaleCurtis, 2017/02/17 18:23:23]

cmath? for modf?

http://en.cppreference.com/w/cpp/numeric/math/modf

[tommi (sloooow) - chröme, 2017/02/17 22:21:27]

Done.

 #include <memory>
 
 #include "base/logging.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/trace_event/trace_event.h"
 #include "media/audio/audio_device_description.h"
 #include "media/audio/win/audio_manager_win.h"
 #include "media/audio/win/avrt_wrapper_win.h"
 #include "media/audio/win/core_audio_util_win.h"
+#include "media/base/audio_block_fifo.h"
 #include "media/base/audio_bus.h"
+#include "media/base/channel_layout.h"
+#include "media/base/limits.h"
 
 using base::win::ScopedComPtr;
 using base::win::ScopedCOMInitializer;
 
 namespace media {
 
 WASAPIAudioInputStream::WASAPIAudioInputStream(AudioManagerWin* manager,
                                                const AudioParameters& params,
                                                const std::string& device_id)
-    : manager_(manager),
-      device_id_(device_id),
-      audio_bus_(media::AudioBus::Create(params)) {
+    : manager_(manager), device_id_(device_id) {
   DCHECK(manager_);
   DCHECK(!device_id_.empty());
 
   // Load the Avrt DLL if not already loaded. Required to support MMCSS.
   bool avrt_init = avrt::Initialize();
   DCHECK(avrt_init) << "Failed to load the Avrt.dll";
 
   // Set up the desired capture format specified by the client.
   format_.nSamplesPerSec = params.sample_rate();
   format_.wFormatTag = WAVE_FORMAT_PCM;
@@ skipping 76 matching lines @@
   // set during construction.
   if (!DesiredFormatIsSupported()) {
     open_result_ = OPEN_RESULT_FORMAT_NOT_SUPPORTED;
     ReportOpenResult();
     return false;
   }
 
   // Initialize the audio stream between the client and the device using
   // shared mode and a lowest possible glitch-free latency.
   hr = InitializeAudioEngine();
+  if (SUCCEEDED(hr) && converter_)
+    open_result_ = OPEN_RESULT_OK_WITH_RESAMPLING;
   ReportOpenResult();  // Report before we assign a value to |opened_|.
   opened_ = SUCCEEDED(hr);
-  DCHECK(open_result_ == OPEN_RESULT_OK || !opened_);
 
   return opened_;
 }
 
 void WASAPIAudioInputStream::Start(AudioInputCallback* callback) {
   DCHECK(CalledOnValidThread());
   DCHECK(callback);
   DLOG_IF(ERROR, !opened_) << "Open() has not been called successfully";
   if (!opened_)
     return;
@@ skipping 81 matching lines @@
   started_ = false;
   sink_ = NULL;
 }
 
 void WASAPIAudioInputStream::Close() {
   DVLOG(1) << "WASAPIAudioInputStream::Close()";
   // It is valid to call Close() before calling open or Start().
   // It is also valid to call Close() after Start() has been called.
   Stop();
 
+  if (converter_)
+    converter_->RemoveInput(this);
+
   // Inform the audio manager that we have been closed. This will cause our
   // destruction.
   manager_->ReleaseInputStream(this);
 }
 
 double WASAPIAudioInputStream::GetMaxVolume() {
   // Verify that Open() has been called succesfully, to ensure that an audio
   // session exists and that an ISimpleAudioVolume interface has been created.
   DLOG_IF(ERROR, !opened_) << "Open() has not been called successfully";
   if (!opened_)
@@ skipping 73 matching lines @@
     // to reduced QoS at high load.
     DWORD err = GetLastError();
     LOG(WARNING) << "Failed to enable MMCSS (error code=" << err << ").";
   }
 
   // Allocate a buffer with a size that enables us to take care of cases like:
   // 1) The recorded buffer size is smaller, or does not match exactly with,
   //    the selected packet size used in each callback.
   // 2) The selected buffer size is larger than the recorded buffer size in
   //    each event.
-  size_t buffer_frame_index = 0;
-  size_t capture_buffer_size =
-      std::max(2 * endpoint_buffer_size_frames_ * frame_size_,
-               2 * packet_size_frames_ * frame_size_);
-  std::unique_ptr<uint8_t[]> capture_buffer(new uint8_t[capture_buffer_size]);
+  // In the case where no resampling is required, a single buffer should be
+  // enough but in case we get buffers that don't match exactly, we'll go with
+  // two. Same applies if we need to resample and the buffer ratio is perfect.
+  // However if the buffer ratio is imperfect, we will need 3 buffers to safely
+  // be able to buffer up data in cases where a conversion requires two audio
+  // buffers (and we need to be able to write to the third one).
+  DCHECK(!fifo_);
+  const int buffers_required =
+      converter_ && imperfect_buffer_size_conversion_ ? 3 : 2;
+  fifo_.reset(new AudioBlockFifo(format_.nChannels, packet_size_frames_,
+                                 buffers_required));
+
+  DVLOG(1) << "AudioBlockFifo needs " << buffers_required << " buffers";
 
   LARGE_INTEGER now_count = {};
   bool recording = true;
   bool error = false;
   double volume = GetVolume();
   HANDLE wait_array[2] = {stop_capture_event_.Get(),
                           audio_samples_ready_event_.Get()};
 
   base::win::ScopedComPtr<IAudioClock> audio_clock;
   audio_client_->GetService(__uuidof(IAudioClock), audio_clock.ReceiveVoid());
@@ skipping 34 matching lines @@
         if (audio_clock) {
           // The reported timestamp from GetBuffer is not as reliable as the
           // clock from the client.  We've seen timestamps reported for
           // USB audio devices, be off by several days.  Furthermore we've
           // seen them jump back in time every 2 seconds or so.
           audio_clock->GetPosition(&device_position,
                                    &first_audio_frame_timestamp);
         }
 
         if (num_frames_to_read != 0) {
-          size_t pos = buffer_frame_index * frame_size_;
-          size_t num_bytes = num_frames_to_read * frame_size_;
-          DCHECK_GE(capture_buffer_size, pos + num_bytes);
-
           if (flags & AUDCLNT_BUFFERFLAGS_SILENT) {
-            // Clear out the local buffer since silence is reported.
-            memset(&capture_buffer[pos], 0, num_bytes);
+            fifo_->PushSilence(num_frames_to_read);
           } else {
-            // Copy captured data from audio engine buffer to local buffer.
-            memcpy(&capture_buffer[pos], data_ptr, num_bytes);
+            fifo_->Push(data_ptr, num_frames_to_read,
+                        format_.wBitsPerSample / 8);
           }
-
-          buffer_frame_index += num_frames_to_read;
         }
 
         hr = audio_capture_client_->ReleaseBuffer(num_frames_to_read);
         DLOG_IF(ERROR, FAILED(hr)) << "Failed to release capture buffer";
 
         // Derive a delay estimate for the captured audio packet.
         // The value contains two parts (A+B), where A is the delay of the
         // first audio frame in the packet and B is the extra delay
         // contained in any stored data. Unit is in audio frames.
         QueryPerformanceCounter(&now_count);
         // first_audio_frame_timestamp will be 0 if we didn't get a timestamp.
         double audio_delay_frames =
             first_audio_frame_timestamp == 0
                 ? num_frames_to_read
                 : ((perf_count_to_100ns_units_ * now_count.QuadPart -
                     first_audio_frame_timestamp) /
                    10000.0) *
                           ms_to_frame_count_ +
-                      buffer_frame_index - num_frames_to_read;
+                      fifo_->GetAvailableFrames() - num_frames_to_read;
 
         // Get a cached AGC volume level which is updated once every second
         // on the audio manager thread. Note that, |volume| is also updated
         // each time SetVolume() is called through IPC by the render-side AGC.
         GetAgcVolume(&volume);
 
         // Deliver captured data to the registered consumer using a packet
         // size which was specified at construction.
         uint32_t delay_frames = static_cast<uint32_t>(audio_delay_frames + 0.5);
-        while (buffer_frame_index >= packet_size_frames_) {
-          // Copy data to audio bus to match the OnData interface.
-          uint8_t* audio_data =
-              reinterpret_cast<uint8_t*>(capture_buffer.get());
-          audio_bus_->FromInterleaved(audio_data, audio_bus_->frames(),
-                                      format_.wBitsPerSample / 8);
+        while (fifo_->available_blocks()) {
+          if (converter_) {
+            if (imperfect_buffer_size_conversion_ &&
+                fifo_->available_blocks() == 1) {
+              // Special case. We need to buffer up more audio before we can
+              // convert or else we'll suffer an underrun.
+              break;
+            }
+            converter_->ConvertWithDelay(delay_frames, convert_bus_.get());
+            sink_->OnData(this, convert_bus_.get(), delay_frames * frame_size_,
+                          volume);
+          } else {
+            sink_->OnData(this, fifo_->Consume(), delay_frames * frame_size_,
+                          volume);
+          }
 
-          // Deliver data packet, delay estimation and volume level to
-          // the user.
-          sink_->OnData(this, audio_bus_.get(), delay_frames * frame_size_,
-                        volume);
-
-          // Store parts of the recorded data which can't be delivered
-          // using the current packet size. The stored section will be used
-          // either in the next while-loop iteration or in the next
-          // capture event.
-          // TODO(tommi): If this data will be used in the next capture
-          // event, we will report incorrect delay estimates because
-          // we'll use the one for the captured data that time around
-          // (i.e. in the future).
-          memmove(&capture_buffer[0], &capture_buffer[packet_size_bytes_],
-                  (buffer_frame_index - packet_size_frames_) * frame_size_);
-
-          DCHECK_GE(buffer_frame_index, packet_size_frames_);
-          buffer_frame_index -= packet_size_frames_;
           if (delay_frames > packet_size_frames_) {
             delay_frames -= packet_size_frames_;
           } else {
             delay_frames = 0;
           }
         }
       } break;
       default:
         error = true;
         break;
     }
   }
 
   if (recording && error) {
     // TODO(henrika): perhaps it worth improving the cleanup here by e.g.
     // stopping the audio client, joining the thread etc.?
     NOTREACHED() << "WASAPI capturing failed with error code "
                  << GetLastError();
   }
 
   // Disable MMCSS.
   if (mm_task && !avrt::AvRevertMmThreadCharacteristics(mm_task)) {
     PLOG(WARNING) << "Failed to disable MMCSS";
   }
+
+  fifo_.reset();
 }
 
 void WASAPIAudioInputStream::HandleError(HRESULT err) {
   NOTREACHED() << "Error code: " << err;
   if (sink_)
     sink_->OnError(this);
 }
 
 HRESULT WASAPIAudioInputStream::SetCaptureDevice() {
   DCHECK_EQ(OPEN_RESULT_OK, open_result_);
@@ skipping 104 matching lines @@
   // engine can convert between a standard PCM sample size used by the
   // application and the floating-point samples that the engine uses for its
   // internal processing. However, the format for an application stream
   // typically must have the same number of channels and the same sample
   // rate as the stream format used by the device.
   // Many audio devices support both PCM and non-PCM stream formats. However,
   // the audio engine can mix only PCM streams.
   base::win::ScopedCoMem<WAVEFORMATEX> closest_match;
   HRESULT hr = audio_client_->IsFormatSupported(AUDCLNT_SHAREMODE_SHARED,
                                                 &format_, &closest_match);
-  DLOG_IF(ERROR, hr == S_FALSE) << "Format is not supported "
-                                << "but a closest match exists.";
+  DLOG_IF(ERROR, hr == S_FALSE)
+      << "Format is not supported but a closest match exists.";
+
+  if (hr == S_FALSE &&
+      closest_match->nSamplesPerSec >= limits::kMinSampleRate &&
+      closest_match->nSamplesPerSec <= limits::kMaxSampleRate) {
+    DVLOG(1) << "Audio capture data conversion needed.";
+    // Ideally, we want a 1:1 ratio between the buffers we get and the buffers
+    // we give to OnData so that each buffer we receive from the OS can be
+    // directly converted to a buffer that matches with what was asked for.
+    const double buffer_ratio =
+        format_.nSamplesPerSec / static_cast<double>(packet_size_frames_);
+    double new_frames_per_buffer = closest_match->nSamplesPerSec / buffer_ratio;
+
+    const AudioParameters input(
+        AudioParameters::AUDIO_PCM_LOW_LATENCY,
+        GuessChannelLayout(closest_match->nChannels),
+        closest_match->nSamplesPerSec,
+        // We need to be careful here to not pick the closest wBitsPerSample

[DaleCurtis, 2017/02/17 18:23:23]

Missed feedback from previous patchset? Re: dead parameter; just needs to be 8,
16, 32.

[tommi (sloooow) - chröme, 2017/02/17 22:21:27]

Ah, sorry, yes. Removed the comment and now use the closest_match bitrate.
Also added DCHECKs for GuessChannelLayout.

[DaleCurtis, 2017/02/17 22:29:23]

You do need to check that it's 8, 16, or 32, otherwise we don't have
FromInterleaved() support which result in silence being injected during the
deinterleave step.

[tommi (sloooow) - chröme, 2017/02/17 23:14:02]

iow, 24 is not supported?  I'll add the checks.  I did test with 8, 16 and 32
btw.

[DaleCurtis, 2017/02/17 23:30:54]

24, if it's actually packed into 3 bytes is not supported. I don't think I've
ever seen it done that way, 4 bytes is supported for 24bit.

[tommi (sloooow) - chröme, 2017/02/18 11:29:49]

Done.

+        // match as we need to use the PCM format (which might not be what
+        // closeest_match->wFormat is) and the internal resampler doesn't
+        // support all formats we might get here.  So, we stick to the
+        // wBitsPerSample that was asked for originally (most likely 16).
+        format_.wBitsPerSample, static_cast<int>(new_frames_per_buffer));
+
+    const AudioParameters output(AudioParameters::AUDIO_PCM_LOW_LATENCY,
+                                 GuessChannelLayout(format_.nChannels),
+                                 format_.nSamplesPerSec, format_.wBitsPerSample,
+                                 packet_size_frames_);
+
+    converter_.reset(new AudioConverter(input, output, false));
+    converter_->AddInput(this);
+    converter_->PrimeWithSilence();
+    convert_bus_ = AudioBus::Create(output);
+
+    // Now change the format we're going to ask for to better match with what
+    // the OS can provide.  If we succeed in opening the stream with these
+    // params, we can take care of the required resampling.
+    format_.nSamplesPerSec = closest_match->nSamplesPerSec;
+    format_.nChannels = closest_match->nChannels;
+    format_.nBlockAlign = (format_.wBitsPerSample / 8) * format_.nChannels;
+    format_.nAvgBytesPerSec = format_.nSamplesPerSec * format_.nBlockAlign;
+
+    // Update our packet size assumptions based on the new format.
+    const auto new_bytes_per_buffer = static_cast<int>(new_frames_per_buffer) *
+                                      format_.nChannels *

[DaleCurtis, 2017/02/17 18:23:23]

Replace terms with * format_.nBlockAlign?

[tommi (sloooow) - chröme, 2017/02/17 22:21:27]

Done.

+                                      (format_.wBitsPerSample / 8);
+    packet_size_frames_ = new_bytes_per_buffer / format_.nBlockAlign;
+    packet_size_bytes_ = new_bytes_per_buffer;
+    frame_size_ = format_.nBlockAlign;
+    ms_to_frame_count_ = static_cast<double>(format_.nSamplesPerSec) / 1000.0;
+
+    imperfect_buffer_size_conversion_ =
+        modf(new_frames_per_buffer, &new_frames_per_buffer) != 0.0;

[DaleCurtis, 2017/02/17 18:23:23]

std:: ? 

[tommi (sloooow) - chröme, 2017/02/17 22:21:27]

Done.

+    DVLOG_IF(1, imperfect_buffer_size_conversion_)
+        << "Audio capture data conversion: Need to inject fifo";
+
+    // Indicate that we're good to go with a close match.
+    hr = S_OK;
+  }
+
   return (hr == S_OK);
 }
 
 HRESULT WASAPIAudioInputStream::InitializeAudioEngine() {
   DCHECK_EQ(OPEN_RESULT_OK, open_result_);
   DWORD flags;
   // Use event-driven mode only fo regular input devices. For loopback the
   // EVENTCALLBACK flag is specified when intializing
   // |audio_render_client_for_loopback_|.
   if (device_id_ == AudioDeviceDescription::kLoopbackInputDeviceId ||
@@ skipping 123 matching lines @@
 
   return hr;
 }
 
 void WASAPIAudioInputStream::ReportOpenResult() const {
   DCHECK(!opened_);  // This method must be called before we set this flag.
   UMA_HISTOGRAM_ENUMERATION("Media.Audio.Capture.Win.Open", open_result_,
                             OPEN_RESULT_MAX + 1);
 }
 
+double WASAPIAudioInputStream::ProvideInput(AudioBus* audio_bus,
+                                            uint32_t frames_delayed) {
+  fifo_->Consume()->CopyTo(audio_bus);
+  return 1.0;
+}
+
 }  // namespace media