Add basic resample support to WASAPIAudioInputStream.

media/audio/win/audio_low_latency_input_win.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/audio/win/audio_low_latency_input_win.h"
 
 #include <memory>
 
 #include "base/logging.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/trace_event/trace_event.h"
 #include "media/audio/audio_device_description.h"
 #include "media/audio/win/audio_manager_win.h"
 #include "media/audio/win/avrt_wrapper_win.h"
 #include "media/audio/win/core_audio_util_win.h"
 #include "media/base/audio_bus.h"
+#include "media/base/audio_fifo.h"
+#include "media/base/channel_layout.h"
+#include "media/base/limits.h"
+#include "media/base/multi_channel_resampler.h"
 
 using base::win::ScopedComPtr;
 using base::win::ScopedCOMInitializer;
 
 namespace media {
 
 WASAPIAudioInputStream::WASAPIAudioInputStream(AudioManagerWin* manager,
                                                const AudioParameters& params,
                                                const std::string& device_id)
     : manager_(manager),
@@ skipping 88 matching lines @@
   // set during construction.
   if (!DesiredFormatIsSupported()) {
     open_result_ = OPEN_RESULT_FORMAT_NOT_SUPPORTED;
     ReportOpenResult();
     return false;
   }
 
   // Initialize the audio stream between the client and the device using
   // shared mode and a lowest possible glitch-free latency.
   hr = InitializeAudioEngine();
+  if (SUCCEEDED(hr) && converter_)
+    open_result_ = OPEN_RESULT_OK_WITH_RESAMPLING;
   ReportOpenResult();  // Report before we assign a value to |opened_|.
   opened_ = SUCCEEDED(hr);
-  DCHECK(open_result_ == OPEN_RESULT_OK || !opened_);
 
   return opened_;
 }
 
 void WASAPIAudioInputStream::Start(AudioInputCallback* callback) {
   DCHECK(CalledOnValidThread());
   DCHECK(callback);
   DLOG_IF(ERROR, !opened_) << "Open() has not been called successfully";
   if (!opened_)
     return;
@@ skipping 81 matching lines @@
   started_ = false;
   sink_ = NULL;
 }
 
 void WASAPIAudioInputStream::Close() {
   DVLOG(1) << "WASAPIAudioInputStream::Close()";
   // It is valid to call Close() before calling open or Start().
   // It is also valid to call Close() after Start() has been called.
   Stop();
 
+  if (converter_)
+    converter_->RemoveInput(this);
+
   // Inform the audio manager that we have been closed. This will cause our
   // destruction.
   manager_->ReleaseInputStream(this);
 }
 
 double WASAPIAudioInputStream::GetMaxVolume() {
   // Verify that Open() has been called succesfully, to ensure that an audio
   // session exists and that an ISimpleAudioVolume interface has been created.
   DLOG_IF(ERROR, !opened_) << "Open() has not been called successfully";
   if (!opened_)
@@ skipping 177 matching lines @@
         // each time SetVolume() is called through IPC by the render-side AGC.
         GetAgcVolume(&volume);
 
         // Deliver captured data to the registered consumer using a packet
         // size which was specified at construction.
         uint32_t delay_frames = static_cast<uint32_t>(audio_delay_frames + 0.5);
         while (buffer_frame_index >= packet_size_frames_) {
           // Copy data to audio bus to match the OnData interface.
           uint8_t* audio_data =
               reinterpret_cast<uint8_t*>(capture_buffer.get());
-          audio_bus_->FromInterleaved(audio_data, audio_bus_->frames(),
-                                      format_.wBitsPerSample / 8);
 
-          // Deliver data packet, delay estimation and volume level to
-          // the user.
-          sink_->OnData(this, audio_bus_.get(), delay_frames * frame_size_,
-                        volume);
+          bool issue_callback = true;
+          if (converter_) {
+            convert_bus_->FromInterleaved(audio_data, packet_size_frames_,
+                                          format_.wBitsPerSample / 8);
+            if (convert_fifo_) {
+              convert_fifo_->Push(convert_bus_.get());
+              // Since we have a fifo, we know that we have one in order to
+              // avoid underruns.  The size of the fifo will be large enough
+              // to hold two buffers from the audio layer, but the minimum
+              // number of frames required in order to safely be able to
+              // convert data, will be one more frame than the buffer size
+              // we have (one frame more will cover a larger time period than
+              // the buffer size as requested by the client, and is only needed
+              // when we reach the point where there would otherwise be an
+              // underrun).
+              issue_callback =
+                  (convert_fifo_->frames() >= (convert_bus_->frames() + 1));
+              if (issue_callback) {
+                data_was_converted_ = 0;
+                converter_->ConvertWithDelay(delay_frames, audio_bus_.get());
+                DCHECK(data_was_converted_ >= 0 || data_was_converted_ < 2);
+              }
+            } else {
+              data_was_converted_ = 0;
+              converter_->ConvertWithDelay(delay_frames, audio_bus_.get());
+              DCHECK_EQ(1, data_was_converted_);
+            }
+          } else {
+            audio_bus_->FromInterleaved(audio_data, audio_bus_->frames(),
+                                        format_.wBitsPerSample / 8);
+          }
+
+          if (issue_callback) {
+            // Deliver data packet, delay estimation and volume level to
+            // the user.
+            sink_->OnData(this, audio_bus_.get(), delay_frames * frame_size_,
+                          volume);
+          }
 
           // Store parts of the recorded data which can't be delivered
           // using the current packet size. The stored section will be used
           // either in the next while-loop iteration or in the next
           // capture event.
           // TODO(tommi): If this data will be used in the next capture
           // event, we will report incorrect delay estimates because
           // we'll use the one for the captured data that time around
           // (i.e. in the future).
           memmove(&capture_buffer[0], &capture_buffer[packet_size_bytes_],
@@ skipping 142 matching lines @@
   // engine can convert between a standard PCM sample size used by the
   // application and the floating-point samples that the engine uses for its
   // internal processing. However, the format for an application stream
   // typically must have the same number of channels and the same sample
   // rate as the stream format used by the device.
   // Many audio devices support both PCM and non-PCM stream formats. However,
   // the audio engine can mix only PCM streams.
   base::win::ScopedCoMem<WAVEFORMATEX> closest_match;
   HRESULT hr = audio_client_->IsFormatSupported(AUDCLNT_SHAREMODE_SHARED,
                                                 &format_, &closest_match);
-  DLOG_IF(ERROR, hr == S_FALSE) << "Format is not supported "
-                                << "but a closest match exists.";
+  DLOG_IF(ERROR, hr == S_FALSE)
+      << "Format is not supported but a closest match exists.";
+
+  if (hr == S_FALSE &&
+      closest_match->nSamplesPerSec >= limits::kMinSampleRate &&
+      closest_match->nSamplesPerSec <= limits::kMaxSampleRate) {
+    DVLOG(1) << "Audio capture data conversion needed.";
+    // Ideally, we want a 1:1 ratio between the buffers we get and the buffers
+    // we give to OnData so that each buffer we receive from the OS can be
+    // directly converted to a buffer that matches with what was asked for.
+    const double buffer_ratio =
+        format_.nSamplesPerSec / static_cast<double>(audio_bus_->frames());
+    const size_t new_frames_per_buffer =
+        static_cast<size_t>(closest_match->nSamplesPerSec / buffer_ratio);
+
+    const AudioParameters input(
+        AudioParameters::AUDIO_PCM_LOW_LATENCY,
+        GuessChannelLayout(closest_match->nChannels),
+        closest_match->nSamplesPerSec,
+        // We need to be careful here to not pick the closest wBitsPerSample
+        // match as we need to use the PCM format (which might not be what
+        // closeest_match->wFormat is) and the internal resampler doesn't
+        // support all formats we might get here.  So, we stick to the
+        // wBitsPerSample that was asked for originally (most likely 16).
+        format_.wBitsPerSample, new_frames_per_buffer);
+
+    const AudioParameters output(AudioParameters::AUDIO_PCM_LOW_LATENCY,
+                                 GuessChannelLayout(format_.nChannels),
+                                 format_.nSamplesPerSec, format_.wBitsPerSample,
+                                 audio_bus_->frames());
+
+    converter_.reset(new AudioConverter(input, output, false));
+    converter_->AddInput(this);
+    converter_->PrimeWithSilence();
+    convert_bus_ = AudioBus::Create(input);
+
+    // Now change the format we're going to ask for to better match with what
+    // the OS can provide.  If we succeed in opening the stream with these
+    // params, we can take care of the required resampling.
+    format_.nSamplesPerSec = closest_match->nSamplesPerSec;
+    format_.nChannels = closest_match->nChannels;
+    format_.nBlockAlign = (format_.wBitsPerSample / 8) * format_.nChannels;
+    format_.nAvgBytesPerSec = format_.nSamplesPerSec * format_.nBlockAlign;
+
+    // Update our packet size assumptions based on the new format.
+    const auto new_bytes_per_buffer = convert_bus_->frames() *
+                                      format_.nChannels *
+                                      (format_.wBitsPerSample / 8);
+    packet_size_frames_ = new_bytes_per_buffer / format_.nBlockAlign;
+    packet_size_bytes_ = new_bytes_per_buffer;
+    frame_size_ = format_.nBlockAlign;
+    ms_to_frame_count_ = static_cast<double>(format_.nSamplesPerSec) / 1000.0;
+
+    // Check if we'll need to inject an intermediery buffer to avoid

[DaleCurtis, 2017/02/16 19:59:48]

You could also determine this by checking if |new_frames_per_buffer| has a
fractional part.

[tommi (sloooow) - chröme, 2017/02/17 17:09:21]

Of course! done.

+    // occasional underruns. This can happen if the buffers don't represent
+    // an equal time period.
+    const double buffer_ratio2 = closest_match->nSamplesPerSec /
+                                 static_cast<double>(convert_bus_->frames());
+    DCHECK(buffer_ratio <= buffer_ratio2);

[DaleCurtis, 2017/02/16 19:59:48]

DCHECK_LE().

[tommi (sloooow) - chröme, 2017/02/17 17:09:21]

Now removed

+    if (buffer_ratio2 == buffer_ratio) {
+      // The buffer ratio is equal, so nothing further needs to be done.
+      // For every buffer we receive, we'll convert directly to a buffer that
+      // will be delivered to the caller.
+    } else {
+      DVLOG(1) << "Audio capture data conversion: Need to inject fifo";
+      convert_fifo_.reset(
+          new AudioFifo(format_.nChannels, new_frames_per_buffer * 2));
+    }
+
+    // Indicate that we're good to go with a close match.
+    hr = S_OK;
+  }
+
   return (hr == S_OK);
 }
 
 HRESULT WASAPIAudioInputStream::InitializeAudioEngine() {
   DCHECK_EQ(OPEN_RESULT_OK, open_result_);
   DWORD flags;
   // Use event-driven mode only fo regular input devices. For loopback the
   // EVENTCALLBACK flag is specified when intializing
   // |audio_render_client_for_loopback_|.
   if (device_id_ == AudioDeviceDescription::kLoopbackInputDeviceId ||
@@ skipping 123 matching lines @@
 
   return hr;
 }
 
 void WASAPIAudioInputStream::ReportOpenResult() const {
   DCHECK(!opened_);  // This method must be called before we set this flag.
   UMA_HISTOGRAM_ENUMERATION("Media.Audio.Capture.Win.Open", open_result_,
                             OPEN_RESULT_MAX + 1);
 }
 
+double WASAPIAudioInputStream::ProvideInput(AudioBus* audio_bus,
+                                            uint32_t frames_delayed) {
+  if (convert_fifo_) {
+    int frames = std::min(convert_fifo_->frames(), audio_bus->frames());
+    convert_fifo_->Consume(audio_bus, 0, frames);
+    LOG_IF(ERROR, frames != audio_bus->frames())
+        << "Wanted " << audio_bus->frames() << " got " << frames;
+  } else {
+    DCHECK(!data_was_converted_);
+    convert_bus_->CopyTo(audio_bus);
+    data_was_converted_ = true;
+  }
+
+  return 1.0;
+}
+
 }  // namespace media