Mixing audio with different latency requirements

content/renderer/media/audio_renderer_mixer_manager.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/audio_renderer_mixer_manager.h"
 
+#include <algorithm>
 #include <string>
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/memory/ptr_util.h"
+#include "base/metrics/histogram_macros.h"
 #include "build/build_config.h"
 #include "content/renderer/media/audio_renderer_sink_cache.h"
 #include "media/audio/audio_device_description.h"
 #include "media/base/audio_hardware_config.h"
 #include "media/base/audio_renderer_mixer.h"
 #include "media/base/audio_renderer_mixer_input.h"
 
+namespace {
+// Calculate mixer output parameters based on input parameters and audio
+// hardware configuration.
+media::AudioParameters GetMixerOutputParams(
+    const media::AudioParameters& input_params,
+    const media::AudioParameters& hardware_params,
+    media::AudioLatency::LatencyType latency) {
+  int output_sample_rate = input_params.sample_rate();
+  bool valid_not_fake_hardware_params =
+      hardware_params.format() != media::AudioParameters::AUDIO_FAKE &&
+      hardware_params.IsValid();
+  int preferred_high_latency_output_bufffer_size = 0;
+
+#if !defined(OS_CHROMEOS)
+  // On ChromeOS as well as when a fake device is used, we can rely on the
+  // playback device to handle resampling, so don't waste cycles on it here.
+  // On other systems if hardware parameters are valid and the device is not
+  // fake, resample to hadrware sample rate. Otherwise, pass the input one and
+  // let the browser side handle automatic fallback.
+  if (valid_not_fake_hardware_params) {
+    output_sample_rate = hardware_params.sample_rate();
+    preferred_high_latency_output_bufffer_size =
+        hardware_params.frames_per_buffer();
+  }
+#endif
+
+  int output_buffer_size = 0;
+
+  switch (latency) {
+    case media::AudioLatency::LATENCY_INTERACTIVE:
+      // WebAudio should provide correct callback size in frames; it does not
+      // depend on the sample rate.
+      output_buffer_size = media::AudioLatency::GetInteractiveBufferSize(
+          hardware_params.frames_per_buffer());
+      break;
+    case media::AudioLatency::LATENCY_RTC:
+      // adjust output buffer size according to the latency requirement.
+      output_buffer_size = media::AudioLatency::GetRtcBufferSize(
+          output_sample_rate, valid_not_fake_hardware_params
+                                  ? hardware_params.frames_per_buffer()
+                                  : 0);
+      break;
+    case media::AudioLatency::LATENCY_PLAYBACK:
+      // adjust output buffer size according to the latency requirement.
+      output_buffer_size = media::AudioLatency::GetHighLatencyBufferSize(
+          output_sample_rate, preferred_high_latency_output_bufffer_size);
+      break;
+    case media::AudioLatency::LATENCY_EXACT_MS:
+    // TODO(olka): add support when WebAudio requires it.
+    default:
+      NOTREACHED();
+  }
+  DCHECK(output_buffer_size);
+
+  // Force to 16-bit output for now since we know that works everywhere;
+  // ChromeOS does not support other bit depths.
+  return media::AudioParameters(input_params.format(),
+                                input_params.channel_layout(),
+                                output_sample_rate, 16, output_buffer_size);
+}
+
+const std::string GetMixerUmaHistogramName(
+    media::AudioLatency::LatencyType latency) {
+  switch (latency) {
+    case media::AudioLatency::LATENCY_EXACT_MS:
+      return "Media.Audio.Render.AudioInputsPerMixer.LatencyExact";
+    case media::AudioLatency::LATENCY_INTERACTIVE:
+      return "Media.Audio.Render.AudioInputsPerMixer.LatencyInteractive";
+    case media::AudioLatency::LATENCY_RTC:
+      return "Media.Audio.Render.AudioInputsPerMixer.LatencyRtc";
+    case media::AudioLatency::LATENCY_PLAYBACK:
+      return "Media.Audio.Render.AudioInputsPerMixer.LatencyPlayback";
+    default:
+      NOTREACHED();
+  }
+}
+}  // namespace
+
 namespace content {
 
 AudioRendererMixerManager::AudioRendererMixerManager(
     std::unique_ptr<AudioRendererSinkCache> sink_cache)
     : sink_cache_(std::move(sink_cache)) {
   DCHECK(sink_cache_);
 }
 
 AudioRendererMixerManager::~AudioRendererMixerManager() {
   // References to AudioRendererMixers may be owned by garbage collected
   // objects.  During process shutdown they may be leaked, so, transitively,
   // |mixers_| may leak (i.e., may be non-empty at this time) as well.

[o1ka, 2016/06/28 15:51:28]

This means mixer destructor is never called and we need to manually log
per-mixer stats here (but see the problem below).

+
+  if (std::size_t subset_value = latency_map_.to_ulong()) {
+    LOCAL_HISTOGRAM_CUSTOM_COUNTS("Media.Audio.Render.AudioMixing.LatencyMap",

[o1ka, 2016/06/28 15:51:28]

The problem is that the destructor for RendererThreadImpl => for
AudioRendererMixerManager is almost never called. Have not figured out what can
be done here yet.

[o1ka, 2016/06/28 16:05:32]

Looks like we need to send updates to the browser each time the value changes,
and log stats on the browser side upon renderer shut down.

[o1ka, 2016/06/28 17:16:08]

Another option is to log the values each time they change. The histogram will be
more ambiguous/harder to interpret (especially "latency subset" one), but it may
be a tradeoff to pay.

+                                  subset_value, 1, 0xff, 0xff);
+  }
 }
 
 // static
 std::unique_ptr<AudioRendererMixerManager> AudioRendererMixerManager::Create() {
   return base::WrapUnique(
       new AudioRendererMixerManager(AudioRendererSinkCache::Create()));
 }
 
 media::AudioRendererMixerInput* AudioRendererMixerManager::CreateInput(
     int source_render_frame_id,
     int session_id,
     const std::string& device_id,
-    const url::Origin& security_origin) {
+    const url::Origin& security_origin,
+    media::AudioLatency::LatencyType latency) {
   // AudioRendererMixerManager lives on the renderer thread and is destroyed on
   // renderer thread destruction, so it's safe to pass its pointer to a mixer
   // input.
   return new media::AudioRendererMixerInput(
       this, source_render_frame_id,
       media::AudioDeviceDescription::UseSessionIdToSelectDevice(session_id,
                                                                 device_id)
           ? GetOutputDeviceInfo(source_render_frame_id, session_id, device_id,
                                 security_origin)
                 .device_id()
           : device_id,
-      security_origin);
+      security_origin, latency);
 }
 
 media::AudioRendererMixer* AudioRendererMixerManager::GetMixer(
     int source_render_frame_id,
-    const media::AudioParameters& params,
+    const media::AudioParameters& input_params,
+    media::AudioLatency::LatencyType latency,
     const std::string& device_id,
     const url::Origin& security_origin,
     media::OutputDeviceStatus* device_status) {
   // Effects are not passed through to output creation, so ensure none are set.
-  DCHECK_EQ(params.effects(), media::AudioParameters::NO_EFFECTS);
+  DCHECK_EQ(input_params.effects(), media::AudioParameters::NO_EFFECTS);
 
-  const MixerKey key(source_render_frame_id, params, device_id,
+  const MixerKey key(source_render_frame_id, input_params, latency, device_id,
                      security_origin);
   base::AutoLock auto_lock(mixers_lock_);
 
+  // Update latency map when the mixer is requested, i.e. there is an attempt to
+  // mix and output audio with a given latency. This is opposite to
+  // CreateInput() which creates a sink which is probably never used for output.
+  latency_map_[latency] = 1;
+
   AudioRendererMixerMap::iterator it = mixers_.find(key);
   if (it != mixers_.end()) {
     if (device_status)
       *device_status = media::OUTPUT_DEVICE_STATUS_OK;
 
     it->second.ref_count++;
+    DVLOG(1) << "Reusing mixer: " << it->second.mixer;
     return it->second.mixer;
   }
 
   scoped_refptr<media::AudioRendererSink> sink =
       sink_cache_->GetSink(source_render_frame_id, device_id, security_origin);
 
   const media::OutputDeviceInfo& device_info = sink->GetOutputDeviceInfo();
   if (device_status)
     *device_status = device_info.device_status();
   if (device_info.device_status() != media::OUTPUT_DEVICE_STATUS_OK) {
     sink_cache_->ReleaseSink(sink.get());
     sink->Stop();
     return nullptr;
   }
 
-  // On ChromeOS as well as when a fake device is used, we can rely on the
-  // playback device to handle resampling, so don't waste cycles on it here.
-  int sample_rate = params.sample_rate();
-  int buffer_size =
-      media::AudioHardwareConfig::GetHighLatencyBufferSize(sample_rate, 0);
-
-#if !defined(OS_CHROMEOS)
-  const media::AudioParameters& hardware_params = device_info.output_params();
-
-  // If we have valid, non-fake hardware parameters, use them.  Otherwise, pass
-  // on the input params and let the browser side handle automatic fallback.
-  if (hardware_params.format() != media::AudioParameters::AUDIO_FAKE &&
-      hardware_params.IsValid()) {
-    sample_rate = hardware_params.sample_rate();
-    buffer_size = media::AudioHardwareConfig::GetHighLatencyBufferSize(
-        sample_rate, hardware_params.frames_per_buffer());
-  }
-#endif
-
-  // Create output parameters based on the audio hardware configuration for
-  // passing on to the output sink.  Force to 16-bit output for now since we
-  // know that works everywhere; ChromeOS does not support other bit depths.
-  media::AudioParameters output_params(
-      media::AudioParameters::AUDIO_PCM_LOW_LATENCY, params.channel_layout(),
-      sample_rate, 16, buffer_size);
-  DCHECK(output_params.IsValid());
-
-  media::AudioRendererMixer* mixer =
-      new media::AudioRendererMixer(output_params, sink);
+  const media::AudioParameters& mixer_output_params =
+      GetMixerOutputParams(input_params, device_info.output_params(), latency);
+  media::AudioRendererMixer* mixer = new media::AudioRendererMixer(
+      mixer_output_params, sink, GetMixerUmaHistogramName(latency));
   AudioRendererMixerReference mixer_reference = {mixer, 1, sink.get()};
   mixers_[key] = mixer_reference;
+  DVLOG(1) << __FUNCTION__ << " mixer: " << mixer << " latency: " << latency
+           << "\n input: " << input_params.AsHumanReadableString()
+           << "\noutput: " << mixer_output_params.AsHumanReadableString();
   return mixer;
 }
 
 void AudioRendererMixerManager::ReturnMixer(
-    int source_render_frame_id,
-    const media::AudioParameters& params,
-    const std::string& device_id,
-    const url::Origin& security_origin) {
-  const MixerKey key(source_render_frame_id, params, device_id,
-                     security_origin);
+    const media::AudioRendererMixer* mixer) {
   base::AutoLock auto_lock(mixers_lock_);
-
-  AudioRendererMixerMap::iterator it = mixers_.find(key);
+  AudioRendererMixerMap::iterator it = std::find_if(
+      mixers_.begin(), mixers_.end(),
+      [mixer](const std::pair<MixerKey, AudioRendererMixerReference>& val) {
+        return val.second.mixer == mixer;
+      });
   DCHECK(it != mixers_.end());
 
   // Only remove the mixer if AudioRendererMixerManager is the last owner.
   it->second.ref_count--;
   if (it->second.ref_count == 0) {
     // The mixer will be deleted now, so release the sink.
     sink_cache_->ReleaseSink(it->second.sink_ptr);
     delete it->second.mixer;
     mixers_.erase(it);
   }
 }
 
 media::OutputDeviceInfo AudioRendererMixerManager::GetOutputDeviceInfo(
     int source_render_frame_id,
     int session_id,
     const std::string& device_id,
     const url::Origin& security_origin) {
   return sink_cache_->GetSinkInfo(source_render_frame_id, session_id, device_id,
                                   security_origin);
 }
 
 AudioRendererMixerManager::MixerKey::MixerKey(
     int source_render_frame_id,
     const media::AudioParameters& params,
+    media::AudioLatency::LatencyType latency,
     const std::string& device_id,
     const url::Origin& security_origin)
     : source_render_frame_id(source_render_frame_id),
       params(params),
+      latency(latency),
       device_id(device_id),
       security_origin(security_origin) {}
 
 AudioRendererMixerManager::MixerKey::MixerKey(const MixerKey& other) = default;
 
 }  // namespace content