[Chromecast] Process streams with different post-processing.

chromecast/media/cma/backend/alsa/stream_mixer_alsa.cc

Index: chromecast/media/cma/backend/alsa/stream_mixer_alsa.cc
diff --git a/chromecast/media/cma/backend/alsa/stream_mixer_alsa.cc b/chromecast/media/cma/backend/alsa/stream_mixer_alsa.cc
index 463a300aba2498a5aca63e6438cf046cc7c816c3..26c1d2bacc109c9c35aa10a9c47c407925c29627 100644
--- a/chromecast/media/cma/backend/alsa/stream_mixer_alsa.cc
+++ b/chromecast/media/cma/backend/alsa/stream_mixer_alsa.cc
@@ -21,6 +21,7 @@
 #include "chromecast/media/cma/backend/alsa/alsa_wrapper.h"
 #include "chromecast/media/cma/backend/alsa/audio_filter_factory.h"
 #include "chromecast/media/cma/backend/alsa/stream_mixer_alsa_input_impl.h"
+#include "chromecast/public/media/audio_device_ids.h"
 #include "media/base/audio_bus.h"
 #include "media/base/media_switches.h"
 
@@ -117,6 +118,16 @@ const float kSilenceSecondsToFilter = 1.0f;
 
 const int64_t kNoTimestamp = std::numeric_limits<int64_t>::min();
 
+const AudioFilterFactory::FilterType kFilterTypes[kNumFilterGroups] = {
+    AudioFilterFactory::MEDIA_FILTER, AudioFilterFactory::SYSTEM_AUDIO_FILTER};
+
+int GetFilterGroup(std::string stream_name) {
+  if (stream_name == kSystemAudioDeviceId) {
+    return 1;
+  }
+  return 0;
+}
+
 int64_t TimespecToMicroseconds(struct timespec time) {
   return static_cast<int64_t>(time.tv_sec) *
              base::Time::kMicrosecondsPerSecond +
@@ -167,6 +178,12 @@ bool GetSwitchValueAsNonNegativeInt(const std::string& switch_name,
   return true;
 }
 
+void VectorAccumulate(const int32_t* source, size_t size, int32_t* dest) {
+  for (size_t i = 0; i < size; ++i) {
+    dest[i] += source[i];

[wzhong, 2017/02/21 16:03:46]

No clipping or saturation?

[bshaya, 2017/02/21 23:30:14]

Done.

+  }
+}
+
 class StreamMixerAlsaInstance : public StreamMixerAlsa {
  public:
   StreamMixerAlsaInstance() {}
@@ -246,10 +263,10 @@ StreamMixerAlsa::StreamMixerAlsa()
           : 0;
 
   // Create filters
-  pre_loopback_filter_ = AudioFilterFactory::MakeAudioFilter(
-      AudioFilterFactory::PRE_LOOPBACK_FILTER);
-  post_loopback_filter_ = AudioFilterFactory::MakeAudioFilter(
-      AudioFilterFactory::POST_LOOPBACK_FILTER);
+  for (int filter = 0; filter < kNumFilterGroups; ++filter) {
+    pre_loopback_filter_[filter] =
+        AudioFilterFactory::MakeAudioFilter(kFilterTypes[filter]);
+  }
 
   DefineAlsaParameters();
 }
@@ -529,14 +546,11 @@ void StreamMixerAlsa::Start() {
   }
 
   // Initialize filters
-  if (pre_loopback_filter_) {
-    pre_loopback_filter_->SetSampleRateAndFormat(
-        output_samples_per_second_, ::media::SampleFormat::kSampleFormatS32);
-  }
-
-  if (post_loopback_filter_) {
-    post_loopback_filter_->SetSampleRateAndFormat(
-        output_samples_per_second_, ::media::SampleFormat::kSampleFormatS32);
+  for (int filter = 0; filter < kNumFilterGroups; ++filter) {
+    if (pre_loopback_filter_[filter]) {
+      pre_loopback_filter_[filter]->SetSampleRateAndFormat(
+          output_samples_per_second_, ::media::SampleFormat::kSampleFormatS32);
+    }
   }
 
   RETURN_REPORT_ERROR(PcmPrepare, pcm_);
@@ -769,12 +783,14 @@ bool StreamMixerAlsa::TryWriteFrames() {
   const int min_frames_in_buffer =
       output_samples_per_second_ * kMinBufferedDataMs / 1000;
   int chunk_size = output_samples_per_second_ * kMaxWriteSizeMs / 1000;
-  std::vector<InputQueue*> active_inputs;
+  std::vector<InputQueue*> active_inputs[kNumFilterGroups];
+  bool is_silence = true;
   for (auto&& input : inputs_) {
     int read_size = input->MaxReadSize();
     if (read_size > 0) {
-      active_inputs.push_back(input.get());
+      active_inputs[GetFilterGroup(input->name())].push_back(input.get());
       chunk_size = std::min(chunk_size, read_size);
+      is_silence = false;
     } else if (input->primary()) {
       if (alsa_->PcmStatus(pcm_, pcm_status_) != 0) {
         LOG(ERROR) << "Failed to get status";
@@ -801,58 +817,107 @@ bool StreamMixerAlsa::TryWriteFrames() {
     }
   }
 
-  if (active_inputs.empty()) {
-    // No inputs have any data to provide. Fill with silence to avoid underrun.
+  if (is_silence) {
+    // No inputs have any data to provide. Push silence to prevent underrun.
     chunk_size = kPreventUnderrunChunkSize;
-    if (!mixed_ || mixed_->frames() < chunk_size) {
-      mixed_ = ::media::AudioBus::Create(kNumOutputChannels, chunk_size);
-    }
+    ResizeBuffersIfNecessary(chunk_size);
+    memset(interleaved_.data(), 0,
+           static_cast<size_t>(chunk_size * kNumOutputChannels) *
+               BytesPerOutputFormatSample());
+  } else {
+    ResizeBuffersIfNecessary(chunk_size);
+  }
 
-    mixed_->Zero();
-    WriteMixedPcm(*mixed_, chunk_size, true /* is_silence */);
-    return true;
+  // If |mixed_|, |temp_|, |interleaved_|, or |interleaved_intermediate_| have
+  // not been allocated, or are too small, allocate a buffer.
+
+  // Mix each group, pass through corresponding audio filter, and
+  // accumulate into |interleaved_|.
+  bool non_zero_data = false;
+  for (int filter_group = 0; filter_group < kNumFilterGroups; ++filter_group) {
+    non_zero_data =
+        non_zero_data ||
+        MixAndFilterGroup(active_inputs[filter_group], filter_group, chunk_size,
+                          non_zero_data /* accumulate */);
   }
 
-  // If |mixed_| has not been allocated, or it is too small, allocate a buffer.
-  if (!mixed_ || mixed_->frames() < chunk_size) {
-    mixed_ = ::media::AudioBus::Create(kNumOutputChannels, chunk_size);
+  WriteMixedPcm(chunk_size);
+  return true;
+}
+
+bool StreamMixerAlsa::MixAndFilterGroup(
+    const std::vector<InputQueue*>& active_inputs,
+    int filter_group,
+    int frames,
+    bool accumulate) {
+  // Mix into group buffer, |mixed_|.
+  mixed_->ZeroFramesPartial(0, frames);
+  bool is_silence = true;

[tianyuwang1, 2017/02/18 00:17:08]

Do you need thi is_silence?  Just use active_inputs.empty() below.

[bshaya, 2017/02/21 23:30:14]

Outdated

+  if (!active_inputs.empty()) {
+    is_silence = false;
+    // Loop through active inputs, polling them for data, and mixing them.
+    for (InputQueue* input : active_inputs) {
+      input->GetResampledData(temp_.get(), frames);
+      for (int c = 0; c < kNumOutputChannels; ++c) {
+        input->VolumeScaleAccumulate(c, temp_->channel(c), frames,
+                                     mixed_->channel(c));
+      }
+    }
   }
 
-  // If |temp_| has not been allocated, or is too small, allocate a buffer.
-  if (!temp_ || temp_->frames() < chunk_size) {
-    temp_ = ::media::AudioBus::Create(kNumOutputChannels, chunk_size);
+  // If no data has been written to |interleaved_| thus far,
+  // we can write to |interleaved_| directly, skipping a vector addition step.
+  std::vector<uint8_t>* interleaved_group = &interleaved_intermediate_;
+  if (!accumulate) {
+    interleaved_group = &interleaved_;
   }
 
-  mixed_->ZeroFramesPartial(0, chunk_size);
+  // Convert to interleaved for post-processing.
+  mixed_->ToInterleaved(frames, BytesPerOutputFormatSample(),
+                        interleaved_group->data());
 
-  // Loop through active inputs, polling them for data, and mixing them.
-  for (InputQueue* input : active_inputs) {
-    input->GetResampledData(temp_.get(), chunk_size);
-    for (int c = 0; c < kNumOutputChannels; ++c) {
-      input->VolumeScaleAccumulate(c, temp_->channel(c), chunk_size,
-                                   mixed_->channel(c));
+  // Ensure that, on onset of silence, at least |kSilenceSecondsToFilter|
+  // second of audio get pushed through the filters to clear any memory.
+  bool filter_frames = true;

[tianyuwang1, 2017/02/18 00:17:08]

filter_frames doesn't seem to change.

[bshaya, 2017/02/21 23:30:14]

Done.

+  if (is_silence) {

[tianyuwang1, 2017/02/18 00:17:08]

move this block up after if (!active_inputs.empty()) {}?

[bshaya, 2017/02/21 23:30:14]

Done.

+    int silence_frames_to_filter =
+        output_samples_per_second_ * kSilenceSecondsToFilter;
+    if (silence_frames_filtered_[filter_group] < silence_frames_to_filter) {
+      silence_frames_filtered_[filter_group] += frames;
+    } else {
+      return false;  // Output will be silence, no need to mix.
     }
+  } else {
+    silence_frames_filtered_[filter_group] = 0;
   }
 
-  WriteMixedPcm(*mixed_, chunk_size, false /* is_silence */);
-  return true;
+  // Filter the mixed group.
+  if (pre_loopback_filter_[filter_group] && filter_frames) {
+    pre_loopback_filter_[filter_group]->ProcessInterleaved(
+        interleaved_group->data(), frames);
+  }
+
+  // Exit if we already wrote to |interleaved_|.
+  if (!accumulate) {
+    return filter_frames;
+  }
+
+  // Mix into final output buffer, |interleaved_|
+  DCHECK_EQ(4, BytesPerOutputFormatSample());
+  VectorAccumulate(reinterpret_cast<int32_t*>(interleaved_group->data()),
+                   frames * kNumOutputChannels,
+                   reinterpret_cast<int32_t*>(interleaved_.data()));
+  return filter_frames;
 }
 
 ssize_t StreamMixerAlsa::BytesPerOutputFormatSample() {
   return alsa_->PcmFormatSize(pcm_format_, 1);
 }
 
-void StreamMixerAlsa::WriteMixedPcm(const ::media::AudioBus& mixed,
-                                    int frames, bool is_silence) {
+void StreamMixerAlsa::WriteMixedPcm(int frames) {
   DCHECK(mixer_task_runner_->BelongsToCurrentThread());
   CHECK_PCM_INITIALIZED();
 
-  size_t interleaved_size = static_cast<size_t>(frames * kNumOutputChannels) *
-                            BytesPerOutputFormatSample();
-  if (interleaved_.size() < interleaved_size) {
-    interleaved_.resize(interleaved_size);
-  }
-
   int64_t expected_playback_time;
   if (rendering_delay_.timestamp_microseconds == kNoTimestamp) {
     expected_playback_time = kNoTimestamp;
@@ -861,39 +926,14 @@ void StreamMixerAlsa::WriteMixedPcm(const ::media::AudioBus& mixed,
                              rendering_delay_.delay_microseconds;
   }
 
-  mixed.ToInterleaved(frames, BytesPerOutputFormatSample(),
-                      interleaved_.data());
-
-  // Ensure that, on onset of silence, at least |kSilenceSecondsToFilter|
-  // second of audio get pushed through the filters to clear any memory.
-  bool filter_frames = true;
-  if (is_silence) {
-    int silence_frames_to_filter =
-        output_samples_per_second_ * kSilenceSecondsToFilter;
-    if (silence_frames_filtered_ < silence_frames_to_filter) {
-      silence_frames_filtered_ += frames;
-    } else {
-      filter_frames = false;
-    }
-  } else {
-    silence_frames_filtered_ = 0;
-  }
-
-  // Filter, send to observers, and post filter
-  if (pre_loopback_filter_ && filter_frames) {
-    pre_loopback_filter_->ProcessInterleaved(interleaved_.data(), frames);
-  }
-
+  size_t interleaved_size = static_cast<size_t>(frames * kNumOutputChannels) *
+                            BytesPerOutputFormatSample();
   for (CastMediaShlib::LoopbackAudioObserver* observer : loopback_observers_) {
     observer->OnLoopbackAudio(expected_playback_time, kSampleFormatS32,
                               output_samples_per_second_, kNumOutputChannels,
                               interleaved_.data(), interleaved_size);
   }
 
-  if (post_loopback_filter_ && filter_frames) {
-    post_loopback_filter_->ProcessInterleaved(interleaved_.data(), frames);
-  }
-
   // If the PCM has been drained it will be in SND_PCM_STATE_SETUP and need
   // to be prepared in order for playback to work.
   if (alsa_->PcmState(pcm_) == SND_PCM_STATE_SETUP) {
@@ -961,5 +1001,24 @@ void StreamMixerAlsa::RemoveLoopbackAudioObserver(
   observer->OnRemoved();
 }
 
+void StreamMixerAlsa::ResizeBuffersIfNecessary(int chunk_size) {
+  if (!mixed_ || mixed_->frames() < chunk_size) {
+    mixed_ = ::media::AudioBus::Create(kNumOutputChannels, chunk_size);
+  }
+  if (!temp_ || temp_->frames() < chunk_size) {
+    temp_ = ::media::AudioBus::Create(kNumOutputChannels, chunk_size);
+  }
+
+  size_t interleaved_size =
+      static_cast<size_t>(chunk_size * kNumOutputChannels) *
+      BytesPerOutputFormatSample();
+  if (interleaved_.size() < interleaved_size) {
+    interleaved_.resize(interleaved_size);
+  }
+  if (interleaved_intermediate_.size() < interleaved_size) {
+    interleaved_intermediate_.resize(interleaved_size);
+  }
+}
+
 }  // namespace media
 }  // namespace chromecast