Add support for channel transforms.

media/base/channel_mixer.cc

+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// MSVC++ requires this to be set before any other includes to get M_SQRT1_2.
+#define _USE_MATH_DEFINES

[scherkus (not reviewing), 2012/10/18 01:30:39]

nit: move closer to where we need it (i.e., after channel_mixer.h include)

[DaleCurtis, 2012/10/18 01:56:44]

Can't do that, since channel_mixer.h and headers it includes might end up
including <math.h>/<cmath> which will break this. 

[scherkus (not reviewing), 2012/10/18 02:24:03]

Gah.

+
+#include "media/base/channel_mixer.h"
+
+#include <algorithm>
+#include <cmath>
+#include <limits>
+
+#include "base/logging.h"
+#include "media/base/audio_bus.h"
+#include "media/base/vector_math.h"
+
+namespace media {
+
+// Scale factor for mixing two channels together; could use 1 / 2 as well.
+// TODO(dalecurtis): Figure out which value we should use here.  Per crogers it
+// may depend on whether we're down mixing from multichannel or upmixing.
+// FFmpeg uses 1 / sqrt(2) so I've left it the same for now.
+static const float kDefaultScale = static_cast<float>(M_SQRT1_2);
+
+static int ValidateLayout(ChannelLayout layout) {
+  CHECK_NE(layout, CHANNEL_LAYOUT_NONE);
+  CHECK_NE(layout, CHANNEL_LAYOUT_UNSUPPORTED);
+  CHECK_NE(layout, CHANNEL_LAYOUT_MAX);
+
+  // Verify there's at least one channel.  Should always be true here by virtue
+  // of not being one of the invalid layouts, but lets double check to be sure.
+  int channel_count = ChannelLayoutToChannelCount(layout);
+  DCHECK_GT(channel_count, 0);
+
+  // If we have more than one channel, verify a symmetric layout for sanity.
+  // The unit test will verify all possible layouts, so this can be a DCHECK.
+  // Symmetry allows simplifying the matrix building code by allowing us to
+  // assume that if one channel of a pair exists, the other will too.
+  if (channel_count > 1) {

[scherkus (not reviewing), 2012/10/18 01:30:39]

is it worth asserting that channel_count == 1 iff CHANNEL_LAYOUT_MONO?

[DaleCurtis, 2012/10/18 05:22:04]

Can't hurt, done.

+    DCHECK((ChannelOrder(layout, LEFT) >= 0 &&

[Chris Rogers, 2012/10/17 19:46:50]

Where's the ChannelOrder() function defined?

[DaleCurtis, 2012/10/17 20:06:47]

http://codereview.chromium.org/11198018/

Will land soon. Just replaces the previous direct access to
kChannelOrderings[][]

+            ChannelOrder(layout, RIGHT) >= 0) ||
+           (ChannelOrder(layout, SIDE_LEFT) >= 0 &&
+            ChannelOrder(layout, SIDE_RIGHT) >= 0) ||
+           (ChannelOrder(layout, BACK_LEFT) >= 0 &&
+            ChannelOrder(layout, BACK_RIGHT) >= 0) ||
+           (ChannelOrder(layout, LEFT_OF_CENTER) >= 0 &&
+            ChannelOrder(layout, RIGHT_OF_CENTER) >= 0) ||
+           (ChannelOrder(layout, STEREO_LEFT) >= 0 &&
+            ChannelOrder(layout, STEREO_RIGHT) >= 0))
+        << "Non-symmetric channel layout encountered.";
+  }
+
+  return channel_count;
+}
+
+ChannelMixer::ChannelMixer(ChannelLayout input, ChannelLayout output)
+    : input_layout_(input),
+      output_layout_(output) {
+  // Stereo downmix should never be the output layout.
+  CHECK_NE(output_layout_, CHANNEL_LAYOUT_STEREO_DOWNMIX);
+
+  int input_channels = ValidateLayout(input_layout_);
+  int output_channels = ValidateLayout(output_layout_);
+
+  // Size out the initial matrix.
+  matrix_.reserve(output_channels);
+  for (int output_ch = 0; output_ch < output_channels; ++output_ch)
+    matrix_.push_back(std::vector<float>(input_channels, 0));
+
+  // Route matching channels and figure out which ones aren't accounted for.
+  for (Channels ch = LEFT; ch < CHANNELS_MAX;
+       ch = static_cast<Channels>(ch + 1)) {
+

[Chris Rogers, 2012/10/17 19:46:50]

extra blank line

[DaleCurtis, 2012/10/18 05:22:04]

Done.

+    int input_ch_index = ChannelOrder(input_layout_, ch);
+    int output_ch_index = ChannelOrder(output_layout_, ch);
+
+    if (input_ch_index < 0)
+      continue;
+
+    if (output_ch_index < 0) {
+      unaccounted_inputs_.push_back(ch);
+      continue;
+    }
+
+    DCHECK_LT(static_cast<size_t>(output_ch_index), matrix_.size());
+    DCHECK_LT(static_cast<size_t>(input_ch_index), matrix_[0].size());
+    matrix_[output_ch_index][input_ch_index] = 1;
+  }
+
+  // If all input channels are accounted for, there's nothing left to do.
+  if (unaccounted_inputs_.empty())
+    return;
+
+  // Mix front LR into center.
+  if (IsUnaccounted(LEFT)) {
+    Mix(LEFT, CENTER, kDefaultScale);
+    Mix(RIGHT, CENTER, kDefaultScale);
+  }

[Chris Rogers, 2012/10/17 19:46:50]

I think this is the wrong approach for stereo -> mono downmix which will clip
mixing down a full-scale stereo mix.  Clipping is bad and will often happen.

I've never seen any commercial audio sound editors do this.  Instead I recommend
0.5

I know it seems inconsistent as compared with, for example, the ITU-R approach
for 5.1 down-mixing.  But 5.1 (and higher) mixes usually are mastered to account
for the possibility that they might often be down-mixed, and thus won't have
such *hot* signals in L/R, etc.  Stereo mixes, on the other hand are almost
always mastered "to the max", and have no extra safety margin to be using a
1/sqrt(2) approach.

[DaleCurtis, 2012/10/17 20:06:47]

Okay, sounds like we just need two special case blocks for stereo -> mono and
mono -> stereo.

[DaleCurtis, 2012/10/18 01:56:44]

Done.

+
+  // Mix center into front LR.
+  if (IsUnaccounted(CENTER)) {
+    MixWithoutAccounting(CENTER, LEFT, kDefaultScale);
+    Mix(CENTER, RIGHT, kDefaultScale);
+  }

[Chris Rogers, 2012/10/17 19:46:50]

This is odd and very much different from what audio systems usually do for the
mono -> stereo up-mix case.  A full-range mono signal will not be full-range
when up-mixed to stereo which is not how it's normally handled by an OS, audio
editors, etc.  Instead I recommend copying mono to both left and right.

[DaleCurtis, 2012/10/17 20:06:47]

Ditto.

[DaleCurtis, 2012/10/18 01:56:44]

Done.

+
+  // Mix back LR into: side LR || back center || front LR || front center.
+  if (IsUnaccounted(BACK_LEFT)) {
+    if (HasOutputChannel(SIDE_LEFT)) {
+      // If we have side LR, mix back LR into side LR, but instead if the input
+      // doesn't have side LR (but output does) copy back LR to side LR.
+      float scale = HasInputChannel(SIDE_LEFT) ? kDefaultScale : 1;
+      Mix(BACK_LEFT, SIDE_LEFT, scale);
+      Mix(BACK_RIGHT, SIDE_RIGHT, scale);
+    } else if (HasOutputChannel(BACK_CENTER)) {
+      // Mix back LR into back center.
+      Mix(BACK_LEFT, BACK_CENTER, kDefaultScale);
+      Mix(BACK_RIGHT, BACK_CENTER, kDefaultScale);
+    } else if (output > CHANNEL_LAYOUT_MONO) {
+      // Mix back LR into front LR.
+      Mix(BACK_LEFT, LEFT, kDefaultScale);
+      Mix(BACK_RIGHT, RIGHT, kDefaultScale);
+    } else {
+      // Mix back LR into front center.
+      Mix(BACK_LEFT, CENTER, kDefaultScale);
+      Mix(BACK_RIGHT, CENTER, kDefaultScale);
+    }
+  }
+
+  // Mix side LR into: back LR || back center || front LR || front center.
+  if (IsUnaccounted(SIDE_LEFT)) {
+    if (HasOutputChannel(BACK_LEFT)) {
+      // If we have back LR, mix side LR into back LR, but instead if the input
+      // doesn't have back LR (but output does) copy side LR to back LR.
+      float scale = HasInputChannel(BACK_LEFT) ? kDefaultScale : 1;
+      Mix(SIDE_LEFT, BACK_LEFT, scale);
+      Mix(SIDE_RIGHT, BACK_RIGHT, scale);
+    } else if (HasOutputChannel(BACK_CENTER)) {
+      // Mix side LR into back center.
+      Mix(SIDE_LEFT, BACK_CENTER, kDefaultScale);
+      Mix(SIDE_RIGHT, BACK_CENTER, kDefaultScale);
+    } else if (output > CHANNEL_LAYOUT_MONO) {
+      // Mix side LR into front LR.
+      Mix(SIDE_LEFT, LEFT, kDefaultScale);
+      Mix(SIDE_RIGHT, RIGHT, kDefaultScale);
+    } else {
+      // Mix side LR into front center.
+      Mix(SIDE_LEFT, CENTER, kDefaultScale);
+      Mix(SIDE_RIGHT, CENTER, kDefaultScale);
+    }
+  }
+
+  // Mix back center into: back LR || side LR || front LR || front center.
+  if (IsUnaccounted(BACK_CENTER)) {
+    if (HasOutputChannel(BACK_LEFT)) {
+      // Mix back center into back LR.
+      MixWithoutAccounting(BACK_CENTER, BACK_LEFT, kDefaultScale);
+      Mix(BACK_CENTER, BACK_RIGHT, kDefaultScale);
+    } else if (HasOutputChannel(SIDE_LEFT)) {
+      // Mix back center into side LR.
+      MixWithoutAccounting(BACK_CENTER, SIDE_LEFT, kDefaultScale);
+      Mix(BACK_CENTER, SIDE_RIGHT, kDefaultScale);
+    } else if (output > CHANNEL_LAYOUT_MONO) {
+      // Mix back center into front LR.
+      // TODO(dalecurtis): Not sure about these values?
+      MixWithoutAccounting(BACK_CENTER, LEFT, kDefaultScale * kDefaultScale);
+      Mix(BACK_CENTER, RIGHT, kDefaultScale * kDefaultScale);
+    } else {
+      // Mix back center into front center.
+      // TODO(dalecurtis): Not sure about these values?
+      Mix(BACK_CENTER, CENTER, kDefaultScale * kDefaultScale);
+    }
+  }
+
+  // Mix LR of center into: front center || front LR.
+  if (IsUnaccounted(LEFT_OF_CENTER)) {
+    if (HasOutputChannel(CENTER)) {
+      // Mix LR of center into front center.
+      Mix(LEFT_OF_CENTER, CENTER, kDefaultScale);
+      Mix(RIGHT_OF_CENTER, CENTER, kDefaultScale);
+    } else {
+      // Mix LR of center into front LR.
+      Mix(LEFT_OF_CENTER, LEFT, kDefaultScale);
+      Mix(RIGHT_OF_CENTER, RIGHT, kDefaultScale);
+    }
+  }
+
+  // Mix LFE into: front LR || front center.
+  if (IsUnaccounted(LFE)) {
+    if (!HasOutputChannel(CENTER)) {
+      // Mix LFE into front LR.
+      MixWithoutAccounting(LFE, LEFT, kDefaultScale);
+      Mix(LFE, RIGHT, kDefaultScale);
+    } else {
+      // Mix LFE into front center.
+      Mix(LFE, CENTER, kDefaultScale);
+    }
+  }
+
+  // All channels should now be accounted for.
+  DCHECK(unaccounted_inputs_.empty());
+}
+
+ChannelMixer::~ChannelMixer() {}
+
+void ChannelMixer::Rematrix(const AudioBus* input, AudioBus* output) {

[Chris Rogers, 2012/10/17 19:46:50]

What is our plan for simple channel swizzling to account for differences between
Chrome/FFmpeg channel orders, and the orders used by each OS?  I'm thinking of
our plan for replacing PCMQueueOutAudioOutputStream::SwizzleLayout(), and
similar...

It seems like we should have a highly efficient AudioBus method to simply swap
the internal pointers around to avoid any memcpy's or touching the memory
itself.

[DaleCurtis, 2012/10/17 20:06:47]

Yes I plan to add an AudioBus::Remap(int ch, int ch) for non-wrapped AudioBus
objects. Each output stream will be responsible for swizzling as necessary. 

+  CHECK_EQ(matrix_.size(), static_cast<size_t>(output->channels()));
+  CHECK_EQ(matrix_[0].size(), static_cast<size_t>(input->channels()));
+  CHECK_EQ(input->frames(), output->frames());
+
+  for (int output_ch = 0; output_ch < output->channels(); ++output_ch) {
+    // Zero initialize |output| so we're accumulating from zero.

[scherkus (not reviewing), 2012/10/18 01:30:39]

silly optimization question... (I think chris asked this in a different way)

for cases where we don't need to accumulate and the scale is 1 can't we get by
w/ a single memcpy?

we can leave it as a follow up TODO

[DaleCurtis, 2012/10/18 01:56:44]

Hmm, we don't need to accumulate when the output layout is a remapping of the
input layout. Are there any other common cases?

For that case, I can set a bool during construction if there are no unaccounted
channels to handle, then simply memcpy.

[scherkus (not reviewing), 2012/10/18 02:24:03]

Not sure.

I was thinking of a situation (for whatever silly reason) where a ChannelMixer
was instantiated for stereo -> stereo, in which case it seems like that'd be a
memcpy() instead of FMAC()... but TBH I'm not sure how often this would come up.

[DaleCurtis, 2012/10/18 05:22:04]

Added memcpy support for remaps, pretty trivial and should save a bunch in many
cases.

+    memset(output->channel(output_ch), 0,
+           output->frames() * sizeof(*output->channel(output_ch)));

[Chris Rogers, 2012/10/17 19:46:50]

This cries out for a AudioBus::ZeroChannel() method

[DaleCurtis, 2012/10/17 20:06:47]

Good idea. Will add.

[DaleCurtis, 2012/10/18 05:22:04]

Actually, output->Zero() outside of the loop is better.

+
+    for (int input_ch = 0; input_ch < input->channels(); ++input_ch) {
+      float scale = matrix_[output_ch][input_ch];
+      // Scale should always be positive.  Don't bother scaling by zero.
+      DCHECK_GE(scale, 0);
+      if (scale > std::numeric_limits<float>::epsilon()) {
+        vector_math::FMAC(input->channel(input_ch), scale, output->frames(),
+                          output->channel(output_ch));

[Chris Rogers, 2012/10/17 19:46:50]

What if the scale is 1, or does FMAC optimize for this case?

[DaleCurtis, 2012/10/17 20:06:47]

FMAC doesn't optimize for this case. I could add a Float-Accumulate method which
skips the multiply step, but I wonder how much that would really save.

+      }
+    }
+  }
+}
+
+void ChannelMixer::AccountFor(Channels ch) {
+  unaccounted_inputs_.erase(std::find(
+      unaccounted_inputs_.begin(), unaccounted_inputs_.end(), ch));
+}
+
+bool ChannelMixer::IsUnaccounted(Channels ch) {
+  return std::find(unaccounted_inputs_.begin(), unaccounted_inputs_.end(),
+                   ch) != unaccounted_inputs_.end();
+}
+
+bool ChannelMixer::HasInputChannel(Channels ch) {
+  return ChannelOrder(input_layout_, ch) >= 0;
+}
+
+bool ChannelMixer::HasOutputChannel(Channels ch) {
+  return ChannelOrder(output_layout_, ch) >= 0;
+}
+
+void ChannelMixer::Mix(Channels input_ch, Channels output_ch, float scale) {
+  MixWithoutAccounting(input_ch, output_ch, scale);
+  AccountFor(input_ch);
+}
+
+void ChannelMixer::MixWithoutAccounting(Channels input_ch, Channels output_ch,
+                                        float scale) {
+  int input_ch_index = ChannelOrder(input_layout_, input_ch);
+  int output_ch_index = ChannelOrder(output_layout_, output_ch);
+
+  DCHECK(IsUnaccounted(input_ch));
+  DCHECK_GE(input_ch_index, 0);
+  DCHECK_GE(output_ch_index, 0);
+
+  matrix_[output_ch_index][input_ch_index] += scale;
+}
+
+}  // namespace media