Complete the implementation of up/down-mixing rules for AudioNode

third_party/WebKit/Source/platform/audio/AudioBus.cpp

 /*
  * Copyright (C) 2010 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *
  * 1.  Redistributions of source code must retain the above copyright
  *     notice, this list of conditions and the following disclaimer.
  * 2.  Redistributions in binary form must reproduce the above copyright
@@ skipping 198 matching lines @@
 {
     for (unsigned i = 0; i < numberOfChannels(); ++i)
         channel(i)->scale(scale);
 }
 
 void AudioBus::copyFrom(const AudioBus& sourceBus, ChannelInterpretation channelInterpretation)
 {
     if (&sourceBus == this)
         return;
 
-    unsigned numberOfSourceChannels = sourceBus.numberOfChannels();
-    unsigned numberOfDestinationChannels = numberOfChannels();
-
-    if (numberOfDestinationChannels == numberOfSourceChannels) {
-        for (unsigned i = 0; i < numberOfSourceChannels; ++i)
-            channel(i)->copyFrom(sourceBus.channel(i));
-    } else {
-        switch (channelInterpretation) {
-        case Speakers:
-            speakersCopyFrom(sourceBus);
-            break;
-        case Discrete:
-            discreteCopyFrom(sourceBus);
-            break;
-        default:
-            ASSERT_NOT_REACHED();
-        }
-    }
+    // Copying bus is equivalent to zeroing and then summing.
+    zero();
+    sumFrom(sourceBus, channelInterpretation);
 }
 
 void AudioBus::sumFrom(const AudioBus& sourceBus, ChannelInterpretation channelInterpretation)
 {
     if (&sourceBus == this)
         return;
 
     unsigned numberOfSourceChannels = sourceBus.numberOfChannels();
     unsigned numberOfDestinationChannels = numberOfChannels();
 
-    if (numberOfDestinationChannels == numberOfSourceChannels) {
+    // If the channel numbers are equal, perform channels-wise summing.
+    if (numberOfSourceChannels == numberOfDestinationChannels) {
         for (unsigned i = 0; i < numberOfSourceChannels; ++i)
             channel(i)->sumFrom(sourceBus.channel(i));
-    } else {
-        switch (channelInterpretation) {
-        case Speakers:
-            speakersSumFrom(sourceBus);
-            break;
-        case Discrete:
-            discreteSumFrom(sourceBus);
-            break;
-        default:
-            ASSERT_NOT_REACHED();
-        }
-    }
-}
-
-void AudioBus::speakersCopyFrom(const AudioBus& sourceBus)
-{
-    // FIXME: Implement down mixing 5.1 to stereo.
-    // https://bugs.webkit.org/show_bug.cgi?id=79192
-
-    unsigned numberOfSourceChannels = sourceBus.numberOfChannels();
-    unsigned numberOfDestinationChannels = numberOfChannels();
-
-    if (numberOfDestinationChannels == 2 && numberOfSourceChannels == 1) {
-        // Handle mono -> stereo case (for now simply copy mono channel into both left and right)
-        // FIXME: Really we should apply an equal-power scaling factor here, since we're effectively panning center...
-        const AudioChannel* sourceChannel = sourceBus.channel(0);
-        channel(0)->copyFrom(sourceChannel);
-        channel(1)->copyFrom(sourceChannel);
-    } else if (numberOfDestinationChannels == 1 && numberOfSourceChannels == 2) {
-        // Handle stereo -> mono case. output = 0.5 * (input.L + input.R).
-        AudioBus& sourceBusSafe = const_cast<AudioBus&>(sourceBus);
-
-        const float* sourceL = sourceBusSafe.channelByType(ChannelLeft)->data();
-        const float* sourceR = sourceBusSafe.channelByType(ChannelRight)->data();
-
-        float* destination = channelByType(ChannelLeft)->mutableData();
-        vadd(sourceL, 1, sourceR, 1, destination, 1, length());
-        float scale = 0.5;
-        vsmul(destination, 1, &scale, destination, 1, length());
-    } else if (numberOfDestinationChannels == 6 && numberOfSourceChannels == 1) {
-        // Handle mono -> 5.1 case, copy mono channel to center.
-        channel(2)->copyFrom(sourceBus.channel(0));
-        channel(0)->zero();
-        channel(1)->zero();
-        channel(3)->zero();
-        channel(4)->zero();
-        channel(5)->zero();
-    } else if (numberOfDestinationChannels == 1 && numberOfSourceChannels == 6) {
-        // Handle 5.1 -> mono case.
-        zero();
-        speakersSumFrom5_1_ToMono(sourceBus);
-    } else {
-        // Fallback for unknown combinations.
-        discreteCopyFrom(sourceBus);
-    }
-}
-
-void AudioBus::speakersSumFrom(const AudioBus& sourceBus)
-{
-    // FIXME: Implement down mixing 5.1 to stereo.
-    // https://bugs.webkit.org/show_bug.cgi?id=79192
-
-    unsigned numberOfSourceChannels = sourceBus.numberOfChannels();
-    unsigned numberOfDestinationChannels = numberOfChannels();
-
-    if (numberOfDestinationChannels == 2 && numberOfSourceChannels == 1) {
-        // Handle mono -> stereo case (summing mono channel into both left and right).
-        const AudioChannel* sourceChannel = sourceBus.channel(0);
-        channel(0)->sumFrom(sourceChannel);
-        channel(1)->sumFrom(sourceChannel);
-    } else if (numberOfDestinationChannels == 1 && numberOfSourceChannels == 2) {
-        // Handle stereo -> mono case. output += 0.5 * (input.L + input.R).
-        AudioBus& sourceBusSafe = const_cast<AudioBus&>(sourceBus);
-
-        const float* sourceL = sourceBusSafe.channelByType(ChannelLeft)->data();
-        const float* sourceR = sourceBusSafe.channelByType(ChannelRight)->data();
-
-        float* destination = channelByType(ChannelLeft)->mutableData();
-        float scale = 0.5;
-        vsma(sourceL, 1, &scale, destination, 1, length());
-        vsma(sourceR, 1, &scale, destination, 1, length());
-    } else if (numberOfDestinationChannels == 6 && numberOfSourceChannels == 1) {
-        // Handle mono -> 5.1 case, sum mono channel into center.
-        channel(2)->sumFrom(sourceBus.channel(0));
-    } else if (numberOfDestinationChannels == 1 && numberOfSourceChannels == 6) {
-        // Handle 5.1 -> mono case.
-        speakersSumFrom5_1_ToMono(sourceBus);
-    } else {
-        // Fallback for unknown combinations.
+
+        return;
+    }
+
+    // Otherwise perform up/down-mix or the discrete transfer based on the
+    // number of channels and the channel interpretation.
+    switch (channelInterpretation) {
+    case Speakers:
+        if (numberOfSourceChannels < numberOfDestinationChannels)
+            sumFromByUpMixing(sourceBus);
+        else
+            sumFromByDownMixing(sourceBus);
+        break;
+    case Discrete:
         discreteSumFrom(sourceBus);
-    }
-}
-
-void AudioBus::speakersSumFrom5_1_ToMono(const AudioBus& sourceBus)
-{
-    AudioBus& sourceBusSafe = const_cast<AudioBus&>(sourceBus);
-
-    const float* sourceL = sourceBusSafe.channelByType(ChannelLeft)->data();
-    const float* sourceR = sourceBusSafe.channelByType(ChannelRight)->data();
-    const float* sourceC = sourceBusSafe.channelByType(ChannelCenter)->data();
-    const float* sourceSL = sourceBusSafe.channelByType(ChannelSurroundLeft)->data();
-    const float* sourceSR = sourceBusSafe.channelByType(ChannelSurroundRight)->data();
-
-    float* destination = channelByType(ChannelLeft)->mutableData();
-
-    AudioFloatArray temp(length());
-    float* tempData = temp.data();
-
-    // Sum in L and R.
-    vadd(sourceL, 1, sourceR, 1, tempData, 1, length());
-    float scale = 0.7071;
-    vsmul(tempData, 1, &scale, tempData, 1, length());
-    vadd(tempData, 1, destination, 1, destination, 1, length());
-
-    // Sum in SL and SR.
-    vadd(sourceSL, 1, sourceSR, 1, tempData, 1, length());
-    scale = 0.5;
-    vsmul(tempData, 1, &scale, tempData, 1, length());
-    vadd(tempData, 1, destination, 1, destination, 1, length());
-
-    // Sum in center.
-    vadd(sourceC, 1, destination, 1, destination, 1, length());
-}
-
-void AudioBus::discreteCopyFrom(const AudioBus& sourceBus)
-{
-    unsigned numberOfSourceChannels = sourceBus.numberOfChannels();
-    unsigned numberOfDestinationChannels = numberOfChannels();
-
-    if (numberOfDestinationChannels < numberOfSourceChannels) {
-        // Down-mix by copying channels and dropping the remaining.
-        for (unsigned i = 0; i < numberOfDestinationChannels; ++i)
-            channel(i)->copyFrom(sourceBus.channel(i));
-    } else if (numberOfDestinationChannels > numberOfSourceChannels) {
-        // Up-mix by copying as many channels as we have, then zeroing remaining channels.
-        for (unsigned i = 0; i < numberOfSourceChannels; ++i)
-            channel(i)->copyFrom(sourceBus.channel(i));
-        for (unsigned i = numberOfSourceChannels; i < numberOfDestinationChannels; ++i)
-            channel(i)->zero();
+        break;
     }
 }
 
 void AudioBus::discreteSumFrom(const AudioBus& sourceBus)
 {
     unsigned numberOfSourceChannels = sourceBus.numberOfChannels();
     unsigned numberOfDestinationChannels = numberOfChannels();
 
     if (numberOfDestinationChannels < numberOfSourceChannels) {
         // Down-mix by summing channels and dropping the remaining.
         for (unsigned i = 0; i < numberOfDestinationChannels; ++i)
             channel(i)->sumFrom(sourceBus.channel(i));
     } else if (numberOfDestinationChannels > numberOfSourceChannels) {
         // Up-mix by summing as many channels as we have.
         for (unsigned i = 0; i < numberOfSourceChannels; ++i)
             channel(i)->sumFrom(sourceBus.channel(i));
     }
 }
 
+void AudioBus::sumFromByUpMixing(const AudioBus& sourceBus)
+{
+    unsigned numberOfSourceChannels = sourceBus.numberOfChannels();
+    unsigned numberOfDestinationChannels = numberOfChannels();
+
+    if ((numberOfSourceChannels == 1 && numberOfDestinationChannels == 2) || (numberOfSourceChannels == 1 && numberOfDestinationChannels == 4)) {

[Raymond Toy, 2016/03/10 18:56:47]

Not something to fix here, but we should probably start limiting the line length
since blink style will become chromium someday. (Gotta remember that myself.)

[hongchan, 2016/03/11 19:48:06]

Acknowledged, but "git cl format" always reverts my effort of wrapping codes in
80 columns.

[Raymond Toy, 2016/03/11 19:53:17]

Oh well. We'll just have to wait for the Great Reformatting to Come.

+        // Up-mixing: 1 -> 2, 1 -> 4
+        //   output.L = input
+        //   output.R = input
+        //   output.SL = 0 (in the case of 1 -> 4)
+        //   output.SR = 0 (in the case of 1 -> 4)
+        const AudioChannel* sourceChannel = sourceBus.channel(0);

[Raymond Toy, 2016/03/10 18:56:47]

Style nit:  In sumFromByDownMixing, you access channels using channelByType()
method.  Should we do the same here?

[hongchan, 2016/03/11 19:48:06]

Done, but what about the sources? Should we change everything to maintain the
consistency?

+        channel(0)->sumFrom(sourceChannel);
+        channel(1)->sumFrom(sourceChannel);
+    } else if (numberOfSourceChannels == 1 && numberOfDestinationChannels == 6) {
+        // Up-mixing: 1 -> 5.1
+        //   output.L = 0
+        //   output.R = 0
+        //   output.C = input (put in center channel)
+        //   output.LFE = 0
+        //   output.SL = 0
+        //   output.SR = 0
+        channel(2)->sumFrom(sourceBus.channel(0));
+    } else if ((numberOfSourceChannels == 2 && numberOfDestinationChannels == 4) || (numberOfSourceChannels == 2 && numberOfDestinationChannels == 6)) {
+        // Up-mixing: 2 -> 4, 2 -> 5.1
+        //   output.L = input.L
+        //   output.R = input.R
+        //   output.C = 0 (in the case of 2 -> 5.1)
+        //   output.LFE = 0 (in the case of 2 -> 5.1)
+        //   output.SL = 0
+        //   output.SR = 0
+        channel(0)->sumFrom(sourceBus.channel(0));
+        channel(1)->sumFrom(sourceBus.channel(1));
+    } else if (numberOfSourceChannels == 4 && numberOfDestinationChannels == 6) {
+        // Up-mixing: 4 -> 5.1
+        //   output.L = input.L
+        //   output.R = input.R
+        //   output.C = 0
+        //   output.LFE = 0
+        //   output.SL = input.SL
+        //   output.SR = input.SR
+        channel(0)->sumFrom(sourceBus.channel(0));
+        channel(1)->sumFrom(sourceBus.channel(1));
+        channel(4)->sumFrom(sourceBus.channel(2));
+        channel(5)->sumFrom(sourceBus.channel(3));
+    } else {
+        // All other cases, fall back to the discrete sum. This will silence the
+        // excessive channels.
+        discreteSumFrom(sourceBus);
+    }
+}
+
+void AudioBus::sumFromByDownMixing(const AudioBus& sourceBus)
+{
+    unsigned numberOfSourceChannels = sourceBus.numberOfChannels();
+    unsigned numberOfDestinationChannels = numberOfChannels();
+
+    if (numberOfSourceChannels == 2 && numberOfDestinationChannels == 1) {
+        // Down-mixing: 2 -> 1
+        //   output = 0.5 * (input.L + input.R)
+        AudioBus& sourceBusConst = const_cast<AudioBus&>(sourceBus);
+        const float* sourceL = sourceBusConst.channelByType(ChannelLeft)->data();
+        const float* sourceR = sourceBusConst.channelByType(ChannelRight)->data();
+
+        float* destination = channelByType(ChannelLeft)->mutableData();
+        float scale = 0.5;
+
+        vsma(sourceL, 1, &scale, destination, 1, length());
+        vsma(sourceR, 1, &scale, destination, 1, length());
+    } else if (numberOfSourceChannels == 4 && numberOfDestinationChannels == 1) {
+        // Down-mixing: 4 -> 1
+        //   output = 0.25 * (input.L + input.R + input.SL + input.SR)
+        AudioBus& sourceBusConst = const_cast<AudioBus&>(sourceBus);
+        const float* sourceL = sourceBusConst.channelByType(ChannelLeft)->data();
+        const float* sourceR = sourceBusConst.channelByType(ChannelRight)->data();
+        const float* sourceSL = sourceBusConst.channelByType(ChannelSurroundLeft)->data();
+        const float* sourceSR = sourceBusConst.channelByType(ChannelSurroundRight)->data();
+
+        float* destination = channelByType(ChannelLeft)->mutableData();
+        float scale = 0.25;
+
+        vsma(sourceL, 1, &scale, destination, 1, length());
+        vsma(sourceR, 1, &scale, destination, 1, length());
+        vsma(sourceSL, 1, &scale, destination, 1, length());
+        vsma(sourceSR, 1, &scale, destination, 1, length());
+    } else if (numberOfSourceChannels == 6 && numberOfDestinationChannels == 1) {
+        // Down-mixing: 5.1 -> 1
+        //   output = sqrt(1/2) * (input.L + input.R) + input.C
+        //            + 0.5 * (input.SL + input.SR)
+        AudioBus& sourceBusConst = const_cast<AudioBus&>(sourceBus);

[Raymond Toy, 2016/03/10 18:56:47]

Seems weird that you have to do a const cast. I'm guessing data() doesn't return
a const float*.  Which seems strange since for the destination you call
mutableData().  A bug in AudioBus?

[hongchan, 2016/03/11 19:48:06]

After the discussion we had, I removed the all const_cast in this method. It
compiles and works correctly.

+        const float* sourceL = sourceBusConst.channelByType(ChannelLeft)->data();
+        const float* sourceR = sourceBusConst.channelByType(ChannelRight)->data();
+        const float* sourceC = sourceBusConst.channelByType(ChannelCenter)->data();
+        const float* sourceSL = sourceBusConst.channelByType(ChannelSurroundLeft)->data();
+        const float* sourceSR = sourceBusConst.channelByType(ChannelSurroundRight)->data();
+
+        float* destination = channelByType(ChannelLeft)->mutableData();
+        float scaleSqrtHalf = sqrtf(0.5);
+        float scaleHalf = 0.5;
+
+        vsma(sourceL, 1, &scaleSqrtHalf, destination, 1, length());
+        vsma(sourceR, 1, &scaleSqrtHalf, destination, 1, length());
+        vadd(sourceC, 1, destination, 1, destination, 1, length());
+        vsma(sourceSL, 1, &scaleHalf, destination, 1, length());
+        vsma(sourceSR, 1, &scaleHalf, destination, 1, length());
+    } else if (numberOfSourceChannels == 4 && numberOfDestinationChannels == 2) {
+        // Down-mixing: 4 -> 2
+        //   output.L = 0.5 * (input.L + input.SL)
+        //   output.R = 0.5 * (input.R + input.SR)
+        AudioBus& sourceBusConst = const_cast<AudioBus&>(sourceBus);
+        const float* sourceL = sourceBusConst.channelByType(ChannelLeft)->data();
+        const float* sourceR = sourceBusConst.channelByType(ChannelRight)->data();
+        const float* sourceSL = sourceBusConst.channelByType(ChannelSurroundLeft)->data();
+        const float* sourceSR = sourceBusConst.channelByType(ChannelSurroundRight)->data();
+
+        float* destinationL = channelByType(ChannelLeft)->mutableData();
+        float* destinationR = channelByType(ChannelRight)->mutableData();
+        float scaleHalf = 0.5;
+
+        vsma(sourceL, 1, &scaleHalf, destinationL, 1, length());
+        vsma(sourceSL, 1, &scaleHalf, destinationL, 1, length());
+        vsma(sourceR, 1, &scaleHalf, destinationR, 1, length());
+        vsma(sourceSR, 1, &scaleHalf, destinationR, 1, length());
+    } else if (numberOfSourceChannels == 6 && numberOfDestinationChannels == 2) {
+        // Down-mixing: 5.1 -> 2
+        //   output.L = input.L + sqrt(1/2) * (input.C + input.SL)
+        //   output.R = input.R + sqrt(1/2) * (input.C + input.SR)
+        AudioBus& sourceBusConst = const_cast<AudioBus&>(sourceBus);
+        const float* sourceL = sourceBusConst.channelByType(ChannelLeft)->data();
+        const float* sourceR = sourceBusConst.channelByType(ChannelRight)->data();
+        const float* sourceC = sourceBusConst.channelByType(ChannelCenter)->data();
+        const float* sourceSL = sourceBusConst.channelByType(ChannelSurroundLeft)->data();
+        const float* sourceSR = sourceBusConst.channelByType(ChannelSurroundRight)->data();
+
+        float* destinationL = channelByType(ChannelLeft)->mutableData();
+        float* destinationR = channelByType(ChannelRight)->mutableData();
+        float scaleSqrtHalf = sqrtf(0.5);
+
+        vadd(sourceL, 1, destinationL, 1, destinationL, 1, length());
+        vsma(sourceC, 1, &scaleSqrtHalf, destinationL, 1, length());
+        vsma(sourceSL, 1, &scaleSqrtHalf, destinationL, 1, length());
+        vadd(sourceR, 1, destinationR, 1, destinationR, 1, length());
+        vsma(sourceC, 1, &scaleSqrtHalf, destinationR, 1, length());
+        vsma(sourceSR, 1, &scaleSqrtHalf, destinationR, 1, length());
+    } else if (numberOfSourceChannels == 6 && numberOfDestinationChannels == 4) {
+        // Down-mixing: 5.1 -> 4
+        //   output.L = input.L + sqrt(1/2) * input.C
+        //   output.R = input.R + sqrt(1/2) * input.C
+        //   output.SL = input.SL
+        //   output.SR = input.SR
+        AudioBus& sourceBusConst = const_cast<AudioBus&>(sourceBus);
+        const float* sourceL = sourceBusConst.channelByType(ChannelLeft)->data();
+        const float* sourceR = sourceBusConst.channelByType(ChannelRight)->data();
+        const float* sourceC = sourceBusConst.channelByType(ChannelCenter)->data();
+
+        float* destinationL = channelByType(ChannelLeft)->mutableData();
+        float* destinationR = channelByType(ChannelRight)->mutableData();
+        float scaleSqrtHalf = sqrtf(0.5);
+
+        vadd(sourceL, 1, destinationL, 1, destinationL, 1, length());
+        vsma(sourceC, 1, &scaleSqrtHalf, destinationL, 1, length());
+        vadd(sourceR, 1, destinationR, 1, destinationR, 1, length());
+        vsma(sourceC, 1, &scaleSqrtHalf, destinationR, 1, length());
+        channel(2)->sumFrom(sourceBus.channel(4));
+        channel(3)->sumFrom(sourceBus.channel(5));
+    } else {
+        // All other cases, fall back to the discrete sum. This will perform
+        // channel-wise sum until the destination channels run out.
+        discreteSumFrom(sourceBus);
+    }
+}
+
 void AudioBus::copyWithGainFrom(const AudioBus &sourceBus, float* lastMixGain, float targetGain)
 {
     if (!topologyMatches(sourceBus)) {
         ASSERT_NOT_REACHED();
         zero();
         return;
     }
 
     if (sourceBus.isSilent()) {
         zero();
@@ skipping 258 matching lines @@
 
     // If the bus needs no conversion then return as is.
     if ((!mixToMono || audioBus->numberOfChannels() == 1) && audioBus->sampleRate() == sampleRate)
         return audioBus;
 
     return AudioBus::createBySampleRateConverting(audioBus.get(), mixToMono, sampleRate);
 }
 
 } // namespace blink