More optimize approach for NEON in VectorMath

Source/platform/audio/VectorMath.cpp

 /*
  * Copyright (C) 2010, Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1.  Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2.  Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
@@ skipping 161 matching lines @@
 
         if (destAligned)
             SSE2_MULT_ADD(load, store)
         else
             SSE2_MULT_ADD(loadu, storeu)
 
         n = tailFrames;
     }
 #elif HAVE(ARM_NEON_INTRINSICS)
     if ((sourceStride == 1) && (destStride == 1)) {
-        int tailFrames = n % 4;
-        const float* endP = destP + n - tailFrames;
+        unsigned tailFrames = n & 15;
+        float32x4_t scaleNum = vdupq_n_f32(*scale);
 
-        float32x4_t k = vdupq_n_f32(*scale);
-        while (destP < endP) {
-            float32x4_t source = vld1q_f32(sourceP);
-            float32x4_t dest = vld1q_f32(destP);
+        for (unsigned loopCount = (n >> 4); loopCount > 0; loopCount--) {
+            float32x4_t dest0 = vld1q_f32(destP);
+            float32x4_t source0 = vld1q_f32(sourceP);
 
-            dest = vmlaq_f32(dest, source, k);
-            vst1q_f32(destP, dest);
+            float32x4_t dest1 = vld1q_f32(destP + 4);
+            float32x4_t source1 = vld1q_f32(sourceP + 4);
 
-            sourceP += 4;
-            destP += 4;
+            float32x4_t dest2 = vld1q_f32(destP + 8);
+            float32x4_t source2 = vld1q_f32(sourceP + 8);
+
+            float32x4_t dest3 = vld1q_f32(destP + 12);
+            float32x4_t source3 = vld1q_f32(sourceP + 12);
+
+            float32x4_t result0 = vmlaq_f32(dest0, source0, scaleNum);
+            float32x4_t result1 = vmlaq_f32(dest1, source1, scaleNum);
+            float32x4_t result2 = vmlaq_f32(dest2, source2, scaleNum);
+            float32x4_t result3 = vmlaq_f32(dest3, source3, scaleNum);
+
+            vst1q_f32(destP, result0);
+            vst1q_f32(destP + 4, result1);
+            vst1q_f32(destP + 8, result2);
+            vst1q_f32(destP + 12, result3);
+
+            sourceP += 16;
+            destP += 16;
         }
         n = tailFrames;
     }
 #endif
     while (n) {
         *destP += *sourceP * *scale;
         sourceP += sourceStride;
         destP += destStride;
         n--;
     }
@@ skipping 46 matching lines @@
         // Non-SSE handling for remaining frames which is less than 4.
         n %= 4;
         while (n) {
             *destP = k * *sourceP;
             sourceP++;
             destP++;
             n--;
         }
     } else { // If strides are not 1, rollback to normal algorithm.
 #elif HAVE(ARM_NEON_INTRINSICS)
-    if ((sourceStride == 1) && (destStride == 1)) {
-        float k = *scale;
-        int tailFrames = n % 4;
-        const float* endP = destP + n - tailFrames;
+    if (sourceStride == 1 && destStride == 1) {
+        unsigned tailFrames = framesToProcess & 15;
+        float32x4_t scaleNum = vdupq_n_f32(*scale);

[Raymond Toy, 2014/09/22 20:00:36]

Is there any performance difference between making 4 copies of the same item in
scaleNum instead of doing a vector multiply by a single float scalar, as done in
the original code?

[KhNo, 2014/09/23 02:07:32]

No performance difference for scaleNum, It is just for making correct type
develver to vmulq_f32 function instead of vmulq_n_f32 that is originally used.

 
-        while (destP < endP) {
-            float32x4_t source = vld1q_f32(sourceP);
-            vst1q_f32(destP, vmulq_n_f32(source, k));
+        for (unsigned loopCount = (framesToProcess >> 4); loopCount > 0; loopCount--) {
+            float32x4_t source0 = vld1q_f32(sourceP);
+            float32x4_t source1 = vld1q_f32(sourceP + 4);
+            float32x4_t source2 = vld1q_f32(sourceP + 8);
+            float32x4_t source3 = vld1q_f32(sourceP + 12);
 
-            sourceP += 4;
-            destP += 4;
+            float32x4_t result0 = vmulq_f32(source0, scaleNum);
+            float32x4_t result1 = vmulq_f32(source1, scaleNum);
+            float32x4_t result2 = vmulq_f32(source2, scaleNum);
+            float32x4_t result3 = vmulq_f32(source3, scaleNum);
+
+            vst1q_f32(destP, result0);
+            vst1q_f32(destP + 4, result1);
+            vst1q_f32(destP + 8, result2);
+            vst1q_f32(destP + 12, result3);
+
+            sourceP += 16;
+            destP += 16;
         }
         n = tailFrames;
     }
 #endif
     float k = *scale;
     while (n--) {
         *destP = k * *sourceP;
         sourceP += sourceStride;
         destP += destStride;
     }
@@ skipping 80 matching lines @@
         n %= 4;
         while (n) {
             *destP = *source1P + *source2P;
             source1P++;
             source2P++;
             destP++;
             n--;
         }
     } else { // if strides are not 1, rollback to normal algorithm
 #elif HAVE(ARM_NEON_INTRINSICS)
-    if ((sourceStride1 ==1) && (sourceStride2 == 1) && (destStride == 1)) {
-        int tailFrames = n % 4;
-        const float* endP = destP + n - tailFrames;
+    if ((sourceStride1 == 1) && (sourceStride2 == 1) && (destStride == 1)) {
+        unsigned tailFrames = framesToProcess & 15;
 
-        while (destP < endP) {
-            float32x4_t source1 = vld1q_f32(source1P);
-            float32x4_t source2 = vld1q_f32(source2P);
-            vst1q_f32(destP, vaddq_f32(source1, source2));
+        for (unsigned loopCount = (framesToProcess >> 4); loopCount > 0; loopCount--) {
+            float32x4_t source10 = vld1q_f32(source1P);
+            float32x4_t source20 = vld1q_f32(source2P);
 
-            source1P += 4;
-            source2P += 4;
-            destP += 4;
+            float32x4_t source11 = vld1q_f32(source1P + 4);
+            float32x4_t source21 = vld1q_f32(source2P + 4);
+
+            float32x4_t source12 = vld1q_f32(source1P + 8);
+            float32x4_t source22 = vld1q_f32(source2P + 8);
+
+            float32x4_t source13 = vld1q_f32(source1P + 12);
+            float32x4_t source23 = vld1q_f32(source2P + 12);
+
+            float32x4_t result0 = vaddq_f32(source10, source20);
+            float32x4_t result1 = vaddq_f32(source11, source21);
+            float32x4_t result2 = vaddq_f32(source12, source22);
+            float32x4_t result3 = vaddq_f32(source13, source23);
+
+            vst1q_f32(destP, result0);
+            vst1q_f32(destP + 4, result1);
+            vst1q_f32(destP + 8, result2);
+            vst1q_f32(destP + 12, result3);
+
+            source1P += 16;
+            source2P += 16;
+            destP += 16;
         }
         n = tailFrames;
     }
 #endif
     while (n--) {
         *destP = *source1P + *source2P;
         source1P += sourceStride1;
         source2P += sourceStride2;
         destP += destStride;
     }
@@ skipping 45 matching lines @@
         else if (source2Aligned && !destAligned) // Source2 is aligned but dest not.
             SSE2_MULT(load, storeu)
         else if (!source2Aligned && destAligned) // Dest is aligned but source2 not.
             SSE2_MULT(loadu, store)
         else // Neither aligned.
             SSE2_MULT(loadu, storeu)
 
         n = tailFrames;
     }
 #elif HAVE(ARM_NEON_INTRINSICS)
-    if ((sourceStride1 ==1) && (sourceStride2 == 1) && (destStride == 1)) {
-        int tailFrames = n % 4;
-        const float* endP = destP + n - tailFrames;
+    if ((sourceStride1 == 1) && (sourceStride2 == 1) && (destStride == 1)) {
+        unsigned tailFrames = n & 15;
 
-        while (destP < endP) {
-            float32x4_t source1 = vld1q_f32(source1P);
-            float32x4_t source2 = vld1q_f32(source2P);
-            vst1q_f32(destP, vmulq_f32(source1, source2));
+        for (unsigned loopCount = (n >> 4); loopCount > 0; loopCount--) {
+            float32x4_t source10 = vld1q_f32(source1P);
+            float32x4_t source20 = vld1q_f32(source2P);

[Raymond Toy, 2014/09/22 20:00:35]

It might be advantageous to load all of the elements from source1 and then
source2 instead of alternating between source1 and source2  Caching might work
out better?

[KhNo, 2014/09/23 02:07:32]

Thanks for review, I think also it is better for cache mechanism.

 
-            source1P += 4;
-            source2P += 4;
-            destP += 4;
+            float32x4_t source11 = vld1q_f32(source1P + 4);
+            float32x4_t source21 = vld1q_f32(source2P + 4);
+
+            float32x4_t source12 = vld1q_f32(source1P + 8);
+            float32x4_t source22 = vld1q_f32(source2P + 8);
+
+            float32x4_t source13 = vld1q_f32(source1P + 12);
+            float32x4_t source23 = vld1q_f32(source2P + 12);
+
+            float32x4_t result0 = vmulq_f32(source10, source20);
+            float32x4_t result1 = vmulq_f32(source11, source21);
+            float32x4_t result2 = vmulq_f32(source12, source22);
+            float32x4_t result3 = vmulq_f32(source13, source23);
+
+            vst1q_f32(destP, result0);
+            vst1q_f32(destP + 4, result1);
+            vst1q_f32(destP + 8, result2);
+            vst1q_f32(destP + 12, result3);
+
+            source1P += 16;
+            source2P += 16;
+            destP += 16;
         }
         n = tailFrames;
     }
 #endif
     while (n) {
         *destP = *source1P * *source2P;
         source1P += sourceStride1;
         source2P += sourceStride2;
         destP += destStride;
         n--;
@@ skipping 85 matching lines @@
         }
 
         // Summarize the SSE results.
         const float* groupSumP = reinterpret_cast<float*>(&mSum);
         sum += groupSumP[0] + groupSumP[1] + groupSumP[2] + groupSumP[3];
 
         n = tailFrames;
     }
 #elif HAVE(ARM_NEON_INTRINSICS)
     if (sourceStride == 1) {
-        int tailFrames = n % 4;
-        const float* endP = sourceP + n - tailFrames;
+        unsigned tailFrames = n & 15;
 
-        float32x4_t fourSum = vdupq_n_f32(0);
-        while (sourceP < endP) {
-            float32x4_t source = vld1q_f32(sourceP);
-            fourSum = vmlaq_f32(fourSum, source, source);
-            sourceP += 4;
+        float32x4_t result0 = vdupq_n_f32(0);
+        float32x4_t result1 = result0;
+        float32x4_t result2 = result0;
+        float32x4_t result3 = result0;
+
+        for (unsigned loopCount = (n >> 4); loopCount > 0; loopCount--) {
+            float32x4_t source0 = vld1q_f32(sourceP);
+            float32x4_t source1 = vld1q_f32(sourceP + 4);
+            float32x4_t source2 = vld1q_f32(sourceP + 8);
+            float32x4_t source3 = vld1q_f32(sourceP + 12);
+
+            result0 = vmlaq_f32(result0, source0, source0);
+            result1 = vmlaq_f32(result1, source1, source1);
+            result2 = vmlaq_f32(result2, source2, source2);
+            result3 = vmlaq_f32(result3, source3, source3);
+
+            sourceP += 16;
         }
-        float32x2_t twoSum = vadd_f32(vget_low_f32(fourSum), vget_high_f32(fourSum));
 
-        float groupSum[2];
-        vst1_f32(groupSum, twoSum);
-        sum += groupSum[0] + groupSum[1];
+        result0 = vaddq_f32(result0, result1);
+        result0 = vaddq_f32(result0, result2);
+        result0 = vaddq_f32(result0, result3);
+
+        sum += vgetq_lane_f32(result0, 0);
+        sum += vgetq_lane_f32(result0, 1);
+        sum += vgetq_lane_f32(result0, 2);
+        sum += vgetq_lane_f32(result0, 3);
 
         n = tailFrames;
     }
 #endif
 
     while (n--) {
         float sample = *sourceP;
         sum += sample * sample;
         sourceP += sourceStride;
     }
@@ skipping 101 matching lines @@
     }
 }
 
 #endif // OS(MACOSX)
 
 } // namespace VectorMath
 
 } // namespace WebCore
 
 #endif // ENABLE(WEB_AUDIO)