SkBlurImageFilter_opts: optimize NEON box_blur_double in separate loops.

src/opts/SkBlurImageFilter_opts.h

 /*
  * Copyright 2015 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 #ifndef SkBlurImageFilter_opts_DEFINED
 #define SkBlurImageFilter_opts_DEFINED
 
@@ skipping 49 matching lines @@
 #define INCREMENT_SUMS(c) sum = _mm_add_epi32(sum, expand(c))
 #define DECREMENT_SUMS(c) sum = _mm_sub_epi32(sum, expand(c))
 #define STORE_SUMS \
     auto result = mullo_epi32(sum, scale); \
     result = _mm_add_epi32(result, half); \
     *dptr = repack(result);
 #define DOUBLE_ROW_OPTIMIZATION
 
 #elif defined(SK_ARM_HAS_NEON)
 
+// val = (sum * scale * 2 + 0x8000) >> 16
+#define STORE_SUMS_DOUBLE \

[mtklein, 2015/10/28 22:31:17]

This just symmetry or are you thinking of expanding this to SSE?

[Stephen White, 2015/10/28 22:38:33]

Well, mostly because it's repeated four times below.

But yeah, it'd be interesting to consider SSE for this. The one thing I don't
like about it is that it produces different pixels than the generic path. But
we're already paying that price on NEON.

+    uint16x8_t resultPixels = vreinterpretq_u16_s16(vqrdmulhq_s16( \
+        vreinterpretq_s16_u16(sum), vreinterpretq_s16_u16(scale))); \
+    if (dstDirection == BlurDirection::kX) { \
+        uint32x2_t px2 = vreinterpret_u32_u8(vmovn_u16(resultPixels)); \
+        vst1_lane_u32(dptr +     0, px2, 0); \
+        vst1_lane_u32(dptr + width, px2, 1); \
+    } else { \
+        vst1_u8((uint8_t*)dptr, vmovn_u16(resultPixels)); \
+    }
+
+#define INCREMENT_SUMS_DOUBLE(p) sum = vaddw_u8(sum, load_2_pixels(p))
+#define DECREMENT_SUMS_DOUBLE(p) sum = vsubw_u8(sum, load_2_pixels(p))
+
 // Fast path for kernel sizes between 2 and 127, working on two rows at a time.
 template<BlurDirection srcDirection, BlurDirection dstDirection>
 void box_blur_double(const SkPMColor** src, int srcStride, SkPMColor** dst, int kernelSize,
                      int leftOffset, int rightOffset, int width, int* height) {
     // Load 2 pixels from adjacent rows.
     auto load_2_pixels = [&](const SkPMColor* s) {
         if (srcDirection == BlurDirection::kX) {
             // 10% faster by adding these 2 prefetches
             SK_PREFETCH(s + 16);
             SK_PREFETCH(s + 16 + srcStride);
             auto one = vld1_lane_u32(s +         0, vdup_n_u32(0), 0),
                  two = vld1_lane_u32(s + srcStride,           one, 1);
             return vreinterpret_u8_u32(two);
         } else {
             return vld1_u8((uint8_t*)s);
         }
     };
-    const int rightBorder = SkMin32(rightOffset + 1, width);
+    int incrementStart = SkMax32(-rightOffset - 1, -width);
+    int incrementEnd = SkMax32(width - rightOffset - 1, 0);
+    int decrementStart = SkMin32(leftOffset, width);
     const int srcStrideX = srcDirection == BlurDirection::kX ? 1 : srcStride;
     const int dstStrideX = dstDirection == BlurDirection::kX ? 1 : *height;
     const int srcStrideY = srcDirection == BlurDirection::kX ? srcStride : 1;
     const int dstStrideY = dstDirection == BlurDirection::kX ? width : 1;
     const uint16x8_t scale = vdupq_n_u16((1 << 15) / kernelSize);
 
     for (; *height >= 2; *height -= 2) {
         uint16x8_t sum = vdupq_n_u16(0);
-        const SkPMColor* p = *src;
-        for (int i = 0; i < rightBorder; i++) {
-            sum = vaddw_u8(sum, load_2_pixels(p));
-            p += srcStrideX;
+        const SkPMColor* lptr = *src;
+        const SkPMColor* rptr = *src;
+        SkPMColor* dptr = *dst;
+        int x;
+        for (x = incrementStart; x < 0; ++x) {
+            INCREMENT_SUMS_DOUBLE(rptr);
+            rptr += srcStrideX;
         }
-
-        const SkPMColor* sptr = *src;
-        SkPMColor* dptr = *dst;
-        for (int x = 0; x < width; x++) {
-            // val = (sum * scale * 2 + 0x8000) >> 16
-            uint16x8_t resultPixels = vreinterpretq_u16_s16(vqrdmulhq_s16(
-                vreinterpretq_s16_u16(sum), vreinterpretq_s16_u16(scale)));
-            if (dstDirection == BlurDirection::kX) {
-                uint32x2_t px2 = vreinterpret_u32_u8(vmovn_u16(resultPixels));
-                vst1_lane_u32(dptr +     0, px2, 0);
-                vst1_lane_u32(dptr + width, px2, 1);
-            } else {
-                vst1_u8((uint8_t*)dptr, vmovn_u16(resultPixels));
-            }
-
-            if (x >= leftOffset) {
-                sum = vsubw_u8(sum, load_2_pixels(sptr - leftOffset * srcStrideX));
-            }
-            if (x + rightOffset + 1 < width) {
-                sum = vaddw_u8(sum, load_2_pixels(sptr + (rightOffset + 1) * srcStrideX));
-            }
-            sptr += srcStrideX;
+        for (; x < decrementStart && x < incrementEnd; ++x) {
+            STORE_SUMS_DOUBLE
             dptr += dstStrideX;
+            INCREMENT_SUMS_DOUBLE(rptr);
+            rptr += srcStrideX;
+        }
+        for (x = decrementStart; x < incrementEnd; ++x) {
+            STORE_SUMS_DOUBLE
+            dptr += dstStrideX;
+            INCREMENT_SUMS_DOUBLE(rptr);
+            rptr += srcStrideX;
+            DECREMENT_SUMS_DOUBLE(lptr);
+            lptr += srcStrideX;
+        }
+        for (x = incrementEnd; x < decrementStart; ++x) {
+            STORE_SUMS_DOUBLE
+            dptr += dstStrideX;
+        }
+        for (; x < width; ++x) {
+            STORE_SUMS_DOUBLE
+            dptr += dstStrideX;
+            DECREMENT_SUMS_DOUBLE(lptr);
+            lptr += srcStrideX;
         }
         *src += srcStrideY * 2;
         *dst += dstStrideY * 2;
     }
 }
 
 // ARGB -> 0A0R 0G0B
 static inline uint16x4_t expand(SkPMColor p) {
     return vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(p))));
 };
@@ skipping 102 matching lines @@
     }
 }
 
 static auto box_blur_xx = &box_blur<BlurDirection::kX, BlurDirection::kX>,
             box_blur_xy = &box_blur<BlurDirection::kX, BlurDirection::kY>,
             box_blur_yx = &box_blur<BlurDirection::kY, BlurDirection::kX>;
 
 }  // namespace SK_OPTS_NS
 
 #endif