OLD | NEW |
(Empty) | |
| 1 /* |
| 2 * Copyright 2015 Google Inc. |
| 3 * |
| 4 * Use of this source code is governed by a BSD-style license that can be |
| 5 * found in the LICENSE file. |
| 6 */ |
| 7 |
| 8 #ifndef SkBlurImageFilter_opts_DEFINED |
| 9 #define SkBlurImageFilter_opts_DEFINED |
| 10 |
| 11 #include "SkColorPriv.h" |
| 12 #include "SkTypes.h" |
| 13 |
| 14 namespace SK_OPTS_NS { |
| 15 |
| 16 enum Direction { kX, kY }; |
| 17 |
| 18 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE2 |
| 19 template<Direction srcDirection, Direction dstDirection> |
| 20 void box_blur(const SkPMColor* src, int srcStride, SkPMColor* dst, int kernelSiz
e, |
| 21 int leftOffset, int rightOffset, int width, int height) { |
| 22 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE41 |
| 23 // ARGB -> 000A 000R 000G 000B |
| 24 auto expand = [](int p) { |
| 25 return _mm_cvtepu8_epi32(_mm_cvtsi32_si128(p)); |
| 26 }; |
| 27 // Axxx Rxxx Gxxx Bxxx -> ARGB |
| 28 auto repack = [](__m128i p) { |
| 29 const char _ = ~0; // Don't care what ends up in these bytes. This zer
os them. |
| 30 p = _mm_shuffle_epi8(p, _mm_set_epi8(_,_,_,_, _,_,_,_, _,_,_,_, 15,11,7,
3)); |
| 31 return _mm_cvtsi128_si32(p); |
| 32 }; |
| 33 |
| 34 #else |
| 35 // ARGB -> 000A 000R 000G 000B |
| 36 auto expand = [](int p) { |
| 37 auto result = _mm_cvtsi32_si128(p); |
| 38 result = _mm_unpacklo_epi8(result, _mm_setzero_si128()); |
| 39 result = _mm_unpacklo_epi16(result, _mm_setzero_si128()); |
| 40 return result; |
| 41 }; |
| 42 // Axxx Rxxx Gxxx Bxxx -> ARGB |
| 43 auto repack = [](__m128i p) { |
| 44 p = _mm_srli_epi32(p, 24); // 000A 000R 000G 000B |
| 45 p = _mm_packs_epi32(p, p); // xxxx xxxx 0A0R 0G0B |
| 46 p = _mm_packus_epi16(p, p); // xxxx xxxx xxxx ARGB |
| 47 return _mm_cvtsi128_si32(p); |
| 48 }; |
| 49 |
| 50 // _mm_mullo_epi32 is not available, so use the standard trick to emulate it
. |
| 51 auto _mm_mullo_epi32 = [](__m128i a, __m128i b) { |
| 52 __m128i p02 = _mm_mul_epu32(a, b), |
| 53 p13 = _mm_mul_epu32(_mm_srli_si128(a, 4), |
| 54 _mm_srli_si128(b, 4)); |
| 55 return _mm_unpacklo_epi32(_mm_shuffle_epi32(p02, _MM_SHUFFLE(0,0,2,0)), |
| 56 _mm_shuffle_epi32(p13, _MM_SHUFFLE(0,0,2,0))); |
| 57 }; |
| 58 #endif |
| 59 const int rightBorder = SkMin32(rightOffset + 1, width); |
| 60 const int srcStrideX = srcDirection == kX ? 1 : srcStride; |
| 61 const int dstStrideX = dstDirection == kX ? 1 : height; |
| 62 const int srcStrideY = srcDirection == kX ? srcStride : 1; |
| 63 const int dstStrideY = dstDirection == kX ? width : 1; |
| 64 const __m128i scale = _mm_set1_epi32((1 << 24) / kernelSize); |
| 65 const __m128i half = _mm_set1_epi32(1 << 23); |
| 66 for (int y = 0; y < height; ++y) { |
| 67 __m128i sum = _mm_setzero_si128(); |
| 68 const SkPMColor* p = src; |
| 69 for (int i = 0; i < rightBorder; ++i) { |
| 70 sum = _mm_add_epi32(sum, expand(*p)); |
| 71 p += srcStrideX; |
| 72 } |
| 73 |
| 74 const SkPMColor* sptr = src; |
| 75 SkColor* dptr = dst; |
| 76 for (int x = 0; x < width; ++x) { |
| 77 // TODO(mtklein): We are working in 8.24 here. Drop to 8.8 when the
kernel is narrow? |
| 78 // Multiply each component by scale (divide by kernel size) and add
half to round. |
| 79 auto result = _mm_mullo_epi32(sum, scale); |
| 80 result = _mm_add_epi32(result, half); |
| 81 |
| 82 // Now pack the top byte of each 32-bit lane back down into one 32-b
it color. |
| 83 // Axxx Rxxx Gxxx Bxxx -> xxxx xxxx xxxx ARGB |
| 84 *dptr = repack(result); |
| 85 |
| 86 // TODO(mtklein): experiment with breaking this loop into 3 parts |
| 87 if (x >= leftOffset) { |
| 88 SkColor l = *(sptr - leftOffset * srcStrideX); |
| 89 sum = _mm_sub_epi32(sum, expand(l)); |
| 90 } |
| 91 if (x + rightOffset + 1 < width) { |
| 92 SkColor r = *(sptr + (rightOffset + 1) * srcStrideX); |
| 93 sum = _mm_add_epi32(sum, expand(r)); |
| 94 } |
| 95 sptr += srcStrideX; |
| 96 if (srcDirection == kY) { |
| 97 // TODO(mtklein): experiment with moving this prefetch forward |
| 98 _mm_prefetch(reinterpret_cast<const char*>(sptr + (rightOffset +
1) * srcStrideX), |
| 99 _MM_HINT_T0); |
| 100 } |
| 101 dptr += dstStrideX; |
| 102 } |
| 103 src += srcStrideY; |
| 104 dst += dstStrideY; |
| 105 } |
| 106 } |
| 107 |
| 108 #elif defined(SK_ARM_HAS_NEON) |
| 109 |
| 110 // Fast path for kernel sizes between 2 and 127, working on two rows at a time. |
| 111 template<Direction srcDirection, Direction dstDirection> |
| 112 void box_blur_double(const SkPMColor** src, int srcStride, SkPMColor** dst, int
kernelSize, |
| 113 int leftOffset, int rightOffset, int width, int* height) { |
| 114 // Load 2 pixels from adjacent rows. |
| 115 auto load_2_pixels = [&](const SkPMColor* s) { |
| 116 if (srcDirection == kX) { |
| 117 // 10% faster by adding these 2 prefetches |
| 118 SK_PREFETCH(s + 16); |
| 119 SK_PREFETCH(s + 16 + srcStride); |
| 120 auto one = vld1_lane_u32(s + 0, vdup_n_u32(0), 0), |
| 121 two = vld1_lane_u32(s + srcStride, one, 1); |
| 122 return vreinterpret_u8_u32(two); |
| 123 } else { |
| 124 return vld1_u8((uint8_t*)s); |
| 125 } |
| 126 }; |
| 127 const int rightBorder = SkMin32(rightOffset + 1, width); |
| 128 const int srcStrideX = srcDirection == kX ? 1 : srcStride; |
| 129 const int dstStrideX = dstDirection == kX ? 1 : *height; |
| 130 const int srcStrideY = srcDirection == kX ? srcStride : 1; |
| 131 const int dstStrideY = dstDirection == kX ? width : 1; |
| 132 const uint16x8_t scale = vdupq_n_u16((1 << 15) / kernelSize); |
| 133 |
| 134 for (; *height >= 2; *height -= 2) { |
| 135 uint16x8_t sum = vdupq_n_u16(0); |
| 136 const SkPMColor* p = *src; |
| 137 for (int i = 0; i < rightBorder; i++) { |
| 138 sum = vaddw_u8(sum, load_2_pixels(p)); |
| 139 p += srcStrideX; |
| 140 } |
| 141 |
| 142 const SkPMColor* sptr = *src; |
| 143 SkPMColor* dptr = *dst; |
| 144 for (int x = 0; x < width; x++) { |
| 145 // val = (sum * scale * 2 + 0x8000) >> 16 |
| 146 uint16x8_t resultPixels = vreinterpretq_u16_s16(vqrdmulhq_s16( |
| 147 vreinterpretq_s16_u16(sum), vreinterpretq_s16_u16(scale))); |
| 148 if (dstDirection == kX) { |
| 149 uint32x2_t px2 = vreinterpret_u32_u8(vmovn_u16(resultPixels)); |
| 150 vst1_lane_u32(dptr + 0, px2, 0); |
| 151 vst1_lane_u32(dptr + width, px2, 1); |
| 152 } else { |
| 153 vst1_u8((uint8_t*)dptr, vmovn_u16(resultPixels)); |
| 154 } |
| 155 |
| 156 if (x >= leftOffset) { |
| 157 sum = vsubw_u8(sum, load_2_pixels(sptr - leftOffset * srcStrideX
)); |
| 158 } |
| 159 if (x + rightOffset + 1 < width) { |
| 160 sum = vaddw_u8(sum, load_2_pixels(sptr + (rightOffset + 1) * src
StrideX)); |
| 161 } |
| 162 sptr += srcStrideX; |
| 163 dptr += dstStrideX; |
| 164 } |
| 165 *src += srcStrideY * 2; |
| 166 *dst += dstStrideY * 2; |
| 167 } |
| 168 } |
| 169 |
| 170 template<Direction srcDirection, Direction dstDirection> |
| 171 void box_blur(const SkPMColor* src, int srcStride, SkPMColor* dst, int kernelSiz
e, |
| 172 int leftOffset, int rightOffset, int width, int height) { |
| 173 // ARGB -> 0A0R 0G0B |
| 174 auto expand = [](uint32_t p) { |
| 175 return vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(p)))); |
| 176 }; |
| 177 const int rightBorder = SkMin32(rightOffset + 1, width); |
| 178 const int srcStrideX = srcDirection == kX ? 1 : srcStride; |
| 179 const int dstStrideX = dstDirection == kX ? 1 : height; |
| 180 const int srcStrideY = srcDirection == kX ? srcStride : 1; |
| 181 const int dstStrideY = dstDirection == kX ? width : 1; |
| 182 const uint32x4_t scale = vdupq_n_u32((1 << 24) / kernelSize); |
| 183 const uint32x4_t half = vdupq_n_u32(1 << 23); |
| 184 |
| 185 if (1 < kernelSize && kernelSize < 128) { |
| 186 box_blur_double<srcDirection, dstDirection>(&src, srcStride, &dst, kerne
lSize, |
| 187 leftOffset, rightOffset, wid
th, &height); |
| 188 } |
| 189 |
| 190 for (; height > 0; height--) { |
| 191 uint32x4_t sum = vdupq_n_u32(0); |
| 192 const SkPMColor* p = src; |
| 193 for (int i = 0; i < rightBorder; ++i) { |
| 194 sum = vaddw_u16(sum, expand(*p)); |
| 195 p += srcStrideX; |
| 196 } |
| 197 |
| 198 const SkPMColor* sptr = src; |
| 199 SkPMColor* dptr = dst; |
| 200 for (int x = 0; x < width; ++x) { |
| 201 // ( half+sumA*scale half+sumR*scale half+sumG*scale half+sumB*scale
) |
| 202 uint32x4_t result = vmlaq_u32(half, sum, scale); |
| 203 |
| 204 // Saturated conversion to 16-bit. |
| 205 // ( AAAA RRRR GGGG BBBB ) -> ( 0A 0R 0G 0B ) |
| 206 uint16x4_t result16 = vqshrn_n_u32(result, 16); |
| 207 |
| 208 // Saturated conversion to 8-bit. |
| 209 // ( 0A 0R 0G 0B ) -> ( 0A 0R 0G 0B 0A 0R 0G 0B ) -> ( A R G B A R G
B ) |
| 210 uint8x8_t result8 = vqshrn_n_u16(vcombine_u16(result16, result16), 8
); |
| 211 |
| 212 // ( A R G B A R G B ) -> ( ARGB ARGB ) -> ( ARGB ) |
| 213 // Store low 32 bits to destination. |
| 214 vst1_lane_u32(dptr, vreinterpret_u32_u8(result8), 0); |
| 215 |
| 216 if (x >= leftOffset) { |
| 217 const SkPMColor* l = sptr - leftOffset * srcStrideX; |
| 218 sum = vsubw_u16(sum, expand(*l)); |
| 219 } |
| 220 if (x + rightOffset + 1 < width) { |
| 221 const SkPMColor* r = sptr + (rightOffset + 1) * srcStrideX; |
| 222 sum = vaddw_u16(sum, expand(*r)); |
| 223 } |
| 224 sptr += srcStrideX; |
| 225 if (srcDirection == kX) { |
| 226 SK_PREFETCH(sptr + (rightOffset + 16) * srcStrideX); |
| 227 } |
| 228 dptr += dstStrideX; |
| 229 } |
| 230 src += srcStrideY; |
| 231 dst += dstStrideY; |
| 232 } |
| 233 } |
| 234 |
| 235 #else // Neither NEON nor >=SSE2. |
| 236 |
| 237 template<Direction srcDirection, Direction dstDirection> |
| 238 static void box_blur(const SkPMColor* src, int srcStride, SkPMColor* dst, int ke
rnelSize, |
| 239 int leftOffset, int rightOffset, int width, int height) { |
| 240 int rightBorder = SkMin32(rightOffset + 1, width); |
| 241 int srcStrideX = srcDirection == kX ? 1 : srcStride; |
| 242 int dstStrideX = dstDirection == kX ? 1 : height; |
| 243 int srcStrideY = srcDirection == kX ? srcStride : 1; |
| 244 int dstStrideY = dstDirection == kX ? width : 1; |
| 245 uint32_t scale = (1 << 24) / kernelSize; |
| 246 uint32_t half = 1 << 23; |
| 247 for (int y = 0; y < height; ++y) { |
| 248 int sumA = 0, sumR = 0, sumG = 0, sumB = 0; |
| 249 const SkPMColor* p = src; |
| 250 for (int i = 0; i < rightBorder; ++i) { |
| 251 sumA += SkGetPackedA32(*p); |
| 252 sumR += SkGetPackedR32(*p); |
| 253 sumG += SkGetPackedG32(*p); |
| 254 sumB += SkGetPackedB32(*p); |
| 255 p += srcStrideX; |
| 256 } |
| 257 |
| 258 const SkPMColor* sptr = src; |
| 259 SkColor* dptr = dst; |
| 260 for (int x = 0; x < width; ++x) { |
| 261 *dptr = SkPackARGB32((sumA * scale + half) >> 24, |
| 262 (sumR * scale + half) >> 24, |
| 263 (sumG * scale + half) >> 24, |
| 264 (sumB * scale + half) >> 24); |
| 265 if (x >= leftOffset) { |
| 266 SkColor l = *(sptr - leftOffset * srcStrideX); |
| 267 sumA -= SkGetPackedA32(l); |
| 268 sumR -= SkGetPackedR32(l); |
| 269 sumG -= SkGetPackedG32(l); |
| 270 sumB -= SkGetPackedB32(l); |
| 271 } |
| 272 if (x + rightOffset + 1 < width) { |
| 273 SkColor r = *(sptr + (rightOffset + 1) * srcStrideX); |
| 274 sumA += SkGetPackedA32(r); |
| 275 sumR += SkGetPackedR32(r); |
| 276 sumG += SkGetPackedG32(r); |
| 277 sumB += SkGetPackedB32(r); |
| 278 } |
| 279 sptr += srcStrideX; |
| 280 if (srcDirection == kY) { |
| 281 SK_PREFETCH(sptr + (rightOffset + 1) * srcStrideX); |
| 282 } |
| 283 dptr += dstStrideX; |
| 284 } |
| 285 src += srcStrideY; |
| 286 dst += dstStrideY; |
| 287 } |
| 288 } |
| 289 |
| 290 #endif |
| 291 |
| 292 } // namespace SK_OPTS_NS |
| 293 |
| 294 #endif |
OLD | NEW |