OLD | NEW |
1 /* | 1 /* |
2 * Copyright 2015 Google Inc. | 2 * Copyright 2015 Google Inc. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license that can be | 4 * Use of this source code is governed by a BSD-style license that can be |
5 * found in the LICENSE file. | 5 * found in the LICENSE file. |
6 */ | 6 */ |
7 | 7 |
8 #ifndef SkBlurImageFilter_opts_DEFINED | 8 #ifndef SkBlurImageFilter_opts_DEFINED |
9 #define SkBlurImageFilter_opts_DEFINED | 9 #define SkBlurImageFilter_opts_DEFINED |
10 | 10 |
11 #include "SkColorPriv.h" | 11 #include "SkColorPriv.h" |
12 #include "SkTypes.h" | 12 #include "SkTypes.h" |
13 | 13 |
14 namespace SK_OPTS_NS { | 14 namespace SK_OPTS_NS { |
15 | 15 |
16 enum Direction { kX, kY }; | 16 enum class BlurDirection { kX, kY }; |
17 | 17 |
18 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE2 | 18 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE2 |
19 template<Direction srcDirection, Direction dstDirection> | 19 template<BlurDirection srcDirection, BlurDirection dstDirection> |
20 void box_blur(const SkPMColor* src, int srcStride, SkPMColor* dst, int kernelSiz
e, | 20 void box_blur(const SkPMColor* src, int srcStride, SkPMColor* dst, int kernelSiz
e, |
21 int leftOffset, int rightOffset, int width, int height) { | 21 int leftOffset, int rightOffset, int width, int height) { |
22 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE41 | 22 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE41 |
23 // ARGB -> 000A 000R 000G 000B | 23 // ARGB -> 000A 000R 000G 000B |
24 auto expand = [](int p) { | 24 auto expand = [](int p) { |
25 return _mm_cvtepu8_epi32(_mm_cvtsi32_si128(p)); | 25 return _mm_cvtepu8_epi32(_mm_cvtsi32_si128(p)); |
26 }; | 26 }; |
27 // Axxx Rxxx Gxxx Bxxx -> ARGB | 27 // Axxx Rxxx Gxxx Bxxx -> ARGB |
28 auto repack = [](__m128i p) { | 28 auto repack = [](__m128i p) { |
29 const char _ = ~0; // Don't care what ends up in these bytes. This zer
os them. | 29 const char _ = ~0; // Don't care what ends up in these bytes. This zer
os them. |
(...skipping 20 matching lines...) Expand all Loading... |
50 // _mm_mullo_epi32 is not available, so use the standard trick to emulate it
. | 50 // _mm_mullo_epi32 is not available, so use the standard trick to emulate it
. |
51 auto _mm_mullo_epi32 = [](__m128i a, __m128i b) { | 51 auto _mm_mullo_epi32 = [](__m128i a, __m128i b) { |
52 __m128i p02 = _mm_mul_epu32(a, b), | 52 __m128i p02 = _mm_mul_epu32(a, b), |
53 p13 = _mm_mul_epu32(_mm_srli_si128(a, 4), | 53 p13 = _mm_mul_epu32(_mm_srli_si128(a, 4), |
54 _mm_srli_si128(b, 4)); | 54 _mm_srli_si128(b, 4)); |
55 return _mm_unpacklo_epi32(_mm_shuffle_epi32(p02, _MM_SHUFFLE(0,0,2,0)), | 55 return _mm_unpacklo_epi32(_mm_shuffle_epi32(p02, _MM_SHUFFLE(0,0,2,0)), |
56 _mm_shuffle_epi32(p13, _MM_SHUFFLE(0,0,2,0))); | 56 _mm_shuffle_epi32(p13, _MM_SHUFFLE(0,0,2,0))); |
57 }; | 57 }; |
58 #endif | 58 #endif |
59 const int rightBorder = SkMin32(rightOffset + 1, width); | 59 const int rightBorder = SkMin32(rightOffset + 1, width); |
60 const int srcStrideX = srcDirection == kX ? 1 : srcStride; | 60 const int srcStrideX = srcDirection == BlurDirection::kX ? 1 : srcStride; |
61 const int dstStrideX = dstDirection == kX ? 1 : height; | 61 const int dstStrideX = dstDirection == BlurDirection::kX ? 1 : height; |
62 const int srcStrideY = srcDirection == kX ? srcStride : 1; | 62 const int srcStrideY = srcDirection == BlurDirection::kX ? srcStride : 1; |
63 const int dstStrideY = dstDirection == kX ? width : 1; | 63 const int dstStrideY = dstDirection == BlurDirection::kX ? width : 1; |
64 const __m128i scale = _mm_set1_epi32((1 << 24) / kernelSize); | 64 const __m128i scale = _mm_set1_epi32((1 << 24) / kernelSize); |
65 const __m128i half = _mm_set1_epi32(1 << 23); | 65 const __m128i half = _mm_set1_epi32(1 << 23); |
66 for (int y = 0; y < height; ++y) { | 66 for (int y = 0; y < height; ++y) { |
67 __m128i sum = _mm_setzero_si128(); | 67 __m128i sum = _mm_setzero_si128(); |
68 const SkPMColor* p = src; | 68 const SkPMColor* p = src; |
69 for (int i = 0; i < rightBorder; ++i) { | 69 for (int i = 0; i < rightBorder; ++i) { |
70 sum = _mm_add_epi32(sum, expand(*p)); | 70 sum = _mm_add_epi32(sum, expand(*p)); |
71 p += srcStrideX; | 71 p += srcStrideX; |
72 } | 72 } |
73 | 73 |
(...skipping 12 matching lines...) Expand all Loading... |
86 // TODO(mtklein): experiment with breaking this loop into 3 parts | 86 // TODO(mtklein): experiment with breaking this loop into 3 parts |
87 if (x >= leftOffset) { | 87 if (x >= leftOffset) { |
88 SkColor l = *(sptr - leftOffset * srcStrideX); | 88 SkColor l = *(sptr - leftOffset * srcStrideX); |
89 sum = _mm_sub_epi32(sum, expand(l)); | 89 sum = _mm_sub_epi32(sum, expand(l)); |
90 } | 90 } |
91 if (x + rightOffset + 1 < width) { | 91 if (x + rightOffset + 1 < width) { |
92 SkColor r = *(sptr + (rightOffset + 1) * srcStrideX); | 92 SkColor r = *(sptr + (rightOffset + 1) * srcStrideX); |
93 sum = _mm_add_epi32(sum, expand(r)); | 93 sum = _mm_add_epi32(sum, expand(r)); |
94 } | 94 } |
95 sptr += srcStrideX; | 95 sptr += srcStrideX; |
96 if (srcDirection == kY) { | 96 if (srcDirection == BlurDirection::kY) { |
97 // TODO(mtklein): experiment with moving this prefetch forward | 97 // TODO(mtklein): experiment with moving this prefetch forward |
98 _mm_prefetch(reinterpret_cast<const char*>(sptr + (rightOffset +
1) * srcStrideX), | 98 _mm_prefetch(reinterpret_cast<const char*>(sptr + (rightOffset +
1) * srcStrideX), |
99 _MM_HINT_T0); | 99 _MM_HINT_T0); |
100 } | 100 } |
101 dptr += dstStrideX; | 101 dptr += dstStrideX; |
102 } | 102 } |
103 src += srcStrideY; | 103 src += srcStrideY; |
104 dst += dstStrideY; | 104 dst += dstStrideY; |
105 } | 105 } |
106 } | 106 } |
107 | 107 |
108 #elif defined(SK_ARM_HAS_NEON) | 108 #elif defined(SK_ARM_HAS_NEON) |
109 | 109 |
110 // Fast path for kernel sizes between 2 and 127, working on two rows at a time. | 110 // Fast path for kernel sizes between 2 and 127, working on two rows at a time. |
111 template<Direction srcDirection, Direction dstDirection> | 111 template<BlurDirection srcDirection, BlurDirection dstDirection> |
112 void box_blur_double(const SkPMColor** src, int srcStride, SkPMColor** dst, int
kernelSize, | 112 void box_blur_double(const SkPMColor** src, int srcStride, SkPMColor** dst, int
kernelSize, |
113 int leftOffset, int rightOffset, int width, int* height) { | 113 int leftOffset, int rightOffset, int width, int* height) { |
114 // Load 2 pixels from adjacent rows. | 114 // Load 2 pixels from adjacent rows. |
115 auto load_2_pixels = [&](const SkPMColor* s) { | 115 auto load_2_pixels = [&](const SkPMColor* s) { |
116 if (srcDirection == kX) { | 116 if (srcDirection == BlurDirection::kX) { |
117 // 10% faster by adding these 2 prefetches | 117 // 10% faster by adding these 2 prefetches |
118 SK_PREFETCH(s + 16); | 118 SK_PREFETCH(s + 16); |
119 SK_PREFETCH(s + 16 + srcStride); | 119 SK_PREFETCH(s + 16 + srcStride); |
120 auto one = vld1_lane_u32(s + 0, vdup_n_u32(0), 0), | 120 auto one = vld1_lane_u32(s + 0, vdup_n_u32(0), 0), |
121 two = vld1_lane_u32(s + srcStride, one, 1); | 121 two = vld1_lane_u32(s + srcStride, one, 1); |
122 return vreinterpret_u8_u32(two); | 122 return vreinterpret_u8_u32(two); |
123 } else { | 123 } else { |
124 return vld1_u8((uint8_t*)s); | 124 return vld1_u8((uint8_t*)s); |
125 } | 125 } |
126 }; | 126 }; |
127 const int rightBorder = SkMin32(rightOffset + 1, width); | 127 const int rightBorder = SkMin32(rightOffset + 1, width); |
128 const int srcStrideX = srcDirection == kX ? 1 : srcStride; | 128 const int srcStrideX = srcDirection == BlurDirection::kX ? 1 : srcStride; |
129 const int dstStrideX = dstDirection == kX ? 1 : *height; | 129 const int dstStrideX = dstDirection == BlurDirection::kX ? 1 : *height; |
130 const int srcStrideY = srcDirection == kX ? srcStride : 1; | 130 const int srcStrideY = srcDirection == BlurDirection::kX ? srcStride : 1; |
131 const int dstStrideY = dstDirection == kX ? width : 1; | 131 const int dstStrideY = dstDirection == BlurDirection::kX ? width : 1; |
132 const uint16x8_t scale = vdupq_n_u16((1 << 15) / kernelSize); | 132 const uint16x8_t scale = vdupq_n_u16((1 << 15) / kernelSize); |
133 | 133 |
134 for (; *height >= 2; *height -= 2) { | 134 for (; *height >= 2; *height -= 2) { |
135 uint16x8_t sum = vdupq_n_u16(0); | 135 uint16x8_t sum = vdupq_n_u16(0); |
136 const SkPMColor* p = *src; | 136 const SkPMColor* p = *src; |
137 for (int i = 0; i < rightBorder; i++) { | 137 for (int i = 0; i < rightBorder; i++) { |
138 sum = vaddw_u8(sum, load_2_pixels(p)); | 138 sum = vaddw_u8(sum, load_2_pixels(p)); |
139 p += srcStrideX; | 139 p += srcStrideX; |
140 } | 140 } |
141 | 141 |
142 const SkPMColor* sptr = *src; | 142 const SkPMColor* sptr = *src; |
143 SkPMColor* dptr = *dst; | 143 SkPMColor* dptr = *dst; |
144 for (int x = 0; x < width; x++) { | 144 for (int x = 0; x < width; x++) { |
145 // val = (sum * scale * 2 + 0x8000) >> 16 | 145 // val = (sum * scale * 2 + 0x8000) >> 16 |
146 uint16x8_t resultPixels = vreinterpretq_u16_s16(vqrdmulhq_s16( | 146 uint16x8_t resultPixels = vreinterpretq_u16_s16(vqrdmulhq_s16( |
147 vreinterpretq_s16_u16(sum), vreinterpretq_s16_u16(scale))); | 147 vreinterpretq_s16_u16(sum), vreinterpretq_s16_u16(scale))); |
148 if (dstDirection == kX) { | 148 if (dstDirection == BlurDirection::kX) { |
149 uint32x2_t px2 = vreinterpret_u32_u8(vmovn_u16(resultPixels)); | 149 uint32x2_t px2 = vreinterpret_u32_u8(vmovn_u16(resultPixels)); |
150 vst1_lane_u32(dptr + 0, px2, 0); | 150 vst1_lane_u32(dptr + 0, px2, 0); |
151 vst1_lane_u32(dptr + width, px2, 1); | 151 vst1_lane_u32(dptr + width, px2, 1); |
152 } else { | 152 } else { |
153 vst1_u8((uint8_t*)dptr, vmovn_u16(resultPixels)); | 153 vst1_u8((uint8_t*)dptr, vmovn_u16(resultPixels)); |
154 } | 154 } |
155 | 155 |
156 if (x >= leftOffset) { | 156 if (x >= leftOffset) { |
157 sum = vsubw_u8(sum, load_2_pixels(sptr - leftOffset * srcStrideX
)); | 157 sum = vsubw_u8(sum, load_2_pixels(sptr - leftOffset * srcStrideX
)); |
158 } | 158 } |
159 if (x + rightOffset + 1 < width) { | 159 if (x + rightOffset + 1 < width) { |
160 sum = vaddw_u8(sum, load_2_pixels(sptr + (rightOffset + 1) * src
StrideX)); | 160 sum = vaddw_u8(sum, load_2_pixels(sptr + (rightOffset + 1) * src
StrideX)); |
161 } | 161 } |
162 sptr += srcStrideX; | 162 sptr += srcStrideX; |
163 dptr += dstStrideX; | 163 dptr += dstStrideX; |
164 } | 164 } |
165 *src += srcStrideY * 2; | 165 *src += srcStrideY * 2; |
166 *dst += dstStrideY * 2; | 166 *dst += dstStrideY * 2; |
167 } | 167 } |
168 } | 168 } |
169 | 169 |
170 template<Direction srcDirection, Direction dstDirection> | 170 template<BlurDirection srcDirection, BlurDirection dstDirection> |
171 void box_blur(const SkPMColor* src, int srcStride, SkPMColor* dst, int kernelSiz
e, | 171 void box_blur(const SkPMColor* src, int srcStride, SkPMColor* dst, int kernelSiz
e, |
172 int leftOffset, int rightOffset, int width, int height) { | 172 int leftOffset, int rightOffset, int width, int height) { |
173 // ARGB -> 0A0R 0G0B | 173 // ARGB -> 0A0R 0G0B |
174 auto expand = [](uint32_t p) { | 174 auto expand = [](uint32_t p) { |
175 return vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(p)))); | 175 return vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(p)))); |
176 }; | 176 }; |
177 const int rightBorder = SkMin32(rightOffset + 1, width); | 177 const int rightBorder = SkMin32(rightOffset + 1, width); |
178 const int srcStrideX = srcDirection == kX ? 1 : srcStride; | 178 const int srcStrideX = srcDirection == BlurDirection::kX ? 1 : srcStride; |
179 const int dstStrideX = dstDirection == kX ? 1 : height; | 179 const int dstStrideX = dstDirection == BlurDirection::kX ? 1 : height; |
180 const int srcStrideY = srcDirection == kX ? srcStride : 1; | 180 const int srcStrideY = srcDirection == BlurDirection::kX ? srcStride : 1; |
181 const int dstStrideY = dstDirection == kX ? width : 1; | 181 const int dstStrideY = dstDirection == BlurDirection::kX ? width : 1; |
182 const uint32x4_t scale = vdupq_n_u32((1 << 24) / kernelSize); | 182 const uint32x4_t scale = vdupq_n_u32((1 << 24) / kernelSize); |
183 const uint32x4_t half = vdupq_n_u32(1 << 23); | 183 const uint32x4_t half = vdupq_n_u32(1 << 23); |
184 | 184 |
185 if (1 < kernelSize && kernelSize < 128) { | 185 if (1 < kernelSize && kernelSize < 128) { |
186 box_blur_double<srcDirection, dstDirection>(&src, srcStride, &dst, kerne
lSize, | 186 box_blur_double<srcDirection, dstDirection>(&src, srcStride, &dst, kerne
lSize, |
187 leftOffset, rightOffset, wid
th, &height); | 187 leftOffset, rightOffset, wid
th, &height); |
188 } | 188 } |
189 | 189 |
190 for (; height > 0; height--) { | 190 for (; height > 0; height--) { |
191 uint32x4_t sum = vdupq_n_u32(0); | 191 uint32x4_t sum = vdupq_n_u32(0); |
(...skipping 23 matching lines...) Expand all Loading... |
215 | 215 |
216 if (x >= leftOffset) { | 216 if (x >= leftOffset) { |
217 const SkPMColor* l = sptr - leftOffset * srcStrideX; | 217 const SkPMColor* l = sptr - leftOffset * srcStrideX; |
218 sum = vsubw_u16(sum, expand(*l)); | 218 sum = vsubw_u16(sum, expand(*l)); |
219 } | 219 } |
220 if (x + rightOffset + 1 < width) { | 220 if (x + rightOffset + 1 < width) { |
221 const SkPMColor* r = sptr + (rightOffset + 1) * srcStrideX; | 221 const SkPMColor* r = sptr + (rightOffset + 1) * srcStrideX; |
222 sum = vaddw_u16(sum, expand(*r)); | 222 sum = vaddw_u16(sum, expand(*r)); |
223 } | 223 } |
224 sptr += srcStrideX; | 224 sptr += srcStrideX; |
225 if (srcDirection == kX) { | 225 if (srcDirection == BlurDirection::kX) { |
226 SK_PREFETCH(sptr + (rightOffset + 16) * srcStrideX); | 226 SK_PREFETCH(sptr + (rightOffset + 16) * srcStrideX); |
227 } | 227 } |
228 dptr += dstStrideX; | 228 dptr += dstStrideX; |
229 } | 229 } |
230 src += srcStrideY; | 230 src += srcStrideY; |
231 dst += dstStrideY; | 231 dst += dstStrideY; |
232 } | 232 } |
233 } | 233 } |
234 | 234 |
235 #else // Neither NEON nor >=SSE2. | 235 #else // Neither NEON nor >=SSE2. |
236 | 236 |
237 template<Direction srcDirection, Direction dstDirection> | 237 template<BlurDirection srcDirection, BlurDirection dstDirection> |
238 static void box_blur(const SkPMColor* src, int srcStride, SkPMColor* dst, int ke
rnelSize, | 238 static void box_blur(const SkPMColor* src, int srcStride, SkPMColor* dst, int ke
rnelSize, |
239 int leftOffset, int rightOffset, int width, int height) { | 239 int leftOffset, int rightOffset, int width, int height) { |
240 int rightBorder = SkMin32(rightOffset + 1, width); | 240 int rightBorder = SkMin32(rightOffset + 1, width); |
241 int srcStrideX = srcDirection == kX ? 1 : srcStride; | 241 int srcStrideX = srcDirection == BlurDirection::kX ? 1 : srcStride; |
242 int dstStrideX = dstDirection == kX ? 1 : height; | 242 int dstStrideX = dstDirection == BlurDirection::kX ? 1 : height; |
243 int srcStrideY = srcDirection == kX ? srcStride : 1; | 243 int srcStrideY = srcDirection == BlurDirection::kX ? srcStride : 1; |
244 int dstStrideY = dstDirection == kX ? width : 1; | 244 int dstStrideY = dstDirection == BlurDirection::kX ? width : 1; |
245 uint32_t scale = (1 << 24) / kernelSize; | 245 uint32_t scale = (1 << 24) / kernelSize; |
246 uint32_t half = 1 << 23; | 246 uint32_t half = 1 << 23; |
247 for (int y = 0; y < height; ++y) { | 247 for (int y = 0; y < height; ++y) { |
248 int sumA = 0, sumR = 0, sumG = 0, sumB = 0; | 248 int sumA = 0, sumR = 0, sumG = 0, sumB = 0; |
249 const SkPMColor* p = src; | 249 const SkPMColor* p = src; |
250 for (int i = 0; i < rightBorder; ++i) { | 250 for (int i = 0; i < rightBorder; ++i) { |
251 sumA += SkGetPackedA32(*p); | 251 sumA += SkGetPackedA32(*p); |
252 sumR += SkGetPackedR32(*p); | 252 sumR += SkGetPackedR32(*p); |
253 sumG += SkGetPackedG32(*p); | 253 sumG += SkGetPackedG32(*p); |
254 sumB += SkGetPackedB32(*p); | 254 sumB += SkGetPackedB32(*p); |
(...skipping 15 matching lines...) Expand all Loading... |
270 sumB -= SkGetPackedB32(l); | 270 sumB -= SkGetPackedB32(l); |
271 } | 271 } |
272 if (x + rightOffset + 1 < width) { | 272 if (x + rightOffset + 1 < width) { |
273 SkColor r = *(sptr + (rightOffset + 1) * srcStrideX); | 273 SkColor r = *(sptr + (rightOffset + 1) * srcStrideX); |
274 sumA += SkGetPackedA32(r); | 274 sumA += SkGetPackedA32(r); |
275 sumR += SkGetPackedR32(r); | 275 sumR += SkGetPackedR32(r); |
276 sumG += SkGetPackedG32(r); | 276 sumG += SkGetPackedG32(r); |
277 sumB += SkGetPackedB32(r); | 277 sumB += SkGetPackedB32(r); |
278 } | 278 } |
279 sptr += srcStrideX; | 279 sptr += srcStrideX; |
280 if (srcDirection == kY) { | 280 if (srcDirection == BlurDirection::kY) { |
281 SK_PREFETCH(sptr + (rightOffset + 1) * srcStrideX); | 281 SK_PREFETCH(sptr + (rightOffset + 1) * srcStrideX); |
282 } | 282 } |
283 dptr += dstStrideX; | 283 dptr += dstStrideX; |
284 } | 284 } |
285 src += srcStrideY; | 285 src += srcStrideY; |
286 dst += dstStrideY; | 286 dst += dstStrideY; |
287 } | 287 } |
288 } | 288 } |
289 | 289 |
290 #endif | 290 #endif |
291 | 291 |
| 292 static auto box_blur_xx = &box_blur<BlurDirection::kX, BlurDirection::kX>, |
| 293 box_blur_xy = &box_blur<BlurDirection::kX, BlurDirection::kY>, |
| 294 box_blur_yx = &box_blur<BlurDirection::kY, BlurDirection::kX>; |
| 295 |
292 } // namespace SK_OPTS_NS | 296 } // namespace SK_OPTS_NS |
293 | 297 |
294 #endif | 298 #endif |
OLD | NEW |