| OLD | NEW |
|---|
| (Empty) |
| 1 /* NEON optimized code (C) COPYRIGHT 2009 Motorola | |
| 2 * | |
| 3 * Use of this source code is governed by a BSD-style license that can be | |
| 4 * found in the LICENSE file. | |
| 5 */ | |
| 6 | |
| 7 /* | |
| 8 * Modifications done in-house at Motorola | |
| 9 * | |
| 10 * this is a clone of SkBitmapProcState_matrix.h | |
| 11 * and has been tuned to work with the NEON unit. | |
| 12 * | |
| 13 * Still going back and forth between whether this approach | |
| 14 * (clone the entire SkBitmapProcState_matrix.h file or | |
| 15 * if I should put just the modified routines in here and | |
| 16 * then use a construct like #define DONT_DO_THIS_FUNCTION or | |
| 17 * something like that... | |
| 18 * | |
| 19 * This is for the ClampX_ClampY instance | |
| 20 * | |
| 21 */ | |
| 22 | |
| 23 | |
| 24 #include <arm_neon.h> | |
| 25 | |
| 26 /* | |
| 27 * This has been modified on the knowledge that (at the time) | |
| 28 * we had the following macro definitions in the parent file | |
| 29 * | |
| 30 * #define MAKENAME(suffix) ClampX_ClampY ## suffix | |
| 31 * #define TILEX_PROCF(fx, max) SkClampMax((fx) >> 16, max) | |
| 32 * #define TILEY_PROCF(fy, max) SkClampMax((fy) >> 16, max) | |
| 33 * #define TILEX_LOW_BITS(fx, max) (((fx) >> 12) & 0xF) | |
| 34 * #define TILEY_LOW_BITS(fy, max) (((fy) >> 12) & 0xF) | |
| 35 * #define CHECK_FOR_DECAL | |
| 36 */ | |
| 37 | |
| 38 /* SkClampMax(val,max) -- bound to 0..max */ | |
| 39 | |
| 40 #define SCALE_NOFILTER_NAME MAKENAME(_nofilter_scale) | |
| 41 #define SCALE_FILTER_NAME MAKENAME(_filter_scale) | |
| 42 #define AFFINE_NOFILTER_NAME MAKENAME(_nofilter_affine) | |
| 43 #define AFFINE_FILTER_NAME MAKENAME(_filter_affine) | |
| 44 #define PERSP_NOFILTER_NAME MAKENAME(_nofilter_persp) | |
| 45 #define PERSP_FILTER_NAME MAKENAME(_filter_persp) | |
| 46 | |
| 47 #define PACK_FILTER_X_NAME MAKENAME(_pack_filter_x) | |
| 48 #define PACK_FILTER_Y_NAME MAKENAME(_pack_filter_y) | |
| 49 | |
| 50 #ifndef PREAMBLE | |
| 51 #define PREAMBLE(state) | |
| 52 #define PREAMBLE_PARAM_X | |
| 53 #define PREAMBLE_PARAM_Y | |
| 54 #define PREAMBLE_ARG_X | |
| 55 #define PREAMBLE_ARG_Y | |
| 56 #endif | |
| 57 | |
| 58 static void SCALE_NOFILTER_NAME(const SkBitmapProcState& s, | |
| 59 uint32_t xy[], int count, int x, int y) { | |
| 60 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | | |
| 61 SkMatrix::kScale_Mask)) == 0); | |
| 62 | |
| 63 PREAMBLE(s); | |
| 64 // we store y, x, x, x, x, x | |
| 65 | |
| 66 const unsigned maxX = s.fBitmap->width() - 1; | |
| 67 SkFixed fx; | |
| 68 { | |
| 69 SkPoint pt; | |
| 70 s.fInvProc(s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, | |
| 71 SkIntToScalar(y) + SK_ScalarHalf, &pt); | |
| 72 fx = SkScalarToFixed(pt.fY); | |
| 73 const unsigned maxY = s.fBitmap->height() - 1; | |
| 74 *xy++ = TILEY_PROCF(fx, maxY); | |
| 75 fx = SkScalarToFixed(pt.fX); | |
| 76 } | |
| 77 | |
| 78 if (0 == maxX) { | |
| 79 // all of the following X values must be 0 | |
| 80 memset(xy, 0, count * sizeof(uint16_t)); | |
| 81 return; | |
| 82 } | |
| 83 | |
| 84 const SkFixed dx = s.fInvSx; | |
| 85 | |
| 86 #ifdef CHECK_FOR_DECAL | |
| 87 // test if we don't need to apply the tile proc | |
| 88 if ((unsigned)(fx >> 16) <= maxX && | |
| 89 (unsigned)((fx + dx * (count - 1)) >> 16) <= maxX) { | |
| 90 decal_nofilter_scale_neon(xy, fx, dx, count); | |
| 91 return; | |
| 92 } | |
| 93 #endif | |
| 94 | |
| 95 int i; | |
| 96 | |
| 97 /* very much like done in decal_nofilter, but with | |
| 98 * an extra clamping function applied. | |
| 99 * TILEX_PROCF(fx,max) SkClampMax((fx)>>16, max) | |
| 100 */ | |
| 101 if (count >= 8) { | |
| 102 /* SkFixed is 16.16 fixed point */ | |
| 103 SkFixed dx2 = dx+dx; | |
| 104 SkFixed dx4 = dx2+dx2; | |
| 105 SkFixed dx8 = dx4+dx4; | |
| 106 | |
| 107 /* now build fx/fx+dx/fx+2dx/fx+3dx */ | |
| 108 SkFixed fx1, fx2, fx3; | |
| 109 int32x4_t lbase, hbase; | |
| 110 int16_t *dst16 = (int16_t *)xy; | |
| 111 | |
| 112 fx1 = fx+dx; | |
| 113 fx2 = fx1+dx; | |
| 114 fx3 = fx2+dx; | |
| 115 | |
| 116 /* build my template(s) */ | |
| 117 /* avoid the 'lbase unitialized' warning */ | |
| 118 lbase = vdupq_n_s32(fx); | |
| 119 lbase = vsetq_lane_s32(fx1, lbase, 1); | |
| 120 lbase = vsetq_lane_s32(fx2, lbase, 2); | |
| 121 lbase = vsetq_lane_s32(fx3, lbase, 3); | |
| 122 | |
| 123 hbase = vaddq_s32(lbase, vdupq_n_s32(dx4)); | |
| 124 | |
| 125 /* store & bump */ | |
| 126 do { | |
| 127 int32x4_t lout; | |
| 128 int32x4_t hout; | |
| 129 int16x8_t hi16; | |
| 130 | |
| 131 /* get the hi 16s of all those 32s */ | |
| 132 lout = lbase; | |
| 133 hout = hbase; | |
| 134 /* this sets up all lout's then all hout's in hout */ | |
| 135 asm ("vuzpq.16 %q0, %q1" : "+w" (lout), "+w" (hout)); | |
| 136 hi16 = vreinterpretq_s16_s32(hout); | |
| 137 | |
| 138 /* clamp & output */ | |
| 139 hi16 = vmaxq_s16(hi16, vdupq_n_s16(0)); | |
| 140 hi16 = vminq_s16(hi16, vdupq_n_s16(maxX)); | |
| 141 vst1q_s16(dst16, hi16); | |
| 142 | |
| 143 /* but preserving base & on to the next */ | |
| 144 lbase = vaddq_s32 (lbase, vdupq_n_s32(dx8)); | |
| 145 hbase = vaddq_s32 (hbase, vdupq_n_s32(dx8)); | |
| 146 dst16 += 8; | |
| 147 count -= 8; | |
| 148 fx += dx8; | |
| 149 } while (count >= 8); | |
| 150 xy = (uint32_t *) dst16; | |
| 151 } | |
| 152 | |
| 153 uint16_t* xx = (uint16_t*)xy; | |
| 154 for (i = count; i > 0; --i) { | |
| 155 *xx++ = TILEX_PROCF(fx, maxX); fx += dx; | |
| 156 } | |
| 157 } | |
| 158 | |
| 159 // note: we could special-case on a matrix which is skewed in X but not Y. | |
| 160 // this would require a more general setup thatn SCALE does, but could use | |
| 161 // SCALE's inner loop that only looks at dx | |
| 162 | |
| 163 static void AFFINE_NOFILTER_NAME(const SkBitmapProcState& s, | |
| 164 uint32_t xy[], int count, int x, int y) { | |
| 165 SkASSERT(s.fInvType & SkMatrix::kAffine_Mask); | |
| 166 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | | |
| 167 SkMatrix::kScale_Mask | | |
| 168 SkMatrix::kAffine_Mask)) == 0); | |
| 169 | |
| 170 PREAMBLE(s); | |
| 171 SkPoint srcPt; | |
| 172 s.fInvProc(s.fInvMatrix, | |
| 173 SkIntToScalar(x) + SK_ScalarHalf, | |
| 174 SkIntToScalar(y) + SK_ScalarHalf, &srcPt); | |
| 175 | |
| 176 SkFixed fx = SkScalarToFixed(srcPt.fX); | |
| 177 SkFixed fy = SkScalarToFixed(srcPt.fY); | |
| 178 SkFixed dx = s.fInvSx; | |
| 179 SkFixed dy = s.fInvKy; | |
| 180 int maxX = s.fBitmap->width() - 1; | |
| 181 int maxY = s.fBitmap->height() - 1; | |
| 182 | |
| 183 /* NEON lets us do an 8x unrolling */ | |
| 184 if (count >= 8) { | |
| 185 /* SkFixed is 16.16 fixed point */ | |
| 186 SkFixed dx4 = dx * 4; | |
| 187 SkFixed dy4 = dy * 4; | |
| 188 SkFixed dx8 = dx * 8; | |
| 189 SkFixed dy8 = dy * 8; | |
| 190 | |
| 191 int32x4_t xbase, ybase; | |
| 192 int32x4_t x2base, y2base; | |
| 193 int16_t *dst16 = (int16_t *) xy; | |
| 194 | |
| 195 /* my sets of maxx/maxy for clamping */ | |
| 196 int32_t maxpair = (maxX&0xffff) | ((maxY&0xffff)<<16); | |
| 197 int16x8_t maxXY = vreinterpretq_s16_s32(vdupq_n_s32(maxpair)); | |
| 198 | |
| 199 /* now build fx/fx+dx/fx+2dx/fx+3dx */ | |
| 200 /* avoid the 'xbase unitialized' warning...*/ | |
| 201 xbase = vdupq_n_s32(fx); | |
| 202 xbase = vsetq_lane_s32(fx+dx, xbase, 1); | |
| 203 xbase = vsetq_lane_s32(fx+dx+dx, xbase, 2); | |
| 204 xbase = vsetq_lane_s32(fx+dx+dx+dx, xbase, 3); | |
| 205 | |
| 206 /* same for fy */ | |
| 207 /* avoid the 'ybase unitialized' warning...*/ | |
| 208 ybase = vdupq_n_s32(fy); | |
| 209 ybase = vsetq_lane_s32(fy+dy, ybase, 1); | |
| 210 ybase = vsetq_lane_s32(fy+dy+dy, ybase, 2); | |
| 211 ybase = vsetq_lane_s32(fy+dy+dy+dy, ybase, 3); | |
| 212 | |
| 213 x2base = vaddq_s32(xbase, vdupq_n_s32(dx4)); | |
| 214 y2base = vaddq_s32(ybase, vdupq_n_s32(dy4)); | |
| 215 | |
| 216 /* store & bump */ | |
| 217 do { | |
| 218 int32x4_t xout, yout; | |
| 219 int32x4_t x2out, y2out; | |
| 220 int16x8_t hi16, hi16_2; | |
| 221 | |
| 222 xout = xbase; | |
| 223 yout = ybase; | |
| 224 | |
| 225 /* overlay y's low16 with hi16 from x */ | |
| 226 /* so we properly shifted xyxyxyxy */ | |
| 227 yout = vsriq_n_s32(yout, xout, 16); | |
| 228 hi16 = vreinterpretq_s16_s32 (yout); | |
| 229 | |
| 230 /* do the clamping; both guys get 0's */ | |
| 231 hi16 = vmaxq_s16 (hi16, vdupq_n_s16(0)); | |
| 232 hi16 = vminq_s16 (hi16, maxXY); | |
| 233 | |
| 234 vst1q_s16 (dst16, hi16); | |
| 235 | |
| 236 /* and for the other 4 pieces of this iteration */ | |
| 237 x2out = x2base; | |
| 238 y2out = y2base; | |
| 239 | |
| 240 /* overlay y's low16 with hi16 from x */ | |
| 241 /* so we properly shifted xyxyxyxy */ | |
| 242 y2out = vsriq_n_s32(y2out, x2out, 16); | |
| 243 hi16_2 = vreinterpretq_s16_s32 (y2out); | |
| 244 | |
| 245 /* do the clamping; both guys get 0's */ | |
| 246 hi16_2 = vmaxq_s16 (hi16_2, vdupq_n_s16(0)); | |
| 247 hi16_2 = vminq_s16 (hi16_2, maxXY); | |
| 248 | |
| 249 /* RBE: gcc regenerates dst16+8 all the time instead | |
| 250 * of folding it into an addressing mode. *sigh* */ | |
| 251 vst1q_s16 (dst16+8, hi16_2); | |
| 252 | |
| 253 /* moving base and on to the next */ | |
| 254 xbase = vaddq_s32 (xbase, vdupq_n_s32 (dx8)); | |
| 255 ybase = vaddq_s32 (ybase, vdupq_n_s32 (dy8)); | |
| 256 x2base = vaddq_s32 (x2base, vdupq_n_s32 (dx8)); | |
| 257 y2base = vaddq_s32 (y2base, vdupq_n_s32 (dy8)); | |
| 258 | |
| 259 dst16 += 16; /* 8x32 aka 16x16 */ | |
| 260 count -= 8; | |
| 261 fx += dx8; | |
| 262 fy += dy8; | |
| 263 } while (count >= 8); | |
| 264 xy = (uint32_t *) dst16; | |
| 265 } | |
| 266 | |
| 267 for (int i = count; i > 0; --i) { | |
| 268 *xy++ = (TILEY_PROCF(fy, maxY) << 16) | TILEX_PROCF(fx, maxX); | |
| 269 fx += dx; fy += dy; | |
| 270 } | |
| 271 } | |
| 272 | |
| 273 #undef DEBUG_PERSP_NOFILTER | |
| 274 | |
| 275 static void PERSP_NOFILTER_NAME(const SkBitmapProcState& s, | |
| 276 uint32_t* SK_RESTRICT xy, | |
| 277 int count, int x, int y) { | |
| 278 SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask); | |
| 279 | |
| 280 PREAMBLE(s); | |
| 281 /* max{X,Y} are int here, but later shown/assumed to fit in 16 bits */ | |
| 282 int maxX = s.fBitmap->width() - 1; | |
| 283 int maxY = s.fBitmap->height() - 1; | |
| 284 | |
| 285 SkPerspIter iter(s.fInvMatrix, | |
| 286 SkIntToScalar(x) + SK_ScalarHalf, | |
| 287 SkIntToScalar(y) + SK_ScalarHalf, count); | |
| 288 | |
| 289 while ((count = iter.next()) != 0) { | |
| 290 const SkFixed* SK_RESTRICT srcXY = iter.getXY(); | |
| 291 | |
| 292 #if defined(DEBUG_PERSP_NOFILTER) | |
| 293 /* debugging stuff */ | |
| 294 const SkFixed *end_srcXY = srcXY + (count*2); | |
| 295 uint32_t *end_xy = xy + (count); | |
| 296 const SkFixed *base_srcXY = srcXY; | |
| 297 uint32_t *base_xy = xy; | |
| 298 int base_count = count; | |
| 299 #endif | |
| 300 | |
| 301 #if 1 | |
| 302 // 2009/9/30: crashes in ApiDemos - Views - Animation - 3D Transition | |
| 303 // 2009/10/9: reworked to avoid illegal (but allowed by gas) insn | |
| 304 | |
| 305 /* srcXY is a batch of 32 bit numbers X0,Y0,X1,Y1... | |
| 306 * but we immediately discard the low 16 bits... | |
| 307 * so what we're going to do is vld4, which will give us | |
| 308 * xlo,xhi,ylo,yhi distribution and we can ignore the 'lo' | |
| 309 * parts.... | |
| 310 */ | |
| 311 if (count >= 8) { | |
| 312 int16_t *mysrc = (int16_t *) srcXY; | |
| 313 int16_t *mydst = (int16_t *) xy; | |
| 314 int16x4_t maxX4 = vdup_n_s16((int16_t)maxX); | |
| 315 int16x4_t maxY4 = vdup_n_s16((int16_t)maxY); | |
| 316 int16x4_t zero4 = vdup_n_s16(0); | |
| 317 | |
| 318 /* The constructs with local blocks for register assignments | |
| 319 * and asm() instructions is to make keep any hard register | |
| 320 * assignments to as small a scope as possible. and to avoid | |
| 321 * burning call-preserved hard registers on the vld/vst | |
| 322 * instructions. | |
| 323 */ | |
| 324 | |
| 325 do { | |
| 326 int16x4_t xhi, yhi; | |
| 327 int16x4_t x2hi, y2hi; | |
| 328 | |
| 329 /* vld4 does the de-interleaving for us */ | |
| 330 { | |
| 331 register int16x4_t t_xlo asm("d0"); | |
| 332 register int16x4_t t_xhi asm("d1"); | |
| 333 register int16x4_t t_ylo asm("d2"); | |
| 334 register int16x4_t t_yhi asm("d3"); | |
| 335 | |
| 336 asm ("vld4.16 {d0-d3},[%4] /* xlo=%P0 xhi=%P1 ylo=%P2 yh
i=%P3 */" | |
| 337 : "=w" (t_xlo), "=w" (t_xhi), "=w" (t_ylo), "=w" (t_yhi) | |
| 338 : "r" (mysrc) | |
| 339 ); | |
| 340 xhi = t_xhi; | |
| 341 yhi = t_yhi; | |
| 342 } | |
| 343 | |
| 344 /* clamp X>>16 (aka xhi) to 0..maxX */ | |
| 345 xhi = vmax_s16(xhi, zero4); /* now 0.. */ | |
| 346 xhi = vmin_s16(xhi, maxX4); /* now 0..maxX */ | |
| 347 | |
| 348 /* clamp Y>>16 (aka yhi) to 0..maxY */ | |
| 349 yhi = vmax_s16(yhi, zero4); /* now 0.. */ | |
| 350 yhi = vmin_s16(yhi, maxY4); /* now 0..maxY */ | |
| 351 | |
| 352 /* deal with the second set of numbers */ | |
| 353 { | |
| 354 register int16x4_t t_xlo asm("d4"); | |
| 355 register int16x4_t t_xhi asm("d5"); | |
| 356 register int16x4_t t_ylo asm("d6"); | |
| 357 register int16x4_t t_yhi asm("d7"); | |
| 358 | |
| 359 /* offset == 256 bits == 32 bytes == 8 longs == 16 shorts */ | |
| 360 asm ("vld4.16 {d4-d7},[%4] /* xlo=%P0 xhi=%P1 ylo=%P2 yh
i=%P3 */" | |
| 361 : "=w" (t_xlo), "=w" (t_xhi), "=w" (t_ylo), "=w" (t_yhi) | |
| 362 : "r" (mysrc+16) | |
| 363 ); | |
| 364 x2hi = t_xhi; | |
| 365 y2hi = t_yhi; | |
| 366 } | |
| 367 | |
| 368 /* clamp the second 4 here */ | |
| 369 | |
| 370 if (0) { extern void rbe(void); rbe(); } | |
| 371 | |
| 372 /* clamp X>>16 (aka xhi) to 0..maxX */ | |
| 373 x2hi = vmax_s16(x2hi, zero4); /* now 0.. */ | |
| 374 x2hi = vmin_s16(x2hi, maxX4); /* now 0..maxX */ | |
| 375 | |
| 376 /* clamp Y>>16 (aka yhi) to 0..maxY */ | |
| 377 y2hi = vmax_s16(y2hi, zero4); /* now 0.. */ | |
| 378 y2hi = vmin_s16(y2hi, maxY4); /* now 0..maxY */ | |
| 379 | |
| 380 /* we're storing as {x,y}s: x is [0], y is [1] */ | |
| 381 /* we'll use vst2 to make this happen */ | |
| 382 | |
| 383 { | |
| 384 register int16x4_t out_x asm("d16") = xhi; | |
| 385 register int16x4_t out_y asm("d17") = yhi; | |
| 386 | |
| 387 asm ("vst2.16 {d16-d17},[%2] /* xlo=%P0 xhi=%P1 */" | |
| 388 : | |
| 389 : "w" (out_x), "w" (out_y), "r" (mydst) | |
| 390 ); | |
| 391 } | |
| 392 { | |
| 393 register int16x4_t out_x asm("d18") = x2hi; | |
| 394 register int16x4_t out_y asm("d19") = y2hi; | |
| 395 | |
| 396 asm ("vst2.16 {d18-d19},[%2] /* xlo=%P0 xhi=%P1 */" | |
| 397 : | |
| 398 : "w" (out_x), "w" (out_y), "r" (mydst+8) | |
| 399 ); | |
| 400 } | |
| 401 | |
| 402 /* XXX: gcc isn't interleaving these with the NEON ops | |
| 403 * but i think that all the scoreboarding works out */ | |
| 404 count -= 8; /* 8 iterations */ | |
| 405 mysrc += 32; /* 16 longs, aka 32 shorts */ | |
| 406 mydst += 16; /* 16 shorts, aka 8 longs */ | |
| 407 } while (count >= 8); | |
| 408 /* get xy and srcXY fixed up */ | |
| 409 srcXY = (const SkFixed *) mysrc; | |
| 410 xy = (uint32_t *) mydst; | |
| 411 } | |
| 412 #endif | |
| 413 | |
| 414 while (--count >= 0) { | |
| 415 *xy++ = (TILEY_PROCF(srcXY[1], maxY) << 16) | | |
| 416 TILEX_PROCF(srcXY[0], maxX); | |
| 417 srcXY += 2; | |
| 418 } | |
| 419 | |
| 420 #if defined(DEBUG_PERSP_NOFILTER) | |
| 421 /* for checking our NEON-produced results against vanilla code */ | |
| 422 { | |
| 423 int bad = (-1); | |
| 424 for (int i = 0; i < base_count; i++) { | |
| 425 uint32_t val; | |
| 426 val = (TILEY_PROCF (base_srcXY[i * 2 + 1], maxY) << 16) | | |
| 427 TILEX_PROCF (base_srcXY[i * 2 + 0], maxX); | |
| 428 | |
| 429 if (val != base_xy[i]) { | |
| 430 bad = i; | |
| 431 break; | |
| 432 } | |
| 433 } | |
| 434 if (bad >= 0) { | |
| 435 SkDebugf("clamp-nofilter-persp failed piece %d\n", bad); | |
| 436 SkDebugf(" maxX %08x maxY %08x\n", maxX, maxY); | |
| 437 bad -= (bad & 0x7); /* align */ | |
| 438 for (int i = bad; i < bad + 8; i++) { | |
| 439 uint32_t val; | |
| 440 val = (TILEY_PROCF (base_srcXY[i * 2 + 1], maxY) << 16) | | |
| 441 TILEX_PROCF (base_srcXY[i * 2 + 0], maxX); | |
| 442 | |
| 443 SkDebugf("%d: got %08x want %08x srcXY[0] %08x srcXY[1] %08x\n", | |
| 444 i, base_xy[i], val, base_srcXY[i * 2 + 0], | |
| 445 base_srcXY[i * 2 + 1]); | |
| 446 } | |
| 447 SkDebugf ("---\n"); | |
| 448 } | |
| 449 | |
| 450 if (end_xy != xy) { | |
| 451 SkDebugf("xy ended at %08x, should be %08x\n", xy, end_xy); | |
| 452 } | |
| 453 if (end_srcXY != srcXY) { | |
| 454 SkDebugf("srcXY ended at %08x, should be %08x\n", srcXY, | |
| 455 end_srcXY); | |
| 456 } | |
| 457 } | |
| 458 #endif | |
| 459 } | |
| 460 } | |
| 461 | |
| 462 #undef DEBUG_PERSP_NOFILTER | |
| 463 | |
| 464 ////////////////////////////////////////////////////////////////////////////// | |
| 465 | |
| 466 static inline uint32_t PACK_FILTER_Y_NAME(SkFixed f, unsigned max, | |
| 467 SkFixed one PREAMBLE_PARAM_Y) { | |
| 468 unsigned i = TILEY_PROCF(f, max); | |
| 469 i = (i << 4) | TILEY_LOW_BITS(f, max); | |
| 470 return (i << 14) | (TILEY_PROCF((f + one), max)); | |
| 471 } | |
| 472 | |
| 473 static inline uint32_t PACK_FILTER_X_NAME(SkFixed f, unsigned max, | |
| 474 SkFixed one PREAMBLE_PARAM_X) { | |
| 475 unsigned i = TILEX_PROCF(f, max); | |
| 476 i = (i << 4) | TILEX_LOW_BITS(f, max); | |
| 477 return (i << 14) | (TILEX_PROCF((f + one), max)); | |
| 478 } | |
| 479 | |
| 480 static void SCALE_FILTER_NAME(const SkBitmapProcState& s, | |
| 481 uint32_t xy[], int count, int x, int y) { | |
| 482 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | | |
| 483 SkMatrix::kScale_Mask)) == 0); | |
| 484 SkASSERT(s.fInvKy == 0); | |
| 485 | |
| 486 PREAMBLE(s); | |
| 487 | |
| 488 const unsigned maxX = s.fBitmap->width() - 1; | |
| 489 const SkFixed one = s.fFilterOneX; | |
| 490 const SkFixed dx = s.fInvSx; | |
| 491 SkFixed fx; | |
| 492 | |
| 493 { | |
| 494 SkPoint pt; | |
| 495 s.fInvProc(s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, | |
| 496 SkIntToScalar(y) + SK_ScalarHalf, &pt); | |
| 497 const SkFixed fy = SkScalarToFixed(pt.fY) - (s.fFilterOneY >> 1); | |
| 498 const unsigned maxY = s.fBitmap->height() - 1; | |
| 499 // compute our two Y values up front | |
| 500 *xy++ = PACK_FILTER_Y_NAME(fy, maxY, s.fFilterOneY PREAMBLE_ARG_Y); | |
| 501 // now initialize fx | |
| 502 fx = SkScalarToFixed(pt.fX) - (one >> 1); | |
| 503 } | |
| 504 | |
| 505 #ifdef CHECK_FOR_DECAL | |
| 506 // test if we don't need to apply the tile proc | |
| 507 if (dx > 0 && | |
| 508 (unsigned)(fx >> 16) <= maxX && | |
| 509 (unsigned)((fx + dx * (count - 1)) >> 16) < maxX) { | |
| 510 decal_filter_scale_neon(xy, fx, dx, count); | |
| 511 } else | |
| 512 #endif | |
| 513 | |
| 514 if (count >= 4) { | |
| 515 int32x4_t wide_one, wide_fx, wide_fx1, wide_i, wide_lo; | |
| 516 #if 0 | |
| 517 /* verification hooks -- see below */ | |
| 518 SkFixed debug_fx = fx; | |
| 519 int count_done = 0; | |
| 520 #endif | |
| 521 | |
| 522 wide_fx = vdupq_n_s32(fx); | |
| 523 wide_fx = vsetq_lane_s32(fx+dx, wide_fx, 1); | |
| 524 wide_fx = vsetq_lane_s32(fx+dx+dx, wide_fx, 2); | |
| 525 wide_fx = vsetq_lane_s32(fx+dx+dx+dx, wide_fx, 3); | |
| 526 | |
| 527 wide_one = vdupq_n_s32(one); | |
| 528 | |
| 529 while (count >= 4) { | |
| 530 /* original expands to: | |
| 531 * unsigned i = SkClampMax((f) >> 16, max); | |
| 532 * i = (i << 4) | (((f) >> 12) & 0xF); | |
| 533 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max)); | |
| 534 */ | |
| 535 | |
| 536 /* i = SkClampMax(f>>16, maxX) */ | |
| 537 wide_i = vmaxq_s32(vshrq_n_s32(wide_fx,16), vdupq_n_s32(0)); | |
| 538 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxX)); | |
| 539 | |
| 540 /* i<<4 | TILEX_LOW_BITS(fx) */ | |
| 541 wide_lo = vshrq_n_s32(wide_fx, 12); | |
| 542 wide_i = vsliq_n_s32(wide_lo, wide_i, 4); | |
| 543 | |
| 544 /* i<<14 */ | |
| 545 wide_i = vshlq_n_s32(wide_i, 14); | |
| 546 | |
| 547 /* SkClampMax(((f + one)) >> 16, max) */ | |
| 548 wide_fx1 = vaddq_s32(wide_fx, wide_one); | |
| 549 wide_fx1 = vmaxq_s32(vshrq_n_s32(wide_fx1,16), vdupq_n_s32(0)); | |
| 550 wide_fx1 = vminq_s32(wide_fx1, vdupq_n_s32(maxX)); | |
| 551 | |
| 552 /* final combination */ | |
| 553 wide_i = vorrq_s32(wide_i, wide_fx1); | |
| 554 | |
| 555 vst1q_u32(xy, vreinterpretq_u32_s32(wide_i)); | |
| 556 | |
| 557 #if 0 | |
| 558 /* having a verification hook is a good idea */ | |
| 559 /* use debug_fx, debug_fx+dx, etc. */ | |
| 560 | |
| 561 for (int i=0;i<4;i++) { | |
| 562 uint32_t want = PACK_FILTER_X_NAME(debug_fx, maxX, one PREAMBLE_ARG_
X); | |
| 563 if (xy[i] != want) | |
| 564 { | |
| 565 /* print a nastygram */ | |
| 566 SkDebugf("clamp-filter-scale fails\n"); | |
| 567 SkDebugf("got %08x want %08x\n", xy[i], want); | |
| 568 SkDebugf("fx %08x debug_fx %08x dx %08x done %d\n", | |
| 569 fx, debug_fx, dx, count_done); | |
| 570 SkDebugf(" maxX %08x one %08x\n", maxX, one); | |
| 571 | |
| 572 } | |
| 573 debug_fx += dx; | |
| 574 count_done++; | |
| 575 } | |
| 576 #endif | |
| 577 wide_fx += vdupq_n_s32(dx+dx+dx+dx); | |
| 578 fx += dx+dx+dx+dx; | |
| 579 xy += 4; | |
| 580 count -= 4; | |
| 581 } | |
| 582 } | |
| 583 | |
| 584 while (--count >= 0) { | |
| 585 *xy++ = PACK_FILTER_X_NAME(fx, maxX, one PREAMBLE_ARG_X); | |
| 586 fx += dx; | |
| 587 } | |
| 588 } | |
| 589 | |
| 590 static void AFFINE_FILTER_NAME(const SkBitmapProcState& s, | |
| 591 uint32_t xy[], int count, int x, int y) { | |
| 592 SkASSERT(s.fInvType & SkMatrix::kAffine_Mask); | |
| 593 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | | |
| 594 SkMatrix::kScale_Mask | | |
| 595 SkMatrix::kAffine_Mask)) == 0); | |
| 596 | |
| 597 PREAMBLE(s); | |
| 598 SkPoint srcPt; | |
| 599 s.fInvProc(s.fInvMatrix, | |
| 600 SkIntToScalar(x) + SK_ScalarHalf, | |
| 601 SkIntToScalar(y) + SK_ScalarHalf, &srcPt); | |
| 602 | |
| 603 SkFixed oneX = s.fFilterOneX; | |
| 604 SkFixed oneY = s.fFilterOneY; | |
| 605 SkFixed fx = SkScalarToFixed(srcPt.fX) - (oneX >> 1); | |
| 606 SkFixed fy = SkScalarToFixed(srcPt.fY) - (oneY >> 1); | |
| 607 SkFixed dx = s.fInvSx; | |
| 608 SkFixed dy = s.fInvKy; | |
| 609 unsigned maxX = s.fBitmap->width() - 1; | |
| 610 unsigned maxY = s.fBitmap->height() - 1; | |
| 611 | |
| 612 if (count >= 4) { | |
| 613 int32x4_t wide_i, wide_lo; | |
| 614 int32x4_t wide_fx, wide_onex, wide_fx1; | |
| 615 int32x4_t wide_fy, wide_oney, wide_fy1; | |
| 616 | |
| 617 #undef AFFINE_DEBUG | |
| 618 #if defined(AFFINE_DEBUG) | |
| 619 SkFixed fyp = fy; | |
| 620 SkFixed fxp = fx; | |
| 621 uint32_t *xyp = xy; | |
| 622 int count_done = 0; | |
| 623 #endif | |
| 624 | |
| 625 wide_fx = vdupq_n_s32(fx); | |
| 626 wide_fx = vsetq_lane_s32(fx+dx, wide_fx, 1); | |
| 627 wide_fx = vsetq_lane_s32(fx+dx+dx, wide_fx, 2); | |
| 628 wide_fx = vsetq_lane_s32(fx+dx+dx+dx, wide_fx, 3); | |
| 629 | |
| 630 wide_fy = vdupq_n_s32(fy); | |
| 631 wide_fy = vsetq_lane_s32(fy+dy, wide_fy, 1); | |
| 632 wide_fy = vsetq_lane_s32(fy+dy+dy, wide_fy, 2); | |
| 633 wide_fy = vsetq_lane_s32(fy+dy+dy+dy, wide_fy, 3); | |
| 634 | |
| 635 wide_onex = vdupq_n_s32(oneX); | |
| 636 wide_oney = vdupq_n_s32(oneY); | |
| 637 | |
| 638 while (count >= 4) { | |
| 639 int32x4_t wide_x; | |
| 640 int32x4_t wide_y; | |
| 641 | |
| 642 /* do the X side, then the Y side, then interleave them */ | |
| 643 | |
| 644 /* original expands to: | |
| 645 * unsigned i = SkClampMax((f) >> 16, max); | |
| 646 * i = (i << 4) | (((f) >> 12) & 0xF); | |
| 647 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max)); | |
| 648 */ | |
| 649 | |
| 650 /* i = SkClampMax(f>>16, maxX) */ | |
| 651 wide_i = vmaxq_s32(vshrq_n_s32(wide_fx,16), vdupq_n_s32(0)); | |
| 652 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxX)); | |
| 653 | |
| 654 /* i<<4 | TILEX_LOW_BITS(fx) */ | |
| 655 wide_lo = vshrq_n_s32(wide_fx, 12); | |
| 656 wide_i = vsliq_n_s32(wide_lo, wide_i, 4); | |
| 657 | |
| 658 /* i<<14 */ | |
| 659 wide_i = vshlq_n_s32(wide_i, 14); | |
| 660 | |
| 661 /* SkClampMax(((f + one)) >> 16, max) */ | |
| 662 wide_fx1 = vaddq_s32(wide_fx, wide_onex); | |
| 663 wide_fx1 = vmaxq_s32(vshrq_n_s32(wide_fx1,16), vdupq_n_s32(0)); | |
| 664 wide_fx1 = vminq_s32(wide_fx1, vdupq_n_s32(maxX)); | |
| 665 | |
| 666 /* final combination */ | |
| 667 wide_x = vorrq_s32(wide_i, wide_fx1); | |
| 668 | |
| 669 /* And now the Y side */ | |
| 670 | |
| 671 /* i = SkClampMax(f>>16, maxX) */ | |
| 672 wide_i = vmaxq_s32(vshrq_n_s32(wide_fy,16), vdupq_n_s32(0)); | |
| 673 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxY)); | |
| 674 | |
| 675 /* i<<4 | TILEX_LOW_BITS(fx) */ | |
| 676 wide_lo = vshrq_n_s32(wide_fy, 12); | |
| 677 wide_i = vsliq_n_s32(wide_lo, wide_i, 4); | |
| 678 | |
| 679 /* i<<14 */ | |
| 680 wide_i = vshlq_n_s32(wide_i, 14); | |
| 681 | |
| 682 /* SkClampMax(((f + one)) >> 16, max) */ | |
| 683 wide_fy1 = vaddq_s32(wide_fy, wide_oney); | |
| 684 wide_fy1 = vmaxq_s32(vshrq_n_s32(wide_fy1,16), vdupq_n_s32(0)); | |
| 685 wide_fy1 = vminq_s32(wide_fy1, vdupq_n_s32(maxY)); | |
| 686 | |
| 687 /* final combination */ | |
| 688 wide_y = vorrq_s32(wide_i, wide_fy1); | |
| 689 | |
| 690 /* interleave as YXYXYXYX as part of the storing */ | |
| 691 { | |
| 692 /* vst2.32 needs side-by-side registers */ | |
| 693 register int32x4_t t_x asm("q1"); | |
| 694 register int32x4_t t_y asm("q0"); | |
| 695 | |
| 696 t_x = wide_x; t_y = wide_y; | |
| 697 asm ("vst2.32 {q0-q1},[%2] /* y=%q0 x=%q1 */" | |
| 698 : | |
| 699 : "w" (t_y), "w" (t_x), "r" (xy) | |
| 700 ); | |
| 701 } | |
| 702 | |
| 703 #if defined(AFFINE_DEBUG) | |
| 704 /* make sure we're good here -- check the 4 we just output */ | |
| 705 for (int i = 0; i<4;i++) { | |
| 706 uint32_t val; | |
| 707 val = PACK_FILTER_Y_NAME(fyp, maxY, oneY PREAMBLE_ARG_Y); | |
| 708 if (val != xy[i*2+0]) { | |
| 709 /* print a nastygram */ | |
| 710 SkDebugf("clamp-filter-affine fails\n"); | |
| 711 SkDebugf("[bad-y] got %08x want %08x\n", xy[i*2+0], val); | |
| 712 SkDebugf("fy %08x fxp %08x fyp %08x dx %08x dy %08x done %d\n", | |
| 713 fy, fxp, fyp, dx, dy, count_done); | |
| 714 SkDebugf(" maxY %08x oneY %08x\n", maxY, oneY); | |
| 715 } | |
| 716 val = PACK_FILTER_X_NAME(fxp, maxX, oneX PREAMBLE_ARG_X); | |
| 717 if (val != xy[i*2+1]) { | |
| 718 /* print a nastygram */ | |
| 719 SkDebugf("clamp-filter-affine fails\n"); | |
| 720 SkDebugf("[bad-x] got %08x want %08x\n", xy[i*2+1], val); | |
| 721 SkDebugf("fx %08x fxp %08x fyp %08x dx %08x dy %08x done %d\n", | |
| 722 fx, fxp, fyp, dx, dy, count_done); | |
| 723 SkDebugf(" maxX %08x one %08x\n", maxX, oneX); | |
| 724 } | |
| 725 fyp += dy; | |
| 726 fxp += dx; | |
| 727 count_done++; | |
| 728 } | |
| 729 #endif | |
| 730 | |
| 731 wide_fx += vdupq_n_s32(dx+dx+dx+dx); | |
| 732 fx += dx+dx+dx+dx; | |
| 733 wide_fy += vdupq_n_s32(dy+dy+dy+dy); | |
| 734 fy += dy+dy+dy+dy; | |
| 735 xy += 8; /* 4 x's, 4 y's */ | |
| 736 count -= 4; | |
| 737 } | |
| 738 } | |
| 739 | |
| 740 while (--count >= 0) { | |
| 741 /* NB: writing Y/X */ | |
| 742 *xy++ = PACK_FILTER_Y_NAME(fy, maxY, oneY PREAMBLE_ARG_Y); | |
| 743 fy += dy; | |
| 744 *xy++ = PACK_FILTER_X_NAME(fx, maxX, oneX PREAMBLE_ARG_X); | |
| 745 fx += dx; | |
| 746 } | |
| 747 } | |
| 748 | |
| 749 static void PERSP_FILTER_NAME(const SkBitmapProcState& s, | |
| 750 uint32_t* SK_RESTRICT xy, int count, | |
| 751 int x, int y) { | |
| 752 SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask); | |
| 753 | |
| 754 PREAMBLE(s); | |
| 755 unsigned maxX = s.fBitmap->width() - 1; | |
| 756 unsigned maxY = s.fBitmap->height() - 1; | |
| 757 SkFixed oneX = s.fFilterOneX; | |
| 758 SkFixed oneY = s.fFilterOneY; | |
| 759 | |
| 760 SkPerspIter iter(s.fInvMatrix, | |
| 761 SkIntToScalar(x) + SK_ScalarHalf, | |
| 762 SkIntToScalar(y) + SK_ScalarHalf, count); | |
| 763 | |
| 764 while ((count = iter.next()) != 0) { | |
| 765 const SkFixed* SK_RESTRICT srcXY = iter.getXY(); | |
| 766 | |
| 767 if (count >= 4) { | |
| 768 int32x4_t wide_i, wide_lo; | |
| 769 int32x4_t wide_fx1; | |
| 770 int32x4_t wide_fy1; | |
| 771 int32x4_t wide_x, wide_y; | |
| 772 | |
| 773 while (count >= 4) { | |
| 774 /* RBE: it's good, but: | |
| 775 * -- we spill a constant that could be easily regnerated | |
| 776 * [perhaps tweak gcc's NEON constant costs?] | |
| 777 */ | |
| 778 | |
| 779 /* load src: x-y-x-y-x-y-x-y */ | |
| 780 { | |
| 781 register int32x4_t q0 asm ("q0"); | |
| 782 register int32x4_t q1 asm ("q1"); | |
| 783 asm ("vld2.32 {q0-q1},[%2] /* x=%q0 y=%q1 */" | |
| 784 : "=w" (q0), "=w" (q1) | |
| 785 : "r" (srcXY)); | |
| 786 wide_x = q0; wide_y = q1; | |
| 787 } | |
| 788 | |
| 789 /* do the X side, then the Y side, then interleave them */ | |
| 790 | |
| 791 wide_x = vsubq_s32(wide_x, vdupq_n_s32 (oneX>>1)); | |
| 792 | |
| 793 /* original expands to: | |
| 794 * unsigned i = SkClampMax((f) >> 16, max); | |
| 795 * i = (i << 4) | (((f) >> 12) & 0xF); | |
| 796 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max)); | |
| 797 */ | |
| 798 | |
| 799 /* i = SkClampMax(f>>16, maxX) */ | |
| 800 wide_i = vmaxq_s32 (vshrq_n_s32 (wide_x, 16), vdupq_n_s32 (0)); | |
| 801 wide_i = vminq_s32 (wide_i, vdupq_n_s32 (maxX)); | |
| 802 | |
| 803 /* i<<4 | TILEX_LOW_BITS(fx) */ | |
| 804 wide_lo = vshrq_n_s32 (wide_x, 12); | |
| 805 wide_i = vsliq_n_s32 (wide_lo, wide_i, 4); | |
| 806 | |
| 807 /* i<<14 */ | |
| 808 wide_i = vshlq_n_s32 (wide_i, 14); | |
| 809 | |
| 810 /* SkClampMax(((f + one)) >> 16, max) */ | |
| 811 wide_fx1 = vaddq_s32 (wide_x, vdupq_n_s32(oneX)); | |
| 812 wide_fx1 = vmaxq_s32 (vshrq_n_s32 (wide_fx1, 16), vdupq_n_s32 (0
)); | |
| 813 wide_fx1 = vminq_s32 (wide_fx1, vdupq_n_s32 (maxX)); | |
| 814 | |
| 815 /* final combination */ | |
| 816 wide_x = vorrq_s32 (wide_i, wide_fx1); | |
| 817 | |
| 818 | |
| 819 /* And now the Y side */ | |
| 820 | |
| 821 wide_y = vsubq_s32(wide_y, vdupq_n_s32 (oneY>>1)); | |
| 822 | |
| 823 /* i = SkClampMax(f>>16, maxX) */ | |
| 824 wide_i = vmaxq_s32 (vshrq_n_s32 (wide_y, 16), vdupq_n_s32 (0)); | |
| 825 wide_i = vminq_s32 (wide_i, vdupq_n_s32 (maxY)); | |
| 826 | |
| 827 /* i<<4 | TILEX_LOW_BITS(fx) */ | |
| 828 wide_lo = vshrq_n_s32 (wide_y, 12); | |
| 829 wide_i = vsliq_n_s32 (wide_lo, wide_i, 4); | |
| 830 | |
| 831 /* i<<14 */ | |
| 832 wide_i = vshlq_n_s32 (wide_i, 14); | |
| 833 | |
| 834 /* SkClampMax(((f + one)) >> 16, max) */ | |
| 835 | |
| 836 /* wide_fy1_1 and wide_fy1_2 are just temporary variables to | |
| 837 * work-around an ICE in debug */ | |
| 838 int32x4_t wide_fy1_1 = vaddq_s32 (wide_y, vdupq_n_s32(oneY)); | |
| 839 int32x4_t wide_fy1_2 = vmaxq_s32 (vshrq_n_s32 (wide_fy1_1, 16), | |
| 840 vdupq_n_s32 (0)); | |
| 841 wide_fy1 = vminq_s32 (wide_fy1_2, vdupq_n_s32 (maxY)); | |
| 842 | |
| 843 /* final combination */ | |
| 844 wide_y = vorrq_s32 (wide_i, wide_fy1); | |
| 845 | |
| 846 /* switch them around; have to do it this way to get them | |
| 847 * in the proper registers to match our instruction */ | |
| 848 | |
| 849 /* iteration bookkeeping, ahead of the asm() for scheduling */ | |
| 850 srcXY += 2*4; | |
| 851 count -= 4; | |
| 852 | |
| 853 /* store interleaved as y-x-y-x-y-x-y-x (NB != read order) */ | |
| 854 { | |
| 855 register int32x4_t q0 asm ("q0") = wide_y; | |
| 856 register int32x4_t q1 asm ("q1") = wide_x; | |
| 857 | |
| 858 asm ("vst2.32 {q0-q1},[%2] /* y=%q0 x=%q1 */" | |
| 859 : | |
| 860 : "w" (q0), "w" (q1), "r" (xy)); | |
| 861 } | |
| 862 | |
| 863 /* on to the next iteration */ | |
| 864 /* count, srcXY are handled above */ | |
| 865 xy += 2*4; | |
| 866 } | |
| 867 } | |
| 868 | |
| 869 /* was do-while; NEON code invalidates original count>0 assumption */ | |
| 870 while (--count >= 0) { | |
| 871 /* NB: we read x/y, we write y/x */ | |
| 872 *xy++ = PACK_FILTER_Y_NAME(srcXY[1] - (oneY >> 1), maxY, | |
| 873 oneY PREAMBLE_ARG_Y); | |
| 874 *xy++ = PACK_FILTER_X_NAME(srcXY[0] - (oneX >> 1), maxX, | |
| 875 oneX PREAMBLE_ARG_X); | |
| 876 srcXY += 2; | |
| 877 } | |
| 878 } | |
| 879 } | |
| 880 | |
| 881 const SkBitmapProcState::MatrixProc MAKENAME(_Procs)[] = { | |
| 882 SCALE_NOFILTER_NAME, | |
| 883 SCALE_FILTER_NAME, | |
| 884 AFFINE_NOFILTER_NAME, | |
| 885 AFFINE_FILTER_NAME, | |
| 886 PERSP_NOFILTER_NAME, | |
| 887 PERSP_FILTER_NAME | |
| 888 }; | |
| 889 | |
| 890 #undef MAKENAME | |
| 891 #undef TILEX_PROCF | |
| 892 #undef TILEY_PROCF | |
| 893 #ifdef CHECK_FOR_DECAL | |
| 894 #undef CHECK_FOR_DECAL | |
| 895 #endif | |
| 896 | |
| 897 #undef SCALE_NOFILTER_NAME | |
| 898 #undef SCALE_FILTER_NAME | |
| 899 #undef AFFINE_NOFILTER_NAME | |
| 900 #undef AFFINE_FILTER_NAME | |
| 901 #undef PERSP_NOFILTER_NAME | |
| 902 #undef PERSP_FILTER_NAME | |
| 903 | |
| 904 #undef PREAMBLE | |
| 905 #undef PREAMBLE_PARAM_X | |
| 906 #undef PREAMBLE_PARAM_Y | |
| 907 #undef PREAMBLE_ARG_X | |
| 908 #undef PREAMBLE_ARG_Y | |
| 909 | |
| 910 #undef TILEX_LOW_BITS | |
| 911 #undef TILEY_LOW_BITS | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 23835006:
ARM Skia NEON patches - 20 - New improved BitmapProcState code (Closed)
Base URL: https://skia.googlecode.com/svn/trunk