ARM Skia NEON patches - 39 - arm64 565 blitters

src/opts/SkBlitRow_opts_arm_neon.cpp

 /*
  * Copyright 2012 The Android Open Source Project
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 #include "SkBlitRow_opts_arm_neon.h"
 
 #include "SkBlitMask.h"
 #include "SkBlitRow.h"
 #include "SkColorPriv.h"
 #include "SkDither.h"
 #include "SkMathPriv.h"
 #include "SkUtils.h"
 
 #include "SkColor_opts_neon.h"
 #include <arm_neon.h>
 
-#ifdef SK_CPU_ARM32
+#ifdef SK_CPU_ARM64
+static inline uint8x8x4_t sk_vld4_u8_arm64_3(const SkPMColor* SK_RESTRICT & src) {
+    uint8x8x4_t vsrc;
+    uint8x8_t vsrc_0, vsrc_1, vsrc_2;
+
+    asm (
+        "ld4    {v0.8b - v3.8b}, [%[src]], #32 \t\n"
+        "mov    %[vsrc0].8b, v0.8b             \t\n"
+        "mov    %[vsrc1].8b, v1.8b             \t\n"
+        "mov    %[vsrc2].8b, v2.8b             \t\n"
+        : [vsrc0] "=w" (vsrc_0), [vsrc1] "=w" (vsrc_1),
+          [vsrc2] "=w" (vsrc_2), [src] "+&r" (src)
+        : : "v0", "v1", "v2", "v3"
+    );
+
+    vsrc.val[0] = vsrc_0;
+    vsrc.val[1] = vsrc_1;
+    vsrc.val[2] = vsrc_2;
+
+    return vsrc;
+}
+
+static inline uint8x8x4_t sk_vld4_u8_arm64_4(const SkPMColor* SK_RESTRICT & src) {
+    uint8x8x4_t vsrc;
+    uint8x8_t vsrc_0, vsrc_1, vsrc_2, vsrc_3;
+
+    asm (
+        "ld4    {v0.8b - v3.8b}, [%[src]], #32 \t\n"
+        "mov    %[vsrc0].8b, v0.8b             \t\n"
+        "mov    %[vsrc1].8b, v1.8b             \t\n"
+        "mov    %[vsrc2].8b, v2.8b             \t\n"
+        "mov    %[vsrc3].8b, v3.8b             \t\n"
+        : [vsrc0] "=w" (vsrc_0), [vsrc1] "=w" (vsrc_1),
+          [vsrc2] "=w" (vsrc_2), [vsrc3] "=w" (vsrc_3),
+          [src] "+&r" (src)
+        : : "v0", "v1", "v2", "v3"
+    );
+
+    vsrc.val[0] = vsrc_0;
+    vsrc.val[1] = vsrc_1;
+    vsrc.val[2] = vsrc_2;
+    vsrc.val[3] = vsrc_3;
+
+    return vsrc;
+}
+#endif
+
 void S32_D565_Opaque_neon(uint16_t* SK_RESTRICT dst,
                            const SkPMColor* SK_RESTRICT src, int count,
                            U8CPU alpha, int /*x*/, int /*y*/) {
     SkASSERT(255 == alpha);
 
     while (count >= 8) {
         uint8x8x4_t vsrc;
         uint16x8_t vdst;
 
         // Load
+#ifdef SK_CPU_ARM64
+        vsrc = sk_vld4_u8_arm64_3(src);
+#else
         vsrc = vld4_u8((uint8_t*)src);
+        src += 8;
+#endif
 
         // Convert src to 565
         vdst = SkPixel32ToPixel16_neon8(vsrc);
 
         // Store
         vst1q_u16(dst, vdst);
 
         // Prepare next iteration
         dst += 8;
-        src += 8;
         count -= 8;
     };
 
     // Leftovers
     while (count > 0) {
         SkPMColor c = *src++;
         SkPMColorAssert(c);
         *dst = SkPixel32ToPixel16_ToU16(c);
         dst++;
         count--;
     };
 }
 
 void S32_D565_Blend_neon(uint16_t* SK_RESTRICT dst,
                           const SkPMColor* SK_RESTRICT src, int count,
                           U8CPU alpha, int /*x*/, int /*y*/) {
     SkASSERT(255 > alpha);
 
     uint16x8_t vmask_blue, vscale;
 
     // prepare constants
     vscale = vdupq_n_u16(SkAlpha255To256(alpha));
     vmask_blue = vmovq_n_u16(0x1F);
 
     while (count >= 8) {
+        uint8x8x4_t vsrc;
         uint16x8_t vdst, vdst_r, vdst_g, vdst_b;
         uint16x8_t vres_r, vres_g, vres_b;
-        uint8x8_t vsrc_r, vsrc_g, vsrc_b;
 
         // Load src
+#ifdef SK_CPU_ARM64
+        vsrc = sk_vld4_u8_arm64_3(src);
+#else
         {
         register uint8x8_t d0 asm("d0");
         register uint8x8_t d1 asm("d1");
         register uint8x8_t d2 asm("d2");
         register uint8x8_t d3 asm("d3");
 
         asm (
             "vld4.8    {d0-d3},[%[src]]!"
             : "=w" (d0), "=w" (d1), "=w" (d2), "=w" (d3), [src] "+&r" (src)
             :
         );
-        vsrc_g = d1;
-#if SK_PMCOLOR_BYTE_ORDER(B,G,R,A)
-        vsrc_r = d2; vsrc_b = d0;
-#elif SK_PMCOLOR_BYTE_ORDER(R,G,B,A)
-        vsrc_r = d0; vsrc_b = d2;
+        vsrc.val[0] = d0;
+        vsrc.val[1] = d1;
+        vsrc.val[2] = d2;
+        }
 #endif
-        }
 
         // Load and unpack dst
         vdst = vld1q_u16(dst);
         vdst_g = vshlq_n_u16(vdst, 5);        // shift green to top of lanes
         vdst_b = vandq_u16(vdst, vmask_blue); // extract blue
         vdst_r = vshrq_n_u16(vdst, 6+5);      // extract red
         vdst_g = vshrq_n_u16(vdst_g, 5+5);    // extract green
 
-        // Shift src to 565
-        vsrc_r = vshr_n_u8(vsrc_r, 3);    // shift red to 565 range
-        vsrc_g = vshr_n_u8(vsrc_g, 2);    // shift green to 565 range
-        vsrc_b = vshr_n_u8(vsrc_b, 3);    // shift blue to 565 range
+        // Shift src to 565 range
+        vsrc.val[NEON_R] = vshr_n_u8(vsrc.val[NEON_R], 3);
+        vsrc.val[NEON_G] = vshr_n_u8(vsrc.val[NEON_G], 2);
+        vsrc.val[NEON_B] = vshr_n_u8(vsrc.val[NEON_B], 3);
 
         // Scale src - dst
-        vres_r = vmovl_u8(vsrc_r) - vdst_r;
-        vres_g = vmovl_u8(vsrc_g) - vdst_g;
-        vres_b = vmovl_u8(vsrc_b) - vdst_b;
+        vres_r = vmovl_u8(vsrc.val[NEON_R]) - vdst_r;
+        vres_g = vmovl_u8(vsrc.val[NEON_G]) - vdst_g;
+        vres_b = vmovl_u8(vsrc.val[NEON_B]) - vdst_b;
 
         vres_r = vshrq_n_u16(vres_r * vscale, 8);
         vres_g = vshrq_n_u16(vres_g * vscale, 8);
         vres_b = vshrq_n_u16(vres_b * vscale, 8);
 
         vres_r += vdst_r;
         vres_g += vdst_g;
         vres_b += vdst_b;
 
         // Combine
@@ skipping 12 matching lines @@
             SkPMColorAssert(c);
             uint16_t d = *dst;
             *dst++ = SkPackRGB16(
                     SkAlphaBlend(SkPacked32ToR16(c), SkGetPackedR16(d), scale),
                     SkAlphaBlend(SkPacked32ToG16(c), SkGetPackedG16(d), scale),
                     SkAlphaBlend(SkPacked32ToB16(c), SkGetPackedB16(d), scale));
         } while (--count != 0);
     }
 }
 
+#ifdef SK_CPU_ARM32
 void S32A_D565_Opaque_neon(uint16_t* SK_RESTRICT dst,
                            const SkPMColor* SK_RESTRICT src, int count,
                            U8CPU alpha, int /*x*/, int /*y*/) {
     SkASSERT(255 == alpha);
 
     if (count >= 8) {
         uint16_t* SK_RESTRICT keep_dst = 0;
 
         asm volatile (
                       "ands       ip, %[count], #7            \n\t"
@@ skipping 157 matching lines @@
 
                       "21:                                        \n\t"
                       : [count] "+r" (count)
                       : [dst] "r" (dst), [keep_dst] "r" (keep_dst), [src] "r" (src)
                       : "ip", "cc", "memory", "d0","d1","d2","d3","d4","d5","d6","d7",
                       "d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28","d29",
                       "d30","d31"
                       );
     }
 }
+#endif
 
 static inline uint16x8_t SkDiv255Round_neon8(uint16x8_t prod) {
     prod += vdupq_n_u16(128);
     prod += vshrq_n_u16(prod, 8);
     return vshrq_n_u16(prod, 8);
 }
 
 void S32A_D565_Blend_neon(uint16_t* SK_RESTRICT dst,
                           const SkPMColor* SK_RESTRICT src, int count,
                           U8CPU alpha, int /*x*/, int /*y*/) {
@@ skipping 13 matching lines @@
         valpha = vdup_n_u8(alpha);
         vmask_blue = vmovq_n_u16(SK_B16_MASK);
 
         do {
             uint16x8_t vdst, vdst_r, vdst_g, vdst_b;
             uint16x8_t vres_a, vres_r, vres_g, vres_b;
             uint8x8x4_t vsrc;
 
             // load pixels
             vdst = vld1q_u16(dst);
+#ifdef SK_CPU_ARM64
+            vsrc = sk_vld4_u8_arm64_4(src);
+#else
 #if (__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 6))

[mtklein, 2014/06/06 14:41:17]

Think it makes sense to follow up and do the same for 32-bit side, given that we
have what looks like a couple different strategies here too?  Then all the code
can call sk_vld4_u8_3 or sk_vld4_u8_4 as needed?

             asm (
                 "vld4.u8 %h[vsrc], [%[src]]!"
                 : [vsrc] "=w" (vsrc), [src] "+&r" (src)
                 : :
             );
 #else
             register uint8x8_t d0 asm("d0");
             register uint8x8_t d1 asm("d1");
             register uint8x8_t d2 asm("d2");
             register uint8x8_t d3 asm("d3");
 
             asm volatile (
                 "vld4.u8    {d0-d3},[%[src]]!;"
                 : "=w" (d0), "=w" (d1), "=w" (d2), "=w" (d3),
                   [src] "+&r" (src)
                 : :
             );
             vsrc.val[0] = d0;
             vsrc.val[1] = d1;
             vsrc.val[2] = d2;
             vsrc.val[3] = d3;
 #endif
+#endif // #ifdef SK_CPU_ARM64
 
 
             // deinterleave dst
             vdst_g = vshlq_n_u16(vdst, SK_R16_BITS);        // shift green to top of lanes
             vdst_b = vdst & vmask_blue;                     // extract blue
             vdst_r = vshrq_n_u16(vdst, SK_R16_SHIFT);       // extract red
             vdst_g = vshrq_n_u16(vdst_g, SK_R16_BITS + SK_B16_BITS); // extract green
 
             // shift src to 565
             vsrc.val[NEON_R] = vshr_n_u8(vsrc.val[NEON_R], 8 - SK_R16_BITS);
@@ skipping 79 matching lines @@
         const uint8_t *dstart = &gDitherMatrix_Neon[(y&3)*12 + (x&3)];
 
         uint8x8_t vdither = vld1_u8(dstart);         // load dither values
         uint8x8_t vdither_g = vshr_n_u8(vdither, 1); // calc. green dither values
 
         int16x8_t vscale = vdupq_n_s16(scale);        // duplicate scale into neon reg
         uint16x8_t vmask_b = vdupq_n_u16(0x1F);         // set up blue mask
 
         do {
 
+            uint8x8x4_t vsrc;
             uint8x8_t vsrc_r, vsrc_g, vsrc_b;
             uint8x8_t vsrc565_r, vsrc565_g, vsrc565_b;
             uint16x8_t vsrc_dit_r, vsrc_dit_g, vsrc_dit_b;
             uint16x8_t vsrc_res_r, vsrc_res_g, vsrc_res_b;
             uint16x8_t vdst;
             uint16x8_t vdst_r, vdst_g, vdst_b;
             int16x8_t vres_r, vres_g, vres_b;
             int8x8_t vres8_r, vres8_g, vres8_b;
 
             // Load source and add dither
+#ifdef SK_CPU_ARM64
+            vsrc = sk_vld4_u8_arm64_3(src);
+#else
             {
             register uint8x8_t d0 asm("d0");
             register uint8x8_t d1 asm("d1");
             register uint8x8_t d2 asm("d2");
             register uint8x8_t d3 asm("d3");
 
             asm (
-                "vld4.8    {d0-d3},[%[src]]!  /* r=%P0 g=%P1 b=%P2 a=%P3 */"
+                "vld4.8    {d0-d3},[%[src]]! "
                 : "=w" (d0), "=w" (d1), "=w" (d2), "=w" (d3), [src] "+&r" (src)
                 :
             );
-            vsrc_g = d1;
-#if SK_PMCOLOR_BYTE_ORDER(B,G,R,A)
-            vsrc_r = d2; vsrc_b = d0;
-#elif SK_PMCOLOR_BYTE_ORDER(R,G,B,A)
-            vsrc_r = d0; vsrc_b = d2;
+            vsrc.val[0] = d0;
+            vsrc.val[1] = d1;
+            vsrc.val[2] = d2;
+            }
 #endif
-            }
+            vsrc_r = vsrc.val[NEON_R];
+            vsrc_g = vsrc.val[NEON_G];
+            vsrc_b = vsrc.val[NEON_B];
 
             vsrc565_g = vshr_n_u8(vsrc_g, 6); // calc. green >> 6
             vsrc565_r = vshr_n_u8(vsrc_r, 5); // calc. red >> 5
             vsrc565_b = vshr_n_u8(vsrc_b, 5); // calc. blue >> 5
 
             vsrc_dit_g = vaddl_u8(vsrc_g, vdither_g); // add in dither to green and widen
             vsrc_dit_r = vaddl_u8(vsrc_r, vdither);   // add in dither to red and widen
             vsrc_dit_b = vaddl_u8(vsrc_b, vdither);   // add in dither to blue and widen
 
             vsrc_dit_r = vsubw_u8(vsrc_dit_r, vsrc565_r);  // sub shifted red from result
@@ skipping 60 matching lines @@
             sb = SkDITHER_B32To565(sb, dither);
 
             uint16_t d = *dst;
             *dst++ = SkPackRGB16(SkAlphaBlend(sr, SkGetPackedR16(d), scale),
                                  SkAlphaBlend(sg, SkGetPackedG16(d), scale),
                                  SkAlphaBlend(sb, SkGetPackedB16(d), scale));
             DITHER_INC_X(x);
         } while (--count != 0);
     }
 }
-#endif
 
 void S32A_Opaque_BlitRow32_neon(SkPMColor* SK_RESTRICT dst,
                                 const SkPMColor* SK_RESTRICT src,
                                 int count, U8CPU alpha) {
 
     SkASSERT(255 == alpha);
     if (count > 0) {
 
 
     uint8x8_t alpha_mask;
@@ skipping 450 matching lines @@
     uint16_t *pc = (uint16_t*) p;
     sprintf(buf,"%8s:", str);
     len = (len / sizeof(uint16_t));    /* passed as bytes */
     for(i=0;i<len;i++) {
         sprintf(tbuf, " %04x", pc[i]);
         strcat(buf, tbuf);
     }
     SkDebugf("%s\n", buf);
 }
 #endif
+#endif // #ifdef SK_CPU_ARM32
 
 void S32A_D565_Opaque_Dither_neon (uint16_t * SK_RESTRICT dst,
                                    const SkPMColor* SK_RESTRICT src,
                                    int count, U8CPU alpha, int x, int y) {
     SkASSERT(255 == alpha);
 
 #define    UNROLL    8
 
     if (count >= UNROLL) {
 
 #if defined(DEBUG_OPAQUE_DITHER)
     uint16_t tmpbuf[UNROLL];
     int td[UNROLL];
     int tdv[UNROLL];
     int ta[UNROLL];
     int tap[UNROLL];
     uint16_t in_dst[UNROLL];
     int offset = 0;
     int noisy = 0;
 #endif
 
     uint8x8_t dbase;
     const uint8_t *dstart = &gDitherMatrix_Neon[(y&3)*12 + (x&3)];
     dbase = vld1_u8(dstart);
 
         do {
+        uint8x8x4_t vsrc;
         uint8x8_t sr, sg, sb, sa, d;
         uint16x8_t dst8, scale8, alpha8;
         uint16x8_t dst_r, dst_g, dst_b;
 
 #if defined(DEBUG_OPAQUE_DITHER)
         // calculate 8 elements worth into a temp buffer
         {
         int my_y = y;
         int my_x = x;
         SkPMColor* my_src = (SkPMColor*)src;
@@ skipping 30 matching lines @@
                 tmpbuf[i] = *my_dst;
                 ta[i] = tdv[i] = td[i] = 0xbeef;
             }
             in_dst[i] = *my_dst;
             my_dst += 1;
             DITHER_INC_X(my_x);
         }
         }
 #endif
 
-
+#ifdef SK_CPU_ARM64
+        vsrc = sk_vld4_u8_arm64_4(src);
+#else
         {
         register uint8x8_t d0 asm("d0");
         register uint8x8_t d1 asm("d1");
         register uint8x8_t d2 asm("d2");
         register uint8x8_t d3 asm("d3");
 
-        asm ("vld4.8    {d0-d3},[%[src]]!  /* r=%P0 g=%P1 b=%P2 a=%P3 */"
+        asm ("vld4.8    {d0-d3},[%[src]]! "
             : "=w" (d0), "=w" (d1), "=w" (d2), "=w" (d3), [src] "+r" (src)
             :
         );
-#if SK_PMCOLOR_BYTE_ORDER(B,G,R,A)
-            sr = d2; sg = d1; sb = d0; sa = d3;
-#elif SK_PMCOLOR_BYTE_ORDER(R,G,B,A)
-            sr = d0; sg = d1; sb = d2; sa = d3;
+        vsrc.val[0] = d0;
+        vsrc.val[1] = d1;
+        vsrc.val[2] = d2;
+        vsrc.val[3] = d3;
+        }
 #endif
-        }
+        sa = vsrc.val[NEON_A];
+        sr = vsrc.val[NEON_R];
+        sg = vsrc.val[NEON_G];
+        sb = vsrc.val[NEON_B];
 
         /* calculate 'd', which will be 0..7
          * dbase[] is 0..7; alpha is 0..256; 16 bits suffice
          */
         alpha8 = vmovl_u8(dbase);
         alpha8 = vmlal_u8(alpha8, sa, dbase);
         d = vshrn_n_u16(alpha8, 8);    // narrowing too
 
         // sr = sr - (sr>>5) + d
         /* watching for 8-bit overflow.  d is 0..7; risky range of
@@ skipping 123 matching lines @@
 #define    UNROLL    8
     if (count >= UNROLL) {
     uint8x8_t d;
     const uint8_t *dstart = &gDitherMatrix_Neon[(y&3)*12 + (x&3)];
     d = vld1_u8(dstart);
 
     while (count >= UNROLL) {
         uint8x8_t sr, sg, sb;
         uint16x8_t dr, dg, db;
         uint16x8_t dst8;
+        uint8x8x4_t vsrc;
 
+#ifdef SK_CPU_ARM64
+        vsrc = sk_vld4_u8_arm64_3(src);
+#else
         {
         register uint8x8_t d0 asm("d0");
         register uint8x8_t d1 asm("d1");
         register uint8x8_t d2 asm("d2");
         register uint8x8_t d3 asm("d3");
 
         asm (
-            "vld4.8    {d0-d3},[%[src]]!  /* r=%P0 g=%P1 b=%P2 a=%P3 */"
+            "vld4.8    {d0-d3},[%[src]]! "
             : "=w" (d0), "=w" (d1), "=w" (d2), "=w" (d3), [src] "+&r" (src)
             :
         );
-        sg = d1;
-#if SK_PMCOLOR_BYTE_ORDER(B,G,R,A)
-        sr = d2; sb = d0;
-#elif SK_PMCOLOR_BYTE_ORDER(R,G,B,A)
-        sr = d0; sb = d2;
+        vsrc.val[0] = d0;
+        vsrc.val[1] = d1;
+        vsrc.val[2] = d2;
+        }
 #endif
-        }
+        sr = vsrc.val[NEON_R];
+        sg = vsrc.val[NEON_G];
+        sb = vsrc.val[NEON_B];
+
         /* XXX: if we want to prefetch, hide it in the above asm()
          * using the gcc __builtin_prefetch(), the prefetch will
          * fall to the bottom of the loop -- it won't stick up
          * at the top of the loop, just after the vld4.
          */
 
         // sr = sr - (sr>>5) + d
         sr = vsub_u8(sr, vshr_n_u8(sr, 5));
         dr = vaddl_u8(sr, d);
 
@@ skipping 47 matching lines @@
             SkPMColor c = *src++;
             SkPMColorAssert(c);
             SkASSERT(SkGetPackedA32(c) == 255);
 
             unsigned dither = DITHER_VALUE(x);
             *dst++ = SkDitherRGB32To565(c, dither);
             DITHER_INC_X(x);
         } while (--count != 0);
     }
 }
-#endif
 
 void Color32_arm_neon(SkPMColor* dst, const SkPMColor* src, int count,
                       SkPMColor color) {
     if (count <= 0) {
         return;
     }
 
     if (0 == color) {
         if (src != dst) {
             memcpy(dst, src, count * sizeof(SkPMColor));
@@ skipping 85 matching lines @@
         *dst = color + SkAlphaMulQ(*src, scale);
         src += 1;
         dst += 1;
         count--;
     }
 }
 
 ///////////////////////////////////////////////////////////////////////////////
 
 const SkBlitRow::Proc sk_blitrow_platform_565_procs_arm_neon[] = {
-#ifdef SK_CPU_ARM32
     // no dither
     S32_D565_Opaque_neon,
     S32_D565_Blend_neon,
+#ifdef SK_CPU_ARM32
     S32A_D565_Opaque_neon,
+#else
+    NULL,
+#endif
     S32A_D565_Blend_neon,
 
     // dither
     S32_D565_Opaque_Dither_neon,
     S32_D565_Blend_Dither_neon,
     S32A_D565_Opaque_Dither_neon,
     NULL,   // S32A_D565_Blend_Dither
-#else
-    NULL, NULL, NULL, NULL,
-    NULL, NULL, NULL, NULL
-#endif
 };
 
 const SkBlitRow::Proc32 sk_blitrow_platform_32_procs_arm_neon[] = {
     NULL,   // S32_Opaque,
     S32_Blend_BlitRow32_neon,        // S32_Blend,
     /*
      * We have two choices for S32A_Opaque procs. The one reads the src alpha
      * value and attempts to optimize accordingly.  The optimization is
      * sensitive to the source content and is not a win in all cases. For
      * example, if there are a lot of transitions between the alpha states,
      * the performance will almost certainly be worse.  However, for many
      * common cases the performance is equivalent or better than the standard
      * case where we do not inspect the src alpha.
      */
 #if SK_A32_SHIFT == 24
     // This proc assumes the alpha value occupies bits 24-32 of each SkPMColor
     S32A_Opaque_BlitRow32_neon_src_alpha,   // S32A_Opaque,
 #else
     S32A_Opaque_BlitRow32_neon,     // S32A_Opaque,
 #endif
 #ifdef SK_CPU_ARM32
     S32A_Blend_BlitRow32_neon        // S32A_Blend
 #else
     NULL
 #endif
 };