ARM Skia NEON patches - 35 - First AArch64 support

src/opts/SkXfermode_opts_arm_neon.cpp

 #include "SkXfermode.h"
 #include "SkXfermode_proccoeff.h"
 #include "SkColorPriv.h"
 
 #include <arm_neon.h>
 #include "SkColor_opts_neon.h"
 #include "SkXfermode_opts_arm_neon.h"
 
 #define SkAlphaMulAlpha(a, b)   SkMulDiv255Round(a, b)
 
@@ skipping 23 matching lines @@
     ret = vaddq_u16(ret, vshrq_n_u16(ret, 8));
 
     ret = vshrq_n_u16(ret, 8);
 
     return ret;
 }
 
 static inline uint8x8_t SkDiv255Round_neon8_32_8(int32x4_t p1, int32x4_t p2) {
     uint16x8_t tmp;
 
+#ifdef SK_CPU_ARM64
+    tmp = vmovn_high_u32(vmovn_u32(vreinterpretq_u32_s32(p1)),
+                         vreinterpretq_u32_s32(p2));
+#else
     tmp = vcombine_u16(vmovn_u32(vreinterpretq_u32_s32(p1)),
                        vmovn_u32(vreinterpretq_u32_s32(p2)));
+#endif
 
     tmp += vdupq_n_u16(128);
     tmp += vshrq_n_u16(tmp, 8);
 
     return vshrn_n_u16(tmp, 8);
 }
 
 static inline uint16x8_t SkDiv255Round_neon8_16_16(uint16x8_t prod) {
     prod += vdupq_n_u16(128);
     prod += vshrq_n_u16(prod, 8);
 
     return vshrq_n_u16(prod, 8);
 }
 
 static inline uint8x8_t clamp_div255round_simd8_32(int32x4_t val1, int32x4_t val2) {
     uint8x8_t ret;
     uint32x4_t cmp1, cmp2;
     uint16x8_t cmp16;
     uint8x8_t cmp8, cmp8_1;
 
     // Test if <= 0
     cmp1 = vcleq_s32(val1, vdupq_n_s32(0));
     cmp2 = vcleq_s32(val2, vdupq_n_s32(0));
+#ifdef SK_CPU_ARM64
+    cmp16 = vmovn_high_u32(vmovn_u32(cmp1), cmp2);
+#else
     cmp16 = vcombine_u16(vmovn_u32(cmp1), vmovn_u32(cmp2));
+#endif
     cmp8_1 = vmovn_u16(cmp16);
 
     // Init to zero
     ret = vdup_n_u8(0);
 
     // Test if >= 255*255
     cmp1 = vcgeq_s32(val1, vdupq_n_s32(255*255));
     cmp2 = vcgeq_s32(val2, vdupq_n_s32(255*255));
+#ifdef SK_CPU_ARM64
+    cmp16 = vmovn_high_u32(vmovn_u32(cmp1), cmp2);
+#else
     cmp16 = vcombine_u16(vmovn_u32(cmp1), vmovn_u32(cmp2));
+#endif
     cmp8 = vmovn_u16(cmp16);
 
     // Insert 255 where true
     ret = vbsl_u8(cmp8, vdup_n_u8(255), ret);
 
     // Calc SkDiv255Round
     uint8x8_t div = SkDiv255Round_neon8_32_8(val1, val2);
 
     // Insert where false and previous test false
     cmp8 = cmp8 | cmp8_1;
@@ skipping 313 matching lines @@
     uint16x8_t scda = vmull_u8(sc, da);
     uint16x8_t dcsa = vmull_u8(dc, sa);
     uint16x8_t sada = vmull_u8(sa, da);
 
     // Prepare non common subexpressions
     uint16x8_t dc2, sc2;
     uint32x4_t scdc2_1, scdc2_2;
     if (overlay) {
         dc2 = vshll_n_u8(dc, 1);
         scdc2_1 = vmull_u16(vget_low_u16(dc2), vget_low_u16(vmovl_u8(sc)));
+#ifdef SK_CPU_ARM64
+        scdc2_2 = vmull_high_u16(dc2, vmovl_u8(sc));
+#else
         scdc2_2 = vmull_u16(vget_high_u16(dc2), vget_high_u16(vmovl_u8(sc)));
+#endif
     } else {
         sc2 = vshll_n_u8(sc, 1);
         scdc2_1 = vmull_u16(vget_low_u16(sc2), vget_low_u16(vmovl_u8(dc)));
+#ifdef SK_CPU_ARM64
+        scdc2_2 = vmull_high_u16(sc2, vmovl_u8(dc));
+#else
         scdc2_2 = vmull_u16(vget_high_u16(sc2), vget_high_u16(vmovl_u8(dc)));
+#endif
     }
 
     // Calc COM
     int32x4_t com1, com2;
     com1 = vreinterpretq_s32_u32(
                 vmull_u16(vget_low_u16(const255), vget_low_u16(sc_plus_dc)));
     com2 = vreinterpretq_s32_u32(
+#ifdef SK_CPU_ARM64
+                vmull_high_u16(const255, sc_plus_dc));
+#else
                 vmull_u16(vget_high_u16(const255), vget_high_u16(sc_plus_dc)));
+#endif
 
     // Calc SUB
     int32x4_t sub1, sub2;
     sub1 = vreinterpretq_s32_u32(vaddl_u16(vget_low_u16(scda), vget_low_u16(dcsa)));
+#ifdef SK_CPU_ARM64
+    sub2 = vreinterpretq_s32_u32(vaddl_high_u16(scda, dcsa));
+#else
     sub2 = vreinterpretq_s32_u32(vaddl_u16(vget_high_u16(scda), vget_high_u16(dcsa)));
+#endif
     sub1 = vsubq_s32(sub1, vreinterpretq_s32_u32(scdc2_1));
     sub2 = vsubq_s32(sub2, vreinterpretq_s32_u32(scdc2_2));
 
     // Compare 2*dc <= da
     uint16x8_t cmp;
 
     if (overlay) {
         cmp = vcleq_u16(dc2, vmovl_u8(da));
     } else {
         cmp = vcleq_u16(sc2, vmovl_u8(sa));
     }
 
     // Prepare variables
     int32x4_t val1_1, val1_2;
     int32x4_t val2_1, val2_2;
     uint32x4_t cmp1, cmp2;
 
-    cmp1 = vmovl_u16(vget_low_u16(cmp));
-    cmp1 |= vshlq_n_u32(cmp1, 16);
-    cmp2 = vmovl_u16(vget_high_u16(cmp));
-    cmp2 |= vshlq_n_u32(cmp2, 16);
+    // Doing a signed lengthening allows to save a few instructions
+    // thanks to sign extension.
+    cmp1 = vreinterpretq_u32_s32(vmovl_s16(vreinterpret_s16_u16(vget_low_u16(cmp))));
+#ifdef SK_CPU_ARM64
+    cmp2 = vreinterpretq_u32_s32(vmovl_high_s16(vreinterpretq_s16_u16(cmp)));
+#else
+    cmp2 = vreinterpretq_u32_s32(vmovl_s16(vreinterpret_s16_u16(vget_high_u16(cmp))));
+#endif
 
     // Calc COM - SUB
     val1_1 = com1 - sub1;
     val1_2 = com2 - sub2;
 
     // Calc COM + SUB - sa*da
     val2_1 = com1 + sub1;
     val2_2 = com2 + sub2;
 
     val2_1 = vsubq_s32(val2_1, vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(sada))));
+#ifdef SK_CPU_ARM64
+    val2_2 = vsubq_s32(val2_2, vreinterpretq_s32_u32(vmovl_high_u16(sada)));
+#else
     val2_2 = vsubq_s32(val2_2, vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(sada))));
+#endif
 
     // Insert where needed
     val1_1 = vbslq_s32(cmp1, val1_1, val2_1);
     val1_2 = vbslq_s32(cmp2, val1_2, val2_2);
 
     // Call the clamp_div255round function
     return clamp_div255round_simd8_32(val1_1, val1_2);
 }
 
 static inline uint8x8_t overlay_color(uint8x8_t sc, uint8x8_t dc,
@@ skipping 149 matching lines @@
     sc_plus_dc = vaddl_u8(sc, dc);
     scdc = vmull_u8(sc, dc);
 
     /* Prepare constants */
     const255 = vdupq_n_u16(255);
 
     /* Calc the first term */
     term1_1 = vreinterpretq_s32_u32(
                 vmull_u16(vget_low_u16(const255), vget_low_u16(sc_plus_dc)));
     term1_2 = vreinterpretq_s32_u32(
+#ifdef SK_CPU_ARM64
+                vmull_high_u16(const255, sc_plus_dc));
+#else
                 vmull_u16(vget_high_u16(const255), vget_high_u16(sc_plus_dc)));
+#endif
 
     /* Calc the second term */
     term2_1 = vreinterpretq_s32_u32(vshll_n_u16(vget_low_u16(scdc), 1));
+#ifdef SK_CPU_ARM64
+    term2_2 = vreinterpretq_s32_u32(vshll_high_n_u16(scdc, 1));
+#else
     term2_2 = vreinterpretq_s32_u32(vshll_n_u16(vget_high_u16(scdc), 1));
+#endif
 
     return clamp_div255round_simd8_32(term1_1 - term2_1, term1_2 - term2_2);
 }
 
 uint8x8x4_t exclusion_modeproc_neon8(uint8x8x4_t src, uint8x8x4_t dst) {
     uint8x8x4_t ret;
 
     ret.val[NEON_A] = srcover_color(src.val[NEON_A], dst.val[NEON_A]);
     ret.val[NEON_R] = exclusion_color(src.val[NEON_R], dst.val[NEON_R],
                                       src.val[NEON_A], dst.val[NEON_A]);
     ret.val[NEON_G] = exclusion_color(src.val[NEON_G], dst.val[NEON_G],
                                       src.val[NEON_A], dst.val[NEON_A]);
     ret.val[NEON_B] = exclusion_color(src.val[NEON_B], dst.val[NEON_B],
                                       src.val[NEON_A], dst.val[NEON_A]);
 
     return ret;
 }
 
 static inline uint8x8_t blendfunc_multiply_color(uint8x8_t sc, uint8x8_t dc,
                                                  uint8x8_t sa, uint8x8_t da) {
     uint32x4_t val1, val2;
     uint16x8_t scdc, t1, t2;
 
     t1 = vmull_u8(sc, vdup_n_u8(255) - da);
     t2 = vmull_u8(dc, vdup_n_u8(255) - sa);
     scdc = vmull_u8(sc, dc);
 
     val1 = vaddl_u16(vget_low_u16(t1), vget_low_u16(t2));
+#ifdef SK_CPU_ARM64
+    val2 = vaddl_high_u16(t1, t2);
+#else
     val2 = vaddl_u16(vget_high_u16(t1), vget_high_u16(t2));
+#endif
 
     val1 = vaddw_u16(val1, vget_low_u16(scdc));
+#ifdef SK_CPU_ARM64
+    val2 = vaddw_high_u16(val2, scdc);
+#else
     val2 = vaddw_u16(val2, vget_high_u16(scdc));
+#endif
 
     return clamp_div255round_simd8_32(
                 vreinterpretq_s32_u32(val1), vreinterpretq_s32_u32(val2));
 }
 
 uint8x8x4_t multiply_modeproc_neon8(uint8x8x4_t src, uint8x8x4_t dst) {
     uint8x8x4_t ret;
 
     ret.val[NEON_A] = srcover_color(src.val[NEON_A], dst.val[NEON_A]);
     ret.val[NEON_R] = blendfunc_multiply_color(src.val[NEON_R], dst.val[NEON_R],
@@ skipping 23 matching lines @@
 
     SkXfermodeProc proc = this->getProc();
     SkXfermodeProcSIMD procSIMD = reinterpret_cast<SkXfermodeProcSIMD>(fProcSIMD);
     SkASSERT(procSIMD != NULL);
 
     if (NULL == aa) {
         // Unrolled NEON code
         while (count >= 8) {
             uint8x8x4_t vsrc, vdst, vres;
 
+#ifdef SK_CPU_ARM64
+            vsrc = vld4_u8((uint8_t*)src);
+            vdst = vld4_u8((uint8_t*)dst);
+#else
 #if (__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 6))
             asm volatile (
                 "vld4.u8    %h[vsrc], [%[src]]!  \t\n"
                 "vld4.u8    %h[vdst], [%[dst]]   \t\n"
                 : [vsrc] "=w" (vsrc), [vdst] "=w" (vdst), [src] "+&r" (src)
                 : [dst] "r" (dst)
                 :
             );
 #else
             register uint8x8_t d0 asm("d0");
@@ skipping 12 matching lines @@
                   "=w" (d4), "=w" (d5), "=w" (d6), "=w" (d7),
                   [src] "+&r" (src)
                 : [dst] "r" (dst)
                 :
             );
             vsrc.val[0] = d0; vdst.val[0] = d4;
             vsrc.val[1] = d1; vdst.val[1] = d5;
             vsrc.val[2] = d2; vdst.val[2] = d6;
             vsrc.val[3] = d3; vdst.val[3] = d7;
 #endif
+#endif // #ifdef SK_CPU_ARM64
 
             vres = procSIMD(vsrc, vdst);
 
             vst4_u8((uint8_t*)dst, vres);
 
             count -= 8;
             dst += 8;
+#ifdef SK_CPU_ARM64
+            src += 8;
+#endif
         }
         // Leftovers
         for (int i = 0; i < count; i++) {
             dst[i] = proc(src[i], dst[i]);
         }
     } else {
         for (int i = count - 1; i >= 0; --i) {
             unsigned a = aa[i];
             if (0 != a) {
                 SkPMColor dstC = dst[i];
@@ skipping 16 matching lines @@
     SkXfermodeProcSIMD procSIMD = reinterpret_cast<SkXfermodeProcSIMD>(fProcSIMD);
     SkASSERT(procSIMD != NULL);
 
     if (NULL == aa) {
         while(count >= 8) {
             uint16x8_t vdst, vres16;
             uint8x8x4_t vdst32, vsrc, vres;
 
             vdst = vld1q_u16(dst);
 
+#ifdef SK_CPU_ARM64
+            vsrc = vld4_u8((uint8_t*)src);
+#else
 #if (__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 6))
             asm volatile (
                 "vld4.u8    %h[vsrc], [%[src]]!  \t\n"
                 : [vsrc] "=w" (vsrc), [src] "+&r" (src)
                 : :
             );
 #else
             register uint8x8_t d0 asm("d0");
             register uint8x8_t d1 asm("d1");
             register uint8x8_t d2 asm("d2");
             register uint8x8_t d3 asm("d3");
 
             asm volatile (
                 "vld4.u8    {d0-d3},[%[src]]!;"
                 : "=w" (d0), "=w" (d1), "=w" (d2), "=w" (d3),
                   [src] "+&r" (src)
                 : :
             );
             vsrc.val[0] = d0;
             vsrc.val[1] = d1;
             vsrc.val[2] = d2;
             vsrc.val[3] = d3;
 #endif
+#endif // #ifdef SK_CPU_ARM64
 
             vdst32 = SkPixel16ToPixel32_neon8(vdst);
             vres = procSIMD(vsrc, vdst32);
             vres16 = SkPixel32ToPixel16_neon8(vres);
 
             vst1q_u16(dst, vres16);
 
             count -= 8;
             dst += 8;
+#ifdef SK_CPU_ARM64
+            src += 8;
+#endif
         }
         for (int i = 0; i < count; i++) {
             SkPMColor dstC = SkPixel16ToPixel32(dst[i]);
             dst[i] = SkPixel32ToPixel16_ToU16(proc(src[i], dstC));
         }
     } else {
         for (int i = count - 1; i >= 0; --i) {
             unsigned a = aa[i];
             if (0 != a) {
                 SkPMColor dstC = SkPixel16ToPixel32(dst[i]);
@@ skipping 100 matching lines @@
 
     if (procSIMD != NULL) {
         return SkNEW_ARGS(SkNEONProcCoeffXfermode, (rec, mode, procSIMD));
     }
     return NULL;
 }
 
 SkXfermodeProc SkPlatformXfermodeProcFactory_impl_neon(SkXfermode::Mode mode) {
     return gNEONXfermodeProcs1[mode];
 }