Add SSSE3 Optimizations for premul and swap

src/opts/SkSwizzler_opts.h

 /*
  * Copyright 2016 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 #ifndef SkSwizzler_opts_DEFINED
 #define SkSwizzler_opts_DEFINED
 
@@ skipping 156 matching lines @@
         // Store 8 pixels.
         vst4_u8((uint8_t*) dst, bgra);
         src += 8;
         dst += 8;
         count -= 8;
     }
 
     swaprb_xxxa_portable(dst, src, count);
 }
 
+#elif SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSSE3
+
+template <bool kSwapRB>
+static void premul_xxxa_should_swaprb(uint32_t dst[], const uint32_t src[], int count) {

[msarett, 2016/01/18 20:35:05]

There are a lot of different ways to implement this.  Specifically with the
unpacking and packing of pixels.  I'm not certain this is the fastest way, but
it's a start.

+    const __m128i zeros = _mm_setzero_si128();
+    const __m128i _128 = _mm_set1_epi16(128);
+    const __m128i _257 = _mm_set1_epi16(257);
+    const __m128i combine = _mm_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);
+    __m128i split;
+    if (kSwapRB) {
+        split = _mm_set_epi8(15, 3, 7, 11, 14, 2, 6, 10, 13, 1, 5, 9, 12, 0, 4, 8);
+    } else {
+        split = combine;
+    }
+
+    while (count >= 8) {
+        __m128i argb_lo = _mm_loadu_si128((const __m128i*) src);
+        __m128i argb_hi = _mm_loadu_si128((const __m128i*) (src + 4));
+
+        // argb_argb_argb_argb -> aaaa_rrrr_gggg_bbbb

[mtklein, 2016/01/19 15:59:14]

Let's kick some of these comments a little bit higher-level:

// We'll load 8 pixels into 4 registers, each holding a 16-bit component plane.

// First just load the 8 interlaced pixels.
__m128i lo = _mm_loadu_si128(... +0), // bgrabgra bgrabgra
        hi = _mm_loadu_si128(... +4); // BGRABGRA BGRABGRA

// Swizzle them to 8-bit planar.
lo = _mm_shuffle_epi8(lo, planar);    // bbbbgggg rrrraaaa
hi = _mm_shuffle_epi8(hi, planar);    // BBBBGGGG RRRRAAAA
__m128i bg = _mm_unpacklo(...),       // bbbbBBBB ggggGGGG
        ra = _mm_unpackhi(...);       // rrrrRRRR aaaaAAAA

// Unpack to 16-bit planar in four registers.
__m128i b = _mm_unpacklo(...),        // b_b_b_b_ B_B_B_B_
        ...;

// OK, premultiply!  (x+127)/255 == ((x+128)*257)>>16 for 0 <= x <= 255*255.
...

[msarett, 2016/01/19 17:34:38]

Done.

Ugggh, for some reason I thought the rest of this file was written as ARGB (so I
did that on purpose).  But I'm realizing now that it's BGRA.  And I agree that
BGRA is easier to think about.

+        argb_lo = _mm_shuffle_epi8(argb_lo, combine);
+        argb_hi = _mm_shuffle_epi8(argb_hi, combine);
+
+        // aaaa_rrrr_gggg_bbbb -> aaaa_aaaa_rrrr_rrrr
+        __m128i ar = _mm_unpackhi_epi32(argb_lo, argb_hi);
+        // aaaa_rrrr_gggg_bbbb -> gggg_gggg_bbbb_bbbb
+        __m128i gb = _mm_unpacklo_epi32(argb_lo, argb_hi);
+
+        // xxxx_xxxx_yyyy_yyyy -> 0x0x_0x0x_0x0x_0x0x
+        // xxxx_xxxx_yyyy_yyyy -> 0y0y_0y0y_0y0y_0y0y
+        __m128i a = _mm_unpackhi_epi8(ar, zeros);
+        __m128i r = _mm_unpacklo_epi8(ar, zeros);
+        __m128i g = _mm_unpackhi_epi8(gb, zeros);
+        __m128i b = _mm_unpacklo_epi8(gb, zeros);
+
+        // (x + 127) / 255 == ((x + 128) * 257) >> 16 for 0 <= x <= 255 * 255

[msarett, 2016/01/18 20:35:05]

Thanks to Mike for this insight.

+        // Note that _mm_mulhi_epu16 performs the entire (y * 257) >> 16.
+        r = _mm_mulhi_epu16(_mm_add_epi16(_mm_mullo_epi16(a, r), _128), _257);

[mtklein, 2016/01/19 15:59:14]

This may be a matter of personal preference, but you might consider:

auto scale = [](__m128i x, __m128i y) { return _mm_mulhi_epu16(...); };
r = scale(r,a);
g = scale(g,a);
b = scale(b,a);

[msarett, 2016/01/19 17:34:38]

Leaving as is, though I'm kind of indifferent.

I needed to pass references to _128 and _257 to "scale" in order to get it to
compile, and I found it a bit confusing.

+        g = _mm_mulhi_epu16(_mm_add_epi16(_mm_mullo_epi16(a, g), _128), _257);
+        b = _mm_mulhi_epu16(_mm_add_epi16(_mm_mullo_epi16(a, b), _128), _257);
+
+        // aaaa_rrrr_aaaa_rrrr

[mtklein, 2016/01/19 15:59:14]

I think we can do this repacking as something like:

__m128i bg = b | (g << 8)),
        ra = r | (a << 8)),
        lo = unpacklo_epi16(bg, ra),
        hi = unpackhi_epi16(bg, ra);

if (kSwapRB) {
    lo = shuffle_epi8(lo, swapRB)
    hi = shuffle_epi8(hi, swapRB)
}
storeu_si128(... +0, lo)
storeu_si128(... +4, hi)

Does that work?  I think that makes the non-swapRB path a bit shorter, and the
swapRB path no longer.

[msarett, 2016/01/19 17:34:37]

Yes this is better!

Let's even swap BR in the "swizzle to planar step".  Then it is the same cost as
not-swapping.

+        ar = _mm_shuffle_epi32(_mm_packus_epi16(r, a), 0xD8);
+        // gggg_bbbb_gggg_bbbb
+        gb = _mm_shuffle_epi32(_mm_packus_epi16(b, g), 0xD8);
+
+        // aaaa_rrrr_gggg_bbbb
+        argb_lo = _mm_unpacklo_epi64(gb, ar);
+        argb_hi = _mm_unpackhi_epi64(gb, ar);
+
+        // aaaa_rrrr_gggg_bbbb -> argb_argb_argb_argb
+        argb_lo = _mm_shuffle_epi8(argb_lo, split);
+        argb_hi = _mm_shuffle_epi8(argb_hi, split);
+
+        _mm_storeu_si128((__m128i*) dst, argb_lo);
+        _mm_storeu_si128((__m128i*) (dst + 4), argb_hi);
+
+        src += 8;
+        dst += 8;
+        count -= 8;
+    }
+
+    if (count >= 4) {

[mtklein, 2016/01/19 15:59:14]

Reminder to self to circle back here when we're happy with n >= 8.

+        __m128i argb = _mm_loadu_si128((const __m128i*) src);
+
+        // argb_argb_argb_argb -> aaaa_rrrr_gggg_bbbb
+        argb = _mm_shuffle_epi8(argb, combine);
+
+        // aaaa_rrrr_gggg_bbbb -> 0000_aaaa_0000_rrrr
+        __m128i ar = _mm_unpackhi_epi32(argb, zeros);
+        // aaaa_rrrr_gggg_bbbb -> 0000_gggg_0000_bbbb
+        __m128i gb = _mm_unpacklo_epi32(argb, zeros);
+
+        // xxxx_xxxx_yyyy_yyyy -> 0x0x_0x0x_0x0x_0x0x
+        // xxxx_xxxx_yyyy_yyyy -> 0y0y_0y0y_0y0y_0y0y
+        __m128i a = _mm_unpackhi_epi8(ar, zeros);
+        __m128i r = _mm_unpacklo_epi8(ar, zeros);
+        __m128i g = _mm_unpackhi_epi8(gb, zeros);
+        __m128i b = _mm_unpacklo_epi8(gb, zeros);
+
+        // (x + 127) / 255 == ((x + 128) * 257) >> 16 for 0 <= x <= 255 * 255
+        // Note that _mm_mulhi_epu16 performs the entire (y * 257) >> 16.
+        r = _mm_mulhi_epu16(_mm_add_epi16(_mm_mullo_epi16(a, r), _128), _257);
+        g = _mm_mulhi_epu16(_mm_add_epi16(_mm_mullo_epi16(a, g), _128), _257);
+        b = _mm_mulhi_epu16(_mm_add_epi16(_mm_mullo_epi16(a, b), _128), _257);
+
+        // aaaa_rrrr_0000_0000
+        ar = _mm_shuffle_epi32(_mm_packus_epi16(r, a), 0x8F);
+        // 0000_0000_gggg_bbbb
+        gb = _mm_shuffle_epi32(_mm_packus_epi16(b, g), 0xD8);
+
+        // aaaa_rrrr_gggg_bbbb
+        argb = _mm_or_si128(ar, gb);
+
+        // aaaa_rrrr_gggg_bbbb -> argb_argb_argb_argb
+        argb = _mm_shuffle_epi8(argb, split);
+
+        _mm_storeu_si128((__m128i*) dst, argb);
+
+        src += 4;
+        dst += 4;
+        count -= 4;
+    }
+
+    // Call portable code to finish up the tail of [0,4) pixels.
+    auto proc = kSwapRB ? premul_swaprb_xxxa_portable : premul_xxxa_portable;
+    proc(dst, src, count);
+}
+
+static void premul_xxxa(uint32_t dst[], const uint32_t src[], int count) {
+    premul_xxxa_should_swaprb<false>(dst, src, count);
+}
+
+static void premul_swaprb_xxxa(uint32_t dst[], const uint32_t src[], int count) {
+    premul_xxxa_should_swaprb<true>(dst, src, count);
+}
+
+static void swaprb_xxxa(uint32_t dst[], const uint32_t src[], int count) {
+    const __m128i swapRB = _mm_set_epi8(15, 12, 13, 14, 11, 8, 9, 10, 7, 4, 5, 6, 3, 0, 1, 2);

[mtklein, 2016/01/19 15:59:15]

I often find it's easier to read these if you use _mm_setr_foo, so that the
indices go in ascending order:
   _mm_setr_epi8(2,1,0,3, 6,5,4,7, 10,9,8,11, 14,13,12,15);

If you do like them as you've written, they're perfectly fine.

[msarett, 2016/01/19 17:34:37]

I think you're right.

+
+    while (count >= 4) {
+        __m128i argb = _mm_loadu_si128((const __m128i*) src);
+        __m128i abgr = _mm_shuffle_epi8(argb, swapRB);
+        _mm_storeu_si128((__m128i*) dst, abgr);
+
+        src += 4;
+        dst += 4;
+        count -= 4;
+    }
+
+    swaprb_xxxa_portable(dst, src, count);
+}
+
 #else
 
 static void premul_xxxa(uint32_t dst[], const uint32_t src[], int count) {
     premul_xxxa_portable(dst, src, count);
 }
 
 static void premul_swaprb_xxxa(uint32_t dst[], const uint32_t src[], int count) {
     premul_swaprb_xxxa_portable(dst, src, count);
 }
 
 static void swaprb_xxxa(uint32_t dst[], const uint32_t src[], int count) {
     swaprb_xxxa_portable(dst, src, count);
 }
 
 #endif
 
 }
 
 #endif // SkSwizzler_opts_DEFINED