OLD | NEW |
(Empty) | |
| 1 // It is important _not_ to put header guards here. |
| 2 // This file will be intentionally included three times. |
| 3 |
| 4 // Useful reading: |
| 5 // https://software.intel.com/sites/landingpage/IntrinsicsGuide/ |
| 6 |
| 7 #if defined(SK4X_PREAMBLE) |
| 8 // Code in this file may assume SSE and SSE2. |
| 9 #include <emmintrin.h> |
| 10 |
| 11 // It must check for later instruction sets. |
| 12 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE41 |
| 13 #include <immintrin.h> |
| 14 #endif |
| 15 |
| 16 // A little bit of template metaprogramming to map |
| 17 // float to __m128 and int32_t to __m128i. |
| 18 template <typename T> struct SkScalarToSIMD; |
| 19 template <> struct SkScalarToSIMD<float> { typedef __m128 Type; }; |
| 20 template <> struct SkScalarToSIMD<int32_t> { typedef __m128i Type; }; |
| 21 |
| 22 // These are all free, zero instructions. |
| 23 // MSVC insists we use _mm_castA_B(a) instead of (B)a. |
| 24 static __m128 as_4f(__m128i v) { return _mm_castsi128_ps(v); } |
| 25 static __m128 as_4f(__m128 v) { return v ; } |
| 26 static __m128i as_4i(__m128i v) { return v ; } |
| 27 static __m128i as_4i(__m128 v) { return _mm_castps_si128(v); } |
| 28 |
| 29 #elif defined(SK4X_PRIVATE) |
| 30 // It'd be slightly faster to call _mm_cmpeq_epi32() on an unintialized regi
ster and itself, |
| 31 // but that has caused hard to debug issues when compilers recognize dealing
with uninitialized |
| 32 // memory as undefined behavior that can be optimized away. |
| 33 static __m128i True() { return _mm_set1_epi8(~0); } |
| 34 |
| 35 // Leaving these implicit makes the rest of the code below a bit less noisy
to read. |
| 36 Sk4x(__m128i); |
| 37 Sk4x(__m128); |
| 38 |
| 39 Sk4x andNot(const Sk4x&) const; |
| 40 |
| 41 typename SkScalarToSIMD<T>::Type fVec; |
| 42 |
| 43 #else//Method definitions. |
| 44 |
| 45 // Helps to get these in before anything else. |
| 46 template <> inline Sk4f::Sk4x(__m128i v) : fVec(as_4f(v)) {} |
| 47 template <> inline Sk4f::Sk4x(__m128 v) : fVec( v ) {} |
| 48 template <> inline Sk4i::Sk4x(__m128i v) : fVec( v ) {} |
| 49 template <> inline Sk4i::Sk4x(__m128 v) : fVec(as_4i(v)) {} |
| 50 |
| 51 // Next, methods whose implementation is the same for Sk4f and Sk4i. |
| 52 template <typename T> Sk4x<T>::Sk4x() {} |
| 53 template <typename T> Sk4x<T>::Sk4x(const Sk4x& other) { *this = other; } |
| 54 template <typename T> Sk4x<T>& Sk4x<T>::operator=(const Sk4x<T>& other) { |
| 55 fVec = other.fVec; |
| 56 return *this; |
| 57 } |
| 58 |
| 59 // We pun in these _mm_shuffle_* methods a little to use the fastest / most avai
lable methods. |
| 60 // They're all bit-preserving operations so it shouldn't matter. |
| 61 |
| 62 template <typename T> |
| 63 Sk4x<T> Sk4x<T>::zwxy() const { return _mm_shuffle_epi32(as_4i(fVec), _MM_SHUFFL
E(1,0,3,2)); } |
| 64 |
| 65 template <typename T> |
| 66 Sk4x<T> Sk4x<T>::XYAB(const Sk4x<T>& a, const Sk4x<T>& b) { |
| 67 return _mm_movelh_ps(as_4f(a.fVec), as_4f(b.fVec)); |
| 68 } |
| 69 |
| 70 template <typename T> |
| 71 Sk4x<T> Sk4x<T>::ZWCD(const Sk4x<T>& a, const Sk4x<T>& b) { |
| 72 return _mm_movehl_ps(as_4f(b.fVec), as_4f(a.fVec)); |
| 73 } |
| 74 |
| 75 // Now we'll write all Sk4f specific methods. This M() macro will remove some n
oise. |
| 76 #define M(...) template <> inline __VA_ARGS__ Sk4f:: |
| 77 |
| 78 M() Sk4x(float a, float b, float c, float d) : fVec(_mm_set_ps(d,c,b,a)) {} |
| 79 |
| 80 M(Sk4f) Load (const float fs[4]) { return _mm_loadu_ps(fs); } |
| 81 M(Sk4f) LoadAligned(const float fs[4]) { return _mm_load_ps (fs); } |
| 82 |
| 83 M(void) store (float fs[4]) const { _mm_storeu_ps(fs, fVec); } |
| 84 M(void) storeAligned(float fs[4]) const { _mm_store_ps (fs, fVec); } |
| 85 |
| 86 template <> template <> |
| 87 Sk4i Sk4f::reinterpret<Sk4i>() const { return as_4i(fVec); } |
| 88 |
| 89 template <> template <> |
| 90 Sk4i Sk4f::cast<Sk4i>() const { return _mm_cvtps_epi32(fVec); } |
| 91 |
| 92 // We're going to try a little experiment here and skip allTrue(), anyTrue(), an
d bit-manipulators |
| 93 // for Sk4f. Code that calls them probably does so accidentally. |
| 94 // Ask mtklein to fill these in if you really need them. |
| 95 |
| 96 M(Sk4f) add (const Sk4f& o) const { return _mm_add_ps(fVec, o.fVec); } |
| 97 M(Sk4f) subtract(const Sk4f& o) const { return _mm_sub_ps(fVec, o.fVec); } |
| 98 M(Sk4f) multiply(const Sk4f& o) const { return _mm_mul_ps(fVec, o.fVec); } |
| 99 M(Sk4f) divide (const Sk4f& o) const { return _mm_div_ps(fVec, o.fVec); } |
| 100 |
| 101 M(Sk4i) equal (const Sk4f& o) const { return _mm_cmpeq_ps (fVec, o.fVe
c); } |
| 102 M(Sk4i) notEqual (const Sk4f& o) const { return _mm_cmpneq_ps(fVec, o.fVe
c); } |
| 103 M(Sk4i) lessThan (const Sk4f& o) const { return _mm_cmplt_ps (fVec, o.fVe
c); } |
| 104 M(Sk4i) greaterThan (const Sk4f& o) const { return _mm_cmpgt_ps (fVec, o.fVe
c); } |
| 105 M(Sk4i) lessThanEqual (const Sk4f& o) const { return _mm_cmple_ps (fVec, o.fVe
c); } |
| 106 M(Sk4i) greaterThanEqual(const Sk4f& o) const { return _mm_cmpge_ps (fVec, o.fVe
c); } |
| 107 |
| 108 M(Sk4f) Min(const Sk4f& a, const Sk4f& b) { return _mm_min_ps(a.fVec, b.fVec); } |
| 109 M(Sk4f) Max(const Sk4f& a, const Sk4f& b) { return _mm_max_ps(a.fVec, b.fVec); } |
| 110 |
| 111 // Now we'll write all the Sk4i specific methods. Same deal for M(). |
| 112 #undef M |
| 113 #define M(...) template <> inline __VA_ARGS__ Sk4i:: |
| 114 |
| 115 M() Sk4x(int32_t a, int32_t b, int32_t c, int32_t d) : fVec(_mm_set_epi32(d,c,b,
a)) {} |
| 116 |
| 117 M(Sk4i) Load (const int32_t is[4]) { return _mm_loadu_si128((const __m128i
*)is); } |
| 118 M(Sk4i) LoadAligned(const int32_t is[4]) { return _mm_load_si128 ((const __m128i
*)is); } |
| 119 |
| 120 M(void) store (int32_t is[4]) const { _mm_storeu_si128((__m128i*)is, fVec)
; } |
| 121 M(void) storeAligned(int32_t is[4]) const { _mm_store_si128 ((__m128i*)is, fVec)
; } |
| 122 |
| 123 template <> template <> |
| 124 Sk4f Sk4i::reinterpret<Sk4f>() const { return as_4f(fVec); } |
| 125 |
| 126 template <> template <> |
| 127 Sk4f Sk4i::cast<Sk4f>() const { return _mm_cvtepi32_ps(fVec); } |
| 128 |
| 129 M(bool) allTrue() const { return 0xf == _mm_movemask_ps(as_4f(fVec)); } |
| 130 M(bool) anyTrue() const { return 0x0 != _mm_movemask_ps(as_4f(fVec)); } |
| 131 |
| 132 M(Sk4i) bitNot() const { return _mm_xor_si128(fVec, True()); } |
| 133 M(Sk4i) bitAnd(const Sk4i& o) const { return _mm_and_si128(fVec, o.fVec); } |
| 134 M(Sk4i) bitOr (const Sk4i& o) const { return _mm_or_si128 (fVec, o.fVec); } |
| 135 |
| 136 M(Sk4i) equal (const Sk4i& o) const { return _mm_cmpeq_epi32 (fVec, o.
fVec); } |
| 137 M(Sk4i) lessThan (const Sk4i& o) const { return _mm_cmplt_epi32 (fVec, o.
fVec); } |
| 138 M(Sk4i) greaterThan (const Sk4i& o) const { return _mm_cmpgt_epi32 (fVec, o.
fVec); } |
| 139 M(Sk4i) notEqual (const Sk4i& o) const { return this-> equal(o).bitN
ot(); } |
| 140 M(Sk4i) lessThanEqual (const Sk4i& o) const { return this->greaterThan(o).bitN
ot(); } |
| 141 M(Sk4i) greaterThanEqual(const Sk4i& o) const { return this-> lessThan(o).bitN
ot(); } |
| 142 |
| 143 M(Sk4i) add (const Sk4i& o) const { return _mm_add_epi32(fVec, o.fVec); } |
| 144 M(Sk4i) subtract(const Sk4i& o) const { return _mm_sub_epi32(fVec, o.fVec); } |
| 145 |
| 146 // SSE doesn't have integer division. Let's see how far we can get without Sk4i
::divide(). |
| 147 |
| 148 // Sk4i's multiply(), Min(), and Max() all improve significantly with SSE4.1. |
| 149 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE41 |
| 150 M(Sk4i) multiply(const Sk4i& o) const { return _mm_mullo_epi32(fVec, o.fVec)
; } |
| 151 M(Sk4i) Min(const Sk4i& a, const Sk4i& b) { return _mm_min_epi32(a.fVec, b.f
Vec); } |
| 152 M(Sk4i) Max(const Sk4i& a, const Sk4i& b) { return _mm_max_epi32(a.fVec, b.f
Vec); } |
| 153 #else |
| 154 M(Sk4i) multiply(const Sk4i& o) const { |
| 155 // First 2 32->64 bit multiplies. |
| 156 __m128i mul02 = _mm_mul_epu32(fVec, o.fVec), |
| 157 mul13 = _mm_mul_epu32(_mm_srli_si128(fVec, 4), _mm_srli_si128(o.
fVec, 4)); |
| 158 // Now recombine the high bits of the two products. |
| 159 return _mm_unpacklo_epi32(_mm_shuffle_epi32(mul02, _MM_SHUFFLE(0,0,2,0))
, |
| 160 _mm_shuffle_epi32(mul13, _MM_SHUFFLE(0,0,2,0))
); |
| 161 } |
| 162 |
| 163 M(Sk4i) andNot(const Sk4i& o) const { return _mm_andnot_si128(o.fVec, fVec);
} |
| 164 |
| 165 M(Sk4i) Min(const Sk4i& a, const Sk4i& b) { |
| 166 Sk4i less = a.lessThan(b); |
| 167 return a.bitAnd(less).bitOr(b.andNot(less)); |
| 168 } |
| 169 M(Sk4i) Max(const Sk4i& a, const Sk4i& b) { |
| 170 Sk4i less = a.lessThan(b); |
| 171 return b.bitAnd(less).bitOr(a.andNot(less)); |
| 172 } |
| 173 #endif |
| 174 |
| 175 #undef M |
| 176 |
| 177 #endif//Method definitions. |
OLD | NEW |