OLD | NEW |
| (Empty) |
1 // It is important _not_ to put header guards here. | |
2 // This file will be intentionally included three times. | |
3 | |
4 // Useful reading: | |
5 // https://software.intel.com/sites/landingpage/IntrinsicsGuide/ | |
6 | |
7 #if defined(SK4X_PREAMBLE) | |
8 // Code in this file may assume SSE and SSE2. | |
9 #include <emmintrin.h> | |
10 | |
11 // It must check for later instruction sets. | |
12 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE41 | |
13 #include <immintrin.h> | |
14 #endif | |
15 | |
16 // A little bit of template metaprogramming to map | |
17 // float to __m128 and int32_t to __m128i. | |
18 template <typename T> struct SkScalarToSIMD; | |
19 template <> struct SkScalarToSIMD<float> { typedef __m128 Type; }; | |
20 template <> struct SkScalarToSIMD<int32_t> { typedef __m128i Type; }; | |
21 | |
22 // These are all free. MSVC insists we use _mm_castA_B(a) instead of (B)a. | |
23 __m128 as_4f(__m128i v) { return _mm_castsi128_ps(v); } | |
24 __m128 as_4f(__m128 v) { return v ; } | |
25 __m128i as_4i(__m128i v) { return v ; } | |
26 __m128i as_4i(__m128 v) { return _mm_castps_si128(v); } | |
27 | |
28 #elif defined(SK4X_PRIVATE) | |
29 // The best (1 op) way to get all -1s in a register. Our compilers are a lit
tle too cautious... | |
30 static __m128i True() { | |
31 #ifdef __GNUC__ | |
32 #pragma GCC diagnostic push | |
33 #pragma GCC diagnostic ignored "-Wuninitialized" | |
34 __m128i uninitialized; | |
35 return _mm_cmpeq_epi32(uninitialized, uninitialized); | |
36 #pragma GCC diagnostic pop | |
37 #else | |
38 // Can't figure out how to suppress C4700 from MSVC. Oh well, we'll be
a little slower. | |
39 __m128i zero = _mm_setzero_si128(); | |
40 return _mm_cmpeq_epi32(zero, zero); | |
41 #endif | |
42 } | |
43 | |
44 // Leaving these implicit makes the rest of the code below a bit less noisy
to read. | |
45 Sk4x(__m128i); | |
46 Sk4x(__m128); | |
47 | |
48 Sk4x andNot(const Sk4x&) const; | |
49 | |
50 typename SkScalarToSIMD<T>::Type fVec; | |
51 | |
52 #else//Method definitions. | |
53 | |
54 // Helps to get these in before anything else. | |
55 template <> inline Sk4f::Sk4x(__m128i v) : fVec(as_4f(v)) {} | |
56 template <> inline Sk4f::Sk4x(__m128 v) : fVec( v ) {} | |
57 template <> inline Sk4i::Sk4x(__m128i v) : fVec( v ) {} | |
58 template <> inline Sk4i::Sk4x(__m128 v) : fVec(as_4i(v)) {} | |
59 | |
60 // Next, methods whose implementation is the same for Sk4f and Sk4i. | |
61 template <typename T> Sk4x<T>::Sk4x() {} | |
62 template <typename T> Sk4x<T>::Sk4x(const Sk4x& other) { *this = other; } | |
63 template <typename T> Sk4x<T>& Sk4x<T>::operator=(const Sk4x<T>& other) { | |
64 fVec = other.fVec; | |
65 return *this; | |
66 } | |
67 | |
68 // We pun in these _mm_shuffle_* methods a little to use the fastest / most avai
lable methods. | |
69 // They're all bit-preserving operations so it shouldn't matter. | |
70 | |
71 template <typename T> | |
72 Sk4x<T> Sk4x<T>::zwxy() const { return _mm_shuffle_epi32(as_4i(fVec), _MM_SHUFFL
E(1,0,3,2)); } | |
73 | |
74 template <typename T> | |
75 Sk4x<T> Sk4x<T>::XYAB(const Sk4x<T>& a, const Sk4x<T>& b) { | |
76 return _mm_movelh_ps(as_4f(a.fVec), as_4f(b.fVec)); | |
77 } | |
78 | |
79 template <typename T> | |
80 Sk4x<T> Sk4x<T>::ZWCD(const Sk4x<T>& a, const Sk4x<T>& b) { | |
81 return _mm_movehl_ps(as_4f(b.fVec), as_4f(a.fVec)); | |
82 } | |
83 | |
84 // Now we'll write all Sk4f specific methods. This M() macro will remove some n
oise. | |
85 #define M(...) template <> inline __VA_ARGS__ Sk4f:: | |
86 | |
87 M() Sk4x(float a, float b, float c, float d) : fVec(_mm_set_ps(d,c,b,a)) {} | |
88 | |
89 M(Sk4f) Load (const float fs[4]) { return _mm_loadu_ps(fs); } | |
90 M(Sk4f) LoadAligned(const float fs[4]) { return _mm_load_ps (fs); } | |
91 | |
92 M(void) store (float fs[4]) const { _mm_storeu_ps(fs, fVec); } | |
93 M(void) storeAligned(float fs[4]) const { _mm_store_ps (fs, fVec); } | |
94 | |
95 template <> template <> | |
96 Sk4i Sk4f::reinterpret<Sk4i>() const { return as_4i(fVec); } | |
97 | |
98 template <> template <> | |
99 Sk4i Sk4f::cast<Sk4i>() const { return _mm_cvtps_epi32(fVec); } | |
100 | |
101 // We're going to try a little experiment here and skip allTrue(), anyTrue(), an
d bit-manipulators | |
102 // for Sk4f. Code that calls them probably does so accidentally. | |
103 // Ask mtklein to fill these in if you really need them. | |
104 | |
105 M(Sk4f) add (const Sk4f& o) const { return _mm_add_ps(fVec, o.fVec); } | |
106 M(Sk4f) subtract(const Sk4f& o) const { return _mm_sub_ps(fVec, o.fVec); } | |
107 M(Sk4f) multiply(const Sk4f& o) const { return _mm_mul_ps(fVec, o.fVec); } | |
108 M(Sk4f) divide (const Sk4f& o) const { return _mm_div_ps(fVec, o.fVec); } | |
109 | |
110 M(Sk4i) equal (const Sk4f& o) const { return _mm_cmpeq_ps (fVec, o.fVe
c); } | |
111 M(Sk4i) notEqual (const Sk4f& o) const { return _mm_cmpneq_ps(fVec, o.fVe
c); } | |
112 M(Sk4i) lessThan (const Sk4f& o) const { return _mm_cmplt_ps (fVec, o.fVe
c); } | |
113 M(Sk4i) greaterThan (const Sk4f& o) const { return _mm_cmpgt_ps (fVec, o.fVe
c); } | |
114 M(Sk4i) lessThanEqual (const Sk4f& o) const { return _mm_cmple_ps (fVec, o.fVe
c); } | |
115 M(Sk4i) greaterThanEqual(const Sk4f& o) const { return _mm_cmpge_ps (fVec, o.fVe
c); } | |
116 | |
117 M(Sk4f) Min(const Sk4f& a, const Sk4f& b) { return _mm_min_ps(a.fVec, b.fVec); } | |
118 M(Sk4f) Max(const Sk4f& a, const Sk4f& b) { return _mm_max_ps(a.fVec, b.fVec); } | |
119 | |
120 // Now we'll write all the Sk4i specific methods. Same deal for M(). | |
121 #undef M | |
122 #define M(...) template <> inline __VA_ARGS__ Sk4i:: | |
123 | |
124 M() Sk4x(int32_t a, int32_t b, int32_t c, int32_t d) : fVec(_mm_set_epi32(d,c,b,
a)) {} | |
125 | |
126 M(Sk4i) Load (const int32_t is[4]) { return _mm_loadu_si128((const __m128i
*)is); } | |
127 M(Sk4i) LoadAligned(const int32_t is[4]) { return _mm_load_si128 ((const __m128i
*)is); } | |
128 | |
129 M(void) store (int32_t is[4]) const { _mm_storeu_si128((__m128i*)is, fVec)
; } | |
130 M(void) storeAligned(int32_t is[4]) const { _mm_store_si128 ((__m128i*)is, fVec)
; } | |
131 | |
132 template <> template <> | |
133 Sk4f Sk4i::reinterpret<Sk4f>() const { return as_4f(fVec); } | |
134 | |
135 template <> template <> | |
136 Sk4f Sk4i::cast<Sk4f>() const { return _mm_cvtepi32_ps(fVec); } | |
137 | |
138 M(bool) allTrue() const { return 0xf == _mm_movemask_ps(as_4f(fVec)); } | |
139 M(bool) anyTrue() const { return 0x0 != _mm_movemask_ps(as_4f(fVec)); } | |
140 | |
141 M(Sk4i) bitNot() const { return _mm_xor_si128(fVec, True()); } | |
142 M(Sk4i) bitAnd(const Sk4i& o) const { return _mm_and_si128(fVec, o.fVec); } | |
143 M(Sk4i) bitOr (const Sk4i& o) const { return _mm_or_si128 (fVec, o.fVec); } | |
144 | |
145 M(Sk4i) equal (const Sk4i& o) const { return _mm_cmpeq_epi32 (fVec, o.
fVec); } | |
146 M(Sk4i) lessThan (const Sk4i& o) const { return _mm_cmplt_epi32 (fVec, o.
fVec); } | |
147 M(Sk4i) greaterThan (const Sk4i& o) const { return _mm_cmpgt_epi32 (fVec, o.
fVec); } | |
148 M(Sk4i) notEqual (const Sk4i& o) const { return this-> equal(o).bitN
ot(); } | |
149 M(Sk4i) lessThanEqual (const Sk4i& o) const { return this->greaterThan(o).bitN
ot(); } | |
150 M(Sk4i) greaterThanEqual(const Sk4i& o) const { return this-> lessThan(o).bitN
ot(); } | |
151 | |
152 M(Sk4i) add (const Sk4i& o) const { return _mm_add_epi32(fVec, o.fVec); } | |
153 M(Sk4i) subtract(const Sk4i& o) const { return _mm_sub_epi32(fVec, o.fVec); } | |
154 | |
155 // SSE doesn't have integer division. Let's see how far we can get without Sk4i
::divide(). | |
156 | |
157 // Sk4i's multiply(), Min(), and Max() all improve significantly with SSE4.1. | |
158 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE41 | |
159 M(Sk4i) multiply(const Sk4i& o) const { return _mm_mullo_epi32(fVec, o.fVec)
; } | |
160 M(Sk4i) Min(const Sk4i& a, const Sk4i& b) { return _mm_min_epi32(a.fVec, b.f
Vec); } | |
161 M(Sk4i) Max(const Sk4i& a, const Sk4i& b) { return _mm_max_epi32(a.fVec, b.f
Vec); } | |
162 #else | |
163 M(Sk4i) multiply(const Sk4i& o) const { | |
164 // First 2 32->64 bit multiplies. | |
165 __m128i mul02 = _mm_mul_epu32(fVec, o.fVec), | |
166 mul13 = _mm_mul_epu32(_mm_srli_si128(fVec, 4), _mm_srli_si128(o.
fVec, 4)); | |
167 // Now recombine the high bits of the two products. | |
168 return _mm_unpacklo_epi32(_mm_shuffle_epi32(mul02, _MM_SHUFFLE(0,0,2,0))
, | |
169 _mm_shuffle_epi32(mul13, _MM_SHUFFLE(0,0,2,0))
); | |
170 } | |
171 | |
172 M(Sk4i) andNot(const Sk4i& o) const { return _mm_andnot_si128(o.fVec, fVec);
} | |
173 | |
174 M(Sk4i) Min(const Sk4i& a, const Sk4i& b) { | |
175 Sk4i less = a.lessThan(b); | |
176 return a.bitAnd(less).bitOr(b.andNot(less)); | |
177 } | |
178 M(Sk4i) Max(const Sk4i& a, const Sk4i& b) { | |
179 Sk4i less = a.lessThan(b); | |
180 return b.bitAnd(less).bitOr(a.andNot(less)); | |
181 } | |
182 #endif | |
183 | |
184 #undef M | |
185 | |
186 #endif//Method definitions. | |
OLD | NEW |