OLD | NEW |
1 /*===---- avxintrin.h - AVX intrinsics -------------------------------------=== | 1 /*===---- avxintrin.h - AVX intrinsics -------------------------------------=== |
2 * | 2 * |
3 * Permission is hereby granted, free of charge, to any person obtaining a copy | 3 * Permission is hereby granted, free of charge, to any person obtaining a copy |
4 * of this software and associated documentation files (the "Software"), to deal | 4 * of this software and associated documentation files (the "Software"), to deal |
5 * in the Software without restriction, including without limitation the rights | 5 * in the Software without restriction, including without limitation the rights |
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | 6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
7 * copies of the Software, and to permit persons to whom the Software is | 7 * copies of the Software, and to permit persons to whom the Software is |
8 * furnished to do so, subject to the following conditions: | 8 * furnished to do so, subject to the following conditions: |
9 * | 9 * |
10 * The above copyright notice and this permission notice shall be included in | 10 * The above copyright notice and this permission notice shall be included in |
(...skipping 287 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
298 return (__m256)__builtin_ia32_vperm2f128_ps256((__v8sf)a, (__v8sf)b, c); | 298 return (__m256)__builtin_ia32_vperm2f128_ps256((__v8sf)a, (__v8sf)b, c); |
299 } | 299 } |
300 | 300 |
301 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) | 301 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) |
302 _mm256_permute2f128_si256(__m256i a, __m256i b, const int c) | 302 _mm256_permute2f128_si256(__m256i a, __m256i b, const int c) |
303 { | 303 { |
304 return (__m256i)__builtin_ia32_vperm2f128_si256((__v8si)a, (__v8si)b, c); | 304 return (__m256i)__builtin_ia32_vperm2f128_si256((__v8si)a, (__v8si)b, c); |
305 } | 305 } |
306 | 306 |
307 /* Vector Blend */ | 307 /* Vector Blend */ |
308 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) | 308 #define _mm256_blend_pd(V1, V2, M) __extension__ ({ \ |
309 _mm256_blend_pd(__m256d a, __m256d b, const int c) | 309 __m256d __V1 = (V1); \ |
310 { | 310 __m256d __V2 = (V2); \ |
311 return (__m256d)__builtin_ia32_blendpd256((__v4df)a, (__v4df)b, c); | 311 (__m256d)__builtin_ia32_blendpd256((__v4df)__V1, (__v4df)__V2, M); }) |
312 } | |
313 | 312 |
314 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) | 313 #define _mm256_blend_ps(V1, V2, M) __extension__ ({ \ |
315 _mm256_blend_ps(__m256 a, __m256 b, const int c) | 314 __m256 __V1 = (V1); \ |
316 { | 315 __m256 __V2 = (V2); \ |
317 return (__m256)__builtin_ia32_blendps256((__v8sf)a, (__v8sf)b, c); | 316 (__m256)__builtin_ia32_blendps256((__v8sf)__V1, (__v8sf)__V2, M); }) |
318 } | |
319 | 317 |
320 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) | 318 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) |
321 _mm256_blendv_pd(__m256d a, __m256d b, __m256d c) | 319 _mm256_blendv_pd(__m256d a, __m256d b, __m256d c) |
322 { | 320 { |
323 return (__m256d)__builtin_ia32_blendvpd256((__v4df)a, (__v4df)b, (__v4df)c); | 321 return (__m256d)__builtin_ia32_blendvpd256((__v4df)a, (__v4df)b, (__v4df)c); |
324 } | 322 } |
325 | 323 |
326 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) | 324 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) |
327 _mm256_blendv_ps(__m256 a, __m256 b, __m256 c) | 325 _mm256_blendv_ps(__m256 a, __m256 b, __m256 c) |
328 { | 326 { |
329 return (__m256)__builtin_ia32_blendvps256((__v8sf)a, (__v8sf)b, (__v8sf)c); | 327 return (__m256)__builtin_ia32_blendvps256((__v8sf)a, (__v8sf)b, (__v8sf)c); |
330 } | 328 } |
331 | 329 |
332 /* Vector Dot Product */ | 330 /* Vector Dot Product */ |
333 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) | 331 #define _mm256_dp_ps(V1, V2, M) __extension__ ({ \ |
334 _mm256_dp_ps(__m256 a, __m256 b, const int c) | 332 __m256 __V1 = (V1); \ |
335 { | 333 __m256 __V2 = (V2); \ |
336 return (__m256)__builtin_ia32_dpps256((__v8sf)a, (__v8sf)b, c); | 334 (__m256)__builtin_ia32_dpps256((__v8sf)__V1, (__v8sf)__V2, M); }) |
337 } | |
338 | 335 |
339 /* Vector shuffle */ | 336 /* Vector shuffle */ |
340 #define _mm256_shuffle_ps(a, b, mask) \ | 337 #define _mm256_shuffle_ps(a, b, mask) __extension__ ({ \ |
341 (__builtin_shufflevector((__v8sf)(a), (__v8sf)(b), \ | 338 __m256 __a = (a); \ |
| 339 __m256 __b = (b); \ |
| 340 (__m256)__builtin_shufflevector((__v8sf)__a, (__v8sf)__b, \ |
342 (mask) & 0x3, ((mask) & 0xc) >> 2, \ | 341 (mask) & 0x3, ((mask) & 0xc) >> 2, \ |
343 (((mask) & 0x30) >> 4) + 8, (((mask) & 0xc0) >> 6) + 8, \ | 342 (((mask) & 0x30) >> 4) + 8, (((mask) & 0xc0) >> 6) + 8, \ |
344 ((mask) & 0x3) + 4, (((mask) & 0xc) >> 2) + 4, \ | 343 ((mask) & 0x3) + 4, (((mask) & 0xc) >> 2) + 4, \ |
345 (((mask) & 0x30) >> 4) + 12, (((mask) & 0xc0) >> 6) + 12)) | 344 (((mask) & 0x30) >> 4) + 12, (((mask) & 0xc0) >> 6) + 12); }) |
346 | 345 |
347 #define _mm256_shuffle_pd(a, b, mask) \ | 346 #define _mm256_shuffle_pd(a, b, mask) __extension__ ({ \ |
348 (__builtin_shufflevector((__v4df)(a), (__v4df)(b), \ | 347 __m256d __a = (a); \ |
| 348 __m256d __b = (b); \ |
| 349 (__m256d)__builtin_shufflevector((__v4df)__a, (__v4df)__b, \ |
349 (mask) & 0x1, \ | 350 (mask) & 0x1, \ |
350 (((mask) & 0x2) >> 1) + 4, \ | 351 (((mask) & 0x2) >> 1) + 4, \ |
351 (((mask) & 0x4) >> 2) + 2, \ | 352 (((mask) & 0x4) >> 2) + 2, \ |
352 (((mask) & 0x8) >> 3) + 6)) | 353 (((mask) & 0x8) >> 3) + 6); }) |
353 | 354 |
354 /* Compare */ | 355 /* Compare */ |
355 #define _CMP_EQ_OQ 0x00 /* Equal (ordered, non-signaling) */ | 356 #define _CMP_EQ_OQ 0x00 /* Equal (ordered, non-signaling) */ |
356 #define _CMP_LT_OS 0x01 /* Less-than (ordered, signaling) */ | 357 #define _CMP_LT_OS 0x01 /* Less-than (ordered, signaling) */ |
357 #define _CMP_LE_OS 0x02 /* Less-than-or-equal (ordered, signaling) */ | 358 #define _CMP_LE_OS 0x02 /* Less-than-or-equal (ordered, signaling) */ |
358 #define _CMP_UNORD_Q 0x03 /* Unordered (non-signaling) */ | 359 #define _CMP_UNORD_Q 0x03 /* Unordered (non-signaling) */ |
359 #define _CMP_NEQ_UQ 0x04 /* Not-equal (unordered, non-signaling) */ | 360 #define _CMP_NEQ_UQ 0x04 /* Not-equal (unordered, non-signaling) */ |
360 #define _CMP_NLT_US 0x05 /* Not-less-than (unordered, signaling) */ | 361 #define _CMP_NLT_US 0x05 /* Not-less-than (unordered, signaling) */ |
361 #define _CMP_NLE_US 0x06 /* Not-less-than-or-equal (unordered, signaling) */ | 362 #define _CMP_NLE_US 0x06 /* Not-less-than-or-equal (unordered, signaling) */ |
362 #define _CMP_ORD_Q 0x07 /* Ordered (nonsignaling) */ | 363 #define _CMP_ORD_Q 0x07 /* Ordered (nonsignaling) */ |
(...skipping 15 matching lines...) Expand all Loading... |
378 #define _CMP_ORD_S 0x17 /* Ordered (signaling) */ | 379 #define _CMP_ORD_S 0x17 /* Ordered (signaling) */ |
379 #define _CMP_EQ_US 0x18 /* Equal (unordered, signaling) */ | 380 #define _CMP_EQ_US 0x18 /* Equal (unordered, signaling) */ |
380 #define _CMP_NGE_UQ 0x19 /* Not-greater-than-or-equal (unord, non-sign) */ | 381 #define _CMP_NGE_UQ 0x19 /* Not-greater-than-or-equal (unord, non-sign) */ |
381 #define _CMP_NGT_UQ 0x1a /* Not-greater-than (unordered, non-signaling) */ | 382 #define _CMP_NGT_UQ 0x1a /* Not-greater-than (unordered, non-signaling) */ |
382 #define _CMP_FALSE_OS 0x1b /* False (ordered, signaling) */ | 383 #define _CMP_FALSE_OS 0x1b /* False (ordered, signaling) */ |
383 #define _CMP_NEQ_OS 0x1c /* Not-equal (ordered, signaling) */ | 384 #define _CMP_NEQ_OS 0x1c /* Not-equal (ordered, signaling) */ |
384 #define _CMP_GE_OQ 0x1d /* Greater-than-or-equal (ordered, non-signaling) */ | 385 #define _CMP_GE_OQ 0x1d /* Greater-than-or-equal (ordered, non-signaling) */ |
385 #define _CMP_GT_OQ 0x1e /* Greater-than (ordered, non-signaling) */ | 386 #define _CMP_GT_OQ 0x1e /* Greater-than (ordered, non-signaling) */ |
386 #define _CMP_TRUE_US 0x1f /* True (unordered, signaling) */ | 387 #define _CMP_TRUE_US 0x1f /* True (unordered, signaling) */ |
387 | 388 |
388 #define _mm_cmp_pd(a, b, c) \ | 389 #define _mm_cmp_pd(a, b, c) __extension__ ({ \ |
389 (__m128d)__builtin_ia32_cmppd((__v2df)(a), (__v2df)(b), (c)) | 390 __m128d __a = (a); \ |
| 391 __m128d __b = (b); \ |
| 392 (__m128d)__builtin_ia32_cmppd((__v2df)__a, (__v2df)__b, (c)); }) |
390 | 393 |
391 #define _mm_cmp_ps(a, b, c) \ | 394 #define _mm_cmp_ps(a, b, c) __extension__ ({ \ |
392 (__m128)__builtin_ia32_cmpps((__v4sf)(a), (__v4sf)(b), (c)) | 395 __m128 __a = (a); \ |
| 396 __m128 __b = (b); \ |
| 397 (__m128)__builtin_ia32_cmpps((__v4sf)__a, (__v4sf)__b, (c)); }) |
393 | 398 |
394 #define _mm256_cmp_pd(a, b, c) \ | 399 #define _mm256_cmp_pd(a, b, c) __extension__ ({ \ |
395 (__m256d)__builtin_ia32_cmppd256((__v4df)(a), (__v4df)(b), (c)) | 400 __m256d __a = (a); \ |
| 401 __m256d __b = (b); \ |
| 402 (__m256d)__builtin_ia32_cmppd256((__v4df)__a, (__v4df)__b, (c)); }) |
396 | 403 |
397 #define _mm256_cmp_ps(a, b, c) \ | 404 #define _mm256_cmp_ps(a, b, c) __extension__ ({ \ |
398 (__m256)__builtin_ia32_cmpps256((__v8sf)(a), (__v8sf)(b), (c)) | 405 __m256 __a = (a); \ |
| 406 __m256 __b = (b); \ |
| 407 (__m256)__builtin_ia32_cmpps256((__v8sf)__a, (__v8sf)__b, (c)); }) |
399 | 408 |
400 #define _mm_cmp_sd(a, b, c) \ | 409 #define _mm_cmp_sd(a, b, c) __extension__ ({ \ |
401 (__m128d)__builtin_ia32_cmpsd((__v2df)(a), (__v2df)(b), (c)) | 410 __m128d __a = (a); \ |
| 411 __m128d __b = (b); \ |
| 412 (__m128d)__builtin_ia32_cmpsd((__v2df)__a, (__v2df)__b, (c)); }) |
402 | 413 |
403 #define _mm_cmp_ss(a, b, c) \ | 414 #define _mm_cmp_ss(a, b, c) __extension__ ({ \ |
404 (__m128)__builtin_ia32_cmpss((__v4sf)(a), (__v4sf)(b), (c)) | 415 __m128 __a = (a); \ |
| 416 __m128 __b = (b); \ |
| 417 (__m128)__builtin_ia32_cmpss((__v4sf)__a, (__v4sf)__b, (c)); }) |
405 | 418 |
406 /* Vector extract */ | 419 /* Vector extract */ |
407 static __inline __m128d __attribute__((__always_inline__, __nodebug__)) | 420 static __inline __m128d __attribute__((__always_inline__, __nodebug__)) |
408 _mm256_extractf128_pd(__m256d a, const int o) | 421 _mm256_extractf128_pd(__m256d a, const int o) |
409 { | 422 { |
410 return (__m128d)__builtin_ia32_vextractf128_pd256((__v4df)a, o); | 423 return (__m128d)__builtin_ia32_vextractf128_pd256((__v4df)a, o); |
411 } | 424 } |
412 | 425 |
413 static __inline __m128 __attribute__((__always_inline__, __nodebug__)) | 426 static __inline __m128 __attribute__((__always_inline__, __nodebug__)) |
414 _mm256_extractf128_ps(__m256 a, const int o) | 427 _mm256_extractf128_ps(__m256 a, const int o) |
(...skipping 714 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1129 __m128 zero = _mm_setzero_ps(); | 1142 __m128 zero = _mm_setzero_ps(); |
1130 return __builtin_shufflevector(in, zero, 0, 1, 2, 3, 4, 4, 4, 4); | 1143 return __builtin_shufflevector(in, zero, 0, 1, 2, 3, 4, 4, 4, 4); |
1131 } | 1144 } |
1132 | 1145 |
1133 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) | 1146 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) |
1134 _mm256_castsi128_si256(__m128i in) | 1147 _mm256_castsi128_si256(__m128i in) |
1135 { | 1148 { |
1136 __m128i zero = _mm_setzero_si128(); | 1149 __m128i zero = _mm_setzero_si128(); |
1137 return __builtin_shufflevector(in, zero, 0, 1, 2, 2); | 1150 return __builtin_shufflevector(in, zero, 0, 1, 2, 2); |
1138 } | 1151 } |
OLD | NEW |