72 lines
4.8 KiB
C
72 lines
4.8 KiB
C
#include "sn_target.h"
|
|
|
|
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
|
|
|
#define SN_MK_FMA_F32_512_CYCLE(i) \
|
|
do { \
|
|
c0 = _mm512_fmadd_ps(a0, _mm512_permute_ps(b0, _FMA_SELECT(i)), c0); \
|
|
c1 = _mm512_fmadd_ps(a1, _mm512_permute_ps(b1, _FMA_SELECT(i)), c1); \
|
|
c2 = _mm512_fmadd_ps(a2, _mm512_permute_ps(b2, _FMA_SELECT(i)), c2); \
|
|
c3 = _mm512_fmadd_ps(a3, _mm512_permute_ps(b3, _FMA_SELECT(i)), c3); \
|
|
} while (0)
|
|
|
|
#define SN_MK_FMA_F32_AVX2_FMA_CYCLE(i) \
|
|
do { \
|
|
c0 = _mm256_fmadd_ps(a0, _mm256_permute_ps(b0, _FMA_SELECT(i)), c0); \
|
|
c1 = _mm256_fmadd_ps(a1, _mm256_permute_ps(b1, _FMA_SELECT(i)), c1); \
|
|
c2 = _mm256_fmadd_ps(a2, _mm256_permute_ps(b2, _FMA_SELECT(i)), c2); \
|
|
c3 = _mm256_fmadd_ps(a3, _mm256_permute_ps(b3, _FMA_SELECT(i)), c3); \
|
|
} while (0)
|
|
|
|
#define SN_MK_FMA_F32_AVX2_CYCLE(i) \
|
|
do { \
|
|
c0 = _mm256_add_ps(_mm256_mul_ps(a0, _mm256_permute_ps(b0, _FMA_SELECT(i))), c0); \
|
|
c1 = _mm256_add_ps(_mm256_mul_ps(a1, _mm256_permute_ps(b1, _FMA_SELECT(i))), c1); \
|
|
c2 = _mm256_add_ps(_mm256_mul_ps(a2, _mm256_permute_ps(b2, _FMA_SELECT(i))), c2); \
|
|
c3 = _mm256_add_ps(_mm256_mul_ps(a3, _mm256_permute_ps(b3, _FMA_SELECT(i))), c3); \
|
|
} while (0)
|
|
|
|
#define SN_MK_FMA_F64_AVX2_CYCLE(i) \
|
|
do { \
|
|
c0 = _mm512_fmadd_ps(a0, _mm512_permutex_pd(b0, _FMA_SELECT(i)), c0); \
|
|
c1 = _mm512_fmadd_ps(a1, _mm512_permutex_pd(b1, _FMA_SELECT(i)), c1); \
|
|
c2 = _mm512_fmadd_ps(a2, _mm512_permutex_pd(b2, _FMA_SELECT(i)), c2); \
|
|
c3 = _mm512_fmadd_ps(a3, _mm512_permutex_pd(b3, _FMA_SELECT(i)), c3); \
|
|
} while (0)
|
|
|
|
// double
|
|
#define SN_MK_FMA_F64_SSE_PHASE(i) \
|
|
do { \
|
|
/* k = 0 */ \
|
|
register __m128d b_lo = _mm_load_pd(right_values + (2 * i + 0)); \
|
|
register __m128d b_hi = _mm_load_pd(right_values + (2 * i + 1)); \
|
|
register __m128d a_lo = _mm_load_pd(left_values + (0 * 2 + 0)); \
|
|
register __m128d a_hi = _mm_load_pd(left_values + (0 * 2 + 1)); \
|
|
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
|
|
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
|
|
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
|
|
\
|
|
/* group = 1 */ \
|
|
a_lo = _mm_load_pd(left_values + (1 * 2 + 0)); \
|
|
a_hi = _mm_load_pd(left_values + (1 * 2 + 1)); \
|
|
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
|
|
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
|
|
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
|
|
\
|
|
/* group = 2 */ \
|
|
a_lo = _mm_load_pd(left_values + (2 * 2 + 0)); \
|
|
a_hi = _mm_load_pd(left_values + (2 * 2 + 1)); \
|
|
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
|
|
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
|
|
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
|
|
\
|
|
/* group = 3 */ \
|
|
a_lo = _mm_load_pd(left_values + (3 * 2 + 0)); \
|
|
a_hi = _mm_load_pd(left_values + (3 * 2 + 1)); \
|
|
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
|
|
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
|
|
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
|
|
} while (0)
|
|
|
|
#endif
|