85 lines
3.3 KiB
C
85 lines
3.3 KiB
C
#include "sn_intrinsic.h"
|
|
|
|
INTERNAL void SN_DECL f32_fma_sse(MAT_PANEL_SSE(int) * left)
|
|
{
|
|
}
|
|
|
|
// ans = left * right + ans
|
|
// right is not transposed
|
|
INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left,
|
|
MAT_KERNEL(float) const *restrict right,
|
|
MAT_KERNEL(float) *restrict ans)
|
|
{
|
|
register __m128 source0 = _mm_load_ps(&left->xmm[0]);
|
|
register __m128 source1 = _mm_load_ps(&left->xmm[1]);
|
|
register __m128 source2 = _mm_load_ps(&left->xmm[2]);
|
|
register __m128 source3 = _mm_load_ps(&left->xmm[3]);
|
|
|
|
register __m128 ans0 = _mm_load_ps(&ans->xmm[0]);
|
|
register __m128 ans1 = _mm_load_ps(&ans->xmm[1]);
|
|
register __m128 ans2 = _mm_load_ps(&ans->xmm[2]);
|
|
register __m128 ans3 = _mm_load_ps(&ans->xmm[3]);
|
|
|
|
register __m128 broadcast;
|
|
for (size_t i = 0; i < 4; i++) {
|
|
broadcast = _mm_load_ps(&right->xmm[i]);
|
|
|
|
ans0 = _mm_add_ps(ans0,
|
|
_mm_mul_ps(source0, _mm_shuffle_ps(broadcast, broadcast, 0x00)));
|
|
ans1 = _mm_add_ps(ans1,
|
|
_mm_mul_ps(source1, _mm_shuffle_ps(broadcast, broadcast, 0x55)));
|
|
ans2 = _mm_add_ps(ans2,
|
|
_mm_mul_ps(source2, _mm_shuffle_ps(broadcast, broadcast, 0xAA)));
|
|
ans3 = _mm_add_ps(ans3,
|
|
_mm_mul_ps(source3, _mm_shuffle_ps(broadcast, broadcast, 0xFF)));
|
|
}
|
|
|
|
_mm_store_ps(&ans->xmm[0], ans0);
|
|
_mm_store_ps(&ans->xmm[1], ans1);
|
|
_mm_store_ps(&ans->xmm[2], ans2);
|
|
_mm_store_ps(&ans->xmm[3], ans3);
|
|
}
|
|
|
|
// ans = left * right + ans
|
|
INTERNAL void SN_DECL f32_fma_avx(MAT_KERNEL(float) const *restrict left,
|
|
MAT_KERNEL(float) const *restrict right,
|
|
MAT_KERNEL(float) *restrict ans)
|
|
{
|
|
register __m256 source0 = _mm256_load_ps(&left->ymm[0]);
|
|
register __m256 source1 = _mm256_load_ps(&left->ymm[1]);
|
|
|
|
register __m256 ans0 = _mm256_load_ps(&ans->ymm[0]);
|
|
register __m256 ans1 = _mm256_load_ps(&ans->ymm[1]);
|
|
|
|
register __m128 bs0, bs1;
|
|
register __m256 broadcast;
|
|
for (size_t i = 0; i < 4; i++) {
|
|
bs0 = _mm_load_ps(&right->xmm[i]);
|
|
|
|
__m128 lo = _mm_shuffle_ps(bs0, bs0, 0x00);
|
|
__m128 hi = _mm_shuffle_ps(bs0, bs0, 0x55);
|
|
broadcast = _mm256_castps128_ps256(lo);
|
|
broadcast = _mm256_insertf128_ps(broadcast, bs0, 1);
|
|
ans0 = _mm256_add_ps(ans0, _mm256_mul_ps(source0, broadcast));
|
|
|
|
lo = _mm_shuffle_ps(bs0, bs0, 0xAA);
|
|
hi = _mm_shuffle_ps(bs0, bs0, 0xFF);
|
|
broadcast = _mm256_castps128_ps256(lo);
|
|
broadcast = _mm256_insertf128_ps(broadcast, bs0, 1);
|
|
ans1 = _mm256_add_ps(ans1, _mm256_mul_ps(source1, broadcast));
|
|
}
|
|
|
|
_mm256_store_ps(&ans->ymm[0], ans0);
|
|
_mm256_store_ps(&ans->ymm[1], ans1);
|
|
}
|
|
|
|
// ans = left * right + ans
|
|
INTERNAL void SN_DECL f32_fma_512(MAT_KERNEL(float) const *restrict left,
|
|
MAT_KERNEL(float) const *restrict right,
|
|
MAT_KERNEL(float) *restrict ans)
|
|
{
|
|
/*
|
|
* to be implemented
|
|
*/
|
|
}
|