youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixing tile fma

Browse Source
This commit is contained in:
Youmu Liu
2026-01-11 15:28:30 +08:00
parent 4be5c1c7c9
commit cd8210d9a1
8 changed files with 198 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-src/m.fma_tile.x86.c
+28 -22
View File
@@ -1,4 +1,4 @@
#include "sn_target.h"
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
@@ -20,7 +20,7 @@ internal static unsafe MatrixKernel<T>* GetKernelAddressUnsafe<T>(
}
*/
#define GET_INDEX(col_len, x, y) col_len *x + y
#define GET_INDEX(col_len, x, y) (col_len * x + y)
/*
for (int k = 0; k < commonK; k += 1) {
@@ -56,7 +56,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
for (size_t k = 0; k < commonK; k++) {
MAT_KERNEL(float)
const *right_cur = right + GET_INDEX(ansHeight, k, j), // GetKernelAddressUnsafe
const *left_cur = left + GET_INDEX(commonK, j, k); // GetKernelAddressUnsafe
const *left_cur = left + GET_INDEX(commonK, i, k); // GetKernelAddressUnsafe
const __m128 a0 = _mm_load_ps((float *)&left_cur->f32_x[0]);
const __m128 a1 = _mm_load_ps((float *)&left_cur->f32_x[1]);
@@ -119,7 +119,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
for (size_t k = 0; k < commonK; k++) {
MAT_KERNEL(float)
const *left_cur = left + GET_INDEX(ansHeight, k, j), // GetKernelAddressUnsafe
const *left_cur = left + GET_INDEX(ansHeight, i, k), // GetKernelAddressUnsafe
const *right_cur =
right + GET_INDEX(ansHeight, j, k); // GetKernelAddressUnsafe
@@ -182,9 +182,9 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
for (size_t k = 0; k < commonK; k++) {
MAT_KERNEL(float)
const *left_cur = left + GET_INDEX(ansHeight, k, j), // GetKernelAddressUnsafe
const *left_cur = left + GET_INDEX(commonK, i, k), // GetKernelAddressUnsafe
const *right_cur =
right + GET_INDEX(ansHeight, j, k); // GetKernelAddressUnsafe
right + GET_INDEX(ansHeight, k, j); // GetKernelAddressUnsafe
register __m256 a01 = _mm256_load_ps((float *)&left_cur->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left_cur->f32_y[1]);
@@ -293,7 +293,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
a_hi = _mm_load_pd(left_cur + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b));\
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b));
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
{
@@ -313,12 +313,18 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
for (size_t k = 0; k < commonK; k++) {
MAT_KERNEL(double)
const *left_cur = left + GET_INDEX(ansHeight, k, j),
const *left_cur = left + GET_INDEX(ansHeight, i, k),
const *right_cur = right + GET_INDEX(commonK, j, k); // GetKernelAddressUnsafe
{DOUBLE_KERNEL_FMA_CYCLE(0)}
{DOUBLE_KERNEL_FMA_CYCLE(1)}
{DOUBLE_KERNEL_FMA_CYCLE(2)}
{
DOUBLE_KERNEL_FMA_CYCLE(0)
}
{
DOUBLE_KERNEL_FMA_CYCLE(1)
}
{
DOUBLE_KERNEL_FMA_CYCLE(2)
}
{
DOUBLE_KERNEL_FMA_CYCLE(3)
}
@@ -353,12 +359,12 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, , fma)
for (size_t k = 0; k < commonK; k++) {
MAT_KERNEL(double)
const *right_cur = right + GET_INDEX(ansHeight, k, j), // GetKernelAddressUnsafe
const *left_cur = left + GET_INDEX(commonK, j, k); // GetKernelAddressUnsafe
const *left_cur = left + GET_INDEX(commonK, i, k); // GetKernelAddressUnsafe
const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_z[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_z[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_cur->f64_z[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_cur->f64_z[3]);
const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_cur->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_cur->f64_y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[0]);
@@ -430,12 +436,12 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma)
for (size_t k = 0; k < commonK; k++) {
MAT_KERNEL(double)
const *right_cur = right + GET_INDEX(ansHeight, k, j), // GetKernelAddressUnsafe
const *left_cur = left + GET_INDEX(commonK, j, k); // GetKernelAddressUnsafe
const *left_cur = left + GET_INDEX(commonK, i, k); // GetKernelAddressUnsafe
const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_z[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_z[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_cur->f64_z[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_cur->f64_z[3]);
const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_cur->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_cur->f64_y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[0]);
@@ -493,7 +499,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, 512, , fma)
for (size_t k = 0; k < commonK; k++) {
MAT_KERNEL(double)
const *left_cur = left + GET_INDEX(ansHeight, k, j), // GetKernelAddressUnsafe
const *left_cur = left + GET_INDEX(ansHeight, i, k), // GetKernelAddressUnsafe
const *right_cur =
right + GET_INDEX(ansHeight, j, k); // GetKernelAddressUnsafe