youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove mat panel, mat_task goes on span like

Browse Source
This commit is contained in:
Youmu Liu
2026-06-10 18:10:52 +08:00
parent 0356f5aff8
commit 09c1cace48
8 changed files with 717 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-head/sn_internal.h
+2
View File
@@ -40,4 +40,6 @@
#define PRIVATE static
#define INTERNAL extern
#define u64 uint64_t
#endif //
StgSharp.Native/sn-head/sn_intrinsic.context.x86.h
+1 -1
View File
@@ -128,7 +128,7 @@
#pragma region matix function
INTERNAL void f32_normalize(sn_vec_f32 * source, sn_vec_f32 * target);
INTERNAL void f32_normalize(VEC_SEGMENT(float) * source, VEC_SEGMENT(float) * target);
SN_MK_PROC_DECL_STD(float, sse, , k_quality);
StgSharp.Native/sn-head/sn_intrinsic.h
+49 -32
View File
@@ -65,11 +65,37 @@ typedef union uint64_u {
typedef struct mat_kernel mat_kernel;
typedef struct mat_panel mat_panel;
typedef union M128{
#if SN_IS_ARCH(x86_64)
__m128 m128;
#endif
float f32[4];
double f64[2];
int32_t i32[4];
int64_t i64[2];
}M128;
typedef union scalar_pack {
uint64_t data[8];
} scalar_pack;
typedef union M256{
#if SN_IS_ARCH(x86_64)
__m256 m256;
#endif
float f32[8];
double f64[4];
int32_t i32[8];
int64_t i64[4];
}M256;
typedef union M512{
#if SN_IS_ARCH(x86_64)
__m512 m512;
#endif
uint64_t u64[8];
uint32_t u32[16];
float f32[16];
double f64[8];
int32_t i32[16];
int64_t i64[8];
}M512;
typedef struct sn_L4_cache_head {
uint64_t reverse_mask_0;
@@ -81,42 +107,33 @@ typedef struct sn_L4_cache_head {
typedef struct sn_L4_handle {
void *ptr;
sn_L4_cache_head head;
sn_L4_cache_head* head;
} sn_L4_handle;
typedef struct matrix_task_pack {
typedef struct matrix_task {
sn_L4_handle buffer_0;
sn_L4_handle buffer_1;
sn_L4_handle buffer_2;
sn_L4_handle buffer_3;
uint64_t reverse_0;
uint64_t reverse_1;
uint64_t reverse_2;
uint64_t reverse_3;
uint64_t reverse_4;
uint64_t reverse_5;
uint64_t reverse_6;
uint64_t reverse_7;
union {
struct {
int offset_0;
int count_0;
int offset_1;
int count_1;
int offset_2;
int count_2;
int offset_3;
int count_3;
uint64_t reverse;
M128 scalar_pack;
};
M512 profile;
};
} matrix_task_pack;
typedef struct matrix_task {
void *mat_0;
void *mat_1;
void *mat_2;
void *scalar;
int32_t x;
int32_t y;
int32_t z;
int32_t common_x;
int32_t common_y;
int32_t common_z;
uint32_t mask;
uint32_t reverse;
scalar_pack padding;
} matrix_task;
#define MAT_TASK(T) matrix_task_pack
#define MAT_TASK(T) matrix_task
#if defined(__cplusplus)
static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mismatch");
@@ -128,7 +145,7 @@ typedef void(SN_DECL *UNI_MK_PROC)(matrix_task *task);
typedef void(SN_DECL *UNI_SK_PROC)(mat_kernel const *left, mat_kernel const *right,
mat_kernel *restrict ans, uint64_t scalar);
typedef void(SN_DECL *VECTORNORMALIZEPROC)(sn_vec_f32 *source, sn_vec_f32 *target);
typedef void(SN_DECL *VECTORNORMALIZEPROC)(VEC_SEGMENT(float) *source, VEC_SEGMENT(float) *target);
typedef void(SN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans);
typedef uint64_t(SN_DECL *FACTORIALROC)(int n);
StgSharp.Native/sn-head/sn_intrinsic.matkernel.h
+29 -112
View File
@@ -12,13 +12,14 @@
#define YMMCOUNT(T) (sizeof(T) / 2)
#define ZMMCOUNT(T) (sizeof(T) / 4)
#define SN_MAT_KERNEL_MEMBER(T) {\
__m128 x[XMMCOUNT(T)]; \
__m256 y[YMMCOUNT(T)]; \
__m512 z[ZMMCOUNT(T)]; \
T m[4][4];\
}
#define SN_MAT_KERNEL_MEMBER(T) \
{ \
__m128 x[XMMCOUNT(T)]; \
__m256 y[YMMCOUNT(T)]; \
__m512 z[ZMMCOUNT(T)]; \
T m[4][4]; \
}
/*
#define SN_DEFINE_MAT_KERNEL_TYPE(name, T) \
typedef union name SN_MAT_KERNEL_MEMBER(T) name
@@ -26,120 +27,36 @@
SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_float, float);
SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_double, double);
SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_uint64_t, uint64_t);
#define MAT_KERNEL_DEFINE(T) \
/* */
#define SN_MAT_KERNEL_DEFINE(T) \
typedef union SN_MAT_KERNEL_TYPE_##T SN_MAT_KERNEL_MEMBER(T) SN_MAT_KERNEL_TYPE_##T
#define SN_MAT_KERNEL_TYPE(T) SN_MAT_KERNEL_TYPE_##T
MAT_KERNEL_DEFINE(float);
MAT_KERNEL_DEFINE(double);
MAT_KERNEL_DEFINE(uint64_t);
#define MAT_KERNEL(T) SN_MAT_KERNEL_TYPE(T)
#define SN_MAT_KERNEL_TYPE(T) SN_MAT_KERNEL_TYPE_##T
#define VEC(T) \
union { \
__m128 f32_x[XMMCOUNT(T) / 4]; \
__m256 f32_y[YMMCOUNT(T) / 4]; \
__m512 f32_z[ZMMCOUNT(T) / 4]; \
\
__m128d f64_x[XMMCOUNT(T) / 4]; \
__m256d f64_y[YMMCOUNT(T) / 4]; \
__m512d f64_z[ZMMCOUNT(T) / 4]; \
T v[4]; \
SN_MAT_KERNEL_DEFINE(float);
SN_MAT_KERNEL_DEFINE(double);
SN_MAT_KERNEL_DEFINE(uint64_t);
#define SN_VEC_SEGMENT_MEMBER(T) \
{ \
__m128 f32_x[XMMCOUNT(T) / 4]; \
__m256 f32_y[YMMCOUNT(T) / 4]; \
__m512 f32_z[ZMMCOUNT(T) / 4]; \
T v[4]; \
}
typedef VEC(float) sn_vec_f32;
#define SN_VEC_SEGMENT_DEFINE(T) \
typedef union SN_VEC_SEGMENT_TYPE_##T SN_VEC_SEGMENT_MEMBER(T) SN_VEC_SEGMENT_TYPE_##T
/* Provide a named alias for the zero kernel type to ensure consistent linkage in C++. */
typedef sn_mat_kernel_uint64_t sn_zero_kernel_u64;
#define VEC_SEGMENT(T) SN_VEC_SEGMENT_TYPE(T)
#define SN_VEC_SEGMENT_TYPE(T) SN_VEC_SEGMENT_TYPE_##T
/* Helpers (all compile-time): */
#define SN_LANES(T_VEC, T) (sizeof(T_VEC) / sizeof(T))
#define SN_MAX(a, b) (((a)) > ((b)) ? (a) : (b))
#define SN_CEIL_DIV(a, b) (((a) + (b) - 1) / (b))
#define SN_BITS(x) ((int)(sizeof(x) * 8))
#define PANEL_MINQ(T_VEC, T) SN_MAX(4, SN_MAX(SN_BITS(T_VEC) / 64, SN_BITS(T_VEC) / SN_BITS(T)))
/* Panel geometry helpers (all compile-time): */
#define PANEL_Q(T, T_VEC, MINQ) SN_MAX(SN_LANES(T_VEC, T), (MINQ))
#define PANEL_CHUNKS(T, T_VEC, MINQ) SN_CEIL_DIV(PANEL_Q(T, T_VEC, MINQ), SN_LANES(T_VEC, T))
/*
Column-major square panel with minimum side Q:
- Let R = lanes per vector = sizeof(T_VEC)/sizeof(T).
- Side length Q = max(R, MINQ) to enforce the requested minimum extent.
- Each column is stored contiguously via CH = ceil(Q/R) vector chunks.
- Memory layout:
vec[col][chunk] is a T_VEC holding up to R scalars of column 'col'
spanning rows chunk*R .. chunk*R + (R - 1).
flat[] mirrors the same column-major order (column stride = Q).
- SIMD view remains integer-typed; cast inside micro-kernels to the
appropriate floating-point/integer vector type on demand.
*/
#define MAT_PANEL_SQ_MIN_NOALIGN(T, T_VEC, MINQ) \
union { \
T_VEC vec[PANEL_Q(T, T_VEC, MINQ)][PANEL_CHUNKS(T, T_VEC, MINQ)]; \
T flat[PANEL_Q(T, T_VEC, MINQ) * PANEL_Q(T, T_VEC, MINQ)]; \
}
/* ISA presets with minimum square side:
- SSE : MINQ=4 (avoid 2x2 for double)
- AVX2 : MINQ computed as max(4, vector_bits/64, vector_bits/element_bits)
- AVX-512: same rule (float/i32 naturally expand to 16)
*/
/* SSE panel layout: column-major buffer holding 4 logical columns with MINQ=4.
Columns: each column packs up to four scalars so ALIGN equals sizeof(__m128).
Alignment: every column boundary lands on a 16-byte multiple for XMM loads. */
#define MAT_PANEL_sse(T) SN_ALIGNAS(16) MAT_PANEL_SQ_MIN_NOALIGN(T, __m128, PANEL_MINQ(__m128, T))
/* AVX2 panel layout: column-major buffer holding 8 logical columns with MINQ per PANEL_MINQ.
Columns: each column stores up to lanes elements via contiguous chunks.
Alignment: column starts are 32-byte aligned so AVX2 loads/stores remain contiguous. */
#define MAT_PANEL_avx(T) SN_ALIGNAS(32) MAT_PANEL_SQ_MIN_NOALIGN(T, __m256, PANEL_MINQ(__m256, T))
/* AVX2/FMA panel layout shares the same geometry as AVX2. */
#define MAT_PANEL_avx_fma(T) \
SN_ALIGNAS(32) MAT_PANEL_SQ_MIN_NOALIGN(T, __m256, PANEL_MINQ(__m256, T))
/* AVX-512 panel layout: column-major buffer holding 8 or more columns depending on T.
Columns: each column expands to ceil(Q/lanes) ZMM chunks.
Alignment: column starts sit on 64-byte boundaries to match AVX-512 streaming access. */
#define MAT_PANEL_512(T) SN_ALIGNAS(64) MAT_PANEL_SQ_MIN_NOALIGN(T, __m512, PANEL_MINQ(__m512, T))
SN_VEC_SEGMENT_DEFINE(float);
SN_VEC_SEGMENT_DEFINE(double);
/* Global zero kernel declaration (defined in dllmain.c), aligned to 64 bytes */
#ifdef __cplusplus
extern "C" {
#endif
extern __declspec(align(64)) sn_zero_kernel_u64 SN_ZERO_KERNEL;
#ifdef __cplusplus
}
#endif
/* Access helpers (all compile-time friendly):
Given Q = PANEL_Q(...), CH = PANEL_CHUNKS(...), R = lanes per vector:
- Column c occupies vec[c][0..CH-1].
- Row r inside column c is located at vec[c][r / R] with lane (r % R).
*/
#define PANEL_COL_VEC(panel_obj, c, chunk) ((panel_obj).vec[(int)(c)][(int)(chunk)])
/* Scalar flat index helper for column-major addressing */
#define PANEL_ELEM(panel_obj, T, T_VEC, MINQ, r, c) \
((panel_obj).flat[(int)(c) * PANEL_Q(T, T_VEC, MINQ) + (int)(r)])
/* API macros */
#define BUILD_PANEL(arch, element) build_panel_##arch##_##element
#define STORE_PANEL(arch, element) store_panel_##arch##_##element
#define DECLARE_BUILD_PANEL(arch, element) \
INTERNAL void SN_DECL BUILD_PANEL(arch, element)( \
MAT_PANEL_##arch(element) *restrict panel, \
MAT_KERNEL(element) const *const matrix, int col_length, int row_length, \
int col_index, int row_index)
#define DECLARE_STORE_PANEL(arch, element) \
INTERNAL void SN_DECL STORE_PANEL(arch, element)(MAT_PANEL_##arch(element) \
const *const panel, \
MAT_KERNEL(element) *restrict matrix, \
int col_length, int row_length, \
int col_index, int row_index)
extern __declspec(align(64)) MAT_KERNEL(u64) SN_ZERO_KERNEL;
#endif /* SN_MATKERNEL */
StgSharp.Native/sn-head/sn_intrinsic.std.h
+30 -49
View File
@@ -46,73 +46,54 @@
#define _SN_MATPROC_RIGHT_ANS_SCALAR 1
#define _SN_MATPROC_ANS_SCALAR 1
#define _SN_MK_EXPAND_RIGHT_ANS \
SN_MAT_KERNEL_LOCAL const *right = (SN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
SN_MAT_KERNEL_LOCAL *restrict ans = (SN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr;
#define _SN_MK_EXPAND_BUFFER_SIZE(i)\
int offset_##i = task->offset_##i;\
int count_##i = task->count_##i;\
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS \
#define _SN_MK_EXPAND_RIGHT_ANS \
SN_MAT_KERNEL_LOCAL const *right = (SN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
SN_MAT_KERNEL_LOCAL *restrict ans = (SN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr;\
_SN_MK_EXPAND_BUFFER_SIZE(1); \
_SN_MK_EXPAND_BUFFER_SIZE(2);
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS \
SN_MAT_KERNEL_LOCAL const *left = (SN_MAT_KERNEL_LOCAL const *)task->buffer_0.ptr; \
SN_MAT_KERNEL_LOCAL const *right = (SN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
SN_MAT_KERNEL_LOCAL *restrict ans = (SN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr;
SN_MAT_KERNEL_LOCAL *restrict ans = (SN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr;\
_SN_MK_EXPAND_BUFFER_SIZE(0); \
_SN_MK_EXPAND_BUFFER_SIZE(1);\
_SN_MK_EXPAND_BUFFER_SIZE(2);
#define _SN_MK_EXPAND_RIGHT_ANS_SCALAR \
#define _SN_MK_EXPAND_RIGHT_ANS_SCALAR \
SN_MAT_KERNEL_LOCAL const *right = (SN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
SN_MAT_KERNEL_LOCAL *restrict ans = (SN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
scalar_pack *restrict scalar = (scalar_pack *restrict)task->scalar;
scalar_pack *restrict scalar = (scalar_pack *restrict)task->scalar;\
_SN_MK_EXPAND_BUFFER_SIZE(1);\
_SN_MK_EXPAND_BUFFER_SIZE(2);
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS_SCALAR \
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS_SCALAR \
SN_MAT_KERNEL_LOCAL const *left = (SN_MAT_KERNEL_LOCAL const *)task->buffer_0.ptr; \
SN_MAT_KERNEL_LOCAL const *right = (SN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
SN_MAT_KERNEL_LOCAL *restrict ans = (SN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
scalar_pack *restrict scalar = (scalar_pack *restrict)task->scalar;
scalar_pack *restrict scalar = (scalar_pack *restrict)task->scalar;\
_SN_MK_EXPAND_BUFFER_SIZE(0);\
_SN_MK_EXPAND_BUFFER_SIZE(1);\
_SN_MK_EXPAND_BUFFER_SIZE(2);
#define _SN_MK_EXPAND_ANS_SCALAR \
#define _SN_MK_EXPAND_ANS_SCALAR \
SN_MAT_KERNEL_LOCAL *restrict ans = (SN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
scalar_pack *restrict scalar = (scalar_pack *restrict)&(task->padding);
scalar_pack *restrict scalar = (scalar_pack *restrict)&(task->padding);\
_SN_MK_EXPAND_BUFFER_SIZE(2);
#define _SN_PARAM_STYLE(T) _SN_MATPROC_##T
#define _SN_MK_EXPAND_PARAM(style) _SN_MK_EXPAND_##style
#pragma endregion
#pragma region mem layout
#define _SN_MATPROC_LAYOUT_BUFFER 1
#define _SN_MATPROC_LAYOUT_TILE 1
#define _SN_MATPROC_LAYOUT_KERNEL 1
#define _MK_MKPROC_EXPAND_INDEX_BUFFER \
size_t offset = UNSAFE_AS(size_t, task->padding.data[0]); \
size_t count = UNSAFE_AS(size_t, task->padding.data[1]);
#define _MK_MKPROC_EXPAND_INDEX_BUFFER \
size_t offset = UNSAFE_AS(size_t, task->padding.data[0]); \
size_t count = UNSAFE_AS(size_t, task->padding.data[1]);
#define _MK_MKPROC_EXPAND_INDEX_KERNEL \
size_t x = UNSAFE_AS(size_t, task->x); \
size_t y = UNSAFE_AS(size_t, task->y); \
size_t common_x = UNSAFE_AS(size_t, task->common_x); \
size_t common_y = UNSAFE_AS(size_t, task->common_y); \
size_t common_z = UNSAFE_AS(size_t, task->common_z);
#define _MK_MKPROC_EXPAND_INDEX_TILE \
size_t x = UNSAFE_AS(size_t, task->x); \
size_t y = UNSAFE_AS(size_t, task->y); \
size_t common_x = UNSAFE_AS(size_t, task->common_x); \
size_t common_y = UNSAFE_AS(size_t, task->common_y); \
size_t common_z = UNSAFE_AS(size_t, task->common_z);
#define _SN_MK_EXPAND_MEM(mem) _MK_MKPROC_EXPAND_INDEX_##mem
#define _SN_MEM_LAYOUT(T) _SN_MATPROC_LAYOUT_##T
#pragma endregion
#define __mk_param_std(mem, param, T) \
typedef MAT_KERNEL(T) SN_MAT_KERNEL_LOCAL; \
matrix_task_pack const *_ = (_SN_PARAM_STYLE(param), (_SN_MEM_LAYOUT(mem)), task); \
_SN_MK_EXPAND_PARAM(param); \
_SN_MK_EXPAND_MEM(mem)
#define __mk_param_std(param, T) \
typedef MAT_KERNEL(T) SN_MAT_KERNEL_LOCAL; \
matrix_task const *_ = (_SN_PARAM_STYLE(param), task); \
_SN_MK_EXPAND_PARAM(param)
#define __sk_param_std(T) \
typedef MAT_KERNEL(T) SN_MAT_KERNEL_LOCAL; \
StgSharp.Native/sn-src/mk.fma.x86.c
+11 -11
View File
@@ -17,28 +17,28 @@
SN_MK_PROC_DECL_STD(float, sse, , fma_pack)
{
__mk_param_std(TILE, RIGHT_ANS, float);
__mk_param_std(RIGHT_ANS, float);
memcpy(ans, right, sizeof(MAT_KERNEL(float)) * common_y * common_z);
}
SN_MK_PROC_DECL_STD(float, avx, , fma_pack)
{
__mk_param_std(TILE, RIGHT_ANS, float);
__mk_param_std(RIGHT_ANS, float);
memcpy(ans, right, sizeof(MAT_KERNEL(float)) * common_y * common_z);
}
SN_MK_PROC_DECL_STD(float, 512, , fma_pack)
{
__mk_param_std(TILE, RIGHT_ANS, float);
__mk_param_std(RIGHT_ANS, float);
memcpy(ans, right, sizeof(MAT_KERNEL(float)) * common_y * common_z);
}
SN_MK_PROC_DECL_STD(float, sse, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
__mk_param_std(LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
@@ -95,7 +95,7 @@ SN_MK_PROC_DECL_STD(float, sse, , fma)
SN_MK_PROC_DECL_STD(float, avx, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
__mk_param_std(LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]);
register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]);
@@ -130,7 +130,7 @@ SN_MK_PROC_DECL_STD(float, avx, , fma)
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
__mk_param_std(LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]);
register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]);
@@ -166,7 +166,7 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
SN_MK_PROC_DECL_STD(float, 512, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
__mk_param_std(LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m512 c0 = _mm512_load_ps((float *)&ans_cur->z[0]);
register __m512 c1 = _mm512_load_ps((float *)&ans_cur->z[0]);
@@ -202,7 +202,7 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
SN_MK_PROC_DECL_STD(double, sse, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
__mk_param_std(LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
@@ -243,7 +243,7 @@ SN_MK_PROC_DECL_STD(double, sse, , fma)
SN_MK_PROC_DECL_STD(double, avx, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
__mk_param_std(LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // tKernelAddressUnsafe
@@ -315,7 +315,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fma)
SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
__mk_param_std(LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
@@ -371,7 +371,7 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
SN_MK_PROC_DECL_STD(double, 512, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
__mk_param_std(LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m512d c0 = _mm512_load_pd((double *)&ans_cur->z[0]);