youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Mat mul mat is now on full function

Browse Source
This commit is contained in:
Youmu Liu
2025-12-16 16:15:30 +08:00
parent 2c7277f3f9
commit 88ed5a95c5
14 changed files with 357 additions and 488 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-head/sn_matkernel.h
+2 -2
View File
@@ -110,9 +110,9 @@ extern __declspec(align(64)) sn_zero_kernel_u64 SN_ZERO_KERNEL;
MAT_KERNEL(element) const *const matrix, int col_length, int row_length, \
int col_index, int row_index)
#define DECLARE_STORE_PANEL(arch, element) \
INTERNAL void SN_DECL STORE_PANEL(arch, element)(MAT_KERNEL(element) *restrict matrix, \
MAT_PANEL_##arch(element) \
INTERNAL void SN_DECL STORE_PANEL(arch, element)(MAT_PANEL_##arch(element) \
const *const panel, \
MAT_KERNEL(element) *restrict matrix, \
int col_length, int row_length, \
int col_index, int row_index)
StgSharp.Native/sn-src/k_basicop.c
+6 -8
View File
@@ -12,19 +12,17 @@ DECLARE_PNL_PROC_ANS(float, sse, clear)
DECLARE_PNL_PROC_ANS(float, avx, clear)
{
register __m256 zero = _mm256_setzero_ps();
PANEL_COL_VEC((*ans), 0, 0) = zero;
PANEL_COL_VEC((*ans), 1, 0) = zero;
PANEL_COL_VEC((*ans), 2, 0) = zero;
PANEL_COL_VEC((*ans), 3, 0) = zero;
for (size_t i = 0; i < 8; i++) {
PANEL_COL_VEC((*ans), i, 0) = zero;
}
}
DECLARE_PNL_PROC_ANS(float, 512, clear)
{
register __m512 zero = _mm512_setzero_ps();
PANEL_COL_VEC((*ans), 0, 0) = zero;
PANEL_COL_VEC((*ans), 1, 0) = zero;
PANEL_COL_VEC((*ans), 2, 0) = zero;
PANEL_COL_VEC((*ans), 3, 0) = zero;
for (size_t i = 0; i < 16; i++) {
PANEL_COL_VEC((*ans), i, 0) = zero;
}
}
// Matrix transpose implementations
StgSharp.Native/sn-src/k_panel.c
+12 -12
View File
@@ -8,16 +8,16 @@ DECLARE_BUILD_PANEL(avx, float)
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
const MAT_KERNEL(float) *const k_lt = matrix + (col_index * row_length + row_index);
const MAT_KERNEL(float) *const k_lt = matrix + (row_index * col_length + col_index);
const MAT_KERNEL(float) *const k_rt =
has_right ? (matrix + ((col_index + 1) * row_length + row_index)) :
has_right ? (matrix + (row_index * col_length + (col_index + 1))) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) *const k_lb =
has_bottom ? (matrix + (col_index * row_length + (row_index + 1))) :
has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((col_index + 1) * row_length + (row_index + 1))) :
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
register __m128 top128, bot128;
@@ -44,7 +44,7 @@ DECLARE_BUILD_PANEL(avx, float)
DECLARE_BUILD_PANEL(sse, float)
{
/* Single kernel -> 4 columns, each is one __m128 column vector. */
const MAT_KERNEL(float) *src = matrix + (col_index * row_length + row_index);
const MAT_KERNEL(float) *src = matrix + (row_index * col_length + col_index);
for (int pc = 0; pc < 4; ++pc) {
PANEL_COL_VEC((*panel), pc, 0) = src->xmm[pc];
@@ -61,7 +61,7 @@ DECLARE_BUILD_PANEL(512, float)
const int col = col_index + kc;
const int row = row_index + kr;
k[kc][kr] = (col < col_length && row < row_length) ?
(matrix + (col * row_length + row)) :
(matrix + (row * col_length + col)) :
zero;
}
}
@@ -97,19 +97,19 @@ DECLARE_STORE_PANEL(avx, float)
const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(float) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (col_index * row_length + row_index)) :
(matrix + (row_index * col_length + col_index)) :
&zero;
MAT_KERNEL(float) *const k_rt =
has_right && (row_index < row_length) ?
(matrix + ((col_index + 1) * row_length + row_index)) :
(matrix + (row_index * col_length + (col_index + 1))) :
&zero;
MAT_KERNEL(float) *const k_lb =
has_bottom && (col_index < col_length) ?
(matrix + (col_index * row_length + (row_index + 1))) :
(matrix + ((row_index + 1) * col_length + col_index)) :
&zero;
MAT_KERNEL(float) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((col_index + 1) * row_length + (row_index + 1))) :
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero;
register __m128 top128, bot128;
@@ -144,7 +144,7 @@ DECLARE_STORE_PANEL(avx, float)
DECLARE_STORE_PANEL(sse, float)
{
MAT_KERNEL(float) *dst = matrix + (col_index * row_length + row_index);
MAT_KERNEL(float) *dst = matrix + (row_index * col_length + col_index);
for (int pc = 0; pc < 4; ++pc) {
const __m128 col = PANEL_COL_VEC((*panel), pc, 0);
@@ -162,7 +162,7 @@ DECLARE_STORE_PANEL(512, float)
const int col = col_index + kc;
const int row = row_index + kr;
k[kc][kr] = (col < col_length && row < row_length) ?
(matrix + (col * row_length + row)) :
(matrix + (row * col_length + col)) :
&zero;
}
}
StgSharp.Native/sn-src/load_intrinsic.c
+4 -1
View File
@@ -65,6 +65,7 @@ void load_intrinsic_sse(sn_intrinsic *context)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->f32_store_panel = STORE_PANEL(sse, float);
context->f32_clear_panel = PNL_PROC(float, sse, clear);
context->f32_build_panel = BUILD_PANEL(sse, float);
context->f32_buffer_add = BUF_PROC(float, sse, add);
@@ -82,6 +83,7 @@ void load_intrinsic_avx(sn_intrinsic *context)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->f32_store_panel = STORE_PANEL(avx, float);
context->f32_clear_panel = PNL_PROC(float, avx, clear);
context->f32_build_panel = BUILD_PANEL(avx, float);
context->f32_buffer_add = BUF_PROC(float, avx, add);
@@ -99,7 +101,8 @@ void load_intrinsic_512(sn_intrinsic *context)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->f32_clear_panel = PNL_PROC(float, sse, clear);
context->f32_store_panel = STORE_PANEL(512, float);
context->f32_clear_panel = PNL_PROC(float, 512, clear);
context->f32_build_panel = BUILD_PANEL(512, float);
context->f32_buffer_add = BUF_PROC(float, 512, add);
context->f32_panel_fma = PNL_PROC(float, 512, fma);