youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

many things

Browse Source
This commit is contained in:
Youmu Liu
2026-02-26 18:05:35 +08:00
parent 1b04196314
commit 15434ed053
15 changed files with 530 additions and 591 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-src/load_intrinsic.x86.c
-2
View File
@@ -14,9 +14,7 @@
#endif // _MSC_VER
PRIVATE void load_intrinsic_sse(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_avx_dual(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_avx(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_512_dual(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID id);
SN_API void load_intrinsic_function(void *intrinsic_context, uint64_t id)
StgSharp.Native/sn-src/m.fma.x86.c
+19 -14
View File
@@ -151,29 +151,34 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
{
static const __m512i b_shift = { .m512i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9, 15,
12, 13, 14 } };
static const __m512i b_shift = { .m512i_i32 = {
// shuff order
0, 1, 2, 3, // col0
5, 6, 7, 4, //col1
10, 11, 8, 9, //col2
15, 12, 13, 14 //col3
} };
register __m512 a = _mm512_load_ps((float *)left);
register __m512 b = _mm512_load_ps((float *)right);
register __m512 c = _mm512_load_ps((float *)ans);
b = _mm512_permutexvar_ps(b_shift, b);
register __m512 a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0));
register __m512 b_perm = _mm512_permute_ps(b, _FMA_SELECT(0));
c = _mm512_fmadd_ps(a_perm, b_perm, c);
register __m512 a_perm0 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0));
register __m512 b_perm0 = _mm512_permute_ps(b, _FMA_SELECT(0));
c = _mm512_fmadd_ps(a_perm0, b_perm0, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(1));
c = _mm512_fmadd_ps(a_perm, b_perm, c);
register __m512 a_perm1 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3));
register __m512 b_perm1 = _mm512_permute_ps(b, _FMA_SELECT(1));
c = _mm512_fmadd_ps(a_perm1, b_perm1, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(2));
c = _mm512_fmadd_ps(a_perm, b_perm, c);
register __m512 a_perm2 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2));
register __m512 b_perm2 = _mm512_permute_ps(b, _FMA_SELECT(2));
c = _mm512_fmadd_ps(a_perm2, b_perm2, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(3));
c = _mm512_fmadd_ps(a_perm, b_perm, c);
register __m512 a_perm3 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1));
register __m512 b_perm3 = _mm512_permute_ps(b, _FMA_SELECT(3));
c = _mm512_fmadd_ps(a_perm3, b_perm3, c);
_mm512_store_ps((float *)ans, c);
}
StgSharp.Native/sn-src/m.fma_tile.x86.c
+7 -2
View File
@@ -201,8 +201,13 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
{
static const __m512i b_shift_source = { .m512i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9,
15, 12, 13, 14 } };
static const __m512i b_shift_source = { .m512i_i32 = {
// shuff order
0, 1, 2, 3, // col0
5, 6, 7, 4, //col1
10, 11, 8, 9, //col2
15, 12, 13, 14 //col3
} };
int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1];
int commonK = enumeration->m128_i32[2];
StgSharp.Native/sn-src/m.panel.x86.c → StgSharp.Native/sn-src/m.panel.x86.old
+301 -301
View File
@@ -1,301 +1,301 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_matkernel.h"
DECLARE_BUILD_PANEL(avx, float)
{
const int Q = 8; /* fixed for AVX float panel */
/* Neighbor kernels; substitute global zero kernel where out-of-bounds */
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
const MAT_KERNEL(float) *const k_lt = matrix + (row_index * col_length + col_index);
const MAT_KERNEL(float) *const k_rt =
has_right ? (matrix + (row_index * col_length + (col_index + 1))) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) *const k_lb =
has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
register __m128 top128, bot128;
register __m256 col256;
for (int pc = 0; pc < Q; ++pc) {
const int is_right_half = (pc >= 4);
const int kc_col = pc & 3; /* column index inside 4x4 kernel */
const MAT_KERNEL(float) *src_top = is_right_half ? k_rt : k_lt;
const MAT_KERNEL(float) *src_bot = is_right_half ? k_rb : k_lb;
/* Vector loads: each kernel column is a compact __m128 */
top128 = src_top->f32_x[kc_col];
bot128 = src_bot->f32_x[kc_col];
/* Combine into an AVX column: low 128 = top, high 128 = bottom */
col256 = _mm256_set_m128(bot128, top128);
PANEL_COL_VEC((*panel), pc, 0) = col256;
}
}
DECLARE_BUILD_PANEL(512, float)
{
const MAT_KERNEL(float) *const zero = (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) * k[4][4];
for (int kc = 0; kc < 4; ++kc) {
for (int kr = 0; kr < 4; ++kr) {
const int col = col_index + kc;
const int row = row_index + kr;
k[kc][kr] = (col < col_length && row < row_length) ?
(matrix + (row * col_length + col)) :
zero;
}
}
/* Each panel column corresponds to a kernel column (0..3) within a block col (0..3). */
for (int pc = 0; pc < 16; ++pc) {
const int blk_col = pc >> 2; /* which kernel column block (0..3) */
const int kc_col = pc & 3; /* column inside a 4x4 kernel */
const __m128 r0 = k[blk_col][0]->f32_x[kc_col];
const __m128 r1 = k[blk_col][1]->f32_x[kc_col];
const __m128 r2 = k[blk_col][2]->f32_x[kc_col];
const __m128 r3 = k[blk_col][3]->f32_x[kc_col];
const __m256 top = _mm256_set_m128(r1, r0);
const __m256 bot = _mm256_set_m128(r3, r2);
__m512 col = _mm512_castps256_ps512(top);
col = _mm512_insertf32x8(col, bot, 1);
PANEL_COL_VEC((*panel), pc, 0) = col;
}
}
DECLARE_BUILD_PANEL(512, double)
{
const int Q = 8; /* max(4, lanes) for double with ZMM -> 8 columns */
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
const MAT_KERNEL(double) *const k_lt = matrix + (row_index * col_length + col_index);
const MAT_KERNEL(double) *const k_rt =
has_right ? (matrix + (row_index * col_length + (col_index + 1))) :
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(double) *const k_lb =
has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) :
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(double) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
for (int pc = 0; pc < Q; ++pc) {
const int is_right_half = (pc >= 4);
const int kc_col = pc & 3; /* column index inside 4x4 kernel */
const MAT_KERNEL(double) *src_top = is_right_half ? k_rt : k_lt;
const MAT_KERNEL(double) *src_bot = is_right_half ? k_rb : k_lb;
const __m256d top256 = src_top->f64_y[kc_col];
const __m256d bot256 = src_bot->f64_y[kc_col];
__m512d col = _mm512_castpd256_pd512(top256);
col = _mm512_insertf64x4(col, bot256, 1);
PANEL_COL_VEC((*panel), pc, 0) = _mm512_castpd_ps(col);
}
}
DECLARE_STORE_PANEL(avx, float)
{
const int Q = 8; /* fixed for AVX float panel */
/* Local dummy kernel to swallow out-of-bounds writes; no init needed */
MAT_KERNEL(float) zero;
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(float) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (row_index * col_length + col_index)) :
&zero;
MAT_KERNEL(float) *const k_rt =
has_right && (row_index < row_length) ?
(matrix + (row_index * col_length + (col_index + 1))) :
&zero;
MAT_KERNEL(float) *const k_lb =
has_bottom && (col_index < col_length) ?
(matrix + ((row_index + 1) * col_length + col_index)) :
&zero;
MAT_KERNEL(float) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero;
register __m128 top128, bot128;
int i = 0;
for (; i < Q / 2; i++) {
const int kc_col = i; /* column index inside 4x4 kernel */
/* Load the panel column vector */
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
/* Split into top/bottom halves */
top128 = _mm256_castps256_ps128(col256); /* low 128 */
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
/* Store back into kernels directly (out-of-bounds writes go to dummy) */
k_rt->f32_x[kc_col] = top128;
k_rb->f32_x[kc_col] = bot128;
}
for (; i < Q; i++) {
const int kc_col = i - 4; /* column index inside 4x4 kernel */
/* Load the panel column vector */
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
/* Split into top/bottom halves */
top128 = _mm256_castps256_ps128(col256); /* low 128 */
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
/* Store back into kernels directly (out-of-bounds writes go to dummy) */
k_lt->f32_x[kc_col] = top128;
k_lb->f32_x[kc_col] = bot128;
}
}
DECLARE_STORE_PANEL(512, float)
{
MAT_KERNEL(float) zero;
MAT_KERNEL(float) * k[4][4];
for (int kc = 0; kc < 4; ++kc) {
for (int kr = 0; kr < 4; ++kr) {
const int col = col_index + kc;
const int row = row_index + kr;
k[kc][kr] = (col < col_length && row < row_length) ?
(matrix + (row * col_length + col)) :
&zero;
}
}
for (int pc = 0; pc < 16; ++pc) {
const int blk_col = pc >> 2; /* which kernel column block (0..3) */
const int kc_col = pc & 3; /* column inside a 4x4 kernel */
const __m512 col = PANEL_COL_VEC((*panel), pc, 0);
const __m256 top = _mm512_castps512_ps256(col);
const __m256 bot = _mm512_extractf32x8_ps(col, 1);
const __m128 r0 = _mm256_castps256_ps128(top);
const __m128 r1 = _mm256_extractf128_ps(top, 1);
const __m128 r2 = _mm256_castps256_ps128(bot);
const __m128 r3 = _mm256_extractf128_ps(bot, 1);
k[blk_col][0]->f32_x[kc_col] = r0;
k[blk_col][1]->f32_x[kc_col] = r1;
k[blk_col][2]->f32_x[kc_col] = r2;
k[blk_col][3]->f32_x[kc_col] = r3;
}
}
DECLARE_STORE_PANEL(sse, double)
{
MAT_KERNEL(double) *dst = matrix + (row_index * col_length + col_index);
for (int pc = 0; pc < 4; ++pc) {
const int base = pc * 2;
dst->f64_x[base] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 0));
dst->f64_x[base + 1] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 1));
}
}
DECLARE_STORE_PANEL(avx, double)
{
const int Q = 8; /* fixed for AVX double panel */
MAT_KERNEL(double) zero;
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (row_index * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rt =
has_right && (row_index < row_length) ?
(matrix + (row_index * col_length + (col_index + 1))) :
&zero;
MAT_KERNEL(double) *const k_lb =
has_bottom && (col_index < col_length) ?
(matrix + ((row_index + 1) * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero;
for (int i = 0; i < Q; ++i) {
const int kc_col = (i < 4) ? i : (i - 4);
const __m256 top_chunk = PANEL_COL_VEC((*panel), i, 0);
const __m256 bot_chunk = PANEL_COL_VEC((*panel), i, 1);
MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt;
MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb;
top_dst->f64_y[kc_col] = _mm256_castps_pd(top_chunk);
bot_dst->f64_y[kc_col] = _mm256_castps_pd(bot_chunk);
}
}
DECLARE_STORE_PANEL(512, double)
{
const int Q = 8; /* max(4, lanes) -> 8 columns, one ZMM per column */
MAT_KERNEL(double) zero;
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (row_index * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rt =
has_right && (row_index < row_length) ?
(matrix + (row_index * col_length + (col_index + 1))) :
&zero;
MAT_KERNEL(double) *const k_lb =
has_bottom && (col_index < col_length) ?
(matrix + ((row_index + 1) * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero;
for (int i = 0; i < Q; ++i) {
const int kc_col = (i < 4) ? i : (i - 4);
const __m512 col = PANEL_COL_VEC((*panel), i, 0);
const __m512d col_pd = _mm512_castps_pd(col);
const __m256d top256 = _mm512_castpd512_pd256(col_pd);
const __m256d bot256 = _mm512_extractf64x4_pd(col_pd, 1);
MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt;
MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb;
top_dst->f64_y[kc_col] = top256;
bot_dst->f64_y[kc_col] = bot256;
}
}
#endif
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_matkernel.h"
DECLARE_BUILD_PANEL(avx, float)
{
const int Q = 8; /* fixed for AVX float panel */
/* Neighbor kernels; substitute global zero kernel where out-of-bounds */
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
const MAT_KERNEL(float) *const k_lt = matrix + (row_index * col_length + col_index);
const MAT_KERNEL(float) *const k_rt =
has_right ? (matrix + (row_index * col_length + (col_index + 1))) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) *const k_lb =
has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
register __m128 top128, bot128;
register __m256 col256;
for (int pc = 0; pc < Q; ++pc) {
const int is_right_half = (pc >= 4);
const int kc_col = pc & 3; /* column index inside 4x4 kernel */
const MAT_KERNEL(float) *src_top = is_right_half ? k_rt : k_lt;
const MAT_KERNEL(float) *src_bot = is_right_half ? k_rb : k_lb;
/* Vector loads: each kernel column is a compact __m128 */
top128 = src_top->f32_x[kc_col];
bot128 = src_bot->f32_x[kc_col];
/* Combine into an AVX column: low 128 = top, high 128 = bottom */
col256 = _mm256_set_m128(bot128, top128);
PANEL_COL_VEC((*panel), pc, 0) = col256;
}
}
DECLARE_BUILD_PANEL(512, float)
{
const MAT_KERNEL(float) *const zero = (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) * k[4][4];
for (int kc = 0; kc < 4; ++kc) {
for (int kr = 0; kr < 4; ++kr) {
const int col = col_index + kc;
const int row = row_index + kr;
k[kc][kr] = (col < col_length && row < row_length) ?
(matrix + (row * col_length + col)) :
zero;
}
}
/* Each panel column corresponds to a kernel column (0..3) within a block col (0..3). */
for (int pc = 0; pc < 16; ++pc) {
const int blk_col = pc >> 2; /* which kernel column block (0..3) */
const int kc_col = pc & 3; /* column inside a 4x4 kernel */
const __m128 r0 = k[blk_col][0]->f32_x[kc_col];
const __m128 r1 = k[blk_col][1]->f32_x[kc_col];
const __m128 r2 = k[blk_col][2]->f32_x[kc_col];
const __m128 r3 = k[blk_col][3]->f32_x[kc_col];
const __m256 top = _mm256_set_m128(r1, r0);
const __m256 bot = _mm256_set_m128(r3, r2);
__m512 col = _mm512_castps256_ps512(top);
col = _mm512_insertf32x8(col, bot, 1);
PANEL_COL_VEC((*panel), pc, 0) = col;
}
}
DECLARE_BUILD_PANEL(512, double)
{
const int Q = 8; /* max(4, lanes) for double with ZMM -> 8 columns */
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
const MAT_KERNEL(double) *const k_lt = matrix + (row_index * col_length + col_index);
const MAT_KERNEL(double) *const k_rt =
has_right ? (matrix + (row_index * col_length + (col_index + 1))) :
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(double) *const k_lb =
has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) :
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(double) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
for (int pc = 0; pc < Q; ++pc) {
const int is_right_half = (pc >= 4);
const int kc_col = pc & 3; /* column index inside 4x4 kernel */
const MAT_KERNEL(double) *src_top = is_right_half ? k_rt : k_lt;
const MAT_KERNEL(double) *src_bot = is_right_half ? k_rb : k_lb;
const __m256d top256 = src_top->f64_y[kc_col];
const __m256d bot256 = src_bot->f64_y[kc_col];
__m512d col = _mm512_castpd256_pd512(top256);
col = _mm512_insertf64x4(col, bot256, 1);
PANEL_COL_VEC((*panel), pc, 0) = _mm512_castpd_ps(col);
}
}
DECLARE_STORE_PANEL(avx, float)
{
const int Q = 8; /* fixed for AVX float panel */
/* Local dummy kernel to swallow out-of-bounds writes; no init needed */
MAT_KERNEL(float) zero;
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(float) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (row_index * col_length + col_index)) :
&zero;
MAT_KERNEL(float) *const k_rt =
has_right && (row_index < row_length) ?
(matrix + (row_index * col_length + (col_index + 1))) :
&zero;
MAT_KERNEL(float) *const k_lb =
has_bottom && (col_index < col_length) ?
(matrix + ((row_index + 1) * col_length + col_index)) :
&zero;
MAT_KERNEL(float) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero;
register __m128 top128, bot128;
int i = 0;
for (; i < Q / 2; i++) {
const int kc_col = i; /* column index inside 4x4 kernel */
/* Load the panel column vector */
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
/* Split into top/bottom halves */
top128 = _mm256_castps256_ps128(col256); /* low 128 */
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
/* Store back into kernels directly (out-of-bounds writes go to dummy) */
k_rt->f32_x[kc_col] = top128;
k_rb->f32_x[kc_col] = bot128;
}
for (; i < Q; i++) {
const int kc_col = i - 4; /* column index inside 4x4 kernel */
/* Load the panel column vector */
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
/* Split into top/bottom halves */
top128 = _mm256_castps256_ps128(col256); /* low 128 */
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
/* Store back into kernels directly (out-of-bounds writes go to dummy) */
k_lt->f32_x[kc_col] = top128;
k_lb->f32_x[kc_col] = bot128;
}
}
DECLARE_STORE_PANEL(512, float)
{
MAT_KERNEL(float) zero;
MAT_KERNEL(float) * k[4][4];
for (int kc = 0; kc < 4; ++kc) {
for (int kr = 0; kr < 4; ++kr) {
const int col = col_index + kc;
const int row = row_index + kr;
k[kc][kr] = (col < col_length && row < row_length) ?
(matrix + (row * col_length + col)) :
&zero;
}
}
for (int pc = 0; pc < 16; ++pc) {
const int blk_col = pc >> 2; /* which kernel column block (0..3) */
const int kc_col = pc & 3; /* column inside a 4x4 kernel */
const __m512 col = PANEL_COL_VEC((*panel), pc, 0);
const __m256 top = _mm512_castps512_ps256(col);
const __m256 bot = _mm512_extractf32x8_ps(col, 1);
const __m128 r0 = _mm256_castps256_ps128(top);
const __m128 r1 = _mm256_extractf128_ps(top, 1);
const __m128 r2 = _mm256_castps256_ps128(bot);
const __m128 r3 = _mm256_extractf128_ps(bot, 1);
k[blk_col][0]->f32_x[kc_col] = r0;
k[blk_col][1]->f32_x[kc_col] = r1;
k[blk_col][2]->f32_x[kc_col] = r2;
k[blk_col][3]->f32_x[kc_col] = r3;
}
}
DECLARE_STORE_PANEL(sse, double)
{
MAT_KERNEL(double) *dst = matrix + (row_index * col_length + col_index);
for (int pc = 0; pc < 4; ++pc) {
const int base = pc * 2;
dst->f64_x[base] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 0));
dst->f64_x[base + 1] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 1));
}
}
DECLARE_STORE_PANEL(avx, double)
{
const int Q = 8; /* fixed for AVX double panel */
MAT_KERNEL(double) zero;
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (row_index * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rt =
has_right && (row_index < row_length) ?
(matrix + (row_index * col_length + (col_index + 1))) :
&zero;
MAT_KERNEL(double) *const k_lb =
has_bottom && (col_index < col_length) ?
(matrix + ((row_index + 1) * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero;
for (int i = 0; i < Q; ++i) {
const int kc_col = (i < 4) ? i : (i - 4);
const __m256 top_chunk = PANEL_COL_VEC((*panel), i, 0);
const __m256 bot_chunk = PANEL_COL_VEC((*panel), i, 1);
MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt;
MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb;
top_dst->f64_y[kc_col] = _mm256_castps_pd(top_chunk);
bot_dst->f64_y[kc_col] = _mm256_castps_pd(bot_chunk);
}
}
DECLARE_STORE_PANEL(512, double)
{
const int Q = 8; /* max(4, lanes) -> 8 columns, one ZMM per column */
MAT_KERNEL(double) zero;
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (row_index * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rt =
has_right && (row_index < row_length) ?
(matrix + (row_index * col_length + (col_index + 1))) :
&zero;
MAT_KERNEL(double) *const k_lb =
has_bottom && (col_index < col_length) ?
(matrix + ((row_index + 1) * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero;
for (int i = 0; i < Q; ++i) {
const int kc_col = (i < 4) ? i : (i - 4);
const __m512 col = PANEL_COL_VEC((*panel), i, 0);
const __m512d col_pd = _mm512_castps_pd(col);
const __m256d top256 = _mm512_castpd512_pd256(col_pd);
const __m256d bot256 = _mm512_extractf64x4_pd(col_pd, 1);
MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt;
MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb;
top_dst->f64_y[kc_col] = top256;
bot_dst->f64_y[kc_col] = bot256;
}
}
#endif
StgSharp.Native/sn-src/simdid.x86.c
+24 -31
View File
@@ -57,24 +57,15 @@ static int xgetbv_has_avx512_state(void)
return ((xcr0 & mask) == mask);
}
static int is_intel_dual_avx512(int family, int model)
/* AVX-512 execution-width classification for AMD, by CPU family.
* Zen4 (0x19): single 256-bit ALU, 512-bit op = two 256-bit micro-ops (dual-pump) → DUAL.
* Zen5+ (>=0x1A): native 512-bit execution units → NATIVE.
* Other/future AMD families → UNKNOWN (conservative). */
static uint64_t amd_avx512_impl(int family)
{
return (family == 6) && (model == 0x55); /* SKX/CLX/CPX */
}
static int is_intel_native_avx512(int family, int model)
{
if (family != 6)
return 0;
switch (model) {
case 0x6A: /* ICX */
case 0x6C: /* ICX stepping */
case 0x8F: /* SPR/EMR */
case 0xCF: /* GNR */
return 1;
default:
return 0;
}
if (family == 0x19) return SIMDID_AVX512_IMPL_DUAL;
if (family >= 0x1A) return SIMDID_AVX512_IMPL_NATIVE;
return SIMDID_AVX512_IMPL_UNKNOWN;
}
/* Returns 1 if this AMD/Hygon CPU uses split-lane YMM (two fused 128-bit execution units).
@@ -110,9 +101,9 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local()
((int *)vendor)[2] = ecx;
vendor[12] = '\0';
const int is_intel = (strcmp(vendor, "GenuineIntel") == 0);
const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0) ||
(strcmp(vendor, "HygonGenuine") == 0);
const int is_via = (strcmp(vendor, "CentaurHauls") == 0) ||
const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0);
const int is_hygon = (strcmp(vendor, "HygonGenuine") == 0); /* 海光: separate from AMD */
const int is_via = (strcmp(vendor, "CentaurHauls") == 0) ||
(memcmp(vendor, " Shanghai ", 12) == 0); /* VIA / Zhaoxin */
/* CPUID.1 */
@@ -128,9 +119,10 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local()
int has_f16c = (ecx & (1 << 29)) != 0;
uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64);
uint64_t manu = SIMDID_PACK_MANU(is_intel ? SIMDID_MANU_INTEL :
is_amd ? SIMDID_MANU_AMD :
is_via ? SIMDID_MANU_VIA :
uint64_t manu = SIMDID_PACK_MANU(is_intel ? SIMDID_MANU_INTEL :
is_amd ? SIMDID_MANU_AMD :
is_hygon ? SIMDID_MANU_HYGON :
is_via ? SIMDID_MANU_VIA :
SIMDID_MANU_UNKNOWN);
uint64_t main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE);
uint64_t avx512_bits = 0;
@@ -202,16 +194,17 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local()
if (has_vbmi2)
avx512_bits |= SIMDID_AVX512_VBMI2;
/* impl classification */
const int glue = is_amd || is_intel_dual_avx512(family, model);
const int native512 = is_intel_native_avx512(family, model);
if (glue) {
avx512_bits |= SIMDID_AVX512_IMPL_DUAL;
} else if (native512) {
/* Execution-width classification:
* Intel all → NATIVE (genuine 512-bit ALU; SKX dual-port is native)
* AMD Zen4 → DUAL (dual-pump 256-bit, Family 0x19)
* AMD Zen5+ → NATIVE (native 512-bit, Family >= 0x1A)
* Hygon/VIA/unknown → UNKNOWN (conservative) */
if (is_intel)
avx512_bits |= SIMDID_AVX512_IMPL_NATIVE;
} else {
else if (is_amd)
avx512_bits |= amd_avx512_impl(family);
else
avx512_bits |= SIMDID_AVX512_IMPL_UNKNOWN;
}
/* main level for AVX512 */
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512);