youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Working on L4, now on swiss. FMA_TILE is re arranged.

Browse Source
This commit is contained in:
Youmu Liu
2026-03-03 18:02:31 +08:00
parent ae1a49d6fb
commit 91e136f81f
12 changed files with 1083 additions and 477 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-head/sn_intrinsic.context.x86.h
+8 -8
View File
@@ -182,15 +182,15 @@ SN_MK_PROC_DECL_STD(double, avx, , fma);
SN_MK_PROC_DECL_STD(double, avx, _fma, fma);
SN_MK_PROC_DECL_STD(double, 512, , fma);
SN_MK_PROC_DECL_STD(float, sse, , fma);
SN_MK_PROC_DECL_STD(float, avx, , fma);
SN_MK_PROC_DECL_STD(float, avx, _fma, fma);
SN_MK_PROC_DECL_STD(float, 512, , fma);
SN_MK_PROC_DECL_STD(float, sse, , tiled_fma);
SN_MK_PROC_DECL_STD(float, avx, , tiled_fma);
SN_MK_PROC_DECL_STD(float, avx, _fma, tiled_fma);
SN_MK_PROC_DECL_STD(float, 512, , tiled_fma);
SN_MK_PROC_DECL_STD(double, sse, , fma);
SN_MK_PROC_DECL_STD(double, avx, , fma);
SN_MK_PROC_DECL_STD(double, avx, _fma, fma);
SN_MK_PROC_DECL_STD(double, 512, , fma);
SN_MK_PROC_DECL_STD(double, sse, , tiled_fma);
SN_MK_PROC_DECL_STD(double, avx, , tiled_fma);
SN_MK_PROC_DECL_STD(double, avx, _fma, tiled_fma);
SN_MK_PROC_DECL_STD(double, 512, , tiled_fma);
#pragma endregion
StgSharp.Native/sn-head/sn_intrinsic.h
+6 -6
View File
@@ -54,12 +54,12 @@ typedef union scalar_pack {
MAT_KERNEL(T) * mat_1; /* Offset 8: 2nd matrix kernel base */ \
MAT_KERNEL(T) * mat_2; /* Offset 16: 3rd matrix kernel base */ \
void *scalar; /* Offset 24: scalar pack */ \
int32_t i; /* Offset 32: row index of the result tile */ \
int32_t j; /* Offset 36: column index of the result tile */ \
int32_t k; /* Offset 40: column index of the result tile */ \
int32_t common_i; /* Offset 44: row index of the result tile */ \
int32_t common_j; /* Offset 48: column index of the result tile */ \
int32_t common_k; /* Offset 52: column index of the result tile */ \
int32_t x; /* Offset 32: row index of the result tile */ \
int32_t y; /* Offset 36: column index of the result tile */ \
int32_t z; /* Offset 40: column index of the result tile */ \
int32_t common_x; /* Offset 44: row index of the result tile */ \
int32_t common_y; /* Offset 48: column index of the result tile */ \
int32_t common_z; /* Offset 52: column index of the result tile */ \
uint32_t mask; /* Offset 56: mask for task schedualing, not useful in C side */ \
uint32_t reverse; /* Offset 60: reserved for future use */ \
scalar_pack padding; /* Offset 64: padding to make the struct size 128 bytes */ \
StgSharp.Native/sn-head/sn_intrinsic.std.h
+28 -22
View File
@@ -30,28 +30,28 @@
#define _SN_MATPROC_RIGHT_ANS_SCALAR 1
#define _SN_MATPROC_ANS_SCALAR 1
#define _SN_MK_EXPAND_RIGHT_ANS \
_mat_kernel_local const *right = mat_task_local->mat_1; \
_mat_kernel_local *restrict ans = mat_task_local->mat_2;
#define _SN_MK_EXPAND_RIGHT_ANS \
SN_MAT_KERNEL_LOCAL const *right = mat_task_local->mat_1; \
SN_MAT_KERNEL_LOCAL *restrict ans = mat_task_local->mat_2;
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS(T) \
_mat_kernel_local const *left = mat_task_local->mat_0; \
_mat_kernel_local const *right = mat_task_local->mat_1; \
_mat_kernel_local *restrict ans = mat_task_local->mat_2;
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS \
SN_MAT_KERNEL_LOCAL const *left = mat_task_local->mat_0; \
SN_MAT_KERNEL_LOCAL const *right = mat_task_local->mat_1; \
SN_MAT_KERNEL_LOCAL *restrict ans = mat_task_local->mat_2;
#define _SN_MK_EXPAND_RIGHT_ANS_SCALAR(T) \
_mat_kernel_local const *right = mat_task_local->mat_1; \
_mat_kernel_local *restrict ans = mat_task_local->mat_2; \
#define _SN_MK_EXPAND_RIGHT_ANS_SCALAR \
SN_MAT_KERNEL_LOCAL const *right = mat_task_local->mat_1; \
SN_MAT_KERNEL_LOCAL *restrict ans = mat_task_local->mat_2; \
scalar_pack *restrict scalar = mat_task_local->scalar;
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS_SCALAR(T) \
_mat_kernel_local const *left = mat_task_local->mat_0; \
_mat_kernel_local const *right = mat_task_local->mat_1; \
_mat_kernel_local *restrict ans = mat_task_local->mat_2; \
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS_SCALAR \
SN_MAT_KERNEL_LOCAL const *left = mat_task_local->mat_0; \
SN_MAT_KERNEL_LOCAL const *right = mat_task_local->mat_1; \
SN_MAT_KERNEL_LOCAL *restrict ans = mat_task_local->mat_2; \
scalar_pack *restrict scalar = mat_task_local->scalar;
#define _SN_MK_EXPAND_ANS_SCALAR(T) \
MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2; \
#define _SN_MK_EXPAND_ANS_SCALAR \
SN_MAT_KERNEL_LOCAL *restrict ans = mat_task_local->mat_2; \
scalar_pack *restrict scalar = &(mat_task_local->padding);
#define _SN_PARAM_STYLE(T) _SN_MATPROC_##T
@@ -70,11 +70,17 @@
size_t count = UNSAFE_AS(size_t, task->padding.data[1]);
#define _MK_MKPROC_EXPAND_INDEX_KERNEL \
size_t i = UNSAFE_AS(size_t, task->i); \
size_t j = UNSAFE_AS(size_t, task->j); \
size_t common_i = UNSAFE_AS(size_t, task->common_i); \
size_t common_j = UNSAFE_AS(size_t, task->common_j); \
size_t common_k = UNSAFE_AS(size_t, task->common_k);
size_t x = UNSAFE_AS(size_t, task->x); \
size_t y = UNSAFE_AS(size_t, task->y); \
size_t common_x = UNSAFE_AS(size_t, task->common_x); \
size_t common_y = UNSAFE_AS(size_t, task->common_y); \
size_t common_z = UNSAFE_AS(size_t, task->common_z);
#define _MK_MKPROC_EXPAND_INDEX_TILE \
size_t x = UNSAFE_AS(size_t, task->x); \
size_t y = UNSAFE_AS(size_t, task->y); \
size_t common_x = UNSAFE_AS(size_t, task->common_x); \
size_t common_y = UNSAFE_AS(size_t, task->common_y); \
size_t common_z = UNSAFE_AS(size_t, task->common_z);
#define _SN_MK_EXPAND_MEM(mem) _MK_MKPROC_EXPAND_INDEX_##mem
#define _SN_MEM_LAYOUT(T) _SN_MATPROC_LAYOUT_##T
@@ -82,7 +88,7 @@
#pragma endregion
#define __mk_param_std(mem, param, T) \
typedef MAT_KERNEL(T) _mat_kernel_local; \
typedef MAT_KERNEL(T) SN_MAT_KERNEL_LOCAL; \
typedef MAT_TASK(T) _mat_task_local; \
_mat_task_local *_ = (_SN_PARAM_STYLE(param), (_SN_MEM_LAYOUT(mem)), task); \
_SN_MK_EXPAND_PARAM(param); \
StgSharp.Native/sn-src/check.ps1
+25
View File
@@ -0,0 +1,25 @@
# 1. 版本 — 已确认 18.1.8
clang-format --version
# 2. 不可见字符扫描 — 用完整路径
$file = Join-Path $PWD "simdid.x86.c"
$bytes = [System.IO.File]::ReadAllBytes($file)
for ($i = 0; $i -lt $bytes.Length - 2; $i++) {
if ($bytes[$i] -eq 0xEF -and $bytes[$i+1] -eq 0xBB -and $bytes[$i+2] -eq 0xBF) {
Write-Host "BOM found at offset $i"
}
if ($bytes[$i] -eq 0xC2 -and $bytes[$i+1] -eq 0xA0) {
Write-Host "NBSP at offset $i"
}
if ($bytes[$i] -eq 0xE2 -and $bytes[$i+1] -eq 0x80 -and
($bytes[$i+2] -eq 0x8B -or $bytes[$i+2] -eq 0x8C -or $bytes[$i+2] -eq 0x8D)) {
Write-Host "Zero-width char at offset $i"
}
}
Write-Host "Scan complete. File size: $($bytes.Length) bytes"
# 3. 格式化对比 — 用 fc.exe 避免 PowerShell 的 Format-Custom 歧义
clang-format -assume-filename=test.c $file > formatted.c
fc.exe $file formatted.c
StgSharp.Native/sn-src/formatted.c
+372
View File
@@ -0,0 +1,372 @@
#include "sn_target.h"
#include "sn_intrinsic.h"
#include "StgSharpNative.h"
static SIMDID global_simdid = { 0 };
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.context.x86.h"
#include <immintrin.h>
#include <string.h>
#if defined(_MSC_VER)
#include <intrin.h>
#else
#include <cpuid.h>
#endif
static void get_cpuid(int leaf, int subleaf, int *eax, int *ebx, int *ecx, int *edx)
{
#if defined(_MSC_VER)
int regs[4];
__cpuidex(regs, leaf, subleaf);
*eax = regs[0];
*ebx = regs[1];
*ecx = regs[2];
*edx = regs[3];
#else
unsigned int a, b, c, d;
__cpuid_count((unsigned int)leaf, (unsigned int)subleaf, a, b, c, d);
*eax = (int)a;
*ebx = (int)b;
*ecx = (int)c;
*edx = (int)d;
#endif
}
static int xgetbv_has_xmm_ymm(void)
{
#if defined(_MSC_VER)
unsigned __int64 xcr0 = _xgetbv(0);
#else
unsigned long long xcr0 = __builtin_ia32_xgetbv(0);
#endif
return ((xcr0 & 0x6) == 0x6);
}
static int xgetbv_has_avx512_state(void)
{
const unsigned long long mask = (1ull << 0) | (1ull << 1) | (1ull << 2) | (1ull << 5) |
(1ull << 6);
#if defined(_MSC_VER)
unsigned __int64 xcr0 = _xgetbv(0);
#else
unsigned long long xcr0 = __builtin_ia32_xgetbv(0);
#endif
return ((xcr0 & mask) == mask);
}
static void decode_family_model(int cpuid1_eax, int *family, int *model)
{
int family_value = 0;
int model_value = 0;
const int base_fam = (cpuid1_eax >> 8) & 0xF;
const int ext_fam = (cpuid1_eax >> 20) & 0xFF;
const int base_model = (cpuid1_eax >> 4) & 0xF;
const int ext_model = (cpuid1_eax >> 16) & 0xF;
if (base_fam == 0xF) {
family_value = base_fam + ext_fam;
} else {
family_value = base_fam;
}
if ((base_fam == 0x6) || (base_fam == 0xF)) {
model_value = (ext_model << 4) | base_model;
} else {
model_value = base_model;
}
*family = family_value;
*model = model_value;
}
/* AVX-512 execution-width classification for AMD, by CPU family.
* Zen4 (0x19): single 256-bit ALU, 512-bit op = two 256-bit micro-ops (dual-pump) → DUAL.
* Zen5+ (>=0x1A): native 512-bit execution units → NATIVE.
* Other/future AMD families → UNKNOWN (conservative). */
static uint64_t amd_avx512_impl(int family)
{
if (family == 0x19) {
return SIMDID_AVX512_IMPL_DUAL;
} else if (family >= 0x1A) {
return SIMDID_AVX512_IMPL_NATIVE;
} else {
return SIMDID_AVX512_IMPL_UNKNOWN;
}
}
/* Returns 1 if this AMD/Hygon CPU uses split-lane YMM (two fused 128-bit execution units).
* - AMD Zen1 (family 0x17, model < 0x30) and Zen+ (0x18-0x2F) use a split/glued path.
* - Hygon Dhyana is architecturally Zen1-based; always split.
* - AMD Zen2+ (family 0x17 model >= 0x30, family 0x19+) have native 256-bit execution. */
static int is_amd_glue_ymm(const char *vendor, int cpuid1_eax)
{
int family = 0;
int model = 0;
if (strcmp(vendor, "HygonGenuine") == 0) {
return 1;
}
if (strcmp(vendor, "AuthenticAMD") != 0) {
return 0;
}
decode_family_model(cpuid1_eax, &family, &model);
return (family == 0x17) && (model < 0x30);
}
SN_API SIMDID SN_DECL sn_get_simd_level_local()
{
SIMDID simdid = { 0 };
int eax = 0;
int ebx = 0;
int ecx = 0;
int edx = 0;
int family = 0;
int model = 0;
get_cpuid(0, 0, &eax, &ebx, &ecx, &edx);
const int max_basic_leaf = eax;
char vendor[13];
memcpy(vendor + 0, &ebx, 4);
memcpy(vendor + 4, &edx, 4);
memcpy(vendor + 8, &ecx, 4);
vendor[12] = '\0';
const int is_intel = (strcmp(vendor, "GenuineIntel") == 0);
const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0);
const int is_hygon = (strcmp(vendor, "HygonGenuine") == 0);
const int is_centaur = (strcmp(vendor, "CentaurHauls") == 0);
const int is_zhaoxin = (memcmp(vendor, " Shanghai ", 12) == 0);
const int is_via = is_centaur || is_zhaoxin;
get_cpuid(1, 0, &eax, &ebx, &ecx, &edx);
const int cpuid1_eax = eax;
decode_family_model(cpuid1_eax, &family, &model);
(void)model;
const int has_sse42 = (ecx & (1 << 20)) != 0;
const int has_avx = (ecx & (1 << 28)) != 0;
const int has_osxsave = (ecx & (1 << 27)) != 0;
const int has_fma = (ecx & (1 << 12)) != 0;
const int has_f16c = (ecx & (1 << 29)) != 0;
const uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64);
uint64_t manu_code = SIMDID_MANU_UNKNOWN;
if (is_intel) {
manu_code = SIMDID_MANU_INTEL;
} else if (is_amd) {
manu_code = SIMDID_MANU_AMD;
} else if (is_hygon) {
manu_code = SIMDID_MANU_HYGON;
} else if (is_via) {
manu_code = SIMDID_MANU_VIA;
}
const uint64_t manu = SIMDID_PACK_MANU(manu_code);
uint64_t main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE);
uint64_t avx_bits = 0;
uint64_t avx512_bits = 0;
uint64_t amx_bits = 0;
uint64_t avx10_bits = 0;
uint64_t uarch_bits = 0;
int has_hybrid = 0;
if (has_sse42) {
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE);
}
if (has_avx && has_osxsave && xgetbv_has_xmm_ymm()) {
avx_bits |= SIMDID_AVX;
if (is_amd_glue_ymm(vendor, cpuid1_eax) || is_via) {
avx_bits |= SIMDID_AVX_DUAL;
}
if (max_basic_leaf >= 7) {
get_cpuid(7, 0, &eax, &ebx, &ecx, &edx);
const int has_avx2 = (ebx & (1 << 5)) != 0;
const int has_avx512f = (ebx & (1 << 16)) != 0;
const int has_avx512dq = (ebx & (1 << 17)) != 0;
const int has_avx512bw = (ebx & (1 << 30)) != 0;
const int has_avx512vl = (ebx & (1u << 31)) != 0;
const int has_avx512_vnni = (ecx & (1 << 11)) != 0;
const int has_vbmi = (ecx & (1 << 1)) != 0;
const int has_vbmi2 = (ecx & (1 << 6)) != 0;
const int has_amx_tile = (edx & (1 << 24)) != 0;
const int has_amx_int8 = (edx & (1 << 25)) != 0;
const int has_amx_bf16 = (edx & (1 << 22)) != 0;
has_hybrid = (edx & (1 << 15)) != 0;
int has_bf16 = 0;
int has_fp16 = 0;
int has_avx10 = 0;
if (eax >= 1) {
get_cpuid(7, 1, &eax, &ebx, &ecx, &edx);
has_bf16 = (eax & (1 << 5)) != 0;
has_fp16 = (eax & (1 << 23)) != 0;
has_avx10 = (edx & (1 << 19)) != 0;
}
if (has_avx2) {
avx_bits |= SIMDID_AVX2;
if (has_fma) {
avx_bits |= SIMDID_AVX_FMA;
}
if (has_f16c) {
avx_bits |= SIMDID_AVX_F16C;
}
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2);
}
if (has_avx512f && xgetbv_has_avx512_state()) {
if (has_avx512bw && has_avx512vl && has_avx512dq) {
avx512_bits |= SIMDID_AVX512_BASE;
}
if (has_avx512_vnni) {
avx512_bits |= SIMDID_AVX512_VNNI;
}
if (has_bf16) {
avx512_bits |= SIMDID_AVX512_BF16;
}
if (has_fp16) {
avx512_bits |= SIMDID_AVX512_FP16;
}
if (has_vbmi) {
avx512_bits |= SIMDID_AVX512_VBMI;
}
if (has_vbmi2) {
avx512_bits |= SIMDID_AVX512_VBMI2;
}
if (is_intel) {
avx512_bits |= SIMDID_AVX512_IMPL_NATIVE;
} else if (is_amd) {
avx512_bits |= amd_avx512_impl(family);
} else {
avx512_bits |= SIMDID_AVX512_IMPL_UNKNOWN;
}
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512);
if (has_amx_tile) {
amx_bits |= SIMDID_AMX_TILE;
if (has_amx_int8) {
amx_bits |= SIMDID_AMX_INT8;
}
if (has_amx_bf16) {
amx_bits |= SIMDID_AMX_BF16;
}
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX);
}
if (has_avx10 && (max_basic_leaf >= 0x24)) {
int a10_eax = 0;
int a10_ebx = 0;
int a10_ecx = 0;
int a10_edx = 0;
int avx10_ver = 0;
int has_256w = 0;
int has_512w = 0;
get_cpuid(0x24, 0, &a10_eax, &a10_ebx, &a10_ecx, &a10_edx);
avx10_ver = a10_ebx & 0xFF;
if (avx10_ver >= 1) {
avx10_bits |= SIMDID_AVX10 | SIMDID_AVX10_V1;
if (avx10_ver >= 2) {
avx10_bits |= SIMDID_AVX10_V2;
}
has_256w = (a10_ebx & (1 << 17)) != 0;
has_512w = (a10_ebx & (1 << 18)) != 0;
if (has_256w) {
avx10_bits |= SIMDID_AVX10_256W;
}
if (has_512w) {
avx10_bits |= SIMDID_AVX10_512W;
}
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10);
}
}
}
}
}
if (has_hybrid && (max_basic_leaf >= 0x1A)) {
int h_eax = 0;
int h_ebx = 0;
int h_ecx = 0;
int h_edx = 0;
get_cpuid(0x1A, 0, &h_eax, &h_ebx, &h_ecx, &h_edx);
{
const unsigned core_type = (unsigned)(h_eax >> 24) & 0xFFU;
if (core_type == 0x40U) {
uarch_bits |= SIMDID_UARCH_HYBRID_P;
} else if (core_type == 0x20U) {
uarch_bits |= SIMDID_UARCH_HYBRID_E;
}
}
} else {
uarch_bits |= SIMDID_UARCH_HYBRID_P | SIMDID_UARCH_HYBRID_E;
}
{
const int has_avx_dual = (avx_bits & SIMDID_AVX_DUAL) != 0;
const uint64_t avx512_impl = avx512_bits & SIMDID_AVX512_IMPL_MASK;
const int has_avx512_dual = avx512_impl == SIMDID_AVX512_IMPL_DUAL;
if (has_avx_dual || has_avx512_dual) {
uarch_bits |= SIMDID_UARCH_SIMD_DUAL;
}
}
{
uint64_t final_mask = root;
final_mask |= manu;
final_mask |= main_lvl;
final_mask |= SIMDID_PACK_AVX512(avx512_bits);
final_mask |= SIMDID_PACK_AMX(amx_bits);
final_mask |= SIMDID_PACK_AVX(avx_bits);
final_mask |= SIMDID_PACK_AVX10(avx10_bits);
final_mask |= SIMDID_PACK_UARCH(uarch_bits);
simdid.mask = final_mask;
}
return simdid;
}
SN_API SIMDID SN_DECL sn_get_simd_level_global()
{
if (global_simdid.mask) {
return global_simdid;
}
global_simdid = sn_get_simd_level_local();
return global_simdid;
}
#else
SN_API SIMDID SN_DECL sn_get_simd_level_local()
{
SIMDID simdid;
simdid.mask = 0;
return simdid;
}
SN_API SIMDID SN_DECL sn_get_simd_level_global()
{
if (global_simdid.mask) {
return global_simdid;
}
global_simdid = sn_get_simd_level_local();
return global_simdid;
}
#endif
StgSharp.Native/sn-src/load_intrinsic.x86.c
+8 -8
View File
@@ -58,7 +58,7 @@ void load_intrinsic_sse(sn_intrinsic *context, SIMDID id)
context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, sse, , sub);
context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, sse, , transpose);
context->mat[F32].ker_fma = STD_MK_PROC_NAME(float, sse, , fma);
context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, sse, , fma);
context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, sse, , tiled_fma);
context->mat[F32].pivot = NULL;
context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, sse, , add);
@@ -67,7 +67,7 @@ void load_intrinsic_sse(sn_intrinsic *context, SIMDID id)
context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, sse, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].ker_fma = STD_MK_PROC_NAME(double, sse, , fma);
context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, sse, , fma);
context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, sse, , tiled_fma);
context->mat[F64].pivot = NULL;
context->vec_f32_normalize_3 = f32_normalize;
@@ -90,8 +90,8 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, avx, , transpose);
context->mat[F32].ker_fma = is_fma ? STD_MK_PROC_NAME(float, avx, , fma) :
STD_MK_PROC_NAME(float, avx, _fma, fma);
context->mat[F32].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(float, avx, , fma) :
STD_MK_PROC_NAME(float, avx, _fma, fma);
context->mat[F32].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(float, avx, , tiled_fma) :
STD_MK_PROC_NAME(float, avx, _fma, tiled_fma);
context->mat[F32].pivot = NULL;
context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, avx, , add);
@@ -101,8 +101,8 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].ker_fma = is_fma ? STD_MK_PROC_NAME(double, avx, , fma) :
STD_MK_PROC_NAME(double, avx, _fma, fma);
context->mat[F64].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(double, avx, , fma) :
STD_MK_PROC_NAME(double, avx, _fma, fma);
context->mat[F64].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(double, avx, , tiled_fma) :
STD_MK_PROC_NAME(double, avx, _fma, tiled_fma);
context->mat[F64].pivot = NULL;
context->vec_f32_normalize_3 = f32_normalize;
@@ -120,7 +120,7 @@ void load_intrinsic_512(sn_intrinsic *context, SIMDID id)
context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, 512, , sub);
context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, 512, , transpose);
context->mat[F32].ker_fma = STD_MK_PROC_NAME(float, 512, , fma);
context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, 512, , fma);
context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, 512, , tiled_fma);
context->mat[F32].pivot = NULL;
context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, 512, , add);
@@ -129,7 +129,7 @@ void load_intrinsic_512(sn_intrinsic *context, SIMDID id)
context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, 512, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].ker_fma = STD_MK_PROC_NAME(double, 512, , fma);
context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, 512, , fma);
context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, 512, , tiled_fma);
context->mat[F64].pivot = NULL;
context->vec_f32_normalize_3 = f32_normalize;
StgSharp.Native/sn-src/m.fma_tile.x86.c
+68 -106
View File
@@ -12,17 +12,12 @@
// float
SN_MK_PROC_DECL_STD(float, sse, , fma)
SN_MK_PROC_DECL_STD(float, sse, , tiled_fma)
{
__mk_param_std();
int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1];
int commonK = enumeration->m128_i32[2];
int ansHeight = enumeration->m128_i32[3];
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
MAT_KERNEL(float)
const *ans_cur = ans + GET_INDEX(ansHeight, i, j); // GetKernelAddressUnsafe
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m128 c0 = _mm_load_ps((float *)&ans_cur->f32_x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans_cur->f32_x[1]);
@@ -30,11 +25,9 @@ SN_MK_PROC_DECL_STD(float, sse, , fma)
register __m128 c3 = _mm_load_ps((float *)&ans_cur->f32_x[3]);
register __m128 bcol;
for (size_t k = 0; k < commonK; k++) {
MAT_KERNEL(float)
const *right_cur = right + GET_INDEX(commonK, i, k), // GetKernelAddressUnsafe
const *left_cur =
left + GET_INDEX(ansHeight, k, j); // GetKernelAddressUnsafe
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
const __m128 a0 = _mm_load_ps((float *)&left_cur->f32_x[0]);
const __m128 a1 = _mm_load_ps((float *)&left_cur->f32_x[1]);
@@ -76,18 +69,15 @@ SN_MK_PROC_DECL_STD(float, sse, , fma)
_mm_store_ps((float *)&ans_cur->f32_x[3], c3);
}
SN_MK_PROC_DECL_STD(float, avx, , fma)
SN_MK_PROC_DECL_STD(float, avx, , tiled_fma)
{
static const __m256i b01_shift_source = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift_source = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1];
int commonK = enumeration->m128_i32[2];
int ansHeight = enumeration->m128_i32[3];
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
MAT_KERNEL(float)
const *ans_cur = ans + GET_INDEX(ansHeight, i, j); // GetKernelAddressUnsafe
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
@@ -95,11 +85,9 @@ SN_MK_PROC_DECL_STD(float, avx, , fma)
register __m256i b01_shift = _mm256_load_si256(&b01_shift_source);
register __m256i b23_shift = _mm256_load_si256(&b23_shift_source);
for (size_t k = 0; k < commonK; k++) {
MAT_KERNEL(float)
const *left_cur = left + GET_INDEX(ansHeight, i, k), // GetKernelAddressUnsafe
const *right_cur =
right + GET_INDEX(ansHeight, j, k); // GetKernelAddressUnsafe
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
register __m256 a01 = _mm256_load_ps((float *)&left_cur->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left_cur->f32_y[1]);
@@ -139,18 +127,15 @@ SN_MK_PROC_DECL_STD(float, avx, , fma)
_mm256_store_ps((float *)&ans_cur->f32_y[1], c23);
}
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
SN_MK_PROC_DECL_STD(float, avx, _fma, tiled_fma)
{
static const __m256i b01_shift_source = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift_source = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1];
int commonK = enumeration->m128_i32[2];
int ansHeight = enumeration->m128_i32[3];
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
MAT_KERNEL(float)
const *ans_cur = ans + GET_INDEX(ansHeight, i, j); // GetKernelAddressUnsafe
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256 c01 = _mm256_load_ps((float *)&ans_cur->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans_cur->f32_y[1]);
@@ -158,11 +143,9 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
register __m256i b01_shift = _mm256_load_si256(&b01_shift_source);
register __m256i b23_shift = _mm256_load_si256(&b23_shift_source);
for (size_t k = 0; k < commonK; k++) {
MAT_KERNEL(float)
const *left_cur = left + GET_INDEX(ansHeight, k, j), // GetKernelAddressUnsafe
const *right_cur =
right + GET_INDEX(commonK, i, k); // GetKernelAddressUnsafe
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
register __m256 a01 = _mm256_load_ps((float *)&left_cur->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left_cur->f32_y[1]);
@@ -202,7 +185,7 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
_mm256_store_ps((float *)&ans_cur->f32_y[1], c23);
}
SN_MK_PROC_DECL_STD(float, 512, , fma)
SN_MK_PROC_DECL_STD(float, 512, , tiled_fma)
{
static const __m512i b_shift_source = { .m512i_i32 = {
// shuff order
@@ -211,18 +194,13 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
10, 11, 8, 9, //col2
15, 12, 13, 14 //col3
} };
int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1];
int commonK = enumeration->m128_i32[2];
int ansHeight = enumeration->m128_i32[3];
register __m512i b_shift = _mm512_load_si512(&b_shift_source);
register __m512 c = _mm512_load_ps((float *)(ans + GET_INDEX(ansHeight, i, j)));
for (size_t k = 0; k < commonK; k++) {
MAT_KERNEL(double) *const left_cur = left + GET_INDEX(ansHeight, k, j),
const *right_cur =
right + GET_INDEX(commonK, i,
k); // GetKernelAddressUnsafe
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y);
register __m512 c = _mm512_load_ps(ans_cur);
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
register __m512 a = _mm512_load_ps((float *)left_cur);
register __m512 b = _mm512_load_ps((float *)right_cur);
@@ -245,7 +223,7 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
c = _mm512_fmadd_ps(a_perm, b_perm, c);
}
_mm512_store_ps((float *)(ans + GET_INDEX(ansHeight, i, j)), c);
_mm512_store_ps((float *)ans_cur, c);
}
// double
@@ -282,26 +260,25 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
} while (0)
SN_MK_PROC_DECL_STD(double, sse, , fma)
SN_MK_PROC_DECL_STD(double, sse, , tiled_fma)
{
int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1];
int commonK = enumeration->m128_i32[2];
int ansHeight = enumeration->m128_i32[3];
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
register __m128d c0_lo = _mm_load_pd(ans->f64_x + (0 * 2 + 0));
register __m128d c0_hi = _mm_load_pd(ans->f64_x + (0 * 2 + 1));
register __m128d c1_lo = _mm_load_pd(ans->f64_x + (1 * 2 + 0));
register __m128d c1_hi = _mm_load_pd(ans->f64_x + (1 * 2 + 1));
register __m128d c2_lo = _mm_load_pd(ans->f64_x + (2 * 2 + 0));
register __m128d c2_hi = _mm_load_pd(ans->f64_x + (2 * 2 + 1));
register __m128d c3_lo = _mm_load_pd(ans->f64_x + (3 * 2 + 0));
register __m128d c3_hi = _mm_load_pd(ans->f64_x + (3 * 2 + 1));
MAT_KERNEL(double)
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
for (size_t k = 0; k < commonK; k++) {
MAT_KERNEL(double)
const *left_cur = left + GET_INDEX(ansHeight, i, k),
const *right_cur = right + GET_INDEX(commonK, j, k); // GetKernelAddressUnsafe
register __m128d c0_lo = _mm_load_pd(ans_cur->f64_x + (0 * 2 + 0));
register __m128d c0_hi = _mm_load_pd(ans_cur->f64_x + (0 * 2 + 1));
register __m128d c1_lo = _mm_load_pd(ans_cur->f64_x + (1 * 2 + 0));
register __m128d c1_hi = _mm_load_pd(ans_cur->f64_x + (1 * 2 + 1));
register __m128d c2_lo = _mm_load_pd(ans_cur->f64_x + (2 * 2 + 0));
register __m128d c2_hi = _mm_load_pd(ans_cur->f64_x + (2 * 2 + 1));
register __m128d c3_lo = _mm_load_pd(ans_cur->f64_x + (3 * 2 + 0));
register __m128d c3_hi = _mm_load_pd(ans_cur->f64_x + (3 * 2 + 1));
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
DOUBLE_KERNEL_FMA_CYCLE(0);
@@ -312,25 +289,22 @@ SN_MK_PROC_DECL_STD(double, sse, , fma)
DOUBLE_KERNEL_FMA_CYCLE(3);
}
_mm_store_pd(ans->f64_x + (0 * 2 + 0), c0_lo);
_mm_store_pd(ans->f64_x + (0 * 2 + 1), c0_hi);
_mm_store_pd(ans->f64_x + (1 * 2 + 0), c1_lo);
_mm_store_pd(ans->f64_x + (1 * 2 + 1), c1_hi);
_mm_store_pd(ans->f64_x + (2 * 2 + 0), c2_lo);
_mm_store_pd(ans->f64_x + (2 * 2 + 1), c2_hi);
_mm_store_pd(ans->f64_x + (3 * 2 + 0), c3_lo);
_mm_store_pd(ans->f64_x + (3 * 2 + 1), c3_hi);
_mm_store_pd(ans_cur->f64_x + (0 * 2 + 0), c0_lo);
_mm_store_pd(ans_cur->f64_x + (0 * 2 + 1), c0_hi);
_mm_store_pd(ans_cur->f64_x + (1 * 2 + 0), c1_lo);
_mm_store_pd(ans_cur->f64_x + (1 * 2 + 1), c1_hi);
_mm_store_pd(ans_cur->f64_x + (2 * 2 + 0), c2_lo);
_mm_store_pd(ans_cur->f64_x + (2 * 2 + 1), c2_hi);
_mm_store_pd(ans_cur->f64_x + (3 * 2 + 0), c3_lo);
_mm_store_pd(ans_cur->f64_x + (3 * 2 + 1), c3_hi);
}
SN_MK_PROC_DECL_STD(double, avx, , fma)
SN_MK_PROC_DECL_STD(double, avx, , tiled_fma)
{
int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1];
int commonK = enumeration->m128_i32[2];
int ansHeight = enumeration->m128_i32[3];
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
MAT_KERNEL(double)
const *ans_cur = ans + GET_INDEX(ansHeight, i, j); // GetKernelAddressUnsafe
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]);
@@ -338,11 +312,9 @@ SN_MK_PROC_DECL_STD(double, avx, , fma)
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->f64_y[3]);
register __m256d bcol;
for (size_t k = 0; k < commonK; k++) {
MAT_KERNEL(double)
const *right_cur = right + GET_INDEX(commonK, i, k), // GetKernelAddressUnsafe
const *left_cur =
left + GET_INDEX(ansHeight, k, j); // GetKernelAddressUnsafe
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_y[1]);
@@ -400,15 +372,12 @@ SN_MK_PROC_DECL_STD(double, avx, , fma)
_mm256_store_pd((double *)&ans_cur->f64_y[3], c3);
}
SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
SN_MK_PROC_DECL_STD(double, avx, _fma, tiled_fma)
{
int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1];
int commonK = enumeration->m128_i32[2];
int ansHeight = enumeration->m128_i32[3];
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
MAT_KERNEL(double)
const *ans_cur = ans + GET_INDEX(ansHeight, i, j); // GetKernelAddressUnsafe
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]);
@@ -416,11 +385,9 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->f64_y[3]);
register __m256d bcol;
for (size_t k = 0; k < commonK; k++) {
MAT_KERNEL(double)
const *right_cur = right + GET_INDEX(commonK, i, k), // GetKernelAddressUnsafe
const *left_cur =
left + GET_INDEX(ansHeight, k, j); // GetKernelAddressUnsafe
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_y[1]);
@@ -462,18 +429,15 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
_mm256_store_pd((double *)&ans_cur->f64_y[3], c3);
}
SN_MK_PROC_DECL_STD(double, 512, , fma)
SN_MK_PROC_DECL_STD(double, 512, , tiled_fma)
{
static const __m512i b01_shift_source = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m512i b23_shift_source = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1];
int commonK = enumeration->m128_i32[2];
int ansHeight = enumeration->m128_i32[3];
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
MAT_KERNEL(double)
const *ans_cur = ans + GET_INDEX(ansHeight, i, j); // GetKernelAddressUnsafe
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m512d c01 = _mm512_load_pd((double *)&ans->f64_z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans->f64_z[1]);
@@ -481,11 +445,9 @@ SN_MK_PROC_DECL_STD(double, 512, , fma)
register __m512i b01_shift = _mm512_load_si512(&b01_shift_source);
register __m512i b23_shift = _mm512_load_si512(&b23_shift_source);
for (size_t k = 0; k < commonK; k++) {
MAT_KERNEL(double)
const *left_cur = left + GET_INDEX(ansHeight, i, k), // GetKernelAddressUnsafe
const *right_cur =
right + GET_INDEX(ansHeight, j, k); // GetKernelAddressUnsafe
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
register __m512d a01 = _mm512_load_pd((double *)&left_cur->f64_z[0]);
register __m512d a23 = _mm512_load_pd((double *)&left_cur->f64_z[1]);
StgSharp.Native/sn-src/m.solve.x86.c
+2
View File
@@ -10,6 +10,8 @@ static const float quality_threshold = 1e-6f;
SN_MK_PROC_DECL_STD(float, sse, , try_plu)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS_SCALAR, float);
/* Aliases */
MAT_KERNEL(float) *upper = left; /* U */
MAT_KERNEL(float) *lower = right; /* L (unit diag) */
StgSharp.Native/sn-src/simdid.x86.c
+240 -153
View File
@@ -6,7 +6,7 @@ static SIMDID global_simdid = { 0 };
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrincontext.x86.h"
#include "sn_intrinsic.context.x86.h"
#include <immintrin.h>
#include <string.h>
@@ -57,15 +57,44 @@ static int xgetbv_has_avx512_state(void)
return ((xcr0 & mask) == mask);
}
static void decode_family_model(int cpuid1_eax, int *family, int *model)
{
int family_value = 0;
int model_value = 0;
const int base_fam = (cpuid1_eax >> 8) & 0xF;
const int ext_fam = (cpuid1_eax >> 20) & 0xFF;
const int base_model = (cpuid1_eax >> 4) & 0xF;
const int ext_model = (cpuid1_eax >> 16) & 0xF;
if (base_fam == 0xF) {
family_value = base_fam + ext_fam;
} else {
family_value = base_fam;
}
if ((base_fam == 0x6) || (base_fam == 0xF)) {
model_value = (ext_model << 4) | base_model;
} else {
model_value = base_model;
}
*family = family_value;
*model = model_value;
}
/* AVX-512 execution-width classification for AMD, by CPU family.
* Zen4 (0x19): single 256-bit ALU, 512-bit op = two 256-bit micro-ops (dual-pump) → DUAL.
* Zen5+ (>=0x1A): native 512-bit execution units → NATIVE.
* Other/future AMD families → UNKNOWN (conservative). */
static uint64_t amd_avx512_impl(int family)
{
if (family == 0x19) return SIMDID_AVX512_IMPL_DUAL;
if (family >= 0x1A) return SIMDID_AVX512_IMPL_NATIVE;
return SIMDID_AVX512_IMPL_UNKNOWN;
if (family == 0x19) {
return SIMDID_AVX512_IMPL_DUAL;
} else if (family >= 0x1A) {
return SIMDID_AVX512_IMPL_NATIVE;
} else {
return SIMDID_AVX512_IMPL_UNKNOWN;
}
}
/* Returns 1 if this AMD/Hygon CPU uses split-lane YMM (two fused 128-bit execution units).
@@ -74,194 +103,241 @@ static uint64_t amd_avx512_impl(int family)
* - AMD Zen2+ (family 0x17 model >= 0x30, family 0x19+) have native 256-bit execution. */
static int is_amd_glue_ymm(const char *vendor, int cpuid1_eax)
{
if (strcmp(vendor, "HygonGenuine") == 0)
return 1; /* Hygon is Zen1-based; always split YMM */
if (strcmp(vendor, "AuthenticAMD") != 0)
int family = 0;
int model = 0;
if (strcmp(vendor, "HygonGenuine") == 0) {
return 1;
}
if (strcmp(vendor, "AuthenticAMD") != 0) {
return 0;
int base_fam = (cpuid1_eax >> 8) & 0xF;
int ext_fam = (cpuid1_eax >> 20) & 0xFF;
int disp_fam = (base_fam < 0xF) ? base_fam : (base_fam + ext_fam);
int base_mdl = (cpuid1_eax >> 4) & 0xF;
int ext_mdl = (cpuid1_eax >> 16) & 0xF;
int disp_mdl = (ext_mdl << 4) | base_mdl;
return (disp_fam == 0x17) && (disp_mdl < 0x30); /* Zen1/Zen+ */
}
decode_family_model(cpuid1_eax, &family, &model);
return (family == 0x17) && (model < 0x30);
}
SN_API SIMDID SN_DECL sn_get_simd_level_local()
{
SIMDID _simdid = { 0 };
SIMDID simdid = { 0 };
int eax = 0, ebx = 0, ecx = 0, edx = 0;
int eax = 0;
int ebx = 0;
int ecx = 0;
int edx = 0;
int family = 0;
int model = 0;
/* Vendor */
get_cpuid(0, 0, &eax, &ebx, &ecx, &edx);
const int max_basic_leaf = eax;
char vendor[13];
((int *)vendor)[0] = ebx;
((int *)vendor)[1] = edx;
((int *)vendor)[2] = ecx;
memcpy(vendor + 0, &ebx, 4);
memcpy(vendor + 4, &edx, 4);
memcpy(vendor + 8, &ecx, 4);
vendor[12] = '\0';
const int is_intel = (strcmp(vendor, "GenuineIntel") == 0);
const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0);
const int is_hygon = (strcmp(vendor, "HygonGenuine") == 0); /* 海光: separate from AMD */
const int is_via = (strcmp(vendor, "CentaurHauls") == 0) ||
(memcmp(vendor, " Shanghai ", 12) == 0); /* VIA / Zhaoxin */
const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0);
const int is_hygon = (strcmp(vendor, "HygonGenuine") == 0);
const int is_centaur = (strcmp(vendor, "CentaurHauls") == 0);
const int is_zhaoxin = (memcmp(vendor, " Shanghai ", 12) == 0);
const int is_via = is_centaur || is_zhaoxin;
/* CPUID.1 */
get_cpuid(1, 0, &eax, &ebx, &ecx, &edx);
int cpuid1_eax = eax; /* saved for Zen1/Zen+ glue-YMM detection */
int family = ((eax >> 8) & 0xF) + ((eax >> 20) & 0xFF0);
int model = ((eax >> 4) & 0xF) | ((eax >> 12) & 0xF0);
const int cpuid1_eax = eax;
decode_family_model(cpuid1_eax, &family, &model);
(void)model;
int has_sse42 = (ecx & (1 << 20)) != 0;
int has_avx = (ecx & (1 << 28)) != 0;
int has_osxsave = (ecx & (1 << 27)) != 0;
int has_fma = (ecx & (1 << 12)) != 0;
int has_f16c = (ecx & (1 << 29)) != 0;
const int has_sse42 = (ecx & (1 << 20)) != 0;
const int has_avx = (ecx & (1 << 28)) != 0;
const int has_osxsave = (ecx & (1 << 27)) != 0;
const int has_fma = (ecx & (1 << 12)) != 0;
const int has_f16c = (ecx & (1 << 29)) != 0;
const uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64);
uint64_t manu_code = SIMDID_MANU_UNKNOWN;
if (is_intel) {
manu_code = SIMDID_MANU_INTEL;
} else if (is_amd) {
manu_code = SIMDID_MANU_AMD;
} else if (is_hygon) {
manu_code = SIMDID_MANU_HYGON;
} else if (is_via) {
manu_code = SIMDID_MANU_VIA;
}
const uint64_t manu = SIMDID_PACK_MANU(manu_code);
uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64);
uint64_t manu = SIMDID_PACK_MANU(is_intel ? SIMDID_MANU_INTEL :
is_amd ? SIMDID_MANU_AMD :
is_hygon ? SIMDID_MANU_HYGON :
is_via ? SIMDID_MANU_VIA :
SIMDID_MANU_UNKNOWN);
uint64_t main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE);
uint64_t avx_bits = 0;
uint64_t avx512_bits = 0;
uint64_t amx_bits = 0;
uint64_t avx_bits = 0;
uint64_t avx10_bits = 0;
int has_hybrid = 0; /* filled from CPUID.7.0 EDX[15] when AVX present */
uint64_t uarch_bits = 0;
int has_hybrid = 0;
/* Baseline main level */
if (has_sse42)
if (has_sse42) {
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE);
}
if (has_avx && has_osxsave && xgetbv_has_xmm_ymm()) {
avx_bits |= SIMDID_AVX;
/* Glued/split-lane YMM: early AMD Zen1/Zen+, Hygon, and VIA/Zhaoxin */
if (is_amd_glue_ymm(vendor, cpuid1_eax) || is_via)
if (is_amd_glue_ymm(vendor, cpuid1_eax) || is_via) {
avx_bits |= SIMDID_AVX_DUAL;
/* CPUID.7.0 */
get_cpuid(7, 0, &eax, &ebx, &ecx, &edx);
const int has_bmi1 = (ebx & (1 << 3)) != 0;
const int has_bmi2 = (ebx & (1 << 8)) != 0;
const int has_avx2 = (ebx & (1 << 5)) != 0;
const int has_avx512f = (ebx & (1 << 16)) != 0;
const int has_avx512dq = (ebx & (1 << 17)) != 0;
const int has_avx512bw = (ebx & (1 << 30)) != 0;
const int has_avx512vl = (ebx & (1u << 31)) != 0;
const int has_avx512_vnni = (ebx & (1 << 11)) != 0;
const int has_vbmi = (ecx & (1 << 1)) != 0;
const int has_vbmi2 = (ecx & (1 << 6)) != 0;
/* AMX in leaf 7 subleaf 0 EDX bits 24/25/26 — capture before
overwriting regs with subleaf 1 */
const int has_amx_tile = (edx & (1 << 24)) != 0;
const int has_amx_int8 = (edx & (1 << 25)) != 0;
const int has_amx_bf16 = (edx & (1 << 22)) != 0;
has_hybrid = (edx & (1 << 15)) != 0; /* Intel hybrid core topology */
/* leaf 7, subleaf 1 for BF16/FP16/AVX10 convergence */
get_cpuid(7, 1, &eax, &ebx, &ecx, &edx);
const int has_bf16 = (eax & (1 << 5)) != 0; /* AVX512_BF16 */
const int has_fp16 = (eax & (1 << 23)) != 0; /* AVX512_FP16 */
const int has_avx10 = (edx & (1 << 19)) != 0; /* AVX10 convergence */
if (has_avx2) {
avx_bits |= SIMDID_AVX2;
if (has_fma)
avx_bits |= SIMDID_AVX_FMA;
if (has_f16c)
avx_bits |= SIMDID_AVX_F16C;
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2);
}
if (has_avx512f && xgetbv_has_avx512_state()) {
/* Ensure base subset present */
if (has_avx512bw && has_avx512vl && has_avx512dq) {
avx512_bits |= SIMDID_AVX512_BASE;
}
if (has_avx512_vnni)
avx512_bits |= SIMDID_AVX512_VNNI;
if (has_bf16)
avx512_bits |= SIMDID_AVX512_BF16;
if (has_fp16)
avx512_bits |= SIMDID_AVX512_FP16;
if (has_vbmi)
avx512_bits |= SIMDID_AVX512_VBMI;
if (has_vbmi2)
avx512_bits |= SIMDID_AVX512_VBMI2;
if (max_basic_leaf >= 7) {
get_cpuid(7, 0, &eax, &ebx, &ecx, &edx);
/* Execution-width classification:
* Intel all → NATIVE (genuine 512-bit ALU; SKX dual-port is native)
* AMD Zen4 → DUAL (dual-pump 256-bit, Family 0x19)
* AMD Zen5+ → NATIVE (native 512-bit, Family >= 0x1A)
* Hygon/VIA/unknown → UNKNOWN (conservative) */
if (is_intel)
avx512_bits |= SIMDID_AVX512_IMPL_NATIVE;
else if (is_amd)
avx512_bits |= amd_avx512_impl(family);
else
avx512_bits |= SIMDID_AVX512_IMPL_UNKNOWN;
const int has_avx2 = (ebx & (1 << 5)) != 0;
const int has_avx512f = (ebx & (1 << 16)) != 0;
const int has_avx512dq = (ebx & (1 << 17)) != 0;
const int has_avx512bw = (ebx & (1 << 30)) != 0;
const int has_avx512vl = (ebx & (1u << 31)) != 0;
const int has_avx512_vnni = (ecx & (1 << 11)) != 0;
const int has_vbmi = (ecx & (1 << 1)) != 0;
const int has_vbmi2 = (ecx & (1 << 6)) != 0;
/* main level for AVX512 */
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512);
const int has_amx_tile = (edx & (1 << 24)) != 0;
const int has_amx_int8 = (edx & (1 << 25)) != 0;
const int has_amx_bf16 = (edx & (1 << 22)) != 0;
has_hybrid = (edx & (1 << 15)) != 0;
/* AMX promotion if present */
if (has_amx_tile) {
amx_bits |= SIMDID_AMX_TILE;
if (has_amx_int8)
amx_bits |= SIMDID_AMX_INT8;
if (has_amx_bf16)
amx_bits |= SIMDID_AMX_BF16;
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX);
int has_bf16 = 0;
int has_fp16 = 0;
int has_avx10 = 0;
if (eax >= 1) {
get_cpuid(7, 1, &eax, &ebx, &ecx, &edx);
has_bf16 = (eax & (1 << 5)) != 0;
has_fp16 = (eax & (1 << 23)) != 0;
has_avx10 = (edx & (1 << 19)) != 0;
}
/* AVX10 detection — requires AVX10 convergence bit
(leaf 7 subleaf 1 EDX[19]) and leaf 0x24 */
if (has_avx10) {
int a10_eax = 0, a10_ebx = 0, a10_ecx = 0, a10_edx = 0;
get_cpuid(0x24, 0, &a10_eax, &a10_ebx, &a10_ecx, &a10_edx);
const int avx10_ver = a10_ebx & 0xFF;
if (avx10_ver >= 1) {
avx10_bits |= SIMDID_AVX10 | SIMDID_AVX10_V1;
if (avx10_ver >= 2)
avx10_bits |= SIMDID_AVX10_V2;
if (a10_ebx & (1 << 17))
avx10_bits |= SIMDID_AVX10_256W;
if (a10_ebx & (1 << 18))
avx10_bits |= SIMDID_AVX10_512W;
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10);
if (has_avx2) {
avx_bits |= SIMDID_AVX2;
if (has_fma) {
avx_bits |= SIMDID_AVX_FMA;
}
if (has_f16c) {
avx_bits |= SIMDID_AVX_F16C;
}
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2);
}
if (has_avx512f && xgetbv_has_avx512_state()) {
if (has_avx512bw && has_avx512vl && has_avx512dq) {
avx512_bits |= SIMDID_AVX512_BASE;
}
if (has_avx512_vnni) {
avx512_bits |= SIMDID_AVX512_VNNI;
}
if (has_bf16) {
avx512_bits |= SIMDID_AVX512_BF16;
}
if (has_fp16) {
avx512_bits |= SIMDID_AVX512_FP16;
}
if (has_vbmi) {
avx512_bits |= SIMDID_AVX512_VBMI;
}
if (has_vbmi2) {
avx512_bits |= SIMDID_AVX512_VBMI2;
}
if (is_intel) {
avx512_bits |= SIMDID_AVX512_IMPL_NATIVE;
} else if (is_amd) {
avx512_bits |= amd_avx512_impl(family);
} else {
avx512_bits |= SIMDID_AVX512_IMPL_UNKNOWN;
}
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512);
if (has_amx_tile) {
amx_bits |= SIMDID_AMX_TILE;
if (has_amx_int8) {
amx_bits |= SIMDID_AMX_INT8;
}
if (has_amx_bf16) {
amx_bits |= SIMDID_AMX_BF16;
}
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX);
}
if (has_avx10 && (max_basic_leaf >= 0x24)) {
int a10_eax = 0;
int a10_ebx = 0;
int a10_ecx = 0;
int a10_edx = 0;
int avx10_ver = 0;
int has_256w = 0;
int has_512w = 0;
get_cpuid(0x24, 0, &a10_eax, &a10_ebx, &a10_ecx, &a10_edx);
avx10_ver = a10_ebx & 0xFF;
if (avx10_ver >= 1) {
avx10_bits |= SIMDID_AVX10 | SIMDID_AVX10_V1;
if (avx10_ver >= 2) {
avx10_bits |= SIMDID_AVX10_V2;
}
has_256w = (a10_ebx & (1 << 17)) != 0;
has_512w = (a10_ebx & (1 << 18)) != 0;
if (has_256w) {
avx10_bits |= SIMDID_AVX10_256W;
}
if (has_512w) {
avx10_bits |= SIMDID_AVX10_512W;
}
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10);
}
}
}
}
}
/* uArchHi: hybrid core topology and SIMD-dual summary */
uint64_t uarch_bits = 0;
if (has_hybrid) {
int h_eax = 0, h_ebx = 0, h_ecx = 0, h_edx = 0;
if (has_hybrid && (max_basic_leaf >= 0x1A)) {
int h_eax = 0;
int h_ebx = 0;
int h_ecx = 0;
int h_edx = 0;
get_cpuid(0x1A, 0, &h_eax, &h_ebx, &h_ecx, &h_edx);
unsigned core_type = (unsigned)(h_eax >> 24) & 0xFFU;
if (core_type == 0x40) /* Intel Core / P-core → 10 */
uarch_bits |= SIMDID_UARCH_HYBRID_P;
else if (core_type == 0x20) /* Intel Atom / E-core → 01 */
uarch_bits |= SIMDID_UARCH_HYBRID_E;
/* else unknown hybrid core type → topology stays 00 */
{
const unsigned core_type = (unsigned)(h_eax >> 24) & 0xFFU;
if (core_type == 0x40U) {
uarch_bits |= SIMDID_UARCH_HYBRID_P;
} else if (core_type == 0x20U) {
uarch_bits |= SIMDID_UARCH_HYBRID_E;
}
}
} else {
/* Confirmed non-hybrid (unified) → set both bits = 11 (numerically strongest) */
uarch_bits |= SIMDID_UARCH_HYBRID_P | SIMDID_UARCH_HYBRID_E;
}
if ((avx_bits & SIMDID_AVX_DUAL) ||
((avx512_bits & SIMDID_AVX512_IMPL_MASK) == SIMDID_AVX512_IMPL_DUAL))
uarch_bits |= SIMDID_UARCH_SIMD_DUAL;
_simdid.mask = root | manu | main_lvl | SIMDID_PACK_AVX512(avx512_bits) |
SIMDID_PACK_AMX(amx_bits) | SIMDID_PACK_AVX(avx_bits) |
SIMDID_PACK_AVX10(avx10_bits) | SIMDID_PACK_UARCH(uarch_bits);
return _simdid;
{
const int has_avx_dual = (avx_bits & SIMDID_AVX_DUAL) != 0;
const uint64_t avx512_impl = avx512_bits & SIMDID_AVX512_IMPL_MASK;
const int has_avx512_dual = avx512_impl == SIMDID_AVX512_IMPL_DUAL;
if (has_avx_dual || has_avx512_dual) {
uarch_bits |= SIMDID_UARCH_SIMD_DUAL;
}
}
{
uint64_t final_mask = root;
final_mask |= manu;
final_mask |= main_lvl;
final_mask |= SIMDID_PACK_AVX512(avx512_bits);
final_mask |= SIMDID_PACK_AMX(amx_bits);
final_mask |= SIMDID_PACK_AVX(avx_bits);
final_mask |= SIMDID_PACK_AVX10(avx10_bits);
final_mask |= SIMDID_PACK_UARCH(uarch_bits);
simdid.mask = final_mask;
}
return simdid;
}
SN_API SIMDID SN_DECL sn_get_simd_level_global()
@@ -269,17 +345,28 @@ SN_API SIMDID SN_DECL sn_get_simd_level_global()
if (global_simdid.mask) {
return global_simdid;
}
global_simdid = sn_get_simd_level_local();
return global_simdid;
}
#else
SIMDID sn_get_simd_level()
SN_API SIMDID SN_DECL sn_get_simd_level_local()
{
SIMDID simd_id;
simd_id.mask = 0;
return simd_id; /* non-x86: no SIMD level probing */
SIMDID simdid;
simdid.mask = 0;
return simdid;
}
SN_API SIMDID SN_DECL sn_get_simd_level_global()
{
if (global_simdid.mask) {
return global_simdid;
}
global_simdid = sn_get_simd_level_local();
return global_simdid;
}
#endif