235 lines
9.0 KiB
C
235 lines
9.0 KiB
C
#include "sn_target.h"
|
|
|
|
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
|
|
|
#include "sn_intrinsic.h"
|
|
|
|
// -----------------------------------------------------------------------------
|
|
// SIMDID layout helpers (see SIMDID.x86.md)
|
|
// bits [3..0] : Root ISA
|
|
// bits [7..4] : Manufacture (ISA-specific)
|
|
// bits [15..8] : MainLevel (monotonic capability)
|
|
// bits [31..16]: AVX512 Feature
|
|
// bits [39..32]: AMX Feature
|
|
// bits [47..40]: AVX Feature
|
|
// bits [55..48]: AVX10 Feature
|
|
// bits [63..56]: uArchHi / Policy (optional)
|
|
// -----------------------------------------------------------------------------
|
|
#define SIMDID_SHIFT_ROOT 0
|
|
#define SIMDID_SHIFT_MANU 4
|
|
#define SIMDID_SHIFT_MAIN 8
|
|
#define SIMDID_SHIFT_AVX512 16
|
|
#define SIMDID_SHIFT_AMX 32
|
|
#define SIMDID_SHIFT_AVX 40
|
|
#define SIMDID_SHIFT_AVX10 48
|
|
#define SIMDID_SHIFT_UARCH 56
|
|
|
|
#define SIMDID_MASK_ROOT 0xFULL
|
|
#define SIMDID_MASK_MANU (0xFULL << SIMDID_SHIFT_MANU)
|
|
#define SIMDID_MASK_MAIN (0xFFULL << SIMDID_SHIFT_MAIN)
|
|
#define SIMDID_MASK_AVX512 (0xFFFFULL << SIMDID_SHIFT_AVX512)
|
|
#define SIMDID_MASK_AMX (0xFFULL << SIMDID_SHIFT_AMX)
|
|
#define SIMDID_MASK_AVX (0xFFULL << SIMDID_SHIFT_AVX)
|
|
#define SIMDID_MASK_AVX10 (0xFFULL << SIMDID_SHIFT_AVX10)
|
|
#define SIMDID_MASK_UARCH (0xFFULL << SIMDID_SHIFT_UARCH)
|
|
|
|
// Root ISA (low nibble)
|
|
#define SIMDID_ROOT_UNKNOWN 0x0
|
|
#define SIMDID_ROOT_X86_64 0x1
|
|
#define SIMDID_ROOT_AARCH64 0x2
|
|
#define SIMDID_ROOT_PPC64LE 0x3
|
|
#define SIMDID_ROOT_RISCV64 0x4
|
|
#define SIMDID_ROOT_LOONGARCH 0x5
|
|
|
|
// Manufacture (x86-64 mapping; other ISAs may reinterpret)
|
|
#define SIMDID_MANU_UNKNOWN 0x0
|
|
#define SIMDID_MANU_INTEL 0x1
|
|
#define SIMDID_MANU_AMD 0x2
|
|
#define SIMDID_MANU_VIA 0x3
|
|
#define SIMDID_MANU_HYGON 0x4 /* 海光: Zen1-derived, separate from AMD */
|
|
|
|
// MainLevel (ordinal, shared bits 8..15; not bit-flags)
|
|
#define SIMDID_MAIN_LVL_NONE 0x00u
|
|
#define SIMDID_MAIN_LVL_SSE 0x01u /* SSE1..SSE4.2 */
|
|
#define SIMDID_MAIN_LVL_AVX2 0x02u
|
|
#define SIMDID_MAIN_LVL_AVX512 0x03u
|
|
#define SIMDID_MAIN_LVL_AMX 0x04u
|
|
#define SIMDID_MAIN_LVL_AVX10 0x05u
|
|
|
|
#ifndef SIMDID_AVX512_BASE
|
|
#define SIMDID_AVX512_FP16 U << 13) relative bit 13 within [31..16] */
|
|
#define SIMDID_AVX512_BF16 U << 12)
|
|
#define SIMDID_AVX512_VNNI U << 11)
|
|
#define SIMDID_AVX512_VBMI2 U << 10)
|
|
#define SIMDID_AVX512_VBMI U << 9)
|
|
#define SIMDID_AVX512_AI_RESERVED U << 8)
|
|
#define SIMDID_AVX512_BASE U << 4)
|
|
#define SIMDID_AVX512_IMPL_SHIFT
|
|
#define SIMDID_AVX512_IMPL_MASK x3U << SIMDID_AVX512_IMPL_SHIFT)
|
|
#define SIMDID_AVX512_IMPL_UNKNOWN x0U << SIMDID_AVX512_IMPL_SHIFT)
|
|
#define SIMDID_AVX512_IMPL_DUAL x1U << SIMDID_AVX512_IMPL_SHIFT)
|
|
#define SIMDID_AVX512_IMPL_NATIVE x2U << SIMDID_AVX512_IMPL_SHIFT)
|
|
#endif
|
|
|
|
#ifndef SIMDID_AVX_BASE
|
|
#define SIMDID_AVX U << 0) relative bit 0 within [47..40] */
|
|
#define SIMDID_AVX2 U << 1)
|
|
#define SIMDID_AVX_FMA U << 2)
|
|
#define SIMDID_AVX_F16C (1U << 3)
|
|
#define SIMDID_AVX_DUAL ( \
|
|
U << 4) glued/split-lane YMM (two 128-bit units; Zen1, Hygon, VIA/Zhaoxin) */
|
|
#endif
|
|
|
|
#ifndef SIMDID_AMX_TILE
|
|
#define SIMDID_AMX_TILE (1U << 0) relative bit 0 within [39..32] */
|
|
#define SIMDID_AMX_INT8 (1U << 1)
|
|
#define SIMDID_AMX_BF16 (1U << 2)
|
|
#endif
|
|
|
|
#ifndef SIMDID_AVX10_BASE
|
|
#define SIMDID_AVX10 U << 0) /* relative bit 0 within [55..48] */
|
|
#define SIMDID_AVX10_V1 U << 1)
|
|
#define SIMDID_AVX10_V2 U << 2)
|
|
#define SIMDID_AVX10_256W (1U << 3)
|
|
#define SIMDID_AVX10_512W (1U << 4)
|
|
#endif
|
|
|
|
/* uArchHi ([63..56]) bitmask — core topology and SIMD-width execution policy */
|
|
#ifndef SIMDID_UARCH_HYBRID_P
|
|
/* Topology 2-bit field (bits [1..0] of uArchHi byte):
|
|
* 00 = unknown (undetected hybrid type or pre-CPUID.1A hardware)
|
|
* 01 = E-core (efficiency/small core in a hybrid CPU)
|
|
* 10 = P-core (performance/big core in a hybrid CPU)
|
|
* 11 = unified (confirmed non-hybrid; numerically strongest)
|
|
* Higher numeric value == stronger / higher-capability topology. */
|
|
#define SIMDID_UARCH_HYBRID_E << 0) / E-core (efficiency/small) in a hybrid topology */
|
|
#define SIMDID_UARCH_HYBRID_P << 1) / P-core (performance/big) in a hybrid topology */
|
|
#define SIMDID_UARCH_SIMD_DUAL ( \
|
|
<< 2) / SIMD is glued/dual (summary: AVX_DUAL or AVX512_IMPL=DUAL) */
|
|
#endif
|
|
|
|
// Pack helpers (define locally to avoid missing macros when included standalone)
|
|
#ifndef SIMDID_PACK_ROOT
|
|
#define SIMDID_PACK_ROOT(root) (((uint64_t)(root) & 0xFULL) << SIMDID_SHIFT_ROOT)
|
|
#define SIMDID_PACK_MANU(manu) (((uint64_t)(manu) & 0xFULL) << SIMDID_SHIFT_MANU)
|
|
#define SIMDID_PACK_MAIN(main) (((uint64_t)(main) & 0xFFULL) << SIMDID_SHIFT_MAIN)
|
|
#define SIMDID_PACK_AVX512(avx512_bits) \
|
|
(((uint64_t)(avx512_bits) & 0xFFFFULL) << SIMDID_SHIFT_AVX512)
|
|
#define SIMDID_PACK_AMX(amx_bits) (((uint64_t)(amx_bits) & 0xFFULL) << SIMDID_SHIFT_AMX)
|
|
#define SIMDID_PACK_AVX(avx_bits) (((uint64_t)(avx_bits) & 0xFFULL) << SIMDID_SHIFT_AVX)
|
|
#define SIMDID_PACK_AVX10(avx10_bits) (((uint64_t)(avx10_bits) & 0xFFULL) << SIMDID_SHIFT_AVX10)
|
|
#define SIMDID_PACK_UARCH(uarch_hi) (((uint64_t)(uarch_hi) & 0xFFULL) << SIMDID_SHIFT_UARCH)
|
|
#endif
|
|
|
|
/* Pre-shifted helpers for convenience when constructing SIMDID */
|
|
#define SIMDID_MAIN_ENC_NONE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE)
|
|
#define SIMDID_MAIN_ENC_SSE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE)
|
|
#define SIMDID_MAIN_ENC_AVX2 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2)
|
|
#define SIMDID_MAIN_ENC_AVX512 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512)
|
|
#define SIMDID_MAIN_ENC_AMX SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX)
|
|
#define SIMDID_MAIN_ENC_AVX10 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10)
|
|
|
|
#pragma region matix function
|
|
|
|
INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);
|
|
|
|
STD_MK_PROC_DECL(float, sse, , add);
|
|
STD_MK_PROC_DECL(float, avx, , add);
|
|
STD_MK_PROC_DECL(float, 512, , add);
|
|
|
|
STD_MK_PROC_DECL(double, sse, , add);
|
|
STD_MK_PROC_DECL(double, avx, , add);
|
|
STD_MK_PROC_DECL(double, 512, , add);
|
|
|
|
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub);
|
|
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub);
|
|
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub);
|
|
|
|
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub);
|
|
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub);
|
|
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub);
|
|
|
|
DECLARE_KER_PROC_RIGHT_ANS(float, sse, , transpose);
|
|
DECLARE_KER_PROC_RIGHT_ANS(float, avx, , transpose);
|
|
DECLARE_KER_PROC_RIGHT_ANS(float, 512, , transpose);
|
|
|
|
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul);
|
|
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul);
|
|
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul);
|
|
|
|
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul);
|
|
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul);
|
|
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, , scalar_mul);
|
|
|
|
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, , fill);
|
|
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill);
|
|
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill);
|
|
|
|
DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill);
|
|
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill);
|
|
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill);
|
|
|
|
DECLARE_PNL_PROC_ANS(float, sse, , clear);
|
|
DECLARE_PNL_PROC_ANS(float, avx, , clear);
|
|
DECLARE_PNL_PROC_ANS(float, 512, , clear);
|
|
|
|
DECLARE_PNL_PROC_ANS(double, sse, , clear);
|
|
DECLARE_PNL_PROC_ANS(double, avx, , clear);
|
|
DECLARE_PNL_PROC_ANS(double, 512, , clear);
|
|
|
|
DECLARE_KER_PROC_RIGHT_ANS(float, sse, , quality);
|
|
|
|
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma);
|
|
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma);
|
|
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma);
|
|
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma);
|
|
|
|
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
|
|
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
|
|
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
|
|
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
|
|
|
|
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, sse, , fma);
|
|
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, , fma);
|
|
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma);
|
|
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma);
|
|
|
|
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
|
|
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
|
|
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
|
|
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
|
|
|
|
#pragma endregion
|
|
|
|
#pragma region scaler_simd
|
|
|
|
INTERNAL uint64_t SN_DECL factorial_simd_sse(int n);
|
|
|
|
#pragma endregion
|
|
|
|
#pragma region dot_product
|
|
|
|
INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
|
|
__m128 const *vector, __m128 *ans);
|
|
INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
|
__m128 *ans);
|
|
INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
|
__m128 *ans);
|
|
INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
|
__m128 *ans);
|
|
|
|
#pragma endregion
|
|
|
|
#pragma region string
|
|
|
|
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int length);
|
|
INTERNAL int SN_DECL index_pair_sse(short const *stram, uint32_t target, int length);
|
|
|
|
#pragma endregion
|
|
|
|
#else
|
|
|
|
typedef enum intrinsic_level { SIMD0 = 0 } intrinsic_level;
|
|
|
|
#endif
|