124 lines
4.7 KiB
C
124 lines
4.7 KiB
C
#ifndef SN_MATKERNEL
|
|
#define SN_MATKERNEL
|
|
|
|
#define ELEMENTSIZE(T) sizeof(T)
|
|
#define XMMCOUNT(T) sizeof(T)
|
|
#define YMMCOUNT(T) sizeof(T) / 2
|
|
#define ZMMCOUNT(T) sizeof(T) / 4
|
|
|
|
#include <immintrin.h>
|
|
|
|
#define MAT_KERNEL(T) \
|
|
union { \
|
|
__m128 xmm[XMMCOUNT(T) / 4]; \
|
|
__m256 ymm[YMMCOUNT(T) / 4]; \
|
|
__m512 zmm[ZMMCOUNT(T) / 4]; \
|
|
T m[4][4]; \
|
|
}
|
|
|
|
#define VEC(T) \
|
|
union { \
|
|
__m128 xmm[XMMCOUNT(T) / 4]; \
|
|
__m256 ymm[YMMCOUNT(T) / 4]; \
|
|
__m512 zmm[ZMMCOUNT(T) / 4]; \
|
|
T v[4]; \
|
|
}
|
|
|
|
#endif
|
|
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
/* align decoration */
|
|
|
|
#if __cplusplus && defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
|
|
#define ML_ALIGNAS(n) _Alignas(n)
|
|
#elif defined(_MSC_VER)
|
|
#define ML_ALIGNAS(n) __declspec(align(n))
|
|
#else
|
|
#define ML_ALIGNAS(n) __attribute__((aligned(n)))
|
|
#endif
|
|
|
|
/* Round up to the alignment boundary in bytes */
|
|
#ifndef ML_ALIGN_UP
|
|
#define ML_ALIGN_UP(x, a) (((size_t)(x) + ((size_t)(a) - 1)) & ~((size_t)(a) - 1))
|
|
#endif
|
|
|
|
/* Vector register width in bytes */
|
|
#define ML_BYTES_XMM 16u
|
|
#define ML_BYTES_YMM 32u
|
|
#define ML_BYTES_ZMM 64u
|
|
|
|
/* Compute the aligned leading-direction element stride (LEp) */
|
|
#define ML_PANEL_STRIDE_ELEMS(T, LE, ALIGN) \
|
|
(ML_ALIGN_UP((size_t)(LE) * sizeof(T), (size_t)(ALIGN)) / sizeof(T))
|
|
|
|
/* Total element count and total byte count */
|
|
#define ML_PANEL_TOT_ELEMS(T, LE, MINOR, ALIGN) \
|
|
((size_t)ML_PANEL_STRIDE_ELEMS(T, (LE), (ALIGN)) * (size_t)(MINOR))
|
|
|
|
#define ML_TOT_BYTES(T, ELEMS) ((size_t)(ELEMS) * sizeof(T))
|
|
|
|
/* Number of SIMD view slots (ceil by total bytes) */
|
|
#define ML_XMM_COUNT(TOT_BYTES) (((size_t)(TOT_BYTES) + ML_BYTES_XMM - 1) / ML_BYTES_XMM)
|
|
#define ML_YMM_COUNT(TOT_BYTES) (((size_t)(TOT_BYTES) + ML_BYTES_YMM - 1) / ML_BYTES_YMM)
|
|
#define ML_ZMM_COUNT(TOT_BYTES) (((size_t)(TOT_BYTES) + ML_BYTES_ZMM - 1) / ML_BYTES_ZMM)
|
|
|
|
/* ========== Unified "panel" union (no enum/descriptor fields) ==========
|
|
Usage:
|
|
MAT_PANEL(float, 8, 32, 32) P; // LE=8, MINOR=32, aligned to 32 bytes
|
|
Views:
|
|
P.xmm[] / P.ymm[] / P.zmm[] // raw SIMD blocks
|
|
P.blk[MINOR][LEp] // 2D scalar view (includes padding for alignment)
|
|
P.flat[MINOR*LEp] // flat array
|
|
Notes:
|
|
- LE/MINOR/ALIGN must be compile-time constants
|
|
- LEp = ML_PANEL_STRIDE_ELEMS(T, LE, ALIGN)
|
|
- Layout/semantics defined by the calling convention (e.g. blk[k][r] is element r of block k)
|
|
*/
|
|
#define MAT_PANEL(T, LE, MINOR, ALIGN) \
|
|
ML_ALIGNAS(ALIGN) union { \
|
|
/* Raw SIMD view: cast to __m256/__m512d inside the micro-kernel on demand */ \
|
|
__m128i xmm[ML_XMM_COUNT( \
|
|
ML_TOT_BYTES(T, ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))))]; \
|
|
__m256i ymm[ML_YMM_COUNT( \
|
|
ML_TOT_BYTES(T, ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))))]; \
|
|
__m512i zmm[ML_ZMM_COUNT( \
|
|
ML_TOT_BYTES(T, ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))))]; \
|
|
/* Scalar 2D view: minor �� LEp (LE dimension contains alignment padding) */ \
|
|
T blk[(MINOR)][ML_PANEL_STRIDE_ELEMS(T, (LE), (ALIGN))]; \
|
|
/* Flat view */ \
|
|
T flat[ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))]; \
|
|
}
|
|
|
|
/* ---------- SSE preset ---------- */
|
|
#undef PANEL_LE
|
|
#undef PANEL_MINOR
|
|
#undef PANEL_ALIGN
|
|
#define PANEL_LE 4
|
|
#define PANEL_MINOR 32
|
|
#define PANEL_ALIGN 16
|
|
/* ======== SSE preset ========
|
|
4 x 32, 16B aligned
|
|
*/
|
|
#define MAT_PANEL_SSE(T) MAT_PANEL(T, PANEL_LE, PANEL_MINOR, PANEL_ALIGN)
|
|
|
|
/* ---------- AVX2 preset ---------- */
|
|
#undef PANEL_LE
|
|
#undef PANEL_MINOR
|
|
#undef PANEL_ALIGN
|
|
#define PANEL_LE 8
|
|
#define PANEL_MINOR 64
|
|
#define PANEL_ALIGN 32
|
|
#define MAT_PANEL_AVX(T) MAT_PANEL(T, PANEL_LE, PANEL_MINOR, PANEL_ALIGN) /* 8 x 64, 32B aligned */
|
|
|
|
/* ---------- AVX-512 preset ---------- */
|
|
#undef PANEL_LE
|
|
#undef PANEL_MINOR
|
|
#undef PANEL_ALIGN
|
|
#define PANEL_LE 16
|
|
#define PANEL_MINOR 64
|
|
#define PANEL_ALIGN 64
|
|
#define MAT_PANEL_512(T) \
|
|
MAT_PANEL(T, PANEL_LE, PANEL_MINOR, PANEL_ALIGN); /* 16 x 64, 64B aligned */
|