youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

macro gymnastics to implement storage kernel, buffer op and complex compute panel

Browse Source
This commit is contained in:
Youmu Liu
2025-11-13 19:09:27 +08:00
parent 2dfb4ae86a
commit 1417785726
18 changed files with 373 additions and 383 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-head/sn_internal.h
+6 -3
View File
@@ -1,6 +1,3 @@
#ifdef _MSC_VER
#pragma once
#endif
#ifndef SN_INTERNAL
#define SN_INTERNAL
@@ -24,6 +21,12 @@
#define FORCEINLINE
#endif
#if defined(_MSC_VER)
#define SN_ALIGNAS(n) __declspec(align(n))
#else
#define SN_ALIGNAS(n) __attribute__((aligned(n)))
#endif
#define SN_APIDEF
#define PRIVATE static
StgSharp.Native/sn-head/sn_intrinsic.h
+163 -52
View File
@@ -33,71 +33,179 @@ typedef union uint64_u {
typedef struct mat_kernel mat_kernel;
typedef struct mat_panel mat_panel;
typedef union scalar_pack {
uint64_t data[8];
} scalar_pack;
/* Mirrors StgSharp.Mathematics.Numeric.MatrixParallelTaskPackage<T> (128 bytes). */
#define MAT_TASK(T) \
struct { \
MAT_KERNEL(T) * left; /* Offset 0: left matrix kernel base */ \
MAT_KERNEL(T) * right; /* Offset 8: right matrix kernel base */ \
MAT_KERNEL(T) * result; /* Offset 16: result matrix kernel base */ \
int32_t element_size; /* Offset 24: sizeof(T) */ \
int32_t reserved_ptr0; /* Offset 28: reserved/extension slot */ \
int32_t left_prim_offset; /* Offset 32 */ \
int32_t left_prim_stride; /* Offset 36 */ \
int32_t left_sec_offset; /* Offset 40 */ \
int32_t left_sec_stride; /* Offset 44 */ \
int32_t right_prim_offset; /* Offset 48 */ \
int32_t right_prim_stride; /* Offset 52 */ \
int32_t right_sec_offset; /* Offset 56 */ \
int32_t right_sec_stride; /* Offset 60 */ \
int32_t result_prim_offset; /* Offset 64 */ \
int32_t result_prim_stride; /* Offset 68 */ \
int32_t result_sec_offset; /* Offset 72 */ \
int32_t result_sec_stride; /* Offset 76 */ \
scalar_pack *scalar; /* Offset 80 */ \
void *compute_handle; /* Offset 88 */ \
int32_t prim_count; /* Offset 96 */ \
int32_t compute_mode; /* Offset 100 */ \
int32_t sec_count; /* Offset 104 */ \
int32_t reserved_int0; /* Offset 108 */ \
int32_t prim_tile_offset; /* Offset 112 */ \
int32_t sec_tile_offset; /* Offset 116 */ \
int32_t prim_cols_in_tile; /* Offset 120 */ \
int32_t sec_cols_in_tile; /* Offset 124 */ \
}
typedef struct matrix_task matrix_task;
#if defined(__cplusplus)
static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mismatch");
#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
_Static_assert(sizeof(MAT_TASK(float)) == 128, "matrix_parallel_task_package size mismatch");
#endif
#pragma region matrix buffer operation function type
typedef void(SN_DECL *BUF_PROC_AS)(mat_kernel *ans, scalar_pack *scalar, int count);
typedef void(SN_DECL *BUF_PROC_RAS)(mat_kernel *right, mat_kernel *ans, scalar_pack *scalar,
int count);
typedef void(SN_DECL *BUF_PROC_LRAS)(mat_kernel *left, mat_kernel *right, mat_kernel *ans,
scalar_pack *scalar, int count);
typedef void(SN_DECL *BUF_PROC_RA)(mat_kernel *right, mat_kernel *ans, int count);
typedef void(SN_DECL *BUF_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans,
int count);
#define BUF_PROC(T_type, T_arch, T_op_name) T_type##_buf_##T_op_name##_##T_arch
#define DECLARE_BUF_PROC_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, int count)
#define DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
scalar_pack const *scalar, int count)
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, int count)
#define DECLARE_BUF_PROC_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, int count)
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, int count)
#pragma endregion
#pragma region matrix panel function type
typedef void(SN_DECL *PNL_PROC_LRA)(mat_panel *left, mat_panel *right, mat_panel *ans);
typedef void(SN_DECL *PNL_PROC_RA)(mat_panel *right, mat_panel *ans);
typedef void(SN_DECL *PNL_PROC_LRAS)(mat_panel *left, mat_panel *right, mat_panel *ans,
scalar_pack *scalar);
typedef void(SN_DECL *PNL_PROC_RAS)(mat_panel *right, mat_panel *ans, scalar_pack *scalar);
#define PNL_PROC(T_type, T_arch, T_op_name) T_type##_pnl_##T_op_name##_##T_arch
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)
#pragma endregion
#pragma region matrix kernel function type
typedef void(SN_DECL *KER_PROC_AS)(mat_kernel *ans, scalar_pack *scalar);
typedef void(SN_DECL *KER_PROC_RA)(mat_kernel *left, mat_kernel *ans);
typedef void(SN_DECL *KER_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans);
typedef void(SN_DECL *KER_PROC_LAS)(mat_kernel *matrix, mat_kernel *ans, scalar_pack *scalar);
#define KER_PROC(T_type, T_arch, T_op_name) T_type##_kernel_##T_op_name##_##T_arch
#define DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar)
#define DECLARE_KER_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
scalar_pack const *scalar)
#define DECLARE_KER_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans)
#define DECLARE_KER_PROC_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans)
#pragma endregion
typedef void(SN_DECL *VECTORNORMALIZEPROC)(__m128 *source, __m128 *target);
typedef void(SN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans);
typedef void(SN_DECL *PANEL_PROC_AS)(MAT_PANEL_SSE(int));
typedef void(SN_DECL *MAT_PROC_AS)(mat_kernel *ans, scalar_pack *scalar);
typedef void(SN_DECL *MAT_PROC_LA)(mat_kernel *left, mat_kernel *ans);
typedef void(SN_DECL *MAT_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans);
typedef void(SN_DECL *MAT_PROC_LSA)(mat_kernel *matrix, scalar_pack *scalar, mat_kernel *ans);
typedef uint64_t(SN_DECL *FACTORIALROC)(int n);
#pragma region matix function
INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);
INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_add_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_sub_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_transpose_sse(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target);
INTERNAL void SN_DECL f32_transpose_avx(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target);
INTERNAL void SN_DECL f32_transpose_512(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target);
INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans,
scalar_pack const *scalar);
INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans,
scalar_pack const *scalar);
INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans,
scalar_pack const *scalar);
INTERNAL void SN_DECL f32_fill_sse(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler);
INTERNAL void SN_DECL f32_fill_avx(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler);
INTERNAL void SN_DECL f32_fill_512(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, add);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, add);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, add);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, sub);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, sub);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, sub);
DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose);
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul);
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul);
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul);
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill);
#pragma endregion
#pragma region fma
INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_fma_avx(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_fma_512(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, fma);
#pragma endregion
#pragma region scaler_simd
INTERNAL uint64_t SN_DECL factorial_simd_sse(int n);
#pragma endregion
#pragma region dot_product
@@ -115,21 +223,24 @@ INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 cons
#pragma region string
typedef int(SN_DECL *HASH)(byte const *str, int length);
typedef int(SN_DECL *INDEX_PAIR)(short const *stram, uint32_t target, int length);
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int length);
INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int length);
#pragma endregion
typedef struct sn_intrinsic {
int (*city_hash_simplify)(byte *str, int length);
HASH city_hash_simplify;
FACTORIALROC factorial_simd;
int (*index_pair)(short const *str, uint32_t target, int length);
MAT_PROC_LRA f32_add;
MAT_PROC_AS f32_fill;
MAT_PROC_LRA f32_fma;
MAT_PROC_LRA f32_scalar_mul;
MAT_PROC_LRA f32_sub;
MAT_PROC_LA f32_transpose;
INDEX_PAIR index_pair;
KER_PROC_LRA f32_kernel_add;
KER_PROC_AS f32_kernel_fill;
PNL_PROC_LRA f32_panel_fma;
KER_PROC_LRA f32_kernel_scalar_mul;
KER_PROC_LRA f32_kernel_sub;
KER_PROC_RA f32_kernel_transpose;
VECTORNORMALIZEPROC f32_normalize_3;
} sn_intrinsic;
StgSharp.Native/sn-head/sn_matkernel.h
+47 -87
View File
@@ -26,98 +26,58 @@
#endif
#include <stddef.h>
#include <stdint.h>
/* Helpers (all compile-time): */
#define SN_LANES(T_VEC, T) (sizeof(T_VEC) / sizeof(T))
#define SN_MAX(a, b) (((a)) > ((b)) ? (a) : (b))
#define SN_CEIL_DIV(a, b) (((a) + (b) - 1) / (b))
/* align decoration */
/* Panel geometry helpers (all compile-time): */
#define PANEL_Q(T, T_VEC, MINQ) SN_MAX(SN_LANES(T_VEC, T), (MINQ))
#define PANEL_CHUNKS(T, T_VEC, MINQ) SN_CEIL_DIV(PANEL_Q(T, T_VEC, MINQ), SN_LANES(T_VEC, T))
#if __cplusplus && defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
#define ML_ALIGNAS(n) _Alignas(n)
#elif defined(_MSC_VER)
#define ML_ALIGNAS(n) __declspec(align(n))
#else
#define ML_ALIGNAS(n) __attribute__((aligned(n)))
#endif
/* Round up to the alignment boundary in bytes */
#ifndef ML_ALIGN_UP
#define ML_ALIGN_UP(x, a) (((size_t)(x) + ((size_t)(a) - 1)) & ~((size_t)(a) - 1))
#endif
/* Vector register width in bytes */
#define ML_BYTES_XMM 16u
#define ML_BYTES_YMM 32u
#define ML_BYTES_ZMM 64u
/* Compute the aligned leading-direction element stride (LEp) */
#define ML_PANEL_STRIDE_ELEMS(T, LE, ALIGN) \
(ML_ALIGN_UP((size_t)(LE) * sizeof(T), (size_t)(ALIGN)) / sizeof(T))
/* Total element count and total byte count */
#define ML_PANEL_TOT_ELEMS(T, LE, MINOR, ALIGN) \
((size_t)ML_PANEL_STRIDE_ELEMS(T, (LE), (ALIGN)) * (size_t)(MINOR))
#define ML_TOT_BYTES(T, ELEMS) ((size_t)(ELEMS) * sizeof(T))
/* Number of SIMD view slots (ceil by total bytes) */
#define ML_XMM_COUNT(TOT_BYTES) (((size_t)(TOT_BYTES) + ML_BYTES_XMM - 1) / ML_BYTES_XMM)
#define ML_YMM_COUNT(TOT_BYTES) (((size_t)(TOT_BYTES) + ML_BYTES_YMM - 1) / ML_BYTES_YMM)
#define ML_ZMM_COUNT(TOT_BYTES) (((size_t)(TOT_BYTES) + ML_BYTES_ZMM - 1) / ML_BYTES_ZMM)
/* ========== Unified "panel" union (no enum/descriptor fields) ==========
Usage:
MAT_PANEL(float, 8, 32, 32) P; // LE=8, MINOR=32, aligned to 32 bytes
Views:
P.xmm[] / P.ymm[] / P.zmm[] // raw SIMD blocks
P.blk[MINOR][LEp] // 2D scalar view (includes padding for alignment)
P.flat[MINOR*LEp] // flat array
Notes:
- LE/MINOR/ALIGN must be compile-time constants
- LEp = ML_PANEL_STRIDE_ELEMS(T, LE, ALIGN)
- Layout/semantics defined by the calling convention (e.g. blk[k][r] is element r of block k)
/*
Column-major square panel with minimum side Q:
- Let R = lanes per vector = sizeof(T_VEC)/sizeof(T).
- Side length Q = max(R, MINQ) to enforce the requested minimum extent.
- Each column is stored contiguously via CH = ceil(Q/R) vector chunks.
- Memory layout:
vec[col][chunk] is a T_VEC holding up to R scalars of column 'col'
spanning rows chunk*R .. chunk*R + (R - 1).
flat[] mirrors the same column-major order (column stride = Q).
- SIMD view remains integer-typed; cast inside micro-kernels to the
appropriate floating-point/integer vector type on demand.
*/
#define MAT_PANEL(T, LE, MINOR, ALIGN) \
ML_ALIGNAS(ALIGN) union { \
/* Raw SIMD view: cast to __m256/__m512d inside the micro-kernel on demand */ \
__m128i xmm[ML_XMM_COUNT( \
ML_TOT_BYTES(T, ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))))]; \
__m256i ymm[ML_YMM_COUNT( \
ML_TOT_BYTES(T, ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))))]; \
__m512i zmm[ML_ZMM_COUNT( \
ML_TOT_BYTES(T, ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))))]; \
/* Scalar 2D view: minor �� LEp (LE dimension contains alignment padding) */ \
T blk[(MINOR)][ML_PANEL_STRIDE_ELEMS(T, (LE), (ALIGN))]; \
/* Flat view */ \
T flat[ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))]; \
#define MAT_PANEL_SQ_MIN(T, T_VEC, MINQ) \
SN_ALIGNAS(sizeof(T_VEC)) union { \
T_VEC vec[PANEL_Q(T, T_VEC, MINQ)][PANEL_CHUNKS(T, T_VEC, MINQ)]; \
T flat[PANEL_Q(T, T_VEC, MINQ) * PANEL_Q(T, T_VEC, MINQ)]; \
}
/* ---------- SSE preset ---------- */
#undef PANEL_LE
#undef PANEL_MINOR
#undef PANEL_ALIGN
#define PANEL_LE 4
#define PANEL_MINOR 32
#define PANEL_ALIGN 16
/* ======== SSE preset ========
4 x 32, 16B aligned
/* ISA presets with minimum square side:
- SSE : MINQ=4 (avoid 2x2 for double)
- AVX2 : MINQ=8
- AVX-512: MINQ=8 (16 for float/i32 naturally, so this is a no-op there)
*/
#define MAT_PANEL_SSE(T) MAT_PANEL(T, PANEL_LE, PANEL_MINOR, PANEL_ALIGN)
/* ---------- AVX2 preset ---------- */
#undef PANEL_LE
#undef PANEL_MINOR
#undef PANEL_ALIGN
#define PANEL_LE 8
#define PANEL_MINOR 64
#define PANEL_ALIGN 32
#define MAT_PANEL_AVX(T) MAT_PANEL(T, PANEL_LE, PANEL_MINOR, PANEL_ALIGN) /* 8 x 64, 32B aligned */
/* SSE panel layout: column-major buffer holding 4 logical columns with MINQ=4.
Columns: each column packs up to four scalars so ALIGN equals sizeof(__m128).
Alignment: every column boundary lands on a 16-byte multiple for XMM loads. */
#define MAT_PANEL_SSE(T) MAT_PANEL_SQ_MIN(T, __m128, 4)
/* AVX2 panel layout: column-major buffer holding 8 logical columns with MINQ=8.
Columns: each column stores eight elements via two 256-bit chunks when T is 32-bit.
Alignment: column starts are 32-byte aligned so AVX2 loads/stores remain contiguous. */
#define MAT_PANEL_AVX(T) MAT_PANEL_SQ_MIN(T, __m256, 8)
/* AVX-512 panel layout: column-major buffer holding 8 or more columns depending on T.
Columns: each column expands to ceil(Q/lanes) ZMM chunks; default MINQ=8 keeps double-wide tiles.
Alignment: column starts sit on 64-byte boundaries to match AVX-512 streaming access. */
#define MAT_PANEL_512(T) MAT_PANEL_SQ_MIN(T, __m512, 8)
/* ---------- AVX-512 preset ---------- */
#undef PANEL_LE
#undef PANEL_MINOR
#undef PANEL_ALIGN
#define PANEL_LE 16
#define PANEL_MINOR 64
#define PANEL_ALIGN 64
#define MAT_PANEL_512(T) \
MAT_PANEL(T, PANEL_LE, PANEL_MINOR, PANEL_ALIGN); /* 16 x 64, 64B aligned */
/* Access helpers (all compile-time friendly):
Given Q = PANEL_Q(...), CH = PANEL_CHUNKS(...), R = lanes per vector:
- Column c occupies vec[c][0..CH-1].
- Row r inside column c is located at vec[c][r / R] with lane (r % R).
*/
#define PANEL_COL_VEC(panel_obj, c, chunk) ((panel_obj).vec[(int)(c)][(int)(chunk)])
/* Scalar flat index helper for column-major addressing */
#define PANEL_ELEM(panel_obj, T, T_VEC, MINQ, r, c) \
((panel_obj).flat[(int)(c) * PANEL_Q(T, T_VEC, MINQ) + (int)(r)])