#ifndef SSC_INTRIN
#define SSC_INTRIN

#define SSC_INTRIN_HEAD

#include "StgSharpNative.h"
#include "sn_matkernel.h"
#include <immintrin.h>
#include <smmintrin.h>
#include <stdint.h>

#ifdef _MSC_VER
#include <intrin.h>
#include <isa_availability.h>
#endif

#define LASTTWO(x) (x & 3)
#define LAST(x) (x & 3)
#define get_align_begin(x) ((x + 15) & (~15))

#define SSE_INSERT(source, target, reset3, reset2, reset1, reset0)                      \
        (LASTTWO(source - 1) << 6) | (LASTTWO(target - 1) << 4) | (LAST(reset3) << 3) | \
                (LAST(reset2) << 2) | (LAST(reset1) << 1) | LAST(reset0)
#define SSE_INSERT_SIMPLE(source, target) SSE_INSERT(source, target, 0, 0, 0, 0)
#define M256_PERMUTE(s3, s2, s1, s0) \
        ((LASTTWO(s3) << 6) | (LASTTWO(s2) << 4) | (LASTTWO(s1) << 2) | (LASTTWO(s0)))
#define BIT_CVRT(pack, index, T) *(T *)(pack->data + index)
#define zero_vec _mm_setzero_ps()
#define zero_vec_256 _mm256_setzero_ps()

typedef union uint64_u {
        uint64_t u64;
        uint32_t u32[2];
} uint64_u;

typedef struct mat_kernel mat_kernel;

typedef struct mat_panel mat_panel;

typedef union scalar_pack {
        uint64_t data[8];
} scalar_pack;

// define operation style:

#define PANEL_OP 1
#define BUFFER_OP 2
#define KERNEL_OP 3

#define RIGHT_ANS_PARAM 1
#define LEFT_RIGHT_ANS_PARAM 2
#define RIGHT_ANS_SCALAR_PARAM 3
#define LEFT_RIGHT_ANS_SCALAR_PARAM 4
#define ANS_SCALAR_PARAM 5

typedef struct matrix_op_mode {
        int16_t operation_style; /* PANEL_OP, BUFFER_OP, KERNEL_OP */
        int16_t parameter_style; /* RIGHT_ANS_PARAM, LEFT_RIGHT_ANS_PARAM, e.g. */
} matrix_op_mode;

/* Mirrors StgSharp.Mathematics.Numeric.MatrixParallelTaskPackage<T> (128 bytes). */
#define MAT_TASK(T)                                                                \
        struct {                                                                   \
                MAT_KERNEL(T) * left; /* Offset 0: left matrix kernel base */      \
                MAT_KERNEL(T) * right; /* Offset 8: right matrix kernel base */    \
                MAT_KERNEL(T) * result; /* Offset 16: result matrix kernel base */ \
                int32_t element_size; /* Offset 24: sizeof(T) */                   \
                int32_t reserved_ptr0; /* Offset 28: reserved/extension slot */    \
                int32_t left_prim_offset; /* Offset 32 */                          \
                int32_t left_prim_stride; /* Offset 36 */                          \
                int32_t left_sec_offset; /* Offset 40 */                           \
                int32_t left_sec_stride; /* Offset 44 */                           \
                int32_t right_prim_offset; /* Offset 48 */                         \
                int32_t right_prim_stride; /* Offset 52 */                         \
                int32_t right_sec_offset; /* Offset 56 */                          \
                int32_t right_sec_stride; /* Offset 60 */                          \
                int32_t result_prim_offset; /* Offset 64 */                        \
                int32_t result_prim_stride; /* Offset 68 */                        \
                int32_t result_sec_offset; /* Offset 72 */                         \
                int32_t result_sec_stride; /* Offset 76 */                         \
                scalar_pack *scalar; /* Offset 80 */                               \
                void *compute_handle; /* Offset 88 */                              \
                int32_t prim_count; /* Offset 96 */                                \
                matrix_op_mode compute_mode; /* Offset 100 */                      \
                int32_t sec_count; /* Offset 104 */                                \
                int32_t reserved_int0; /* Offset 108 */                            \
                int32_t prim_tile_offset; /* Offset 112 */                         \
                int32_t sec_tile_offset; /* Offset 116 */                          \
                int32_t prim_cols_in_tile; /* Offset 120 */                        \
                int32_t sec_cols_in_tile; /* Offset 124 */                         \
        }
typedef struct matrix_task matrix_task;

#if defined(__cplusplus)
static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mismatch");
#elif defined(__STDC_VERSION__)
 static_assert(sizeof(MAT_TASK(float)) == 128, "matrix_parallel_task_package size mismatch");
#endif

#pragma region matrix buffer operation function type

typedef void(SN_DECL *BUF_PROC_AS)(mat_kernel *ans, scalar_pack *scalar, int count);
typedef void(SN_DECL *BUF_PROC_RAS)(mat_kernel *right, mat_kernel *ans, scalar_pack *scalar,
                                    int count);
typedef void(SN_DECL *BUF_PROC_LRAS)(mat_kernel *left, mat_kernel *right, mat_kernel *ans,
                                     scalar_pack *scalar, int count);
typedef void(SN_DECL *BUF_PROC_RA)(mat_kernel *right, mat_kernel *ans, int count);
typedef void(SN_DECL *BUF_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans,
                                    int count);

#define BUF_PROC(T_type, T_arch, T_op_name) T_type##_buf_##T_op_name##_##T_arch

#define DECLARE_BUF_PROC_ANS_SCALAR(T_type, T_arch, T_op_name)     \
        INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
                MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, size_t count)
#define DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name)               \
        INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)(                 \
                MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
                scalar_pack const *scalar, int count)
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name)        \
        INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)(               \
                MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
                MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, size_t count)
#define DECLARE_BUF_PROC_RIGHT_ANS(T_type, T_arch, T_op_name)      \
        INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
                MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, size_t count)
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name)               \
        INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)(               \
                MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
                MAT_KERNEL(T_type) *restrict ans, size_t count)

#pragma endregion

#pragma region matrix panel function type

typedef void(SN_DECL *PANEL_PROC)(mat_panel *panel, mat_kernel *kernel, int col_length,
                                  int row_length, int col_index, int row_index);
typedef void(SN_DECL *PNL_PROC_LRA)(mat_panel *left, mat_panel *right, mat_panel *ans);
typedef void(SN_DECL *PNL_PROC_RA)(mat_panel *right, mat_panel *ans);
typedef void(SN_DECL *PNL_PROC_LRAS)(mat_panel *left, mat_panel *right, mat_panel *ans,
                                     scalar_pack *scalar);
typedef void(SN_DECL *PNL_PROC_RAS)(mat_panel *right, mat_panel *ans, scalar_pack *scalar);

#define PNL_PROC(T_type, T_arch, T_op_name) T_type##_pnl_##T_op_name##_##T_arch
#define DECLARE_PNL_PROC_ANS(T_type, T_arch, T_op_name) \
        INTERNAL void SN_DECL PNL_PROC(T_type, T_arch,  \
                                       T_op_name)(MAT_PANEL_##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name)                           \
        INTERNAL void SN_DECL PNL_PROC(T_type, T_arch,                                       \
                                       T_op_name)(MAT_PANEL_##T_arch##(T_type) const *left,  \
                                                  MAT_PANEL_##T_arch##(T_type) const *right, \
                                                  MAT_PANEL_##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
        INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)(        \
                MAT_PANEL_##T_arch##(T_type) const *left,                 \
                MAT_PANEL_##T_arch##(T_type) const *right,                \
                MAT_PANEL_##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)

#pragma endregion

#pragma region matrix kernel function type

typedef void(SN_DECL *KER_PROC_AS)(mat_kernel *ans, scalar_pack *scalar);
typedef void(SN_DECL *KER_PROC_RA)(mat_kernel *left, mat_kernel *ans);
typedef void(SN_DECL *KER_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans);
typedef void(SN_DECL *KER_PROC_LAS)(mat_kernel *matrix, mat_kernel *ans, scalar_pack *scalar);

#define KER_PROC(T_type, T_arch, T_op_name) T_type##_kernel_##T_op_name##_##T_arch

#define DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name)        \
        INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)(               \
                MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
                MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar)
#define DECLARE_KER_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name)               \
        INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)(                 \
                MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
                scalar_pack const *scalar)
#define DECLARE_KER_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name)               \
        INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)(               \
                MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
                MAT_KERNEL(T_type) *restrict ans)
#define DECLARE_KER_PROC_RIGHT_ANS(T_type, T_arch, T_op_name)      \
        INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \
                MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans)

#pragma endregion

#pragma region special matrix kernel operation

typedef void(SN_DECL *PIVOTPROC)(mat_kernel const *source, size_t col_begin, size_t col_length,
                                 size_t *restrict ans);
INTERNAL void SN_DECL pivot_float(MAT_KERNEL(float) const *source, size_t col_begin,
                                  size_t col_length, size_t *restrict ans);

#pragma endregion

typedef void(SN_DECL *VECTORNORMALIZEPROC)(__m128 *source, __m128 *target);
typedef void(SN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans);
typedef uint64_t(SN_DECL *FACTORIALROC)(int n);

#pragma region string

typedef int(SN_DECL *HASH)(byte const *str, int length);
typedef int(SN_DECL *INDEX_PAIR)(short const *stram, uint32_t target, int length);

#pragma endregion

typedef struct mat_intrinsic {
        BUF_PROC_LRA buffer_add;
        BUF_PROC_AS buffer_fill;
        BUF_PROC_RAS buffer_scalar_mul;
        BUF_PROC_LRA buffer_sub;
        KER_PROC_RA buffer_transpose;
        PANEL_PROC build_panel;
        PANEL_PROC clear_panel;
        PNL_PROC_LRA panel_fma;
        PIVOTPROC pivot;
        PANEL_PROC store_panel;

} mat_intrinsic;

// Mirrors C# IntrinsicContext field order so managed/unmanaged layouts stay in sync.
typedef struct sn_intrinsic {
        HASH city_hash_simplify;
        FACTORIALROC factorial_simd;
        INDEX_PAIR index_pair;
        VECTORNORMALIZEPROC vec_f32_normalize_3;
        mat_intrinsic mat[4];
} sn_intrinsic;

typedef enum elment_type { F32 = 0, F64 = 1, I32 = 2, I64 = 3 } element_type;

typedef enum most_advanced_instruction {
#ifdef _MSC_VER
        SSE = __IA_SUPPORT_SSE42,
        AVX256 = __IA_SUPPORT_VECTOR256,
        AVX512 = __IA_SUPPORT_VECTOR512,
        NEON
#else
        SSE,
        AVX256,
        AVX512,
        NEON
#endif
} most_advanced_instruction;

#include "sn_target.h"

#if SN_IS_ARCH(SN_ARCH_X86_64)

// -----------------------------------------------------------------------------
// SIMDID layout helpers (see SIMDID.simd.md)
// bits [3..0]  : Root ISA
// bits [7..4]  : Manufacture (ISA-specific)
// bits [15..8] : MainLevel (monotonic capability)
// bits [31..16]: AVX512 Feature
// bits [39..32]: AMX Feature
// bits [63..40]: uArch / Policy (optional)
// -----------------------------------------------------------------------------
#define SIMDID_SHIFT_ROOT 0
#define SIMDID_SHIFT_MANU 4
#define SIMDID_SHIFT_MAIN 8
#define SIMDID_SHIFT_AVX512 16
#define SIMDID_SHIFT_AMX 32
#define SIMDID_SHIFT_UARCH 40

#define SIMDID_MASK_ROOT 0xFULL
#define SIMDID_MASK_MANU (0xFULL << SIMDID_SHIFT_MANU)
#define SIMDID_MASK_MAIN (0xFFULL << SIMDID_SHIFT_MAIN)
#define SIMDID_MASK_AVX512 (0xFFFFULL << SIMDID_SHIFT_AVX512)
#define SIMDID_MASK_AMX (0xFFULL << SIMDID_SHIFT_AMX)
#define SIMDID_MASK_UARCH (0xFFFFFFULL << SIMDID_SHIFT_UARCH)

// Root ISA (low nibble)
#define SIMDID_ROOT_UNKNOWN 0x0
#define SIMDID_ROOT_X86_64 0x1
#define SIMDID_ROOT_AARCH64 0x2
#define SIMDID_ROOT_PPC64LE 0x3
#define SIMDID_ROOT_APPLESILICON 0x4
#define SIMDID_ROOT_RISCV64 0x5
#define SIMDID_ROOT_LOONGARCH 0x6

// Manufacture (x86-64 mapping; other ISAs may reinterpret)
#define SIMDID_MANU_UNKNOWN 0x0
#define SIMDID_MANU_INTEL 0x1
#define SIMDID_MANU_AMD 0x2
#define SIMDID_MANU_VIA 0x3

// MainLevel (ordinal, shared bits 8..15; not bit-flags)
#define SIMDID_MAIN_LVL_NONE 0x00u
#define SIMDID_MAIN_LVL_SSE 0x01u          /* SSE1..SSE4.2 */
#define SIMDID_MAIN_LVL_AVX2 0x02u
#define SIMDID_MAIN_LVL_AVX2_FMA 0x03u
#define SIMDID_MAIN_LVL_AVX2_FMA_F16C 0x04u /* 16-bit scalar/vector ready */
#define SIMDID_MAIN_LVL_AVX512 0x05u
#define SIMDID_MAIN_LVL_AMX 0x06u
#define SIMDID_MAIN_LVL_AVX10 0x07u

/* Pre-shifted helpers for convenience when constructing SIMDID */
#define SIMDID_MAIN_ENC_NONE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE)
#define SIMDID_MAIN_ENC_SSE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE)
#define SIMDID_MAIN_ENC_AVX2 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2)
#define SIMDID_MAIN_ENC_AVX2_FMA SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2_FMA)
#define SIMDID_MAIN_ENC_AVX2_FMA_F16C SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2_FMA_F16C)
#define SIMDID_MAIN_ENC_AVX512 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512)
#define SIMDID_MAIN_ENC_AMX SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX)
#define SIMDID_MAIN_ENC_AVX10 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10)

#pragma region matix function

INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);

DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, add);

DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, add);

DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, sub);

DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, sub);

DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose);

DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul);

DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, scalar_mul);

DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill);

DECLARE_BUF_PROC_ANS_SCALAR(double, sse, fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, fill);

#pragma endregion

#pragma region fma
DECLARE_PNL_PROC_ANS(float, sse, clear);
DECLARE_PNL_PROC_ANS(float, avx, clear);
DECLARE_PNL_PROC_ANS(float, 512, clear);

DECLARE_PNL_PROC_ANS(double, sse, clear);
DECLARE_PNL_PROC_ANS(double, avx, clear);
DECLARE_PNL_PROC_ANS(double, 512, clear);

DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx_fma, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, fma);

DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx_fma, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, 512, fma);

#pragma endregion

#pragma region scaler_simd

INTERNAL uint64_t SN_DECL factorial_simd_sse(int n);

#pragma endregion

#pragma region dot_product

INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
                                             __m128 const *vector, __m128 *ans);
INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
                                 __m128 *ans);
INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
                                 __m128 *ans);
INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
                                 __m128 *ans);

#pragma endregion

#pragma region string

INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int length);
INTERNAL int SN_DECL index_pair_sse(short const *stram, uint32_t target, int length);

#pragma endregion

#else

typedef enum intrinsic_level { SIMD0 = 0 } intrinsic_level;

#endif

#endif