diff --git a/StgSharp.Common/Mathematics/Linear/ElementType.cs b/StgSharp.Common/Mathematics/Linear/ElementType.cs index b570c2d1..4afb6871 100644 --- a/StgSharp.Common/Mathematics/Linear/ElementType.cs +++ b/StgSharp.Common/Mathematics/Linear/ElementType.cs @@ -1,9 +1,9 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="ElementType" // Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -25,15 +25,52 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +#pragma warning disable CA1028 namespace StgSharp.Mathematics.Numeric { - public enum MatrixElementType : int + [StructLayout(LayoutKind.Explicit, Size = 4)] + public struct MatrixElementType { - F32 = 0, - F64 = 1, - I32 = 2, - I64 = 3, + public static readonly MatrixElementType F32 = new(0x03000004); + public static readonly MatrixElementType F64 = new(0x03000008); + public static readonly MatrixElementType I08 = new(0x0100001); + public static readonly MatrixElementType I16 = new(0x0100002); + public static readonly MatrixElementType I32 = new(0x0100004); + public static readonly MatrixElementType I64 = new(0x0100008); + public static readonly MatrixElementType U08 = new(0x0200001); + public static readonly MatrixElementType U16 = new(0x0200002); + public static readonly MatrixElementType U32 = new(0x0200004); + public static readonly MatrixElementType U64 = new(0x0200008); + + [FieldOffset(0)] internal readonly int Mask; + [FieldOffset(3)] public readonly MatrixElementTypeGroup Group; + + [FieldOffset(0)] public readonly ushort Size; + + private MatrixElementType( + int mask + ) + { + Unsafe.SkipInit(out this); + Unsafe.SkipInit(out Size); + Unsafe.SkipInit(out Group); + this.Mask = mask; + } + + } + + public enum MatrixElementTypeGroup : byte + { + + None = 0, + Singed = 1, + Unsigned = 2, + Float = 3, } } +#pragma warning restore CA1028 diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.cs index 19768c5c..0a3ecdfa 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.cs @@ -40,68 +40,6 @@ namespace StgSharp.Mathematics.Numeric public partial class MatrixCompute { - internal static unsafe void ComputeFmaPanelF32( - MatrixParallelTaskPackage* p - ) - { - int f32 = (int)MatrixElementType.F32; - MatrixKernel* leftBuffer = (MatrixKernel*)p->Left; - MatrixKernel* rightBuffer = (MatrixKernel*)p->Right; - MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer; - ref long offsetRef = ref Unsafe.As(ref p->PrimTileOffset); - ref long lengthRef = ref Unsafe.As(ref p->PrimCount); - int ansWidth = p->AnsPrimOffset; - int ansHeight = p->AnsSecOffset; - int commonK = p->LeftPrimOffset; - - M128* enumeration = stackalloc M128[1]; - enumeration->Member(2) = commonK; - enumeration->Member(3) = ansHeight; - - // in this occasion size must be 1 - for (long c = offsetRef; c < offsetRef + lengthRef; c++) - { - int i = (int)(c / ansHeight); - int j = (int)(c % ansHeight); - enumeration->Member(0) = i; - enumeration->Member(1) = j; - NativeIntrinsic.Context.mat[f32].kernel_tile_fma(leftBuffer, rightBuffer, ansBuffer, - enumeration); - } - } - - internal static unsafe void ComputeFmaPanelF64( - MatrixParallelTaskPackage* p - ) - { - int f64 = (int)MatrixElementType.F64; - - // int size = 1; - MatrixKernel* leftBuffer = (MatrixKernel*)p->Left; - MatrixKernel* rightBuffer = (MatrixKernel*)p->Right; - MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer; - ref long offsetRef = ref Unsafe.As(ref p->PrimTileOffset); - ref long lengthRef = ref Unsafe.As(ref p->PrimCount); - int ansWidth = p->AnsPrimOffset; - int ansHeight = p->AnsSecOffset; - int commonK = p->LeftPrimOffset; - - M128* enumeration = stackalloc M128[1]; - enumeration->Member(2) = commonK; - enumeration->Member(3) = ansHeight; - - // in this occasion size must be 1 - for (long c = offsetRef; c < offsetRef + lengthRef; c++) - { - int i = (int)(c / ansHeight); - int j = (int)(c % ansHeight); - enumeration->Member(0) = i; - enumeration->Member(1) = j; - NativeIntrinsic.Context.mat[f64].kernel_tile_fma(leftBuffer, rightBuffer, ansBuffer, - enumeration); - } - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static unsafe MatrixKernel* GetKernelAddressUnsafe( MatrixKernel* buffer, diff --git a/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs b/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs index 2ac93bfe..bd345fb1 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs @@ -71,11 +71,9 @@ namespace StgSharp.Mathematics.Numeric long recommendCount = count / 1024; MatrixParallelThread[] pool = LeadParallel((int)recommendCount); - MatrixParallelWrap handle = new MatrixParallelWrap(pool); - - // TODO set parallel queue for handle - handle.TaskQueue = MatrixParallelQueue.Rent(package); - return handle; + MatrixParallelWrap wrap = new MatrixParallelWrap(pool); + wrap.TaskQueue = MatrixParallelQueue.Rent(package); + return wrap; } } diff --git a/StgSharp.Mathematics/Numeric/MatrixParallel.cs b/StgSharp.Mathematics/Numeric/MatrixParallel.cs index 7ff08f23..b34009eb 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallel.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallel.cs @@ -98,6 +98,7 @@ namespace StgSharp.Mathematics.Numeric } // TODO 构建全局的保底上下文 + foreach (MatrixParallelThread thread in _threads) { } if (contextMap.Keys.Count == 1) { for (int i = 0; i < 4; i++) @@ -113,7 +114,7 @@ namespace StgSharp.Mathematics.Numeric IsInitialized = true; } - [LibraryImport(Native.LibName, EntryPoint = "load_intrinsic_function")] + [LibraryImport(Native.LibName, EntryPoint = "sn_compare_simd")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] private static unsafe partial ulong CompareSIMDID( IntrinsicContext* context, diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs b/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs index 6a49c273..dcfa05f9 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs @@ -52,48 +52,12 @@ namespace StgSharp.Mathematics.Numeric * matrix_op_mode; */ - public enum MatrixOperationSpecial : short + public enum MatrixIndexStyle : int { - SEQ_FMA = -1, - - } - - public enum MatrixIndexStyle : short - { - - SPECIAL = 0, + NONE = 0, BUFFER_INDEX = 1, - - } - - public enum MatrixOperationParam : short - { - - RIGHT_ANS_PARAM = 1, - LEFT_RIGHT_ANS_PARAM = 2, - RIGHT_ANS_SCALAR_PARAM = 3, - LEFT_RIGHT_ANS_SCALAR_PARAM = 4, - ANS_SCALAR_PARAM = 5, - - } - - [StructLayout(LayoutKind.Sequential)] - public struct MatrixOpMode( - MatrixIndexStyle style, - MatrixOperationParam paramLayout - ) - { - - public MatrixIndexStyle OperationStyle = style; - - public MatrixOperationParam ParameterStyle = paramLayout; - - public MatrixOpMode( - MatrixIndexStyle style, - MatrixOperationSpecial paramLayout - ) - : this(style, (MatrixOperationParam)paramLayout) { } + SEQ_FMA = -1, } @@ -166,7 +130,7 @@ namespace StgSharp.Mathematics.Numeric [FieldOffset(96)]public int PrimCount; // 4 [FieldOffset(100)]public int SecCount; // 4 - [FieldOffset(104)]public MatrixOpMode ComputeMode; // 4 + [FieldOffset(104)]public MatrixIndexStyle ComputeMode; // 4 [FieldOffset(108)]private int ReservedInt0; // 4 #endregion diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelThread.Fma.cs b/StgSharp.Mathematics/Numeric/MatrixParallelThread.Fma.cs new file mode 100644 index 00000000..c121964b --- /dev/null +++ b/StgSharp.Mathematics/Numeric/MatrixParallelThread.Fma.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; +using StgSharp.HighPerformance; +using StgSharp.Internal; + +namespace StgSharp.Mathematics.Numeric +{ + public partial class MatrixParallelThread + { + + internal static unsafe void ComputeFmaPanelF32( + MatrixParallelTaskPackage* p + ) + { + int f32 = (int)MatrixElementType.F32; + MatrixKernel* leftBuffer = (MatrixKernel*)p->Left; + MatrixKernel* rightBuffer = (MatrixKernel*)p->Right; + MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer; + ref long offsetRef = ref Unsafe.As(ref p->PrimTileOffset); + ref long lengthRef = ref Unsafe.As(ref p->PrimCount); + int ansWidth = p->AnsPrimOffset; + int ansHeight = p->AnsSecOffset; + int commonK = p->LeftPrimOffset; + + M128* enumeration = stackalloc M128[1]; + enumeration->Member(2) = commonK; + enumeration->Member(3) = ansHeight; + + // in this occasion size must be 1 + for (long c = offsetRef; c < offsetRef + lengthRef; c++) + { + int i = (int)(c / ansHeight); + int j = (int)(c % ansHeight); + enumeration->Member(0) = i; + enumeration->Member(1) = j; + NativeIntrinsic.Context.mat[f32].kernel_tile_fma(leftBuffer, rightBuffer, ansBuffer, + enumeration); + } + } + + internal static unsafe void ComputeFmaPanelF64( + MatrixParallelTaskPackage* p + ) + { + int f64 = (int)MatrixElementType.F64; + + // int size = 1; + MatrixKernel* leftBuffer = (MatrixKernel*)p->Left; + MatrixKernel* rightBuffer = (MatrixKernel*)p->Right; + MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer; + ref long offsetRef = ref Unsafe.As(ref p->PrimTileOffset); + ref long lengthRef = ref Unsafe.As(ref p->PrimCount); + int ansWidth = p->AnsPrimOffset; + int ansHeight = p->AnsSecOffset; + int commonK = p->LeftPrimOffset; + + M128* enumeration = stackalloc M128[1]; + enumeration->Member(2) = commonK; + enumeration->Member(3) = ansHeight; + + // in this occasion size must be 1 + for (long c = offsetRef; c < offsetRef + lengthRef; c++) + { + int i = (int)(c / ansHeight); + int j = (int)(c % ansHeight); + enumeration->Member(0) = i; + enumeration->Member(1) = j; + NativeIntrinsic.Context.mat[f64].kernel_tile_fma(leftBuffer, rightBuffer, ansBuffer, + enumeration); + } + } + } +} diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelThread.Model.cs b/StgSharp.Mathematics/Numeric/MatrixParallelThread.Model.cs index 024dccd4..05e4d438 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelThread.Model.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelThread.Model.cs @@ -34,100 +34,41 @@ namespace StgSharp.Mathematics.Numeric public partial class MatrixParallelThread { - internal static unsafe void BufferComputeBinary( - MatrixParallelTaskPackage* p - ) - { - int size = p->ElementSize * 16; - long count = Unsafe.As(ref p->PrimCount); - long offset = Unsafe.As(ref p->PrimTileOffset) * size; + private const int KernelStealCount = 64; - ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Left + offset), - (void*)(p->Right + offset), - (void*)(p->Answer + offset), - count); - MatrixParallelFactory.Release(p); - } - - internal static unsafe void BufferComputeBinaryScalar( - MatrixParallelTaskPackage* p + internal unsafe void BufferCompute( + MatrixParallelTaskPackage* p ) { + // while () { } int size = p->ElementSize * 16; long count = Unsafe.As(ref p->PrimCount); long offset = Unsafe.As(ref p->PrimTileOffset) * size; ScalarPacket* scalar = p->Scalar; - ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Left + offset), - (void*)(p->Right + offset), - (void*)(p->Answer + offset), - scalar, - count); - MatrixParallelFactory.Release(p); + IntPtr computeHandle = _context[p->ComputeHandle]; + ((delegate* unmanaged[Cdecl])computeHandle)((void*)(p->Left + offset), + (void*)(p->Right + offset), + (void*)(p->Answer + offset), + scalar, + count); } - internal static unsafe void BufferComputeNoOperatorScalar( - MatrixParallelTaskPackage* p + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal unsafe void SpecialBufferCompute( + MatrixParallelTaskPackage* p ) { - int size = p->ElementSize * 16; - long count = Unsafe.As(ref p->PrimCount); - long offset = Unsafe.As(ref p->PrimTileOffset) * size; - ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Answer + offset), - p->Scalar, count); - MatrixParallelFactory.Release(p); - } - - internal static unsafe void BufferComputeUnary( - MatrixParallelTaskPackage* p - ) - { - int size = p->ElementSize * 16; - long count = Unsafe.As(ref p->PrimCount); - long offset = Unsafe.As(ref p->PrimTileOffset) * size; - - ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Right + offset), - (void*)(p->Answer + offset), - count); - MatrixParallelFactory.Release(p); - } - - internal static unsafe void BufferComputeUnaryScalar( - MatrixParallelTaskPackage* p - ) - { - int size = p->ElementSize * 16; - long count = Unsafe.As(ref p->PrimCount); - long offset = Unsafe.As(ref p->PrimTileOffset) * size; - ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Right + offset), - (void*)(p->Answer + offset), - p->Scalar, - count); - MatrixParallelFactory.Release(p); - } - - internal static unsafe void SpecialBufferCompute( - MatrixParallelTaskPackage* p - ) - { - switch ((MatrixOperationSpecial)p->ComputeMode.ParameterStyle) + switch ((MatrixElementType)p->ElementSize) { - case MatrixOperationSpecial.SEQ_FMA: - switch ((MatrixElementType)p->ElementSize) - { - case MatrixElementType.F64: - MatrixCompute.ComputeFmaPanelF64(p); - return; - case MatrixElementType.F32: - MatrixCompute.ComputeFmaPanelF32(p); - break; - default: - break; - } + case MatrixElementType.F64: + ComputeFmaPanelF64(p); + return; + case MatrixElementType.F32: + ComputeFmaPanelF32(p); break; default: break; } - MatrixParallelFactory.Release(p); } } diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs b/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs index 1cc1bde8..7f3e2aea 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs @@ -83,44 +83,6 @@ namespace StgSharp.Mathematics.Numeric GC.SuppressFinalize(this); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void ExecuteTaskPackage( - MatrixParallelQueue taskQueue - ) - { - MatrixParallelTaskPackage* p = taskQueue.Package; - switch (p->ComputeMode.OperationStyle) - { - case MatrixIndexStyle.BUFFER_INDEX: - switch (p->ComputeMode.ParameterStyle) - { - case MatrixOperationParam.RIGHT_ANS_PARAM: - BufferComputeUnary(p); - break; - case MatrixOperationParam.LEFT_RIGHT_ANS_PARAM: - BufferComputeBinary(p); - break; - case MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM: - BufferComputeUnaryScalar(p); - break; - case MatrixOperationParam.LEFT_RIGHT_ANS_SCALAR_PARAM: - BufferComputeBinaryScalar(p); - break; - case MatrixOperationParam.ANS_SCALAR_PARAM: - BufferComputeNoOperatorScalar(p); - break; - default: - break; - } - break; - case MatrixIndexStyle.SPECIAL: - SpecialBufferCompute(p); - break; - default: - break; - } - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Restart() { @@ -182,7 +144,19 @@ namespace StgSharp.Mathematics.Numeric // TODO 将外部线程绑定到当前线程上,并执行任务包 } else { - ExecuteTaskPackage(BindedWrap.TaskQueue); + MatrixParallelQueue taskQueue = BindedWrap.TaskQueue; + MatrixParallelTaskPackage* p = taskQueue.Package; + if (0 > (int)p ->ComputeHandle) { + SpecialBufferCompute(p); + } + switch (p->ComputeMode) + { + case MatrixIndexStyle.BUFFER_INDEX: + BufferCompute(p); + break; + default: + break; + } BindedWrap.ReportThreadAccomplished(); } _managedThread.Priority = AfterWork; diff --git a/StgSharp.Native/SIMDID.x86.md b/StgSharp.Native/SIMDID.x86.md index 795ac0d7..6239c905 100644 --- a/StgSharp.Native/SIMDID.x86.md +++ b/StgSharp.Native/SIMDID.x86.md @@ -55,7 +55,8 @@ For **x86-64**: | 1 | Intel | | 2 | AMD | | 3 | VIA/Zhaoxin | -| 4..15 | Reserved | +| 4 | Hygon (海光) | +| 5..15 | Reserved | For **AArch64**: @@ -111,15 +112,17 @@ AVX512 Feature (bits 31..16) Notes: - **BASE (bit 20)**: Base AVX512 bundle, indicating `F + CD + VL + DQ + BW` are all present. If set to 1, the detecting side must guarantee these subsets are all available. -- **AVX512_IMPL ([23..22], 2-bit)**: Implementation form (width policy). +- **AVX512_IMPL ([23..22], 2-bit)**: 512-bit execution width policy — not port count, not frequency behaviour. | Value | Meaning | |---:|------| -| 0 | Unknown (strategy decided by Manufacture/uArch) | -| 1 | AVX512-DUAL (glue/downclock risk; prefer 256-bit path) | -| 2 | AVX512-NATIVE (full 512-bit ALU; 512-bit path is usable) | +| 0 | Unknown — cannot determine; use conservative 256-bit kernels | +| 1 | DUAL-PUMP — single 256-bit ALU executes 512-bit ops as two consecutive 256-bit micro-ops; ZMM throughput halved (example: AMD Zen4, Family 0x19) | +| 2 | NATIVE — genuine 512-bit execution unit(s); full ZMM throughput (Intel all models including SKX; AMD Zen5 Family 0x1A and later) | | 3 | Reserved | +> **Note**: Intel SKX/CLX/CPX have two 512-bit FMA ports (port-0 + port-5) and are **NATIVE** — genuine 512-bit throughput. Intel downclocking on heavy ZMM workloads is a separate concern not encoded here. DUAL-PUMP applies only where a single 256-bit unit fires twice per 512-bit op (AMD Zen4-style). + - **VBMI/VBMI2/VNNI/BF16/FP16 ([29..25])**: Commonly needed niche or AI-related small-type extensions; set as needed. - **R/AI (bit 24)**: Reserved for future AI or other non-crypto extensions. - **[31..30], [21], [19..16]**: Reserved for future allocation. @@ -243,13 +246,17 @@ Detection: ## 12) Minimal Detection-Side Fill Rules (x86-64) -1) Set `RootISA = x86-64`, `Manufacture = Intel/AMD`. +1) Set `RootISA = x86-64`, `Manufacture = Intel / AMD / Hygon / VIA`. 2) Compute `MainLevel` (per ascending rule). 3) If `MainLevel >= AVX512`: - If `F+CD+VL+DQ+BW` are all present, set `BASE = 1`; otherwise `BASE = 0`. - Set `AVX512_IMPL`: - - AMD: DUAL (per definition) - - Intel: Decide DUAL/NATIVE based on family/model or policy table; if uncertain, use Unknown + - **Intel (all)**: NATIVE — all Intel AVX-512 CPUs have genuine 512-bit execution units + - **AMD Zen4 (Family 0x19)**: DUAL — single 256-bit ALU, dual-pump + - **AMD Zen5+ (Family 0x1A+)**: NATIVE — native full-width 512-bit units + - **AMD other/future**: UNKNOWN (conservative) + - **Hygon**: UNKNOWN (AVX-512 characteristics unclear) + - **VIA/other**: UNKNOWN - When VNNI/BF16/FP16/VBMI/VBMI2 are detected, set the corresponding bits. 4) If `MainLevel >= AMX`: fill the AMX byte. 5) When AVX/AVX2 are present, populate the AVX Feature byte: set `AVX` and `AVX2`; set `FMA` and `F16C` when detected; set `DUAL` when the YMM execution path is split/glued (AMD Zen1/Zen+ Family 0x17 model < 0x30, Hygon all models, VIA/Zhaoxin all models). diff --git a/StgSharp.Native/sn-head/sn_intrincontext.x86.h b/StgSharp.Native/sn-head/sn_intrincontext.x86.h index 7203aa26..0230ce68 100644 --- a/StgSharp.Native/sn-head/sn_intrincontext.x86.h +++ b/StgSharp.Native/sn-head/sn_intrincontext.x86.h @@ -46,6 +46,7 @@ #define SIMDID_MANU_INTEL 0x1 #define SIMDID_MANU_AMD 0x2 #define SIMDID_MANU_VIA 0x3 +#define SIMDID_MANU_HYGON 0x4 /* 海光: Zen1-derived, separate from AMD */ // MainLevel (ordinal, shared bits 8..15; not bit-flags) #define SIMDID_MAIN_LVL_NONE 0x00u diff --git a/StgSharp.Native/sn-src/load_intrinsic.x86.c b/StgSharp.Native/sn-src/load_intrinsic.x86.c index 075e9653..e7d0cac5 100644 --- a/StgSharp.Native/sn-src/load_intrinsic.x86.c +++ b/StgSharp.Native/sn-src/load_intrinsic.x86.c @@ -14,9 +14,7 @@ #endif // _MSC_VER PRIVATE void load_intrinsic_sse(sn_intrinsic *context, SIMDID id); -PRIVATE void load_intrinsic_avx_dual(sn_intrinsic *context, SIMDID id); PRIVATE void load_intrinsic_avx(sn_intrinsic *context, SIMDID id); -PRIVATE void load_intrinsic_512_dual(sn_intrinsic *context, SIMDID id); PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID id); SN_API void load_intrinsic_function(void *intrinsic_context, uint64_t id) diff --git a/StgSharp.Native/sn-src/m.fma.x86.c b/StgSharp.Native/sn-src/m.fma.x86.c index 5069ebe6..118e2362 100644 --- a/StgSharp.Native/sn-src/m.fma.x86.c +++ b/StgSharp.Native/sn-src/m.fma.x86.c @@ -151,29 +151,34 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma) DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma) { - static const __m512i b_shift = { .m512i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9, 15, - 12, 13, 14 } }; + static const __m512i b_shift = { .m512i_i32 = { + // shuff order + 0, 1, 2, 3, // col0 + 5, 6, 7, 4, //col1 + 10, 11, 8, 9, //col2 + 15, 12, 13, 14 //col3 + } }; register __m512 a = _mm512_load_ps((float *)left); register __m512 b = _mm512_load_ps((float *)right); register __m512 c = _mm512_load_ps((float *)ans); b = _mm512_permutexvar_ps(b_shift, b); - register __m512 a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0)); - register __m512 b_perm = _mm512_permute_ps(b, _FMA_SELECT(0)); - c = _mm512_fmadd_ps(a_perm, b_perm, c); + register __m512 a_perm0 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0)); + register __m512 b_perm0 = _mm512_permute_ps(b, _FMA_SELECT(0)); + c = _mm512_fmadd_ps(a_perm0, b_perm0, c); - a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3)); - b_perm = _mm512_permute_ps(b, _FMA_SELECT(1)); - c = _mm512_fmadd_ps(a_perm, b_perm, c); + register __m512 a_perm1 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3)); + register __m512 b_perm1 = _mm512_permute_ps(b, _FMA_SELECT(1)); + c = _mm512_fmadd_ps(a_perm1, b_perm1, c); - a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2)); - b_perm = _mm512_permute_ps(b, _FMA_SELECT(2)); - c = _mm512_fmadd_ps(a_perm, b_perm, c); + register __m512 a_perm2 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2)); + register __m512 b_perm2 = _mm512_permute_ps(b, _FMA_SELECT(2)); + c = _mm512_fmadd_ps(a_perm2, b_perm2, c); - a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1)); - b_perm = _mm512_permute_ps(b, _FMA_SELECT(3)); - c = _mm512_fmadd_ps(a_perm, b_perm, c); + register __m512 a_perm3 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1)); + register __m512 b_perm3 = _mm512_permute_ps(b, _FMA_SELECT(3)); + c = _mm512_fmadd_ps(a_perm3, b_perm3, c); _mm512_store_ps((float *)ans, c); } diff --git a/StgSharp.Native/sn-src/m.fma_tile.x86.c b/StgSharp.Native/sn-src/m.fma_tile.x86.c index 9db7d0de..6af7ebba 100644 --- a/StgSharp.Native/sn-src/m.fma_tile.x86.c +++ b/StgSharp.Native/sn-src/m.fma_tile.x86.c @@ -201,8 +201,13 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma) DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma) { - static const __m512i b_shift_source = { .m512i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9, - 15, 12, 13, 14 } }; + static const __m512i b_shift_source = { .m512i_i32 = { + // shuff order + 0, 1, 2, 3, // col0 + 5, 6, 7, 4, //col1 + 10, 11, 8, 9, //col2 + 15, 12, 13, 14 //col3 + } }; int i = enumeration->m128_i32[0]; int j = enumeration->m128_i32[1]; int commonK = enumeration->m128_i32[2]; diff --git a/StgSharp.Native/sn-src/m.panel.x86.c b/StgSharp.Native/sn-src/m.panel.x86.old similarity index 97% rename from StgSharp.Native/sn-src/m.panel.x86.c rename to StgSharp.Native/sn-src/m.panel.x86.old index d16c50d3..6e477fc1 100644 --- a/StgSharp.Native/sn-src/m.panel.x86.c +++ b/StgSharp.Native/sn-src/m.panel.x86.old @@ -1,301 +1,301 @@ -#include "sn_target.h" - -#if SN_IS_ARCH(SN_ARCH_X86_64) - -#include "sn_matkernel.h" - -DECLARE_BUILD_PANEL(avx, float) -{ - const int Q = 8; /* fixed for AVX float panel */ - - /* Neighbor kernels; substitute global zero kernel where out-of-bounds */ - const int has_right = (col_index + 1) < col_length; - const int has_bottom = (row_index + 1) < row_length; - - const MAT_KERNEL(float) *const k_lt = matrix + (row_index * col_length + col_index); - const MAT_KERNEL(float) *const k_rt = - has_right ? (matrix + (row_index * col_length + (col_index + 1))) : - (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; - const MAT_KERNEL(float) *const k_lb = - has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) : - (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; - const MAT_KERNEL(float) *const k_rb = - (has_right && has_bottom) ? - (matrix + ((row_index + 1) * col_length + (col_index + 1))) : - (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; - - register __m128 top128, bot128; - register __m256 col256; - - for (int pc = 0; pc < Q; ++pc) { - const int is_right_half = (pc >= 4); - const int kc_col = pc & 3; /* column index inside 4x4 kernel */ - - const MAT_KERNEL(float) *src_top = is_right_half ? k_rt : k_lt; - const MAT_KERNEL(float) *src_bot = is_right_half ? k_rb : k_lb; - - /* Vector loads: each kernel column is a compact __m128 */ - top128 = src_top->f32_x[kc_col]; - bot128 = src_bot->f32_x[kc_col]; - - /* Combine into an AVX column: low 128 = top, high 128 = bottom */ - col256 = _mm256_set_m128(bot128, top128); - - PANEL_COL_VEC((*panel), pc, 0) = col256; - } -} - -DECLARE_BUILD_PANEL(512, float) -{ - const MAT_KERNEL(float) *const zero = (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; - const MAT_KERNEL(float) * k[4][4]; - - for (int kc = 0; kc < 4; ++kc) { - for (int kr = 0; kr < 4; ++kr) { - const int col = col_index + kc; - const int row = row_index + kr; - k[kc][kr] = (col < col_length && row < row_length) ? - (matrix + (row * col_length + col)) : - zero; - } - } - - /* Each panel column corresponds to a kernel column (0..3) within a block col (0..3). */ - for (int pc = 0; pc < 16; ++pc) { - const int blk_col = pc >> 2; /* which kernel column block (0..3) */ - const int kc_col = pc & 3; /* column inside a 4x4 kernel */ - - const __m128 r0 = k[blk_col][0]->f32_x[kc_col]; - const __m128 r1 = k[blk_col][1]->f32_x[kc_col]; - const __m128 r2 = k[blk_col][2]->f32_x[kc_col]; - const __m128 r3 = k[blk_col][3]->f32_x[kc_col]; - - const __m256 top = _mm256_set_m128(r1, r0); - const __m256 bot = _mm256_set_m128(r3, r2); - - __m512 col = _mm512_castps256_ps512(top); - col = _mm512_insertf32x8(col, bot, 1); - - PANEL_COL_VEC((*panel), pc, 0) = col; - } -} - -DECLARE_BUILD_PANEL(512, double) -{ - const int Q = 8; /* max(4, lanes) for double with ZMM -> 8 columns */ - - const int has_right = (col_index + 1) < col_length; - const int has_bottom = (row_index + 1) < row_length; - - const MAT_KERNEL(double) *const k_lt = matrix + (row_index * col_length + col_index); - const MAT_KERNEL(double) *const k_rt = - has_right ? (matrix + (row_index * col_length + (col_index + 1))) : - (const MAT_KERNEL(double) *)&SN_ZERO_KERNEL; - const MAT_KERNEL(double) *const k_lb = - has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) : - (const MAT_KERNEL(double) *)&SN_ZERO_KERNEL; - const MAT_KERNEL(double) *const k_rb = - (has_right && has_bottom) ? - (matrix + ((row_index + 1) * col_length + (col_index + 1))) : - (const MAT_KERNEL(double) *)&SN_ZERO_KERNEL; - - for (int pc = 0; pc < Q; ++pc) { - const int is_right_half = (pc >= 4); - const int kc_col = pc & 3; /* column index inside 4x4 kernel */ - - const MAT_KERNEL(double) *src_top = is_right_half ? k_rt : k_lt; - const MAT_KERNEL(double) *src_bot = is_right_half ? k_rb : k_lb; - - const __m256d top256 = src_top->f64_y[kc_col]; - const __m256d bot256 = src_bot->f64_y[kc_col]; - - __m512d col = _mm512_castpd256_pd512(top256); - col = _mm512_insertf64x4(col, bot256, 1); - - PANEL_COL_VEC((*panel), pc, 0) = _mm512_castpd_ps(col); - } -} - -DECLARE_STORE_PANEL(avx, float) -{ - const int Q = 8; /* fixed for AVX float panel */ - - /* Local dummy kernel to swallow out-of-bounds writes; no init needed */ - MAT_KERNEL(float) zero; - - const int has_right = (col_index + 1) < col_length; - const int has_bottom = (row_index + 1) < row_length; - - MAT_KERNEL(float) *const k_lt = (col_index < col_length && row_index < row_length) ? - (matrix + (row_index * col_length + col_index)) : - &zero; - MAT_KERNEL(float) *const k_rt = - has_right && (row_index < row_length) ? - (matrix + (row_index * col_length + (col_index + 1))) : - &zero; - MAT_KERNEL(float) *const k_lb = - has_bottom && (col_index < col_length) ? - (matrix + ((row_index + 1) * col_length + col_index)) : - &zero; - MAT_KERNEL(float) *const k_rb = - (has_right && has_bottom) ? - (matrix + ((row_index + 1) * col_length + (col_index + 1))) : - &zero; - - register __m128 top128, bot128; - int i = 0; - for (; i < Q / 2; i++) { - const int kc_col = i; /* column index inside 4x4 kernel */ - - /* Load the panel column vector */ - const __m256 col256 = PANEL_COL_VEC((*panel), i, 0); - /* Split into top/bottom halves */ - top128 = _mm256_castps256_ps128(col256); /* low 128 */ - bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */ - - /* Store back into kernels directly (out-of-bounds writes go to dummy) */ - k_rt->f32_x[kc_col] = top128; - k_rb->f32_x[kc_col] = bot128; - } - for (; i < Q; i++) { - const int kc_col = i - 4; /* column index inside 4x4 kernel */ - - /* Load the panel column vector */ - const __m256 col256 = PANEL_COL_VEC((*panel), i, 0); - /* Split into top/bottom halves */ - top128 = _mm256_castps256_ps128(col256); /* low 128 */ - bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */ - - /* Store back into kernels directly (out-of-bounds writes go to dummy) */ - k_lt->f32_x[kc_col] = top128; - k_lb->f32_x[kc_col] = bot128; - } -} - -DECLARE_STORE_PANEL(512, float) -{ - MAT_KERNEL(float) zero; - MAT_KERNEL(float) * k[4][4]; - - for (int kc = 0; kc < 4; ++kc) { - for (int kr = 0; kr < 4; ++kr) { - const int col = col_index + kc; - const int row = row_index + kr; - k[kc][kr] = (col < col_length && row < row_length) ? - (matrix + (row * col_length + col)) : - &zero; - } - } - - for (int pc = 0; pc < 16; ++pc) { - const int blk_col = pc >> 2; /* which kernel column block (0..3) */ - const int kc_col = pc & 3; /* column inside a 4x4 kernel */ - - const __m512 col = PANEL_COL_VEC((*panel), pc, 0); - - const __m256 top = _mm512_castps512_ps256(col); - const __m256 bot = _mm512_extractf32x8_ps(col, 1); - - const __m128 r0 = _mm256_castps256_ps128(top); - const __m128 r1 = _mm256_extractf128_ps(top, 1); - const __m128 r2 = _mm256_castps256_ps128(bot); - const __m128 r3 = _mm256_extractf128_ps(bot, 1); - - k[blk_col][0]->f32_x[kc_col] = r0; - k[blk_col][1]->f32_x[kc_col] = r1; - k[blk_col][2]->f32_x[kc_col] = r2; - k[blk_col][3]->f32_x[kc_col] = r3; - } -} - -DECLARE_STORE_PANEL(sse, double) -{ - MAT_KERNEL(double) *dst = matrix + (row_index * col_length + col_index); - - for (int pc = 0; pc < 4; ++pc) { - const int base = pc * 2; - dst->f64_x[base] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 0)); - dst->f64_x[base + 1] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 1)); - } -} - -DECLARE_STORE_PANEL(avx, double) -{ - const int Q = 8; /* fixed for AVX double panel */ - - MAT_KERNEL(double) zero; - - const int has_right = (col_index + 1) < col_length; - const int has_bottom = (row_index + 1) < row_length; - - MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ? - (matrix + (row_index * col_length + col_index)) : - &zero; - MAT_KERNEL(double) *const k_rt = - has_right && (row_index < row_length) ? - (matrix + (row_index * col_length + (col_index + 1))) : - &zero; - MAT_KERNEL(double) *const k_lb = - has_bottom && (col_index < col_length) ? - (matrix + ((row_index + 1) * col_length + col_index)) : - &zero; - MAT_KERNEL(double) *const k_rb = - (has_right && has_bottom) ? - (matrix + ((row_index + 1) * col_length + (col_index + 1))) : - &zero; - - for (int i = 0; i < Q; ++i) { - const int kc_col = (i < 4) ? i : (i - 4); - const __m256 top_chunk = PANEL_COL_VEC((*panel), i, 0); - const __m256 bot_chunk = PANEL_COL_VEC((*panel), i, 1); - - MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt; - MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb; - - top_dst->f64_y[kc_col] = _mm256_castps_pd(top_chunk); - bot_dst->f64_y[kc_col] = _mm256_castps_pd(bot_chunk); - } -} - -DECLARE_STORE_PANEL(512, double) -{ - const int Q = 8; /* max(4, lanes) -> 8 columns, one ZMM per column */ - - MAT_KERNEL(double) zero; - - const int has_right = (col_index + 1) < col_length; - const int has_bottom = (row_index + 1) < row_length; - - MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ? - (matrix + (row_index * col_length + col_index)) : - &zero; - MAT_KERNEL(double) *const k_rt = - has_right && (row_index < row_length) ? - (matrix + (row_index * col_length + (col_index + 1))) : - &zero; - MAT_KERNEL(double) *const k_lb = - has_bottom && (col_index < col_length) ? - (matrix + ((row_index + 1) * col_length + col_index)) : - &zero; - MAT_KERNEL(double) *const k_rb = - (has_right && has_bottom) ? - (matrix + ((row_index + 1) * col_length + (col_index + 1))) : - &zero; - - for (int i = 0; i < Q; ++i) { - const int kc_col = (i < 4) ? i : (i - 4); - const __m512 col = PANEL_COL_VEC((*panel), i, 0); - const __m512d col_pd = _mm512_castps_pd(col); - - const __m256d top256 = _mm512_castpd512_pd256(col_pd); - const __m256d bot256 = _mm512_extractf64x4_pd(col_pd, 1); - - MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt; - MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb; - - top_dst->f64_y[kc_col] = top256; - bot_dst->f64_y[kc_col] = bot256; - } -} - -#endif +#include "sn_target.h" + +#if SN_IS_ARCH(SN_ARCH_X86_64) + +#include "sn_matkernel.h" + +DECLARE_BUILD_PANEL(avx, float) +{ + const int Q = 8; /* fixed for AVX float panel */ + + /* Neighbor kernels; substitute global zero kernel where out-of-bounds */ + const int has_right = (col_index + 1) < col_length; + const int has_bottom = (row_index + 1) < row_length; + + const MAT_KERNEL(float) *const k_lt = matrix + (row_index * col_length + col_index); + const MAT_KERNEL(float) *const k_rt = + has_right ? (matrix + (row_index * col_length + (col_index + 1))) : + (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; + const MAT_KERNEL(float) *const k_lb = + has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) : + (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; + const MAT_KERNEL(float) *const k_rb = + (has_right && has_bottom) ? + (matrix + ((row_index + 1) * col_length + (col_index + 1))) : + (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; + + register __m128 top128, bot128; + register __m256 col256; + + for (int pc = 0; pc < Q; ++pc) { + const int is_right_half = (pc >= 4); + const int kc_col = pc & 3; /* column index inside 4x4 kernel */ + + const MAT_KERNEL(float) *src_top = is_right_half ? k_rt : k_lt; + const MAT_KERNEL(float) *src_bot = is_right_half ? k_rb : k_lb; + + /* Vector loads: each kernel column is a compact __m128 */ + top128 = src_top->f32_x[kc_col]; + bot128 = src_bot->f32_x[kc_col]; + + /* Combine into an AVX column: low 128 = top, high 128 = bottom */ + col256 = _mm256_set_m128(bot128, top128); + + PANEL_COL_VEC((*panel), pc, 0) = col256; + } +} + +DECLARE_BUILD_PANEL(512, float) +{ + const MAT_KERNEL(float) *const zero = (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; + const MAT_KERNEL(float) * k[4][4]; + + for (int kc = 0; kc < 4; ++kc) { + for (int kr = 0; kr < 4; ++kr) { + const int col = col_index + kc; + const int row = row_index + kr; + k[kc][kr] = (col < col_length && row < row_length) ? + (matrix + (row * col_length + col)) : + zero; + } + } + + /* Each panel column corresponds to a kernel column (0..3) within a block col (0..3). */ + for (int pc = 0; pc < 16; ++pc) { + const int blk_col = pc >> 2; /* which kernel column block (0..3) */ + const int kc_col = pc & 3; /* column inside a 4x4 kernel */ + + const __m128 r0 = k[blk_col][0]->f32_x[kc_col]; + const __m128 r1 = k[blk_col][1]->f32_x[kc_col]; + const __m128 r2 = k[blk_col][2]->f32_x[kc_col]; + const __m128 r3 = k[blk_col][3]->f32_x[kc_col]; + + const __m256 top = _mm256_set_m128(r1, r0); + const __m256 bot = _mm256_set_m128(r3, r2); + + __m512 col = _mm512_castps256_ps512(top); + col = _mm512_insertf32x8(col, bot, 1); + + PANEL_COL_VEC((*panel), pc, 0) = col; + } +} + +DECLARE_BUILD_PANEL(512, double) +{ + const int Q = 8; /* max(4, lanes) for double with ZMM -> 8 columns */ + + const int has_right = (col_index + 1) < col_length; + const int has_bottom = (row_index + 1) < row_length; + + const MAT_KERNEL(double) *const k_lt = matrix + (row_index * col_length + col_index); + const MAT_KERNEL(double) *const k_rt = + has_right ? (matrix + (row_index * col_length + (col_index + 1))) : + (const MAT_KERNEL(double) *)&SN_ZERO_KERNEL; + const MAT_KERNEL(double) *const k_lb = + has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) : + (const MAT_KERNEL(double) *)&SN_ZERO_KERNEL; + const MAT_KERNEL(double) *const k_rb = + (has_right && has_bottom) ? + (matrix + ((row_index + 1) * col_length + (col_index + 1))) : + (const MAT_KERNEL(double) *)&SN_ZERO_KERNEL; + + for (int pc = 0; pc < Q; ++pc) { + const int is_right_half = (pc >= 4); + const int kc_col = pc & 3; /* column index inside 4x4 kernel */ + + const MAT_KERNEL(double) *src_top = is_right_half ? k_rt : k_lt; + const MAT_KERNEL(double) *src_bot = is_right_half ? k_rb : k_lb; + + const __m256d top256 = src_top->f64_y[kc_col]; + const __m256d bot256 = src_bot->f64_y[kc_col]; + + __m512d col = _mm512_castpd256_pd512(top256); + col = _mm512_insertf64x4(col, bot256, 1); + + PANEL_COL_VEC((*panel), pc, 0) = _mm512_castpd_ps(col); + } +} + +DECLARE_STORE_PANEL(avx, float) +{ + const int Q = 8; /* fixed for AVX float panel */ + + /* Local dummy kernel to swallow out-of-bounds writes; no init needed */ + MAT_KERNEL(float) zero; + + const int has_right = (col_index + 1) < col_length; + const int has_bottom = (row_index + 1) < row_length; + + MAT_KERNEL(float) *const k_lt = (col_index < col_length && row_index < row_length) ? + (matrix + (row_index * col_length + col_index)) : + &zero; + MAT_KERNEL(float) *const k_rt = + has_right && (row_index < row_length) ? + (matrix + (row_index * col_length + (col_index + 1))) : + &zero; + MAT_KERNEL(float) *const k_lb = + has_bottom && (col_index < col_length) ? + (matrix + ((row_index + 1) * col_length + col_index)) : + &zero; + MAT_KERNEL(float) *const k_rb = + (has_right && has_bottom) ? + (matrix + ((row_index + 1) * col_length + (col_index + 1))) : + &zero; + + register __m128 top128, bot128; + int i = 0; + for (; i < Q / 2; i++) { + const int kc_col = i; /* column index inside 4x4 kernel */ + + /* Load the panel column vector */ + const __m256 col256 = PANEL_COL_VEC((*panel), i, 0); + /* Split into top/bottom halves */ + top128 = _mm256_castps256_ps128(col256); /* low 128 */ + bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */ + + /* Store back into kernels directly (out-of-bounds writes go to dummy) */ + k_rt->f32_x[kc_col] = top128; + k_rb->f32_x[kc_col] = bot128; + } + for (; i < Q; i++) { + const int kc_col = i - 4; /* column index inside 4x4 kernel */ + + /* Load the panel column vector */ + const __m256 col256 = PANEL_COL_VEC((*panel), i, 0); + /* Split into top/bottom halves */ + top128 = _mm256_castps256_ps128(col256); /* low 128 */ + bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */ + + /* Store back into kernels directly (out-of-bounds writes go to dummy) */ + k_lt->f32_x[kc_col] = top128; + k_lb->f32_x[kc_col] = bot128; + } +} + +DECLARE_STORE_PANEL(512, float) +{ + MAT_KERNEL(float) zero; + MAT_KERNEL(float) * k[4][4]; + + for (int kc = 0; kc < 4; ++kc) { + for (int kr = 0; kr < 4; ++kr) { + const int col = col_index + kc; + const int row = row_index + kr; + k[kc][kr] = (col < col_length && row < row_length) ? + (matrix + (row * col_length + col)) : + &zero; + } + } + + for (int pc = 0; pc < 16; ++pc) { + const int blk_col = pc >> 2; /* which kernel column block (0..3) */ + const int kc_col = pc & 3; /* column inside a 4x4 kernel */ + + const __m512 col = PANEL_COL_VEC((*panel), pc, 0); + + const __m256 top = _mm512_castps512_ps256(col); + const __m256 bot = _mm512_extractf32x8_ps(col, 1); + + const __m128 r0 = _mm256_castps256_ps128(top); + const __m128 r1 = _mm256_extractf128_ps(top, 1); + const __m128 r2 = _mm256_castps256_ps128(bot); + const __m128 r3 = _mm256_extractf128_ps(bot, 1); + + k[blk_col][0]->f32_x[kc_col] = r0; + k[blk_col][1]->f32_x[kc_col] = r1; + k[blk_col][2]->f32_x[kc_col] = r2; + k[blk_col][3]->f32_x[kc_col] = r3; + } +} + +DECLARE_STORE_PANEL(sse, double) +{ + MAT_KERNEL(double) *dst = matrix + (row_index * col_length + col_index); + + for (int pc = 0; pc < 4; ++pc) { + const int base = pc * 2; + dst->f64_x[base] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 0)); + dst->f64_x[base + 1] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 1)); + } +} + +DECLARE_STORE_PANEL(avx, double) +{ + const int Q = 8; /* fixed for AVX double panel */ + + MAT_KERNEL(double) zero; + + const int has_right = (col_index + 1) < col_length; + const int has_bottom = (row_index + 1) < row_length; + + MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ? + (matrix + (row_index * col_length + col_index)) : + &zero; + MAT_KERNEL(double) *const k_rt = + has_right && (row_index < row_length) ? + (matrix + (row_index * col_length + (col_index + 1))) : + &zero; + MAT_KERNEL(double) *const k_lb = + has_bottom && (col_index < col_length) ? + (matrix + ((row_index + 1) * col_length + col_index)) : + &zero; + MAT_KERNEL(double) *const k_rb = + (has_right && has_bottom) ? + (matrix + ((row_index + 1) * col_length + (col_index + 1))) : + &zero; + + for (int i = 0; i < Q; ++i) { + const int kc_col = (i < 4) ? i : (i - 4); + const __m256 top_chunk = PANEL_COL_VEC((*panel), i, 0); + const __m256 bot_chunk = PANEL_COL_VEC((*panel), i, 1); + + MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt; + MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb; + + top_dst->f64_y[kc_col] = _mm256_castps_pd(top_chunk); + bot_dst->f64_y[kc_col] = _mm256_castps_pd(bot_chunk); + } +} + +DECLARE_STORE_PANEL(512, double) +{ + const int Q = 8; /* max(4, lanes) -> 8 columns, one ZMM per column */ + + MAT_KERNEL(double) zero; + + const int has_right = (col_index + 1) < col_length; + const int has_bottom = (row_index + 1) < row_length; + + MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ? + (matrix + (row_index * col_length + col_index)) : + &zero; + MAT_KERNEL(double) *const k_rt = + has_right && (row_index < row_length) ? + (matrix + (row_index * col_length + (col_index + 1))) : + &zero; + MAT_KERNEL(double) *const k_lb = + has_bottom && (col_index < col_length) ? + (matrix + ((row_index + 1) * col_length + col_index)) : + &zero; + MAT_KERNEL(double) *const k_rb = + (has_right && has_bottom) ? + (matrix + ((row_index + 1) * col_length + (col_index + 1))) : + &zero; + + for (int i = 0; i < Q; ++i) { + const int kc_col = (i < 4) ? i : (i - 4); + const __m512 col = PANEL_COL_VEC((*panel), i, 0); + const __m512d col_pd = _mm512_castps_pd(col); + + const __m256d top256 = _mm512_castpd512_pd256(col_pd); + const __m256d bot256 = _mm512_extractf64x4_pd(col_pd, 1); + + MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt; + MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb; + + top_dst->f64_y[kc_col] = top256; + bot_dst->f64_y[kc_col] = bot256; + } +} + +#endif diff --git a/StgSharp.Native/sn-src/simdid.x86.c b/StgSharp.Native/sn-src/simdid.x86.c index 7b9c39fd..b852d787 100644 --- a/StgSharp.Native/sn-src/simdid.x86.c +++ b/StgSharp.Native/sn-src/simdid.x86.c @@ -57,24 +57,15 @@ static int xgetbv_has_avx512_state(void) return ((xcr0 & mask) == mask); } -static int is_intel_dual_avx512(int family, int model) +/* AVX-512 execution-width classification for AMD, by CPU family. + * Zen4 (0x19): single 256-bit ALU, 512-bit op = two 256-bit micro-ops (dual-pump) → DUAL. + * Zen5+ (>=0x1A): native 512-bit execution units → NATIVE. + * Other/future AMD families → UNKNOWN (conservative). */ +static uint64_t amd_avx512_impl(int family) { - return (family == 6) && (model == 0x55); /* SKX/CLX/CPX */ -} - -static int is_intel_native_avx512(int family, int model) -{ - if (family != 6) - return 0; - switch (model) { - case 0x6A: /* ICX */ - case 0x6C: /* ICX stepping */ - case 0x8F: /* SPR/EMR */ - case 0xCF: /* GNR */ - return 1; - default: - return 0; - } + if (family == 0x19) return SIMDID_AVX512_IMPL_DUAL; + if (family >= 0x1A) return SIMDID_AVX512_IMPL_NATIVE; + return SIMDID_AVX512_IMPL_UNKNOWN; } /* Returns 1 if this AMD/Hygon CPU uses split-lane YMM (two fused 128-bit execution units). @@ -110,9 +101,9 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local() ((int *)vendor)[2] = ecx; vendor[12] = '\0'; const int is_intel = (strcmp(vendor, "GenuineIntel") == 0); - const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0) || - (strcmp(vendor, "HygonGenuine") == 0); - const int is_via = (strcmp(vendor, "CentaurHauls") == 0) || + const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0); + const int is_hygon = (strcmp(vendor, "HygonGenuine") == 0); /* 海光: separate from AMD */ + const int is_via = (strcmp(vendor, "CentaurHauls") == 0) || (memcmp(vendor, " Shanghai ", 12) == 0); /* VIA / Zhaoxin */ /* CPUID.1 */ @@ -128,9 +119,10 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local() int has_f16c = (ecx & (1 << 29)) != 0; uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64); - uint64_t manu = SIMDID_PACK_MANU(is_intel ? SIMDID_MANU_INTEL : - is_amd ? SIMDID_MANU_AMD : - is_via ? SIMDID_MANU_VIA : + uint64_t manu = SIMDID_PACK_MANU(is_intel ? SIMDID_MANU_INTEL : + is_amd ? SIMDID_MANU_AMD : + is_hygon ? SIMDID_MANU_HYGON : + is_via ? SIMDID_MANU_VIA : SIMDID_MANU_UNKNOWN); uint64_t main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE); uint64_t avx512_bits = 0; @@ -202,16 +194,17 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local() if (has_vbmi2) avx512_bits |= SIMDID_AVX512_VBMI2; - /* impl classification */ - const int glue = is_amd || is_intel_dual_avx512(family, model); - const int native512 = is_intel_native_avx512(family, model); - if (glue) { - avx512_bits |= SIMDID_AVX512_IMPL_DUAL; - } else if (native512) { + /* Execution-width classification: + * Intel all → NATIVE (genuine 512-bit ALU; SKX dual-port is native) + * AMD Zen4 → DUAL (dual-pump 256-bit, Family 0x19) + * AMD Zen5+ → NATIVE (native 512-bit, Family >= 0x1A) + * Hygon/VIA/unknown → UNKNOWN (conservative) */ + if (is_intel) avx512_bits |= SIMDID_AVX512_IMPL_NATIVE; - } else { + else if (is_amd) + avx512_bits |= amd_avx512_impl(family); + else avx512_bits |= SIMDID_AVX512_IMPL_UNKNOWN; - } /* main level for AVX512 */ main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512);