youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

many things

Browse Source
This commit is contained in:
Youmu Liu
2026-02-26 18:05:35 +08:00
parent 1b04196314
commit 15434ed053
15 changed files with 530 additions and 591 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/Mathematics/Linear/ElementType.cs
+45 -8
View File
@@ -1,9 +1,9 @@
//----------------------------------------------------------------------- //-----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="ElementType" // file="ElementType"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload Space // AuthorGroup: Nitload
// Copyright (c) Nitload Space. All rights reserved. // Copyright (c) Nitload. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -25,15 +25,52 @@
// //
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
#pragma warning disable CA1028
namespace StgSharp.Mathematics.Numeric namespace StgSharp.Mathematics.Numeric
{ {
public enum MatrixElementType : int [StructLayout(LayoutKind.Explicit, Size = 4)]
public struct MatrixElementType
{ {
F32 = 0, public static readonly MatrixElementType F32 = new(0x03000004);
F64 = 1, public static readonly MatrixElementType F64 = new(0x03000008);
I32 = 2, public static readonly MatrixElementType I08 = new(0x0100001);
I64 = 3, public static readonly MatrixElementType I16 = new(0x0100002);
public static readonly MatrixElementType I32 = new(0x0100004);
public static readonly MatrixElementType I64 = new(0x0100008);
public static readonly MatrixElementType U08 = new(0x0200001);
public static readonly MatrixElementType U16 = new(0x0200002);
public static readonly MatrixElementType U32 = new(0x0200004);
public static readonly MatrixElementType U64 = new(0x0200008);
[FieldOffset(0)] internal readonly int Mask;
[FieldOffset(3)] public readonly MatrixElementTypeGroup Group;
[FieldOffset(0)] public readonly ushort Size;
private MatrixElementType(
int mask
)
{
Unsafe.SkipInit(out this);
Unsafe.SkipInit(out Size);
Unsafe.SkipInit(out Group);
this.Mask = mask;
}
}
public enum MatrixElementTypeGroup : byte
{
None = 0,
Singed = 1,
Unsigned = 2,
Float = 3,
} }
} }
#pragma warning restore CA1028
StgSharp.Mathematics/Numeric/MatrixCompute.cs
-62
View File
@@ -40,68 +40,6 @@ namespace StgSharp.Mathematics.Numeric
public partial class MatrixCompute public partial class MatrixCompute
{ {
internal static unsafe void ComputeFmaPanelF32(
MatrixParallelTaskPackage* p
)
{
int f32 = (int)MatrixElementType.F32;
MatrixKernel* leftBuffer = (MatrixKernel*)p->Left;
MatrixKernel* rightBuffer = (MatrixKernel*)p->Right;
MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer;
ref long offsetRef = ref Unsafe.As<int, long>(ref p->PrimTileOffset);
ref long lengthRef = ref Unsafe.As<int, long>(ref p->PrimCount);
int ansWidth = p->AnsPrimOffset;
int ansHeight = p->AnsSecOffset;
int commonK = p->LeftPrimOffset;
M128* enumeration = stackalloc M128[1];
enumeration->Member<int>(2) = commonK;
enumeration->Member<int>(3) = ansHeight;
// in this occasion size must be 1
for (long c = offsetRef; c < offsetRef + lengthRef; c++)
{
int i = (int)(c / ansHeight);
int j = (int)(c % ansHeight);
enumeration->Member<int>(0) = i;
enumeration->Member<int>(1) = j;
NativeIntrinsic.Context.mat[f32].kernel_tile_fma(leftBuffer, rightBuffer, ansBuffer,
enumeration);
}
}
internal static unsafe void ComputeFmaPanelF64(
MatrixParallelTaskPackage* p
)
{
int f64 = (int)MatrixElementType.F64;
// int size = 1;
MatrixKernel* leftBuffer = (MatrixKernel*)p->Left;
MatrixKernel* rightBuffer = (MatrixKernel*)p->Right;
MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer;
ref long offsetRef = ref Unsafe.As<int, long>(ref p->PrimTileOffset);
ref long lengthRef = ref Unsafe.As<int, long>(ref p->PrimCount);
int ansWidth = p->AnsPrimOffset;
int ansHeight = p->AnsSecOffset;
int commonK = p->LeftPrimOffset;
M128* enumeration = stackalloc M128[1];
enumeration->Member<int>(2) = commonK;
enumeration->Member<int>(3) = ansHeight;
// in this occasion size must be 1
for (long c = offsetRef; c < offsetRef + lengthRef; c++)
{
int i = (int)(c / ansHeight);
int j = (int)(c % ansHeight);
enumeration->Member<int>(0) = i;
enumeration->Member<int>(1) = j;
NativeIntrinsic.Context.mat[f64].kernel_tile_fma(leftBuffer, rightBuffer, ansBuffer,
enumeration);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static unsafe MatrixKernel* GetKernelAddressUnsafe<T>( internal static unsafe MatrixKernel* GetKernelAddressUnsafe<T>(
MatrixKernel<T>* buffer, MatrixKernel<T>* buffer,
StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs
+3 -5
View File
@@ -71,11 +71,9 @@ namespace StgSharp.Mathematics.Numeric
long recommendCount = count / 1024; long recommendCount = count / 1024;
MatrixParallelThread[] pool = LeadParallel((int)recommendCount); MatrixParallelThread[] pool = LeadParallel((int)recommendCount);
MatrixParallelWrap handle = new MatrixParallelWrap(pool); MatrixParallelWrap wrap = new MatrixParallelWrap(pool);
wrap.TaskQueue = MatrixParallelQueue.Rent(package);
// TODO set parallel queue for handle return wrap;
handle.TaskQueue = MatrixParallelQueue.Rent(package);
return handle;
} }
} }
StgSharp.Mathematics/Numeric/MatrixParallel.cs
+2 -1
View File
@@ -98,6 +98,7 @@ namespace StgSharp.Mathematics.Numeric
} }
// TODO 构建全局的保底上下文 // TODO 构建全局的保底上下文
foreach (MatrixParallelThread thread in _threads) { }
if (contextMap.Keys.Count == 1) if (contextMap.Keys.Count == 1)
{ {
for (int i = 0; i < 4; i++) for (int i = 0; i < 4; i++)
@@ -113,7 +114,7 @@ namespace StgSharp.Mathematics.Numeric
IsInitialized = true; IsInitialized = true;
} }
[LibraryImport(Native.LibName, EntryPoint = "load_intrinsic_function")] [LibraryImport(Native.LibName, EntryPoint = "sn_compare_simd")]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
private static unsafe partial ulong CompareSIMDID( private static unsafe partial ulong CompareSIMDID(
IntrinsicContext* context, IntrinsicContext* context,
StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs
+4 -40
View File
@@ -52,48 +52,12 @@ namespace StgSharp.Mathematics.Numeric
* matrix_op_mode; * matrix_op_mode;
*/ */
public enum MatrixOperationSpecial : short public enum MatrixIndexStyle : int
{ {
SEQ_FMA = -1, NONE = 0,
}
public enum MatrixIndexStyle : short
{
SPECIAL = 0,
BUFFER_INDEX = 1, BUFFER_INDEX = 1,
SEQ_FMA = -1,
}
public enum MatrixOperationParam : short
{
RIGHT_ANS_PARAM = 1,
LEFT_RIGHT_ANS_PARAM = 2,
RIGHT_ANS_SCALAR_PARAM = 3,
LEFT_RIGHT_ANS_SCALAR_PARAM = 4,
ANS_SCALAR_PARAM = 5,
}
[StructLayout(LayoutKind.Sequential)]
public struct MatrixOpMode(
MatrixIndexStyle style,
MatrixOperationParam paramLayout
)
{
public MatrixIndexStyle OperationStyle = style;
public MatrixOperationParam ParameterStyle = paramLayout;
public MatrixOpMode(
MatrixIndexStyle style,
MatrixOperationSpecial paramLayout
)
: this(style, (MatrixOperationParam)paramLayout) { }
} }
@@ -166,7 +130,7 @@ namespace StgSharp.Mathematics.Numeric
[FieldOffset(96)]public int PrimCount; // 4 [FieldOffset(96)]public int PrimCount; // 4
[FieldOffset(100)]public int SecCount; // 4 [FieldOffset(100)]public int SecCount; // 4
[FieldOffset(104)]public MatrixOpMode ComputeMode; // 4 [FieldOffset(104)]public MatrixIndexStyle ComputeMode; // 4
[FieldOffset(108)]private int ReservedInt0; // 4 [FieldOffset(108)]private int ReservedInt0; // 4
#endregion #endregion
StgSharp.Mathematics/Numeric/MatrixParallelThread.Fma.cs
+77
View File
@@ -0,0 +1,77 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using StgSharp.HighPerformance;
using StgSharp.Internal;
namespace StgSharp.Mathematics.Numeric
{
public partial class MatrixParallelThread
{
internal static unsafe void ComputeFmaPanelF32(
MatrixParallelTaskPackage* p
)
{
int f32 = (int)MatrixElementType.F32;
MatrixKernel* leftBuffer = (MatrixKernel*)p->Left;
MatrixKernel* rightBuffer = (MatrixKernel*)p->Right;
MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer;
ref long offsetRef = ref Unsafe.As<int, long>(ref p->PrimTileOffset);
ref long lengthRef = ref Unsafe.As<int, long>(ref p->PrimCount);
int ansWidth = p->AnsPrimOffset;
int ansHeight = p->AnsSecOffset;
int commonK = p->LeftPrimOffset;
M128* enumeration = stackalloc M128[1];
enumeration->Member<int>(2) = commonK;
enumeration->Member<int>(3) = ansHeight;
// in this occasion size must be 1
for (long c = offsetRef; c < offsetRef + lengthRef; c++)
{
int i = (int)(c / ansHeight);
int j = (int)(c % ansHeight);
enumeration->Member<int>(0) = i;
enumeration->Member<int>(1) = j;
NativeIntrinsic.Context.mat[f32].kernel_tile_fma(leftBuffer, rightBuffer, ansBuffer,
enumeration);
}
}
internal static unsafe void ComputeFmaPanelF64(
MatrixParallelTaskPackage* p
)
{
int f64 = (int)MatrixElementType.F64;
// int size = 1;
MatrixKernel* leftBuffer = (MatrixKernel*)p->Left;
MatrixKernel* rightBuffer = (MatrixKernel*)p->Right;
MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer;
ref long offsetRef = ref Unsafe.As<int, long>(ref p->PrimTileOffset);
ref long lengthRef = ref Unsafe.As<int, long>(ref p->PrimCount);
int ansWidth = p->AnsPrimOffset;
int ansHeight = p->AnsSecOffset;
int commonK = p->LeftPrimOffset;
M128* enumeration = stackalloc M128[1];
enumeration->Member<int>(2) = commonK;
enumeration->Member<int>(3) = ansHeight;
// in this occasion size must be 1
for (long c = offsetRef; c < offsetRef + lengthRef; c++)
{
int i = (int)(c / ansHeight);
int j = (int)(c % ansHeight);
enumeration->Member<int>(0) = i;
enumeration->Member<int>(1) = j;
NativeIntrinsic.Context.mat[f64].kernel_tile_fma(leftBuffer, rightBuffer, ansBuffer,
enumeration);
}
}
}
}
StgSharp.Mathematics/Numeric/MatrixParallelThread.Model.cs
+19 -78
View File
@@ -34,100 +34,41 @@ namespace StgSharp.Mathematics.Numeric
public partial class MatrixParallelThread public partial class MatrixParallelThread
{ {
internal static unsafe void BufferComputeBinary( private const int KernelStealCount = 64;
MatrixParallelTaskPackage* p
)
{
int size = p->ElementSize * 16;
long count = Unsafe.As<int, long>(ref p->PrimCount);
long offset = Unsafe.As<int, long>(ref p->PrimTileOffset) * size;
((delegate* unmanaged[Cdecl]<void*, void*, void*, long, void>)p->ComputeHandle)((void*)(p->Left + offset), internal unsafe void BufferCompute(
(void*)(p->Right + offset), MatrixParallelTaskPackage* p
(void*)(p->Answer + offset),
count);
MatrixParallelFactory.Release(p);
}
internal static unsafe void BufferComputeBinaryScalar(
MatrixParallelTaskPackage* p
) )
{ {
// while () { }
int size = p->ElementSize * 16; int size = p->ElementSize * 16;
long count = Unsafe.As<int, long>(ref p->PrimCount); long count = Unsafe.As<int, long>(ref p->PrimCount);
long offset = Unsafe.As<int, long>(ref p->PrimTileOffset) * size; long offset = Unsafe.As<int, long>(ref p->PrimTileOffset) * size;
ScalarPacket* scalar = p->Scalar; ScalarPacket* scalar = p->Scalar;
((delegate* unmanaged[Cdecl]<void*, void*, void*, ScalarPacket*, long, void>)p->ComputeHandle)((void*)(p->Left + offset), IntPtr computeHandle = _context[p->ComputeHandle];
(void*)(p->Right + offset), ((delegate* unmanaged[Cdecl]<void*, void*, void*, void*, long, void>)computeHandle)((void*)(p->Left + offset),
(void*)(p->Answer + offset), (void*)(p->Right + offset),
scalar, (void*)(p->Answer + offset),
count); scalar,
MatrixParallelFactory.Release(p); count);
} }
internal static unsafe void BufferComputeNoOperatorScalar( [MethodImpl(MethodImplOptions.AggressiveInlining)]
MatrixParallelTaskPackage* p internal unsafe void SpecialBufferCompute(
MatrixParallelTaskPackage* p
) )
{ {
int size = p->ElementSize * 16; switch ((MatrixElementType)p->ElementSize)
long count = Unsafe.As<int, long>(ref p->PrimCount);
long offset = Unsafe.As<int, long>(ref p->PrimTileOffset) * size;
((delegate* unmanaged[Cdecl]<void*, ScalarPacket*, long, void>)p->ComputeHandle)((void*)(p->Answer + offset),
p->Scalar, count);
MatrixParallelFactory.Release(p);
}
internal static unsafe void BufferComputeUnary(
MatrixParallelTaskPackage* p
)
{
int size = p->ElementSize * 16;
long count = Unsafe.As<int, long>(ref p->PrimCount);
long offset = Unsafe.As<int, long>(ref p->PrimTileOffset) * size;
((delegate* unmanaged[Cdecl]<void*, void*, long, void>)p->ComputeHandle)((void*)(p->Right + offset),
(void*)(p->Answer + offset),
count);
MatrixParallelFactory.Release(p);
}
internal static unsafe void BufferComputeUnaryScalar(
MatrixParallelTaskPackage* p
)
{
int size = p->ElementSize * 16;
long count = Unsafe.As<int, long>(ref p->PrimCount);
long offset = Unsafe.As<int, long>(ref p->PrimTileOffset) * size;
((delegate* unmanaged[Cdecl]<void*, void*, ScalarPacket*, long, void>)p->ComputeHandle)((void*)(p->Right + offset),
(void*)(p->Answer + offset),
p->Scalar,
count);
MatrixParallelFactory.Release(p);
}
internal static unsafe void SpecialBufferCompute(
MatrixParallelTaskPackage* p
)
{
switch ((MatrixOperationSpecial)p->ComputeMode.ParameterStyle)
{ {
case MatrixOperationSpecial.SEQ_FMA: case MatrixElementType.F64:
switch ((MatrixElementType)p->ElementSize) ComputeFmaPanelF64(p);
{ return;
case MatrixElementType.F64: case MatrixElementType.F32:
MatrixCompute.ComputeFmaPanelF64(p); ComputeFmaPanelF32(p);
return;
case MatrixElementType.F32:
MatrixCompute.ComputeFmaPanelF32(p);
break;
default:
break;
}
break; break;
default: default:
break; break;
} }
MatrixParallelFactory.Release(p);
} }
} }
StgSharp.Mathematics/Numeric/MatrixParallelThread.cs
+13 -39
View File
@@ -83,44 +83,6 @@ namespace StgSharp.Mathematics.Numeric
GC.SuppressFinalize(this); GC.SuppressFinalize(this);
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static unsafe void ExecuteTaskPackage(
MatrixParallelQueue taskQueue
)
{
MatrixParallelTaskPackage* p = taskQueue.Package;
switch (p->ComputeMode.OperationStyle)
{
case MatrixIndexStyle.BUFFER_INDEX:
switch (p->ComputeMode.ParameterStyle)
{
case MatrixOperationParam.RIGHT_ANS_PARAM:
BufferComputeUnary(p);
break;
case MatrixOperationParam.LEFT_RIGHT_ANS_PARAM:
BufferComputeBinary(p);
break;
case MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM:
BufferComputeUnaryScalar(p);
break;
case MatrixOperationParam.LEFT_RIGHT_ANS_SCALAR_PARAM:
BufferComputeBinaryScalar(p);
break;
case MatrixOperationParam.ANS_SCALAR_PARAM:
BufferComputeNoOperatorScalar(p);
break;
default:
break;
}
break;
case MatrixIndexStyle.SPECIAL:
SpecialBufferCompute(p);
break;
default:
break;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Restart() public void Restart()
{ {
@@ -182,7 +144,19 @@ namespace StgSharp.Mathematics.Numeric
// TODO 将外部线程绑定到当前线程上，并执行任务包 // TODO 将外部线程绑定到当前线程上，并执行任务包
} else } else
{ {
ExecuteTaskPackage(BindedWrap.TaskQueue); MatrixParallelQueue taskQueue = BindedWrap.TaskQueue;
MatrixParallelTaskPackage* p = taskQueue.Package;
if (0 > (int)p ->ComputeHandle) {
SpecialBufferCompute(p);
}
switch (p->ComputeMode)
{
case MatrixIndexStyle.BUFFER_INDEX:
BufferCompute(p);
break;
default:
break;
}
BindedWrap.ReportThreadAccomplished(); BindedWrap.ReportThreadAccomplished();
} }
_managedThread.Priority = AfterWork; _managedThread.Priority = AfterWork;
StgSharp.Native/SIMDID.x86.md
+15 -8
View File
@@ -55,7 +55,8 @@ For **x86-64**:
| 1 | Intel | | 1 | Intel |
| 2 | AMD | | 2 | AMD |
| 3 | VIA/Zhaoxin | | 3 | VIA/Zhaoxin |
| 4..15 | Reserved | | 4 | Hygon (海光) |
| 5..15 | Reserved |
For **AArch64**: For **AArch64**:
@@ -111,15 +112,17 @@ AVX512 Feature (bits 31..16)
Notes: Notes:
- **BASE (bit 20)**: Base AVX512 bundle, indicating `F + CD + VL + DQ + BW` are all present. If set to 1, the detecting side must guarantee these subsets are all available. - **BASE (bit 20)**: Base AVX512 bundle, indicating `F + CD + VL + DQ + BW` are all present. If set to 1, the detecting side must guarantee these subsets are all available.
- **AVX512_IMPL ([23..22], 2-bit)**: Implementation form (width policy). - **AVX512_IMPL ([23..22], 2-bit)**: 512-bit execution width policy — not port count, not frequency behaviour.
| Value | Meaning | | Value | Meaning |
|---:|------| |---:|------|
| 0 | Unknown (strategy decided by Manufacture/uArch) | | 0 | Unknown — cannot determine; use conservative 256-bit kernels |
| 1 | AVX512-DUAL (glue/downclock risk; prefer 256-bit path) | | 1 | DUAL-PUMP — single 256-bit ALU executes 512-bit ops as two consecutive 256-bit micro-ops; ZMM throughput halved (example: AMD Zen4, Family 0x19) |
| 2 | AVX512-NATIVE (full 512-bit ALU; 512-bit path is usable) | | 2 | NATIVE — genuine 512-bit execution unit(s); full ZMM throughput (Intel all models including SKX; AMD Zen5 Family 0x1A and later) |
| 3 | Reserved | | 3 | Reserved |
> **Note**: Intel SKX/CLX/CPX have two 512-bit FMA ports (port-0 + port-5) and are **NATIVE** — genuine 512-bit throughput. Intel downclocking on heavy ZMM workloads is a separate concern not encoded here. DUAL-PUMP applies only where a single 256-bit unit fires twice per 512-bit op (AMD Zen4-style).
- **VBMI/VBMI2/VNNI/BF16/FP16 ([29..25])**: Commonly needed niche or AI-related small-type extensions; set as needed. - **VBMI/VBMI2/VNNI/BF16/FP16 ([29..25])**: Commonly needed niche or AI-related small-type extensions; set as needed.
- **R/AI (bit 24)**: Reserved for future AI or other non-crypto extensions. - **R/AI (bit 24)**: Reserved for future AI or other non-crypto extensions.
- **[31..30], [21], [19..16]**: Reserved for future allocation. - **[31..30], [21], [19..16]**: Reserved for future allocation.
@@ -243,13 +246,17 @@ Detection:
## 12) Minimal Detection-Side Fill Rules (x86-64) ## 12) Minimal Detection-Side Fill Rules (x86-64)
1) Set `RootISA = x86-64`, `Manufacture = Intel/AMD`. 1) Set `RootISA = x86-64`, `Manufacture = Intel / AMD / Hygon / VIA`.
2) Compute `MainLevel` (per ascending rule). 2) Compute `MainLevel` (per ascending rule).
3) If `MainLevel >= AVX512`: 3) If `MainLevel >= AVX512`:
- If `F+CD+VL+DQ+BW` are all present, set `BASE = 1`; otherwise `BASE = 0`. - If `F+CD+VL+DQ+BW` are all present, set `BASE = 1`; otherwise `BASE = 0`.
- Set `AVX512_IMPL`: - Set `AVX512_IMPL`:
- AMD: DUAL (per definition) - **Intel (all)**: NATIVE — all Intel AVX-512 CPUs have genuine 512-bit execution units
- Intel: Decide DUAL/NATIVE based on family/model or policy table; if uncertain, use Unknown - **AMD Zen4 (Family 0x19)**: DUAL — single 256-bit ALU, dual-pump
- **AMD Zen5+ (Family 0x1A+)**: NATIVE — native full-width 512-bit units
- **AMD other/future**: UNKNOWN (conservative)
- **Hygon**: UNKNOWN (AVX-512 characteristics unclear)
- **VIA/other**: UNKNOWN
- When VNNI/BF16/FP16/VBMI/VBMI2 are detected, set the corresponding bits. - When VNNI/BF16/FP16/VBMI/VBMI2 are detected, set the corresponding bits.
4) If `MainLevel >= AMX`: fill the AMX byte. 4) If `MainLevel >= AMX`: fill the AMX byte.
5) When AVX/AVX2 are present, populate the AVX Feature byte: set `AVX` and `AVX2`; set `FMA` and `F16C` when detected; set `DUAL` when the YMM execution path is split/glued (AMD Zen1/Zen+ Family 0x17 model < 0x30, Hygon all models, VIA/Zhaoxin all models). 5) When AVX/AVX2 are present, populate the AVX Feature byte: set `AVX` and `AVX2`; set `FMA` and `F16C` when detected; set `DUAL` when the YMM execution path is split/glued (AMD Zen1/Zen+ Family 0x17 model < 0x30, Hygon all models, VIA/Zhaoxin all models).
StgSharp.Native/sn-head/sn_intrincontext.x86.h
+1
View File
@@ -46,6 +46,7 @@
#define SIMDID_MANU_INTEL 0x1 #define SIMDID_MANU_INTEL 0x1
#define SIMDID_MANU_AMD 0x2 #define SIMDID_MANU_AMD 0x2
#define SIMDID_MANU_VIA 0x3 #define SIMDID_MANU_VIA 0x3
#define SIMDID_MANU_HYGON 0x4 /* 海光: Zen1-derived, separate from AMD */
// MainLevel (ordinal, shared bits 8..15; not bit-flags) // MainLevel (ordinal, shared bits 8..15; not bit-flags)
#define SIMDID_MAIN_LVL_NONE 0x00u #define SIMDID_MAIN_LVL_NONE 0x00u
StgSharp.Native/sn-src/load_intrinsic.x86.c
-2
View File
@@ -14,9 +14,7 @@
#endif // _MSC_VER #endif // _MSC_VER
PRIVATE void load_intrinsic_sse(sn_intrinsic *context, SIMDID id); PRIVATE void load_intrinsic_sse(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_avx_dual(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_avx(sn_intrinsic *context, SIMDID id); PRIVATE void load_intrinsic_avx(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_512_dual(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID id); PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID id);
SN_API void load_intrinsic_function(void *intrinsic_context, uint64_t id) SN_API void load_intrinsic_function(void *intrinsic_context, uint64_t id)
StgSharp.Native/sn-src/m.fma.x86.c
+19 -14
View File
@@ -151,29 +151,34 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma) DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
{ {
static const __m512i b_shift = { .m512i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9, 15, static const __m512i b_shift = { .m512i_i32 = {
12, 13, 14 } }; // shuff order
0, 1, 2, 3, // col0
5, 6, 7, 4, //col1
10, 11, 8, 9, //col2
15, 12, 13, 14 //col3
} };
register __m512 a = _mm512_load_ps((float *)left); register __m512 a = _mm512_load_ps((float *)left);
register __m512 b = _mm512_load_ps((float *)right); register __m512 b = _mm512_load_ps((float *)right);
register __m512 c = _mm512_load_ps((float *)ans); register __m512 c = _mm512_load_ps((float *)ans);
b = _mm512_permutexvar_ps(b_shift, b); b = _mm512_permutexvar_ps(b_shift, b);
register __m512 a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0)); register __m512 a_perm0 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0));
register __m512 b_perm = _mm512_permute_ps(b, _FMA_SELECT(0)); register __m512 b_perm0 = _mm512_permute_ps(b, _FMA_SELECT(0));
c = _mm512_fmadd_ps(a_perm, b_perm, c); c = _mm512_fmadd_ps(a_perm0, b_perm0, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3)); register __m512 a_perm1 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(1)); register __m512 b_perm1 = _mm512_permute_ps(b, _FMA_SELECT(1));
c = _mm512_fmadd_ps(a_perm, b_perm, c); c = _mm512_fmadd_ps(a_perm1, b_perm1, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2)); register __m512 a_perm2 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(2)); register __m512 b_perm2 = _mm512_permute_ps(b, _FMA_SELECT(2));
c = _mm512_fmadd_ps(a_perm, b_perm, c); c = _mm512_fmadd_ps(a_perm2, b_perm2, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1)); register __m512 a_perm3 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(3)); register __m512 b_perm3 = _mm512_permute_ps(b, _FMA_SELECT(3));
c = _mm512_fmadd_ps(a_perm, b_perm, c); c = _mm512_fmadd_ps(a_perm3, b_perm3, c);
_mm512_store_ps((float *)ans, c); _mm512_store_ps((float *)ans, c);
} }
StgSharp.Native/sn-src/m.fma_tile.x86.c
+7 -2
View File
@@ -201,8 +201,13 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma) DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
{ {
static const __m512i b_shift_source = { .m512i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9, static const __m512i b_shift_source = { .m512i_i32 = {
15, 12, 13, 14 } }; // shuff order
0, 1, 2, 3, // col0
5, 6, 7, 4, //col1
10, 11, 8, 9, //col2
15, 12, 13, 14 //col3
} };
int i = enumeration->m128_i32[0]; int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1]; int j = enumeration->m128_i32[1];
int commonK = enumeration->m128_i32[2]; int commonK = enumeration->m128_i32[2];
StgSharp.Native/sn-src/m.panel.x86.c → StgSharp.Native/sn-src/m.panel.x86.old
+301 -301
View File
@@ -1,301 +1,301 @@
#include "sn_target.h" #include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64) #if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_matkernel.h" #include "sn_matkernel.h"
DECLARE_BUILD_PANEL(avx, float) DECLARE_BUILD_PANEL(avx, float)
{ {
const int Q = 8; /* fixed for AVX float panel */ const int Q = 8; /* fixed for AVX float panel */
/* Neighbor kernels; substitute global zero kernel where out-of-bounds */ /* Neighbor kernels; substitute global zero kernel where out-of-bounds */
const int has_right = (col_index + 1) < col_length; const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length; const int has_bottom = (row_index + 1) < row_length;
const MAT_KERNEL(float) *const k_lt = matrix + (row_index * col_length + col_index); const MAT_KERNEL(float) *const k_lt = matrix + (row_index * col_length + col_index);
const MAT_KERNEL(float) *const k_rt = const MAT_KERNEL(float) *const k_rt =
has_right ? (matrix + (row_index * col_length + (col_index + 1))) : has_right ? (matrix + (row_index * col_length + (col_index + 1))) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) *const k_lb = const MAT_KERNEL(float) *const k_lb =
has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) : has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) *const k_rb = const MAT_KERNEL(float) *const k_rb =
(has_right && has_bottom) ? (has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) : (matrix + ((row_index + 1) * col_length + (col_index + 1))) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
register __m128 top128, bot128; register __m128 top128, bot128;
register __m256 col256; register __m256 col256;
for (int pc = 0; pc < Q; ++pc) { for (int pc = 0; pc < Q; ++pc) {
const int is_right_half = (pc >= 4); const int is_right_half = (pc >= 4);
const int kc_col = pc & 3; /* column index inside 4x4 kernel */ const int kc_col = pc & 3; /* column index inside 4x4 kernel */
const MAT_KERNEL(float) *src_top = is_right_half ? k_rt : k_lt; const MAT_KERNEL(float) *src_top = is_right_half ? k_rt : k_lt;
const MAT_KERNEL(float) *src_bot = is_right_half ? k_rb : k_lb; const MAT_KERNEL(float) *src_bot = is_right_half ? k_rb : k_lb;
/* Vector loads: each kernel column is a compact __m128 */ /* Vector loads: each kernel column is a compact __m128 */
top128 = src_top->f32_x[kc_col]; top128 = src_top->f32_x[kc_col];
bot128 = src_bot->f32_x[kc_col]; bot128 = src_bot->f32_x[kc_col];
/* Combine into an AVX column: low 128 = top, high 128 = bottom */ /* Combine into an AVX column: low 128 = top, high 128 = bottom */
col256 = _mm256_set_m128(bot128, top128); col256 = _mm256_set_m128(bot128, top128);
PANEL_COL_VEC((*panel), pc, 0) = col256; PANEL_COL_VEC((*panel), pc, 0) = col256;
} }
} }
DECLARE_BUILD_PANEL(512, float) DECLARE_BUILD_PANEL(512, float)
{ {
const MAT_KERNEL(float) *const zero = (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; const MAT_KERNEL(float) *const zero = (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) * k[4][4]; const MAT_KERNEL(float) * k[4][4];
for (int kc = 0; kc < 4; ++kc) { for (int kc = 0; kc < 4; ++kc) {
for (int kr = 0; kr < 4; ++kr) { for (int kr = 0; kr < 4; ++kr) {
const int col = col_index + kc; const int col = col_index + kc;
const int row = row_index + kr; const int row = row_index + kr;
k[kc][kr] = (col < col_length && row < row_length) ? k[kc][kr] = (col < col_length && row < row_length) ?
(matrix + (row * col_length + col)) : (matrix + (row * col_length + col)) :
zero; zero;
} }
} }
/* Each panel column corresponds to a kernel column (0..3) within a block col (0..3). */ /* Each panel column corresponds to a kernel column (0..3) within a block col (0..3). */
for (int pc = 0; pc < 16; ++pc) { for (int pc = 0; pc < 16; ++pc) {
const int blk_col = pc >> 2; /* which kernel column block (0..3) */ const int blk_col = pc >> 2; /* which kernel column block (0..3) */
const int kc_col = pc & 3; /* column inside a 4x4 kernel */ const int kc_col = pc & 3; /* column inside a 4x4 kernel */
const __m128 r0 = k[blk_col][0]->f32_x[kc_col]; const __m128 r0 = k[blk_col][0]->f32_x[kc_col];
const __m128 r1 = k[blk_col][1]->f32_x[kc_col]; const __m128 r1 = k[blk_col][1]->f32_x[kc_col];
const __m128 r2 = k[blk_col][2]->f32_x[kc_col]; const __m128 r2 = k[blk_col][2]->f32_x[kc_col];
const __m128 r3 = k[blk_col][3]->f32_x[kc_col]; const __m128 r3 = k[blk_col][3]->f32_x[kc_col];
const __m256 top = _mm256_set_m128(r1, r0); const __m256 top = _mm256_set_m128(r1, r0);
const __m256 bot = _mm256_set_m128(r3, r2); const __m256 bot = _mm256_set_m128(r3, r2);
__m512 col = _mm512_castps256_ps512(top); __m512 col = _mm512_castps256_ps512(top);
col = _mm512_insertf32x8(col, bot, 1); col = _mm512_insertf32x8(col, bot, 1);
PANEL_COL_VEC((*panel), pc, 0) = col; PANEL_COL_VEC((*panel), pc, 0) = col;
} }
} }
DECLARE_BUILD_PANEL(512, double) DECLARE_BUILD_PANEL(512, double)
{ {
const int Q = 8; /* max(4, lanes) for double with ZMM -> 8 columns */ const int Q = 8; /* max(4, lanes) for double with ZMM -> 8 columns */
const int has_right = (col_index + 1) < col_length; const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length; const int has_bottom = (row_index + 1) < row_length;
const MAT_KERNEL(double) *const k_lt = matrix + (row_index * col_length + col_index); const MAT_KERNEL(double) *const k_lt = matrix + (row_index * col_length + col_index);
const MAT_KERNEL(double) *const k_rt = const MAT_KERNEL(double) *const k_rt =
has_right ? (matrix + (row_index * col_length + (col_index + 1))) : has_right ? (matrix + (row_index * col_length + (col_index + 1))) :
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL; (const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(double) *const k_lb = const MAT_KERNEL(double) *const k_lb =
has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) : has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) :
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL; (const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(double) *const k_rb = const MAT_KERNEL(double) *const k_rb =
(has_right && has_bottom) ? (has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) : (matrix + ((row_index + 1) * col_length + (col_index + 1))) :
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL; (const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
for (int pc = 0; pc < Q; ++pc) { for (int pc = 0; pc < Q; ++pc) {
const int is_right_half = (pc >= 4); const int is_right_half = (pc >= 4);
const int kc_col = pc & 3; /* column index inside 4x4 kernel */ const int kc_col = pc & 3; /* column index inside 4x4 kernel */
const MAT_KERNEL(double) *src_top = is_right_half ? k_rt : k_lt; const MAT_KERNEL(double) *src_top = is_right_half ? k_rt : k_lt;
const MAT_KERNEL(double) *src_bot = is_right_half ? k_rb : k_lb; const MAT_KERNEL(double) *src_bot = is_right_half ? k_rb : k_lb;
const __m256d top256 = src_top->f64_y[kc_col]; const __m256d top256 = src_top->f64_y[kc_col];
const __m256d bot256 = src_bot->f64_y[kc_col]; const __m256d bot256 = src_bot->f64_y[kc_col];
__m512d col = _mm512_castpd256_pd512(top256); __m512d col = _mm512_castpd256_pd512(top256);
col = _mm512_insertf64x4(col, bot256, 1); col = _mm512_insertf64x4(col, bot256, 1);
PANEL_COL_VEC((*panel), pc, 0) = _mm512_castpd_ps(col); PANEL_COL_VEC((*panel), pc, 0) = _mm512_castpd_ps(col);
} }
} }
DECLARE_STORE_PANEL(avx, float) DECLARE_STORE_PANEL(avx, float)
{ {
const int Q = 8; /* fixed for AVX float panel */ const int Q = 8; /* fixed for AVX float panel */
/* Local dummy kernel to swallow out-of-bounds writes; no init needed */ /* Local dummy kernel to swallow out-of-bounds writes; no init needed */
MAT_KERNEL(float) zero; MAT_KERNEL(float) zero;
const int has_right = (col_index + 1) < col_length; const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length; const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(float) *const k_lt = (col_index < col_length && row_index < row_length) ? MAT_KERNEL(float) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (row_index * col_length + col_index)) : (matrix + (row_index * col_length + col_index)) :
&zero; &zero;
MAT_KERNEL(float) *const k_rt = MAT_KERNEL(float) *const k_rt =
has_right && (row_index < row_length) ? has_right && (row_index < row_length) ?
(matrix + (row_index * col_length + (col_index + 1))) : (matrix + (row_index * col_length + (col_index + 1))) :
&zero; &zero;
MAT_KERNEL(float) *const k_lb = MAT_KERNEL(float) *const k_lb =
has_bottom && (col_index < col_length) ? has_bottom && (col_index < col_length) ?
(matrix + ((row_index + 1) * col_length + col_index)) : (matrix + ((row_index + 1) * col_length + col_index)) :
&zero; &zero;
MAT_KERNEL(float) *const k_rb = MAT_KERNEL(float) *const k_rb =
(has_right && has_bottom) ? (has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) : (matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero; &zero;
register __m128 top128, bot128; register __m128 top128, bot128;
int i = 0; int i = 0;
for (; i < Q / 2; i++) { for (; i < Q / 2; i++) {
const int kc_col = i; /* column index inside 4x4 kernel */ const int kc_col = i; /* column index inside 4x4 kernel */
/* Load the panel column vector */ /* Load the panel column vector */
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0); const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
/* Split into top/bottom halves */ /* Split into top/bottom halves */
top128 = _mm256_castps256_ps128(col256); /* low 128 */ top128 = _mm256_castps256_ps128(col256); /* low 128 */
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */ bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
/* Store back into kernels directly (out-of-bounds writes go to dummy) */ /* Store back into kernels directly (out-of-bounds writes go to dummy) */
k_rt->f32_x[kc_col] = top128; k_rt->f32_x[kc_col] = top128;
k_rb->f32_x[kc_col] = bot128; k_rb->f32_x[kc_col] = bot128;
} }
for (; i < Q; i++) { for (; i < Q; i++) {
const int kc_col = i - 4; /* column index inside 4x4 kernel */ const int kc_col = i - 4; /* column index inside 4x4 kernel */
/* Load the panel column vector */ /* Load the panel column vector */
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0); const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
/* Split into top/bottom halves */ /* Split into top/bottom halves */
top128 = _mm256_castps256_ps128(col256); /* low 128 */ top128 = _mm256_castps256_ps128(col256); /* low 128 */
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */ bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
/* Store back into kernels directly (out-of-bounds writes go to dummy) */ /* Store back into kernels directly (out-of-bounds writes go to dummy) */
k_lt->f32_x[kc_col] = top128; k_lt->f32_x[kc_col] = top128;
k_lb->f32_x[kc_col] = bot128; k_lb->f32_x[kc_col] = bot128;
} }
} }
DECLARE_STORE_PANEL(512, float) DECLARE_STORE_PANEL(512, float)
{ {
MAT_KERNEL(float) zero; MAT_KERNEL(float) zero;
MAT_KERNEL(float) * k[4][4]; MAT_KERNEL(float) * k[4][4];
for (int kc = 0; kc < 4; ++kc) { for (int kc = 0; kc < 4; ++kc) {
for (int kr = 0; kr < 4; ++kr) { for (int kr = 0; kr < 4; ++kr) {
const int col = col_index + kc; const int col = col_index + kc;
const int row = row_index + kr; const int row = row_index + kr;
k[kc][kr] = (col < col_length && row < row_length) ? k[kc][kr] = (col < col_length && row < row_length) ?
(matrix + (row * col_length + col)) : (matrix + (row * col_length + col)) :
&zero; &zero;
} }
} }
for (int pc = 0; pc < 16; ++pc) { for (int pc = 0; pc < 16; ++pc) {
const int blk_col = pc >> 2; /* which kernel column block (0..3) */ const int blk_col = pc >> 2; /* which kernel column block (0..3) */
const int kc_col = pc & 3; /* column inside a 4x4 kernel */ const int kc_col = pc & 3; /* column inside a 4x4 kernel */
const __m512 col = PANEL_COL_VEC((*panel), pc, 0); const __m512 col = PANEL_COL_VEC((*panel), pc, 0);
const __m256 top = _mm512_castps512_ps256(col); const __m256 top = _mm512_castps512_ps256(col);
const __m256 bot = _mm512_extractf32x8_ps(col, 1); const __m256 bot = _mm512_extractf32x8_ps(col, 1);
const __m128 r0 = _mm256_castps256_ps128(top); const __m128 r0 = _mm256_castps256_ps128(top);
const __m128 r1 = _mm256_extractf128_ps(top, 1); const __m128 r1 = _mm256_extractf128_ps(top, 1);
const __m128 r2 = _mm256_castps256_ps128(bot); const __m128 r2 = _mm256_castps256_ps128(bot);
const __m128 r3 = _mm256_extractf128_ps(bot, 1); const __m128 r3 = _mm256_extractf128_ps(bot, 1);
k[blk_col][0]->f32_x[kc_col] = r0; k[blk_col][0]->f32_x[kc_col] = r0;
k[blk_col][1]->f32_x[kc_col] = r1; k[blk_col][1]->f32_x[kc_col] = r1;
k[blk_col][2]->f32_x[kc_col] = r2; k[blk_col][2]->f32_x[kc_col] = r2;
k[blk_col][3]->f32_x[kc_col] = r3; k[blk_col][3]->f32_x[kc_col] = r3;
} }
} }
DECLARE_STORE_PANEL(sse, double) DECLARE_STORE_PANEL(sse, double)
{ {
MAT_KERNEL(double) *dst = matrix + (row_index * col_length + col_index); MAT_KERNEL(double) *dst = matrix + (row_index * col_length + col_index);
for (int pc = 0; pc < 4; ++pc) { for (int pc = 0; pc < 4; ++pc) {
const int base = pc * 2; const int base = pc * 2;
dst->f64_x[base] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 0)); dst->f64_x[base] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 0));
dst->f64_x[base + 1] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 1)); dst->f64_x[base + 1] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 1));
} }
} }
DECLARE_STORE_PANEL(avx, double) DECLARE_STORE_PANEL(avx, double)
{ {
const int Q = 8; /* fixed for AVX double panel */ const int Q = 8; /* fixed for AVX double panel */
MAT_KERNEL(double) zero; MAT_KERNEL(double) zero;
const int has_right = (col_index + 1) < col_length; const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length; const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ? MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (row_index * col_length + col_index)) : (matrix + (row_index * col_length + col_index)) :
&zero; &zero;
MAT_KERNEL(double) *const k_rt = MAT_KERNEL(double) *const k_rt =
has_right && (row_index < row_length) ? has_right && (row_index < row_length) ?
(matrix + (row_index * col_length + (col_index + 1))) : (matrix + (row_index * col_length + (col_index + 1))) :
&zero; &zero;
MAT_KERNEL(double) *const k_lb = MAT_KERNEL(double) *const k_lb =
has_bottom && (col_index < col_length) ? has_bottom && (col_index < col_length) ?
(matrix + ((row_index + 1) * col_length + col_index)) : (matrix + ((row_index + 1) * col_length + col_index)) :
&zero; &zero;
MAT_KERNEL(double) *const k_rb = MAT_KERNEL(double) *const k_rb =
(has_right && has_bottom) ? (has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) : (matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero; &zero;
for (int i = 0; i < Q; ++i) { for (int i = 0; i < Q; ++i) {
const int kc_col = (i < 4) ? i : (i - 4); const int kc_col = (i < 4) ? i : (i - 4);
const __m256 top_chunk = PANEL_COL_VEC((*panel), i, 0); const __m256 top_chunk = PANEL_COL_VEC((*panel), i, 0);
const __m256 bot_chunk = PANEL_COL_VEC((*panel), i, 1); const __m256 bot_chunk = PANEL_COL_VEC((*panel), i, 1);
MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt; MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt;
MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb; MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb;
top_dst->f64_y[kc_col] = _mm256_castps_pd(top_chunk); top_dst->f64_y[kc_col] = _mm256_castps_pd(top_chunk);
bot_dst->f64_y[kc_col] = _mm256_castps_pd(bot_chunk); bot_dst->f64_y[kc_col] = _mm256_castps_pd(bot_chunk);
} }
} }
DECLARE_STORE_PANEL(512, double) DECLARE_STORE_PANEL(512, double)
{ {
const int Q = 8; /* max(4, lanes) -> 8 columns, one ZMM per column */ const int Q = 8; /* max(4, lanes) -> 8 columns, one ZMM per column */
MAT_KERNEL(double) zero; MAT_KERNEL(double) zero;
const int has_right = (col_index + 1) < col_length; const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length; const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ? MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (row_index * col_length + col_index)) : (matrix + (row_index * col_length + col_index)) :
&zero; &zero;
MAT_KERNEL(double) *const k_rt = MAT_KERNEL(double) *const k_rt =
has_right && (row_index < row_length) ? has_right && (row_index < row_length) ?
(matrix + (row_index * col_length + (col_index + 1))) : (matrix + (row_index * col_length + (col_index + 1))) :
&zero; &zero;
MAT_KERNEL(double) *const k_lb = MAT_KERNEL(double) *const k_lb =
has_bottom && (col_index < col_length) ? has_bottom && (col_index < col_length) ?
(matrix + ((row_index + 1) * col_length + col_index)) : (matrix + ((row_index + 1) * col_length + col_index)) :
&zero; &zero;
MAT_KERNEL(double) *const k_rb = MAT_KERNEL(double) *const k_rb =
(has_right && has_bottom) ? (has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) : (matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero; &zero;
for (int i = 0; i < Q; ++i) { for (int i = 0; i < Q; ++i) {
const int kc_col = (i < 4) ? i : (i - 4); const int kc_col = (i < 4) ? i : (i - 4);
const __m512 col = PANEL_COL_VEC((*panel), i, 0); const __m512 col = PANEL_COL_VEC((*panel), i, 0);
const __m512d col_pd = _mm512_castps_pd(col); const __m512d col_pd = _mm512_castps_pd(col);
const __m256d top256 = _mm512_castpd512_pd256(col_pd); const __m256d top256 = _mm512_castpd512_pd256(col_pd);
const __m256d bot256 = _mm512_extractf64x4_pd(col_pd, 1); const __m256d bot256 = _mm512_extractf64x4_pd(col_pd, 1);
MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt; MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt;
MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb; MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb;
top_dst->f64_y[kc_col] = top256; top_dst->f64_y[kc_col] = top256;
bot_dst->f64_y[kc_col] = bot256; bot_dst->f64_y[kc_col] = bot256;
} }
} }
#endif #endif
StgSharp.Native/sn-src/simdid.x86.c
+24 -31
View File
@@ -57,24 +57,15 @@ static int xgetbv_has_avx512_state(void)
return ((xcr0 & mask) == mask); return ((xcr0 & mask) == mask);
} }
static int is_intel_dual_avx512(int family, int model) /* AVX-512 execution-width classification for AMD, by CPU family.
* Zen4 (0x19): single 256-bit ALU, 512-bit op = two 256-bit micro-ops (dual-pump) → DUAL.
* Zen5+ (>=0x1A): native 512-bit execution units → NATIVE.
* Other/future AMD families → UNKNOWN (conservative). */
static uint64_t amd_avx512_impl(int family)
{ {
return (family == 6) && (model == 0x55); /* SKX/CLX/CPX */ if (family == 0x19) return SIMDID_AVX512_IMPL_DUAL;
} if (family >= 0x1A) return SIMDID_AVX512_IMPL_NATIVE;
return SIMDID_AVX512_IMPL_UNKNOWN;
static int is_intel_native_avx512(int family, int model)
{
if (family != 6)
return 0;
switch (model) {
case 0x6A: /* ICX */
case 0x6C: /* ICX stepping */
case 0x8F: /* SPR/EMR */
case 0xCF: /* GNR */
return 1;
default:
return 0;
}
} }
/* Returns 1 if this AMD/Hygon CPU uses split-lane YMM (two fused 128-bit execution units). /* Returns 1 if this AMD/Hygon CPU uses split-lane YMM (two fused 128-bit execution units).
@@ -110,9 +101,9 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local()
((int *)vendor)[2] = ecx; ((int *)vendor)[2] = ecx;
vendor[12] = '\0'; vendor[12] = '\0';
const int is_intel = (strcmp(vendor, "GenuineIntel") == 0); const int is_intel = (strcmp(vendor, "GenuineIntel") == 0);
const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0) || const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0);
(strcmp(vendor, "HygonGenuine") == 0); const int is_hygon = (strcmp(vendor, "HygonGenuine") == 0); /* 海光: separate from AMD */
const int is_via = (strcmp(vendor, "CentaurHauls") == 0) || const int is_via = (strcmp(vendor, "CentaurHauls") == 0) ||
(memcmp(vendor, " Shanghai ", 12) == 0); /* VIA / Zhaoxin */ (memcmp(vendor, " Shanghai ", 12) == 0); /* VIA / Zhaoxin */
/* CPUID.1 */ /* CPUID.1 */
@@ -128,9 +119,10 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local()
int has_f16c = (ecx & (1 << 29)) != 0; int has_f16c = (ecx & (1 << 29)) != 0;
uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64); uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64);
uint64_t manu = SIMDID_PACK_MANU(is_intel ? SIMDID_MANU_INTEL : uint64_t manu = SIMDID_PACK_MANU(is_intel ? SIMDID_MANU_INTEL :
is_amd ? SIMDID_MANU_AMD : is_amd ? SIMDID_MANU_AMD :
is_via ? SIMDID_MANU_VIA : is_hygon ? SIMDID_MANU_HYGON :
is_via ? SIMDID_MANU_VIA :
SIMDID_MANU_UNKNOWN); SIMDID_MANU_UNKNOWN);
uint64_t main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE); uint64_t main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE);
uint64_t avx512_bits = 0; uint64_t avx512_bits = 0;
@@ -202,16 +194,17 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local()
if (has_vbmi2) if (has_vbmi2)
avx512_bits |= SIMDID_AVX512_VBMI2; avx512_bits |= SIMDID_AVX512_VBMI2;
/* impl classification */ /* Execution-width classification:
const int glue = is_amd || is_intel_dual_avx512(family, model); * Intel all → NATIVE (genuine 512-bit ALU; SKX dual-port is native)
const int native512 = is_intel_native_avx512(family, model); * AMD Zen4 → DUAL (dual-pump 256-bit, Family 0x19)
if (glue) { * AMD Zen5+ → NATIVE (native 512-bit, Family >= 0x1A)
avx512_bits |= SIMDID_AVX512_IMPL_DUAL; * Hygon/VIA/unknown → UNKNOWN (conservative) */
} else if (native512) { if (is_intel)
avx512_bits |= SIMDID_AVX512_IMPL_NATIVE; avx512_bits |= SIMDID_AVX512_IMPL_NATIVE;
} else { else if (is_amd)
avx512_bits |= amd_avx512_impl(family);
else
avx512_bits |= SIMDID_AVX512_IMPL_UNKNOWN; avx512_bits |= SIMDID_AVX512_IMPL_UNKNOWN;
}
/* main level for AVX512 */ /* main level for AVX512 */
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512); main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512);