youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

many things

Browse Source
This commit is contained in:
Youmu Liu
2026-02-26 18:05:35 +08:00
parent 1b04196314
commit 15434ed053
15 changed files with 530 additions and 591 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/Mathematics/Linear/ElementType.cs
+45 -8
View File
@@ -1,9 +1,9 @@
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="ElementType"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -25,15 +25,52 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
#pragma warning disable CA1028
namespace StgSharp.Mathematics.Numeric
{
public enum MatrixElementType : int
[StructLayout(LayoutKind.Explicit, Size = 4)]
public struct MatrixElementType
{
F32 = 0,
F64 = 1,
I32 = 2,
I64 = 3,
public static readonly MatrixElementType F32 = new(0x03000004);
public static readonly MatrixElementType F64 = new(0x03000008);
public static readonly MatrixElementType I08 = new(0x0100001);
public static readonly MatrixElementType I16 = new(0x0100002);
public static readonly MatrixElementType I32 = new(0x0100004);
public static readonly MatrixElementType I64 = new(0x0100008);
public static readonly MatrixElementType U08 = new(0x0200001);
public static readonly MatrixElementType U16 = new(0x0200002);
public static readonly MatrixElementType U32 = new(0x0200004);
public static readonly MatrixElementType U64 = new(0x0200008);
[FieldOffset(0)] internal readonly int Mask;
[FieldOffset(3)] public readonly MatrixElementTypeGroup Group;
[FieldOffset(0)] public readonly ushort Size;
private MatrixElementType(
int mask
)
{
Unsafe.SkipInit(out this);
Unsafe.SkipInit(out Size);
Unsafe.SkipInit(out Group);
this.Mask = mask;
}
}
public enum MatrixElementTypeGroup : byte
{
None = 0,
Singed = 1,
Unsigned = 2,
Float = 3,
}
}
#pragma warning restore CA1028
StgSharp.Mathematics/Numeric/MatrixCompute.cs
-62
View File
@@ -40,68 +40,6 @@ namespace StgSharp.Mathematics.Numeric
public partial class MatrixCompute
{
internal static unsafe void ComputeFmaPanelF32(
MatrixParallelTaskPackage* p
)
{
int f32 = (int)MatrixElementType.F32;
MatrixKernel* leftBuffer = (MatrixKernel*)p->Left;
MatrixKernel* rightBuffer = (MatrixKernel*)p->Right;
MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer;
ref long offsetRef = ref Unsafe.As<int, long>(ref p->PrimTileOffset);
ref long lengthRef = ref Unsafe.As<int, long>(ref p->PrimCount);
int ansWidth = p->AnsPrimOffset;
int ansHeight = p->AnsSecOffset;
int commonK = p->LeftPrimOffset;
M128* enumeration = stackalloc M128[1];
enumeration->Member<int>(2) = commonK;
enumeration->Member<int>(3) = ansHeight;
// in this occasion size must be 1
for (long c = offsetRef; c < offsetRef + lengthRef; c++)
{
int i = (int)(c / ansHeight);
int j = (int)(c % ansHeight);
enumeration->Member<int>(0) = i;
enumeration->Member<int>(1) = j;
NativeIntrinsic.Context.mat[f32].kernel_tile_fma(leftBuffer, rightBuffer, ansBuffer,
enumeration);
}
}
internal static unsafe void ComputeFmaPanelF64(
MatrixParallelTaskPackage* p
)
{
int f64 = (int)MatrixElementType.F64;
// int size = 1;
MatrixKernel* leftBuffer = (MatrixKernel*)p->Left;
MatrixKernel* rightBuffer = (MatrixKernel*)p->Right;
MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer;
ref long offsetRef = ref Unsafe.As<int, long>(ref p->PrimTileOffset);
ref long lengthRef = ref Unsafe.As<int, long>(ref p->PrimCount);
int ansWidth = p->AnsPrimOffset;
int ansHeight = p->AnsSecOffset;
int commonK = p->LeftPrimOffset;
M128* enumeration = stackalloc M128[1];
enumeration->Member<int>(2) = commonK;
enumeration->Member<int>(3) = ansHeight;
// in this occasion size must be 1
for (long c = offsetRef; c < offsetRef + lengthRef; c++)
{
int i = (int)(c / ansHeight);
int j = (int)(c % ansHeight);
enumeration->Member<int>(0) = i;
enumeration->Member<int>(1) = j;
NativeIntrinsic.Context.mat[f64].kernel_tile_fma(leftBuffer, rightBuffer, ansBuffer,
enumeration);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static unsafe MatrixKernel* GetKernelAddressUnsafe<T>(
MatrixKernel<T>* buffer,
StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs
+3 -5
View File
@@ -71,11 +71,9 @@ namespace StgSharp.Mathematics.Numeric
long recommendCount = count / 1024;
MatrixParallelThread[] pool = LeadParallel((int)recommendCount);
MatrixParallelWrap handle = new MatrixParallelWrap(pool);
// TODO set parallel queue for handle
handle.TaskQueue = MatrixParallelQueue.Rent(package);
return handle;
MatrixParallelWrap wrap = new MatrixParallelWrap(pool);
wrap.TaskQueue = MatrixParallelQueue.Rent(package);
return wrap;
}
}
StgSharp.Mathematics/Numeric/MatrixParallel.cs
+2 -1
View File
@@ -98,6 +98,7 @@ namespace StgSharp.Mathematics.Numeric
}
// TODO 构建全局的保底上下文
foreach (MatrixParallelThread thread in _threads) { }
if (contextMap.Keys.Count == 1)
{
for (int i = 0; i < 4; i++)
@@ -113,7 +114,7 @@ namespace StgSharp.Mathematics.Numeric
IsInitialized = true;
}
[LibraryImport(Native.LibName, EntryPoint = "load_intrinsic_function")]
[LibraryImport(Native.LibName, EntryPoint = "sn_compare_simd")]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
private static unsafe partial ulong CompareSIMDID(
IntrinsicContext* context,
StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs
+4 -40
View File
@@ -52,48 +52,12 @@ namespace StgSharp.Mathematics.Numeric
* matrix_op_mode;
*/
public enum MatrixOperationSpecial : short
public enum MatrixIndexStyle : int
{
SEQ_FMA = -1,
}
public enum MatrixIndexStyle : short
{
SPECIAL = 0,
NONE = 0,
BUFFER_INDEX = 1,
}
public enum MatrixOperationParam : short
{
RIGHT_ANS_PARAM = 1,
LEFT_RIGHT_ANS_PARAM = 2,
RIGHT_ANS_SCALAR_PARAM = 3,
LEFT_RIGHT_ANS_SCALAR_PARAM = 4,
ANS_SCALAR_PARAM = 5,
}
[StructLayout(LayoutKind.Sequential)]
public struct MatrixOpMode(
MatrixIndexStyle style,
MatrixOperationParam paramLayout
)
{
public MatrixIndexStyle OperationStyle = style;
public MatrixOperationParam ParameterStyle = paramLayout;
public MatrixOpMode(
MatrixIndexStyle style,
MatrixOperationSpecial paramLayout
)
: this(style, (MatrixOperationParam)paramLayout) { }
SEQ_FMA = -1,
}
@@ -166,7 +130,7 @@ namespace StgSharp.Mathematics.Numeric
[FieldOffset(96)]public int PrimCount; // 4
[FieldOffset(100)]public int SecCount; // 4
[FieldOffset(104)]public MatrixOpMode ComputeMode; // 4
[FieldOffset(104)]public MatrixIndexStyle ComputeMode; // 4
[FieldOffset(108)]private int ReservedInt0; // 4
#endregion
StgSharp.Mathematics/Numeric/MatrixParallelThread.Fma.cs
+77
View File
@@ -0,0 +1,77 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using StgSharp.HighPerformance;
using StgSharp.Internal;
namespace StgSharp.Mathematics.Numeric
{
public partial class MatrixParallelThread
{
internal static unsafe void ComputeFmaPanelF32(
MatrixParallelTaskPackage* p
)
{
int f32 = (int)MatrixElementType.F32;
MatrixKernel* leftBuffer = (MatrixKernel*)p->Left;
MatrixKernel* rightBuffer = (MatrixKernel*)p->Right;
MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer;
ref long offsetRef = ref Unsafe.As<int, long>(ref p->PrimTileOffset);
ref long lengthRef = ref Unsafe.As<int, long>(ref p->PrimCount);
int ansWidth = p->AnsPrimOffset;
int ansHeight = p->AnsSecOffset;
int commonK = p->LeftPrimOffset;
M128* enumeration = stackalloc M128[1];
enumeration->Member<int>(2) = commonK;
enumeration->Member<int>(3) = ansHeight;
// in this occasion size must be 1
for (long c = offsetRef; c < offsetRef + lengthRef; c++)
{
int i = (int)(c / ansHeight);
int j = (int)(c % ansHeight);
enumeration->Member<int>(0) = i;
enumeration->Member<int>(1) = j;
NativeIntrinsic.Context.mat[f32].kernel_tile_fma(leftBuffer, rightBuffer, ansBuffer,
enumeration);
}
}
internal static unsafe void ComputeFmaPanelF64(
MatrixParallelTaskPackage* p
)
{
int f64 = (int)MatrixElementType.F64;
// int size = 1;
MatrixKernel* leftBuffer = (MatrixKernel*)p->Left;
MatrixKernel* rightBuffer = (MatrixKernel*)p->Right;
MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer;
ref long offsetRef = ref Unsafe.As<int, long>(ref p->PrimTileOffset);
ref long lengthRef = ref Unsafe.As<int, long>(ref p->PrimCount);
int ansWidth = p->AnsPrimOffset;
int ansHeight = p->AnsSecOffset;
int commonK = p->LeftPrimOffset;
M128* enumeration = stackalloc M128[1];
enumeration->Member<int>(2) = commonK;
enumeration->Member<int>(3) = ansHeight;
// in this occasion size must be 1
for (long c = offsetRef; c < offsetRef + lengthRef; c++)
{
int i = (int)(c / ansHeight);
int j = (int)(c % ansHeight);
enumeration->Member<int>(0) = i;
enumeration->Member<int>(1) = j;
NativeIntrinsic.Context.mat[f64].kernel_tile_fma(leftBuffer, rightBuffer, ansBuffer,
enumeration);
}
}
}
}
StgSharp.Mathematics/Numeric/MatrixParallelThread.Model.cs
+19 -78
View File
@@ -34,100 +34,41 @@ namespace StgSharp.Mathematics.Numeric
public partial class MatrixParallelThread
{
internal static unsafe void BufferComputeBinary(
MatrixParallelTaskPackage* p
)
{
int size = p->ElementSize * 16;
long count = Unsafe.As<int, long>(ref p->PrimCount);
long offset = Unsafe.As<int, long>(ref p->PrimTileOffset) * size;
private const int KernelStealCount = 64;
((delegate* unmanaged[Cdecl]<void*, void*, void*, long, void>)p->ComputeHandle)((void*)(p->Left + offset),
(void*)(p->Right + offset),
(void*)(p->Answer + offset),
count);
MatrixParallelFactory.Release(p);
}
internal static unsafe void BufferComputeBinaryScalar(
MatrixParallelTaskPackage* p
internal unsafe void BufferCompute(
MatrixParallelTaskPackage* p
)
{
// while () { }
int size = p->ElementSize * 16;
long count = Unsafe.As<int, long>(ref p->PrimCount);
long offset = Unsafe.As<int, long>(ref p->PrimTileOffset) * size;
ScalarPacket* scalar = p->Scalar;
((delegate* unmanaged[Cdecl]<void*, void*, void*, ScalarPacket*, long, void>)p->ComputeHandle)((void*)(p->Left + offset),
(void*)(p->Right + offset),
(void*)(p->Answer + offset),
scalar,
count);
MatrixParallelFactory.Release(p);
IntPtr computeHandle = _context[p->ComputeHandle];
((delegate* unmanaged[Cdecl]<void*, void*, void*, void*, long, void>)computeHandle)((void*)(p->Left + offset),
(void*)(p->Right + offset),
(void*)(p->Answer + offset),
scalar,
count);
}
internal static unsafe void BufferComputeNoOperatorScalar(
MatrixParallelTaskPackage* p
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal unsafe void SpecialBufferCompute(
MatrixParallelTaskPackage* p
)
{
int size = p->ElementSize * 16;
long count = Unsafe.As<int, long>(ref p->PrimCount);
long offset = Unsafe.As<int, long>(ref p->PrimTileOffset) * size;
((delegate* unmanaged[Cdecl]<void*, ScalarPacket*, long, void>)p->ComputeHandle)((void*)(p->Answer + offset),
p->Scalar, count);
MatrixParallelFactory.Release(p);
}
internal static unsafe void BufferComputeUnary(
MatrixParallelTaskPackage* p
)
{
int size = p->ElementSize * 16;
long count = Unsafe.As<int, long>(ref p->PrimCount);
long offset = Unsafe.As<int, long>(ref p->PrimTileOffset) * size;
((delegate* unmanaged[Cdecl]<void*, void*, long, void>)p->ComputeHandle)((void*)(p->Right + offset),
(void*)(p->Answer + offset),
count);
MatrixParallelFactory.Release(p);
}
internal static unsafe void BufferComputeUnaryScalar(
MatrixParallelTaskPackage* p
)
{
int size = p->ElementSize * 16;
long count = Unsafe.As<int, long>(ref p->PrimCount);
long offset = Unsafe.As<int, long>(ref p->PrimTileOffset) * size;
((delegate* unmanaged[Cdecl]<void*, void*, ScalarPacket*, long, void>)p->ComputeHandle)((void*)(p->Right + offset),
(void*)(p->Answer + offset),
p->Scalar,
count);
MatrixParallelFactory.Release(p);
}
internal static unsafe void SpecialBufferCompute(
MatrixParallelTaskPackage* p
)
{
switch ((MatrixOperationSpecial)p->ComputeMode.ParameterStyle)
switch ((MatrixElementType)p->ElementSize)
{
case MatrixOperationSpecial.SEQ_FMA:
switch ((MatrixElementType)p->ElementSize)
{
case MatrixElementType.F64:
MatrixCompute.ComputeFmaPanelF64(p);
return;
case MatrixElementType.F32:
MatrixCompute.ComputeFmaPanelF32(p);
break;
default:
break;
}
case MatrixElementType.F64:
ComputeFmaPanelF64(p);
return;
case MatrixElementType.F32:
ComputeFmaPanelF32(p);
break;
default:
break;
}
MatrixParallelFactory.Release(p);
}
}
StgSharp.Mathematics/Numeric/MatrixParallelThread.cs
+13 -39
View File
@@ -83,44 +83,6 @@ namespace StgSharp.Mathematics.Numeric
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static unsafe void ExecuteTaskPackage(
MatrixParallelQueue taskQueue
)
{
MatrixParallelTaskPackage* p = taskQueue.Package;
switch (p->ComputeMode.OperationStyle)
{
case MatrixIndexStyle.BUFFER_INDEX:
switch (p->ComputeMode.ParameterStyle)
{
case MatrixOperationParam.RIGHT_ANS_PARAM:
BufferComputeUnary(p);
break;
case MatrixOperationParam.LEFT_RIGHT_ANS_PARAM:
BufferComputeBinary(p);
break;
case MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM:
BufferComputeUnaryScalar(p);
break;
case MatrixOperationParam.LEFT_RIGHT_ANS_SCALAR_PARAM:
BufferComputeBinaryScalar(p);
break;
case MatrixOperationParam.ANS_SCALAR_PARAM:
BufferComputeNoOperatorScalar(p);
break;
default:
break;
}
break;
case MatrixIndexStyle.SPECIAL:
SpecialBufferCompute(p);
break;
default:
break;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Restart()
{
@@ -182,7 +144,19 @@ namespace StgSharp.Mathematics.Numeric
// TODO 将外部线程绑定到当前线程上，并执行任务包
} else
{
ExecuteTaskPackage(BindedWrap.TaskQueue);
MatrixParallelQueue taskQueue = BindedWrap.TaskQueue;
MatrixParallelTaskPackage* p = taskQueue.Package;
if (0 > (int)p ->ComputeHandle) {
SpecialBufferCompute(p);
}
switch (p->ComputeMode)
{
case MatrixIndexStyle.BUFFER_INDEX:
BufferCompute(p);
break;
default:
break;
}
BindedWrap.ReportThreadAccomplished();
}
_managedThread.Priority = AfterWork;
StgSharp.Native/SIMDID.x86.md
+15 -8
View File
@@ -55,7 +55,8 @@ For **x86-64**:
| 1 | Intel |
| 2 | AMD |
| 3 | VIA/Zhaoxin |
| 4..15 | Reserved |
| 4 | Hygon (海光) |
| 5..15 | Reserved |
For **AArch64**:
@@ -111,15 +112,17 @@ AVX512 Feature (bits 31..16)
Notes:
- **BASE (bit 20)**: Base AVX512 bundle, indicating `F + CD + VL + DQ + BW` are all present. If set to 1, the detecting side must guarantee these subsets are all available.
- **AVX512_IMPL ([23..22], 2-bit)**: Implementation form (width policy).
- **AVX512_IMPL ([23..22], 2-bit)**: 512-bit execution width policy — not port count, not frequency behaviour.
| Value | Meaning |
|---:|------|
| 0 | Unknown (strategy decided by Manufacture/uArch) |
| 1 | AVX512-DUAL (glue/downclock risk; prefer 256-bit path) |
| 2 | AVX512-NATIVE (full 512-bit ALU; 512-bit path is usable) |
| 0 | Unknown — cannot determine; use conservative 256-bit kernels |
| 1 | DUAL-PUMP — single 256-bit ALU executes 512-bit ops as two consecutive 256-bit micro-ops; ZMM throughput halved (example: AMD Zen4, Family 0x19) |
| 2 | NATIVE — genuine 512-bit execution unit(s); full ZMM throughput (Intel all models including SKX; AMD Zen5 Family 0x1A and later) |
| 3 | Reserved |
> **Note**: Intel SKX/CLX/CPX have two 512-bit FMA ports (port-0 + port-5) and are **NATIVE** — genuine 512-bit throughput. Intel downclocking on heavy ZMM workloads is a separate concern not encoded here. DUAL-PUMP applies only where a single 256-bit unit fires twice per 512-bit op (AMD Zen4-style).
- **VBMI/VBMI2/VNNI/BF16/FP16 ([29..25])**: Commonly needed niche or AI-related small-type extensions; set as needed.
- **R/AI (bit 24)**: Reserved for future AI or other non-crypto extensions.
- **[31..30], [21], [19..16]**: Reserved for future allocation.
@@ -243,13 +246,17 @@ Detection:
## 12) Minimal Detection-Side Fill Rules (x86-64)
1) Set `RootISA = x86-64`, `Manufacture = Intel/AMD`.
1) Set `RootISA = x86-64`, `Manufacture = Intel / AMD / Hygon / VIA`.
2) Compute `MainLevel` (per ascending rule).
3) If `MainLevel >= AVX512`:
- If `F+CD+VL+DQ+BW` are all present, set `BASE = 1`; otherwise `BASE = 0`.
- Set `AVX512_IMPL`:
- AMD: DUAL (per definition)
- Intel: Decide DUAL/NATIVE based on family/model or policy table; if uncertain, use Unknown
- **Intel (all)**: NATIVE — all Intel AVX-512 CPUs have genuine 512-bit execution units
- **AMD Zen4 (Family 0x19)**: DUAL — single 256-bit ALU, dual-pump
- **AMD Zen5+ (Family 0x1A+)**: NATIVE — native full-width 512-bit units
- **AMD other/future**: UNKNOWN (conservative)
- **Hygon**: UNKNOWN (AVX-512 characteristics unclear)
- **VIA/other**: UNKNOWN
- When VNNI/BF16/FP16/VBMI/VBMI2 are detected, set the corresponding bits.
4) If `MainLevel >= AMX`: fill the AMX byte.
5) When AVX/AVX2 are present, populate the AVX Feature byte: set `AVX` and `AVX2`; set `FMA` and `F16C` when detected; set `DUAL` when the YMM execution path is split/glued (AMD Zen1/Zen+ Family 0x17 model < 0x30, Hygon all models, VIA/Zhaoxin all models).
StgSharp.Native/sn-head/sn_intrincontext.x86.h
+1
View File
@@ -46,6 +46,7 @@
#define SIMDID_MANU_INTEL 0x1
#define SIMDID_MANU_AMD 0x2
#define SIMDID_MANU_VIA 0x3
#define SIMDID_MANU_HYGON 0x4 /* 海光: Zen1-derived, separate from AMD */
// MainLevel (ordinal, shared bits 8..15; not bit-flags)
#define SIMDID_MAIN_LVL_NONE 0x00u
StgSharp.Native/sn-src/load_intrinsic.x86.c
-2
View File
@@ -14,9 +14,7 @@
#endif // _MSC_VER
PRIVATE void load_intrinsic_sse(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_avx_dual(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_avx(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_512_dual(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID id);
SN_API void load_intrinsic_function(void *intrinsic_context, uint64_t id)
StgSharp.Native/sn-src/m.fma.x86.c
+19 -14
View File
@@ -151,29 +151,34 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
{
static const __m512i b_shift = { .m512i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9, 15,
12, 13, 14 } };
static const __m512i b_shift = { .m512i_i32 = {
// shuff order
0, 1, 2, 3, // col0
5, 6, 7, 4, //col1
10, 11, 8, 9, //col2
15, 12, 13, 14 //col3
} };
register __m512 a = _mm512_load_ps((float *)left);
register __m512 b = _mm512_load_ps((float *)right);
register __m512 c = _mm512_load_ps((float *)ans);
b = _mm512_permutexvar_ps(b_shift, b);
register __m512 a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0));
register __m512 b_perm = _mm512_permute_ps(b, _FMA_SELECT(0));
c = _mm512_fmadd_ps(a_perm, b_perm, c);
register __m512 a_perm0 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0));
register __m512 b_perm0 = _mm512_permute_ps(b, _FMA_SELECT(0));
c = _mm512_fmadd_ps(a_perm0, b_perm0, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(1));
c = _mm512_fmadd_ps(a_perm, b_perm, c);
register __m512 a_perm1 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3));
register __m512 b_perm1 = _mm512_permute_ps(b, _FMA_SELECT(1));
c = _mm512_fmadd_ps(a_perm1, b_perm1, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(2));
c = _mm512_fmadd_ps(a_perm, b_perm, c);
register __m512 a_perm2 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2));
register __m512 b_perm2 = _mm512_permute_ps(b, _FMA_SELECT(2));
c = _mm512_fmadd_ps(a_perm2, b_perm2, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(3));
c = _mm512_fmadd_ps(a_perm, b_perm, c);
register __m512 a_perm3 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1));
register __m512 b_perm3 = _mm512_permute_ps(b, _FMA_SELECT(3));
c = _mm512_fmadd_ps(a_perm3, b_perm3, c);
_mm512_store_ps((float *)ans, c);
}
StgSharp.Native/sn-src/m.fma_tile.x86.c
+7 -2
View File
@@ -201,8 +201,13 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
{
static const __m512i b_shift_source = { .m512i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9,
15, 12, 13, 14 } };
static const __m512i b_shift_source = { .m512i_i32 = {
// shuff order
0, 1, 2, 3, // col0
5, 6, 7, 4, //col1
10, 11, 8, 9, //col2
15, 12, 13, 14 //col3
} };
int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1];
int commonK = enumeration->m128_i32[2];
StgSharp.Native/sn-src/m.panel.x86.c → StgSharp.Native/sn-src/m.panel.x86.old
+301 -301
View File
@@ -1,301 +1,301 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_matkernel.h"
DECLARE_BUILD_PANEL(avx, float)
{
const int Q = 8; /* fixed for AVX float panel */
/* Neighbor kernels; substitute global zero kernel where out-of-bounds */
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
const MAT_KERNEL(float) *const k_lt = matrix + (row_index * col_length + col_index);
const MAT_KERNEL(float) *const k_rt =
has_right ? (matrix + (row_index * col_length + (col_index + 1))) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) *const k_lb =
has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
register __m128 top128, bot128;
register __m256 col256;
for (int pc = 0; pc < Q; ++pc) {
const int is_right_half = (pc >= 4);
const int kc_col = pc & 3; /* column index inside 4x4 kernel */
const MAT_KERNEL(float) *src_top = is_right_half ? k_rt : k_lt;
const MAT_KERNEL(float) *src_bot = is_right_half ? k_rb : k_lb;
/* Vector loads: each kernel column is a compact __m128 */
top128 = src_top->f32_x[kc_col];
bot128 = src_bot->f32_x[kc_col];
/* Combine into an AVX column: low 128 = top, high 128 = bottom */
col256 = _mm256_set_m128(bot128, top128);
PANEL_COL_VEC((*panel), pc, 0) = col256;
}
}
DECLARE_BUILD_PANEL(512, float)
{
const MAT_KERNEL(float) *const zero = (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) * k[4][4];
for (int kc = 0; kc < 4; ++kc) {
for (int kr = 0; kr < 4; ++kr) {
const int col = col_index + kc;
const int row = row_index + kr;
k[kc][kr] = (col < col_length && row < row_length) ?
(matrix + (row * col_length + col)) :
zero;
}
}
/* Each panel column corresponds to a kernel column (0..3) within a block col (0..3). */
for (int pc = 0; pc < 16; ++pc) {
const int blk_col = pc >> 2; /* which kernel column block (0..3) */
const int kc_col = pc & 3; /* column inside a 4x4 kernel */
const __m128 r0 = k[blk_col][0]->f32_x[kc_col];
const __m128 r1 = k[blk_col][1]->f32_x[kc_col];
const __m128 r2 = k[blk_col][2]->f32_x[kc_col];
const __m128 r3 = k[blk_col][3]->f32_x[kc_col];
const __m256 top = _mm256_set_m128(r1, r0);
const __m256 bot = _mm256_set_m128(r3, r2);
__m512 col = _mm512_castps256_ps512(top);
col = _mm512_insertf32x8(col, bot, 1);
PANEL_COL_VEC((*panel), pc, 0) = col;
}
}
DECLARE_BUILD_PANEL(512, double)
{
const int Q = 8; /* max(4, lanes) for double with ZMM -> 8 columns */
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
const MAT_KERNEL(double) *const k_lt = matrix + (row_index * col_length + col_index);
const MAT_KERNEL(double) *const k_rt =
has_right ? (matrix + (row_index * col_length + (col_index + 1))) :
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(double) *const k_lb =
has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) :
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(double) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
for (int pc = 0; pc < Q; ++pc) {
const int is_right_half = (pc >= 4);
const int kc_col = pc & 3; /* column index inside 4x4 kernel */
const MAT_KERNEL(double) *src_top = is_right_half ? k_rt : k_lt;
const MAT_KERNEL(double) *src_bot = is_right_half ? k_rb : k_lb;
const __m256d top256 = src_top->f64_y[kc_col];
const __m256d bot256 = src_bot->f64_y[kc_col];
__m512d col = _mm512_castpd256_pd512(top256);
col = _mm512_insertf64x4(col, bot256, 1);
PANEL_COL_VEC((*panel), pc, 0) = _mm512_castpd_ps(col);
}
}
DECLARE_STORE_PANEL(avx, float)
{
const int Q = 8; /* fixed for AVX float panel */
/* Local dummy kernel to swallow out-of-bounds writes; no init needed */
MAT_KERNEL(float) zero;
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(float) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (row_index * col_length + col_index)) :
&zero;
MAT_KERNEL(float) *const k_rt =
has_right && (row_index < row_length) ?
(matrix + (row_index * col_length + (col_index + 1))) :
&zero;
MAT_KERNEL(float) *const k_lb =
has_bottom && (col_index < col_length) ?
(matrix + ((row_index + 1) * col_length + col_index)) :
&zero;
MAT_KERNEL(float) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero;
register __m128 top128, bot128;
int i = 0;
for (; i < Q / 2; i++) {
const int kc_col = i; /* column index inside 4x4 kernel */
/* Load the panel column vector */
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
/* Split into top/bottom halves */
top128 = _mm256_castps256_ps128(col256); /* low 128 */
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
/* Store back into kernels directly (out-of-bounds writes go to dummy) */
k_rt->f32_x[kc_col] = top128;
k_rb->f32_x[kc_col] = bot128;
}
for (; i < Q; i++) {
const int kc_col = i - 4; /* column index inside 4x4 kernel */
/* Load the panel column vector */
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
/* Split into top/bottom halves */
top128 = _mm256_castps256_ps128(col256); /* low 128 */
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
/* Store back into kernels directly (out-of-bounds writes go to dummy) */
k_lt->f32_x[kc_col] = top128;
k_lb->f32_x[kc_col] = bot128;
}
}
DECLARE_STORE_PANEL(512, float)
{
MAT_KERNEL(float) zero;
MAT_KERNEL(float) * k[4][4];
for (int kc = 0; kc < 4; ++kc) {
for (int kr = 0; kr < 4; ++kr) {
const int col = col_index + kc;
const int row = row_index + kr;
k[kc][kr] = (col < col_length && row < row_length) ?
(matrix + (row * col_length + col)) :
&zero;
}
}
for (int pc = 0; pc < 16; ++pc) {
const int blk_col = pc >> 2; /* which kernel column block (0..3) */
const int kc_col = pc & 3; /* column inside a 4x4 kernel */
const __m512 col = PANEL_COL_VEC((*panel), pc, 0);
const __m256 top = _mm512_castps512_ps256(col);
const __m256 bot = _mm512_extractf32x8_ps(col, 1);
const __m128 r0 = _mm256_castps256_ps128(top);
const __m128 r1 = _mm256_extractf128_ps(top, 1);
const __m128 r2 = _mm256_castps256_ps128(bot);
const __m128 r3 = _mm256_extractf128_ps(bot, 1);
k[blk_col][0]->f32_x[kc_col] = r0;
k[blk_col][1]->f32_x[kc_col] = r1;
k[blk_col][2]->f32_x[kc_col] = r2;
k[blk_col][3]->f32_x[kc_col] = r3;
}
}
DECLARE_STORE_PANEL(sse, double)
{
MAT_KERNEL(double) *dst = matrix + (row_index * col_length + col_index);
for (int pc = 0; pc < 4; ++pc) {
const int base = pc * 2;
dst->f64_x[base] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 0));
dst->f64_x[base + 1] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 1));
}
}
DECLARE_STORE_PANEL(avx, double)
{
const int Q = 8; /* fixed for AVX double panel */
MAT_KERNEL(double) zero;
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (row_index * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rt =
has_right && (row_index < row_length) ?
(matrix + (row_index * col_length + (col_index + 1))) :
&zero;
MAT_KERNEL(double) *const k_lb =
has_bottom && (col_index < col_length) ?
(matrix + ((row_index + 1) * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero;
for (int i = 0; i < Q; ++i) {
const int kc_col = (i < 4) ? i : (i - 4);
const __m256 top_chunk = PANEL_COL_VEC((*panel), i, 0);
const __m256 bot_chunk = PANEL_COL_VEC((*panel), i, 1);
MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt;
MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb;
top_dst->f64_y[kc_col] = _mm256_castps_pd(top_chunk);
bot_dst->f64_y[kc_col] = _mm256_castps_pd(bot_chunk);
}
}
DECLARE_STORE_PANEL(512, double)
{
const int Q = 8; /* max(4, lanes) -> 8 columns, one ZMM per column */
MAT_KERNEL(double) zero;
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (row_index * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rt =
has_right && (row_index < row_length) ?
(matrix + (row_index * col_length + (col_index + 1))) :
&zero;
MAT_KERNEL(double) *const k_lb =
has_bottom && (col_index < col_length) ?
(matrix + ((row_index + 1) * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero;
for (int i = 0; i < Q; ++i) {
const int kc_col = (i < 4) ? i : (i - 4);
const __m512 col = PANEL_COL_VEC((*panel), i, 0);
const __m512d col_pd = _mm512_castps_pd(col);
const __m256d top256 = _mm512_castpd512_pd256(col_pd);
const __m256d bot256 = _mm512_extractf64x4_pd(col_pd, 1);
MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt;
MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb;
top_dst->f64_y[kc_col] = top256;
bot_dst->f64_y[kc_col] = bot256;
}
}
#endif
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_matkernel.h"
DECLARE_BUILD_PANEL(avx, float)
{
const int Q = 8; /* fixed for AVX float panel */
/* Neighbor kernels; substitute global zero kernel where out-of-bounds */
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
const MAT_KERNEL(float) *const k_lt = matrix + (row_index * col_length + col_index);
const MAT_KERNEL(float) *const k_rt =
has_right ? (matrix + (row_index * col_length + (col_index + 1))) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) *const k_lb =
has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
register __m128 top128, bot128;
register __m256 col256;
for (int pc = 0; pc < Q; ++pc) {
const int is_right_half = (pc >= 4);
const int kc_col = pc & 3; /* column index inside 4x4 kernel */
const MAT_KERNEL(float) *src_top = is_right_half ? k_rt : k_lt;
const MAT_KERNEL(float) *src_bot = is_right_half ? k_rb : k_lb;
/* Vector loads: each kernel column is a compact __m128 */
top128 = src_top->f32_x[kc_col];
bot128 = src_bot->f32_x[kc_col];
/* Combine into an AVX column: low 128 = top, high 128 = bottom */
col256 = _mm256_set_m128(bot128, top128);
PANEL_COL_VEC((*panel), pc, 0) = col256;
}
}
DECLARE_BUILD_PANEL(512, float)
{
const MAT_KERNEL(float) *const zero = (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) * k[4][4];
for (int kc = 0; kc < 4; ++kc) {
for (int kr = 0; kr < 4; ++kr) {
const int col = col_index + kc;
const int row = row_index + kr;
k[kc][kr] = (col < col_length && row < row_length) ?
(matrix + (row * col_length + col)) :
zero;
}
}
/* Each panel column corresponds to a kernel column (0..3) within a block col (0..3). */
for (int pc = 0; pc < 16; ++pc) {
const int blk_col = pc >> 2; /* which kernel column block (0..3) */
const int kc_col = pc & 3; /* column inside a 4x4 kernel */
const __m128 r0 = k[blk_col][0]->f32_x[kc_col];
const __m128 r1 = k[blk_col][1]->f32_x[kc_col];
const __m128 r2 = k[blk_col][2]->f32_x[kc_col];
const __m128 r3 = k[blk_col][3]->f32_x[kc_col];
const __m256 top = _mm256_set_m128(r1, r0);
const __m256 bot = _mm256_set_m128(r3, r2);
__m512 col = _mm512_castps256_ps512(top);
col = _mm512_insertf32x8(col, bot, 1);
PANEL_COL_VEC((*panel), pc, 0) = col;
}
}
DECLARE_BUILD_PANEL(512, double)
{
const int Q = 8; /* max(4, lanes) for double with ZMM -> 8 columns */
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
const MAT_KERNEL(double) *const k_lt = matrix + (row_index * col_length + col_index);
const MAT_KERNEL(double) *const k_rt =
has_right ? (matrix + (row_index * col_length + (col_index + 1))) :
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(double) *const k_lb =
has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) :
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(double) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
(const MAT_KERNEL(double) *)&SN_ZERO_KERNEL;
for (int pc = 0; pc < Q; ++pc) {
const int is_right_half = (pc >= 4);
const int kc_col = pc & 3; /* column index inside 4x4 kernel */
const MAT_KERNEL(double) *src_top = is_right_half ? k_rt : k_lt;
const MAT_KERNEL(double) *src_bot = is_right_half ? k_rb : k_lb;
const __m256d top256 = src_top->f64_y[kc_col];
const __m256d bot256 = src_bot->f64_y[kc_col];
__m512d col = _mm512_castpd256_pd512(top256);
col = _mm512_insertf64x4(col, bot256, 1);
PANEL_COL_VEC((*panel), pc, 0) = _mm512_castpd_ps(col);
}
}
DECLARE_STORE_PANEL(avx, float)
{
const int Q = 8; /* fixed for AVX float panel */
/* Local dummy kernel to swallow out-of-bounds writes; no init needed */
MAT_KERNEL(float) zero;
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(float) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (row_index * col_length + col_index)) :
&zero;
MAT_KERNEL(float) *const k_rt =
has_right && (row_index < row_length) ?
(matrix + (row_index * col_length + (col_index + 1))) :
&zero;
MAT_KERNEL(float) *const k_lb =
has_bottom && (col_index < col_length) ?
(matrix + ((row_index + 1) * col_length + col_index)) :
&zero;
MAT_KERNEL(float) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero;
register __m128 top128, bot128;
int i = 0;
for (; i < Q / 2; i++) {
const int kc_col = i; /* column index inside 4x4 kernel */
/* Load the panel column vector */
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
/* Split into top/bottom halves */
top128 = _mm256_castps256_ps128(col256); /* low 128 */
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
/* Store back into kernels directly (out-of-bounds writes go to dummy) */
k_rt->f32_x[kc_col] = top128;
k_rb->f32_x[kc_col] = bot128;
}
for (; i < Q; i++) {
const int kc_col = i - 4; /* column index inside 4x4 kernel */
/* Load the panel column vector */
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
/* Split into top/bottom halves */
top128 = _mm256_castps256_ps128(col256); /* low 128 */
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
/* Store back into kernels directly (out-of-bounds writes go to dummy) */
k_lt->f32_x[kc_col] = top128;
k_lb->f32_x[kc_col] = bot128;
}
}
DECLARE_STORE_PANEL(512, float)
{
MAT_KERNEL(float) zero;
MAT_KERNEL(float) * k[4][4];
for (int kc = 0; kc < 4; ++kc) {
for (int kr = 0; kr < 4; ++kr) {
const int col = col_index + kc;
const int row = row_index + kr;
k[kc][kr] = (col < col_length && row < row_length) ?
(matrix + (row * col_length + col)) :
&zero;
}
}
for (int pc = 0; pc < 16; ++pc) {
const int blk_col = pc >> 2; /* which kernel column block (0..3) */
const int kc_col = pc & 3; /* column inside a 4x4 kernel */
const __m512 col = PANEL_COL_VEC((*panel), pc, 0);
const __m256 top = _mm512_castps512_ps256(col);
const __m256 bot = _mm512_extractf32x8_ps(col, 1);
const __m128 r0 = _mm256_castps256_ps128(top);
const __m128 r1 = _mm256_extractf128_ps(top, 1);
const __m128 r2 = _mm256_castps256_ps128(bot);
const __m128 r3 = _mm256_extractf128_ps(bot, 1);
k[blk_col][0]->f32_x[kc_col] = r0;
k[blk_col][1]->f32_x[kc_col] = r1;
k[blk_col][2]->f32_x[kc_col] = r2;
k[blk_col][3]->f32_x[kc_col] = r3;
}
}
DECLARE_STORE_PANEL(sse, double)
{
MAT_KERNEL(double) *dst = matrix + (row_index * col_length + col_index);
for (int pc = 0; pc < 4; ++pc) {
const int base = pc * 2;
dst->f64_x[base] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 0));
dst->f64_x[base + 1] = _mm_castps_pd(PANEL_COL_VEC((*panel), pc, 1));
}
}
DECLARE_STORE_PANEL(avx, double)
{
const int Q = 8; /* fixed for AVX double panel */
MAT_KERNEL(double) zero;
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (row_index * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rt =
has_right && (row_index < row_length) ?
(matrix + (row_index * col_length + (col_index + 1))) :
&zero;
MAT_KERNEL(double) *const k_lb =
has_bottom && (col_index < col_length) ?
(matrix + ((row_index + 1) * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero;
for (int i = 0; i < Q; ++i) {
const int kc_col = (i < 4) ? i : (i - 4);
const __m256 top_chunk = PANEL_COL_VEC((*panel), i, 0);
const __m256 bot_chunk = PANEL_COL_VEC((*panel), i, 1);
MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt;
MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb;
top_dst->f64_y[kc_col] = _mm256_castps_pd(top_chunk);
bot_dst->f64_y[kc_col] = _mm256_castps_pd(bot_chunk);
}
}
DECLARE_STORE_PANEL(512, double)
{
const int Q = 8; /* max(4, lanes) -> 8 columns, one ZMM per column */
MAT_KERNEL(double) zero;
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(double) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (row_index * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rt =
has_right && (row_index < row_length) ?
(matrix + (row_index * col_length + (col_index + 1))) :
&zero;
MAT_KERNEL(double) *const k_lb =
has_bottom && (col_index < col_length) ?
(matrix + ((row_index + 1) * col_length + col_index)) :
&zero;
MAT_KERNEL(double) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((row_index + 1) * col_length + (col_index + 1))) :
&zero;
for (int i = 0; i < Q; ++i) {
const int kc_col = (i < 4) ? i : (i - 4);
const __m512 col = PANEL_COL_VEC((*panel), i, 0);
const __m512d col_pd = _mm512_castps_pd(col);
const __m256d top256 = _mm512_castpd512_pd256(col_pd);
const __m256d bot256 = _mm512_extractf64x4_pd(col_pd, 1);
MAT_KERNEL(double) *const top_dst = (i < 4) ? k_rt : k_lt;
MAT_KERNEL(double) *const bot_dst = (i < 4) ? k_rb : k_lb;
top_dst->f64_y[kc_col] = top256;
bot_dst->f64_y[kc_col] = bot256;
}
}
#endif
StgSharp.Native/sn-src/simdid.x86.c
+24 -31
View File
@@ -57,24 +57,15 @@ static int xgetbv_has_avx512_state(void)
return ((xcr0 & mask) == mask);
}
static int is_intel_dual_avx512(int family, int model)
/* AVX-512 execution-width classification for AMD, by CPU family.
* Zen4 (0x19): single 256-bit ALU, 512-bit op = two 256-bit micro-ops (dual-pump) → DUAL.
* Zen5+ (>=0x1A): native 512-bit execution units → NATIVE.
* Other/future AMD families → UNKNOWN (conservative). */
static uint64_t amd_avx512_impl(int family)
{
return (family == 6) && (model == 0x55); /* SKX/CLX/CPX */
}
static int is_intel_native_avx512(int family, int model)
{
if (family != 6)
return 0;
switch (model) {
case 0x6A: /* ICX */
case 0x6C: /* ICX stepping */
case 0x8F: /* SPR/EMR */
case 0xCF: /* GNR */
return 1;
default:
return 0;
}
if (family == 0x19) return SIMDID_AVX512_IMPL_DUAL;
if (family >= 0x1A) return SIMDID_AVX512_IMPL_NATIVE;
return SIMDID_AVX512_IMPL_UNKNOWN;
}
/* Returns 1 if this AMD/Hygon CPU uses split-lane YMM (two fused 128-bit execution units).
@@ -110,9 +101,9 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local()
((int *)vendor)[2] = ecx;
vendor[12] = '\0';
const int is_intel = (strcmp(vendor, "GenuineIntel") == 0);
const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0) ||
(strcmp(vendor, "HygonGenuine") == 0);
const int is_via = (strcmp(vendor, "CentaurHauls") == 0) ||
const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0);
const int is_hygon = (strcmp(vendor, "HygonGenuine") == 0); /* 海光: separate from AMD */
const int is_via = (strcmp(vendor, "CentaurHauls") == 0) ||
(memcmp(vendor, " Shanghai ", 12) == 0); /* VIA / Zhaoxin */
/* CPUID.1 */
@@ -128,9 +119,10 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local()
int has_f16c = (ecx & (1 << 29)) != 0;
uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64);
uint64_t manu = SIMDID_PACK_MANU(is_intel ? SIMDID_MANU_INTEL :
is_amd ? SIMDID_MANU_AMD :
is_via ? SIMDID_MANU_VIA :
uint64_t manu = SIMDID_PACK_MANU(is_intel ? SIMDID_MANU_INTEL :
is_amd ? SIMDID_MANU_AMD :
is_hygon ? SIMDID_MANU_HYGON :
is_via ? SIMDID_MANU_VIA :
SIMDID_MANU_UNKNOWN);
uint64_t main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE);
uint64_t avx512_bits = 0;
@@ -202,16 +194,17 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local()
if (has_vbmi2)
avx512_bits |= SIMDID_AVX512_VBMI2;
/* impl classification */
const int glue = is_amd || is_intel_dual_avx512(family, model);
const int native512 = is_intel_native_avx512(family, model);
if (glue) {
avx512_bits |= SIMDID_AVX512_IMPL_DUAL;
} else if (native512) {
/* Execution-width classification:
* Intel all → NATIVE (genuine 512-bit ALU; SKX dual-port is native)
* AMD Zen4 → DUAL (dual-pump 256-bit, Family 0x19)
* AMD Zen5+ → NATIVE (native 512-bit, Family >= 0x1A)
* Hygon/VIA/unknown → UNKNOWN (conservative) */
if (is_intel)
avx512_bits |= SIMDID_AVX512_IMPL_NATIVE;
} else {
else if (is_amd)
avx512_bits |= amd_avx512_impl(family);
else
avx512_bits |= SIMDID_AVX512_IMPL_UNKNOWN;
}
/* main level for AVX512 */
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512);