youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better kernel fma

Browse Source
This commit is contained in:
Youmu Liu
2026-01-07 18:06:36 +08:00
parent 3d488abd2e
commit 88af8d8957
11 changed files with 347 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/Environment/Platform.cs
+56 -21
View File
@@ -1,4 +1,4 @@
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="Platform"
// Project: StgSharp
@@ -55,15 +55,28 @@ namespace StgSharp
private static uint markCount = 0;
internal static GraphicAPI API = default;
public static int DefaultSIMDAlignment => NativeIntrinsic.IntrinsicLevel switch
public static unsafe int DefaultSIMDAlignment
{
IntrinsicLevel.SSE => 16,
IntrinsicLevel.AVX2 => 32,
IntrinsicLevel.AVX512 => 64,
IntrinsicLevel.NEON => 16,
IntrinsicLevel.Non => 8,
_ => 8,
};
get
{
SIMDID id = NativeIntrinsic.IntrinsicMask;
switch (id.MaskByte[0] & 0b_00001111)
{
case 1:
return id.MaskByte[1] switch
{
1 => 16,
2 => 32,
3 => 64,
4 => 64,
5 => 64,
_ => 8,
};
default:
return 8;
}
}
}
public static ModuleToInitializeCollection Initialize()
{
@@ -71,31 +84,41 @@ namespace StgSharp
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void LogError(Exception ex)
public static void LogError(
Exception ex
)
{
DefaultLog.InternalWriteLog(ex.Message, LogType.Error);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void LogInfo(string log)
public static void LogInfo(
string log
)
{
DefaultLog.InternalWriteLog(log, LogType.Info);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void LogWarning(string log)
public static void LogWarning(
string log
)
{
DefaultLog.InternalWriteLog(log, LogType.Warning);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void LogWarning(Exception notVerySeriesException)
public static void LogWarning(
Exception notVerySeriesException
)
{
DefaultLog.InternalWriteLog(notVerySeriesException.Message, LogType.Warning);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Mark(bool needConsole)
public static void Mark(
bool needConsole
)
{
markCount++;
if (needConsole) {
@@ -103,7 +126,9 @@ namespace StgSharp
}
}
public static void Terminate(int exitCode)
public static void Terminate(
int exitCode
)
{
MainTimeProvider.StopProvidingTime();
GraphicFramework.glfwTerminate();
@@ -116,7 +141,8 @@ namespace StgSharp
bool isConsole,
[CallerFilePath] string filePath = "",
[CallerMemberName] string callerName = "",
[CallerLineNumber] int lineNumber = 0)
[CallerLineNumber] int lineNumber = 0
)
{
Type t = target.GetType();
string log = $"Tracked object: {nameof(target)}\t\tType:{t.Name}\n";
@@ -147,12 +173,17 @@ namespace StgSharp
/// The severity of current message.
/// </param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void WriteLog(string message, LogType logType)
public static void WriteLog(
string message,
LogType logType
)
{
DefaultLog.InternalWriteLog(message, logType);
}
internal static TDele ConvertAPI<TDele>(IntPtr funcPtr)
internal static TDele ConvertAPI<TDele>(
IntPtr funcPtr
)
{
if (funcPtr == IntPtr.Zero) {
throw new InvalidCastException(
@@ -168,7 +199,10 @@ namespace StgSharp
}
}
internal static Delegate ConvertAPI(IntPtr funcPtr, Type T)
internal static Delegate ConvertAPI(
IntPtr funcPtr,
Type T
)
{
if (funcPtr == IntPtr.Zero) {
throw new InvalidCastException(
@@ -197,12 +231,13 @@ namespace StgSharp
$"Program {Assembly.GetEntryAssembly()!.FullName} on {Environment.MachineName} Started.", LogType.Info);
MainThreadID = System.Environment.CurrentManagedThreadId;
/**/
Dialogue.NeedShowWhenStartup = false;
Dialogue.LoadDialogue();
if (Dialogue.NeedShowWhenStartup) {
Dialogue.CreateDialogueProcessIfNotExist();
}
/**/
NativeIntrinsic.InitIntrinsicContext();
}
StgSharp.Common/HighPerformance/MatrixPanel.cs
+42 -17
View File
@@ -46,13 +46,17 @@ namespace StgSharp.HighPerformance
return (MatrixPanel<T>*)NativeMemory.AlignedAlloc(Size<T>(), AlignmentBytes);
}
public static unsafe void Destroy<T>(MatrixPanel<T>* panel) where T : unmanaged, INumber<T>
public static unsafe void Destroy<T>(
MatrixPanel<T>* panel
) where T : unmanaged, INumber<T>
{
NativeMemory.AlignedFree(panel);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static unsafe Span<T> Elements<T>(ref T head) where T : unmanaged, INumber<T>
public static unsafe Span<T> Elements<T>(
ref T head
) where T : unmanaged, INumber<T>
{
void* p = Unsafe.AsPointer(ref head);
return new Span<T>(p, MaxElementCount<T>());
@@ -92,17 +96,26 @@ namespace StgSharp.HighPerformance
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static int KernelSideCount<T>() where T : unmanaged, INumber<T>
{
return NativeIntrinsic.IntrinsicLevel switch
SIMDID id = NativeIntrinsic.IntrinsicMask;
switch (id.MaskByte[0] & 0b_00001111)
{
IntrinsicLevel.SSE => KernelSideCountSse<T>(),
IntrinsicLevel.AVX2 => KernelSideCountAvx2<T>(),
IntrinsicLevel.AVX512 => KernelSideCountAvx512<T>(),
_ => 0
};
case 1:
return id.MaskByte[1] switch
{
1 => KernelSideCountSse<T>(),
2 => KernelSideCountAvx2<T>(),
3 => KernelSideCountAvx512<T>(),
_ => throw new NotSupportedException("Unsupported SIMD level."),
};
default:
throw new NotSupportedException("Current hardware platform is not supported");
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe nuint Align64(void* p) => ((nuint)p + (AlignmentBytes - 1)) & ~(nuint)(AlignmentBytes - 1);
private static unsafe nuint Align64(
void* p
) => ((nuint)p + (AlignmentBytes - 1)) & ~(nuint)(AlignmentBytes - 1);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int KernelSideCountAvx2<T>() where T : unmanaged, INumber<T>
@@ -123,7 +136,9 @@ namespace StgSharp.HighPerformance
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int PanelSideFromBits<T>(int vectorBits) where T : unmanaged, INumber<T>
private static int PanelSideFromBits<T>(
int vectorBits
) where T : unmanaged, INumber<T>
{
int elemBits = sizeof(T) * 8;
@@ -137,13 +152,20 @@ namespace StgSharp.HighPerformance
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int PanelSideLength<T>() where T : unmanaged, INumber<T>
{
return NativeIntrinsic.IntrinsicLevel switch
SIMDID id = NativeIntrinsic.IntrinsicMask;
switch (id.MaskByte[0] & 0b_00001111)
{
IntrinsicLevel.SSE => PanelSideFromBits<T>(128),
IntrinsicLevel.AVX2 => PanelSideFromBits<T>(256),
IntrinsicLevel.AVX512 => PanelSideFromBits<T>(512),
_ => 0
};
case 1:
return id.MaskByte[1] switch
{
1 => PanelSideFromBits<T>(128),
2 => PanelSideFromBits<T>(256),
3 => PanelSideFromBits<T>(512),
_ => throw new NotSupportedException("Unsupported SIMD level."),
};
default:
throw new NotSupportedException("Current hardware platform is not supported");
}
}
}
@@ -154,7 +176,10 @@ namespace StgSharp.HighPerformance
internal T Head; // first element of the panel buffer
public ref T this[int column, int row]
public ref T this[
int column,
int row
]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => ref Unsafe.Add(ref Head, row * MatrixPanel.ElementSideLength<T>() + column);
StgSharp.Common/Internal/Intrinsic/SIMDID.cs
+51
View File
@@ -0,0 +1,51 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SIMDID"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Internal.Intrinsic
{
[StructLayout(LayoutKind.Explicit)]
internal unsafe struct SIMDID
{
[FieldOffset(0)]public fixed byte MaskByte[16];
[FieldOffset(0)]public ulong Mask;
public SIMDID()
{
Unsafe.SkipInit(out this);
}
}
}
StgSharp.Common/Internal/Intrinsic/intrinsicContextProvider.cs
+11 -18
View File
@@ -39,7 +39,7 @@ namespace StgSharp.Internal
private static readonly IntrinsicContext _intrinsicContext = new();
public static IntrinsicLevel IntrinsicLevel { get; set; }
public static SIMDID IntrinsicMask { get; set; }
internal static IntrinsicContext Context
{
@@ -49,39 +49,32 @@ namespace StgSharp.Internal
internal static void InitIntrinsicContext()
{
int statCode;
ulong statCode = 0;
fixed (IntrinsicContext* cPtr = &_intrinsicContext) {
statCode = LoadIntrinsicFunction(cPtr);
}
switch (statCode)
SIMDID id = new()
{
Mask = statCode
};
switch (id.MaskByte[1])
{
case 0:
Dialogue.PostError(new PlatformNotSupportedException("No SIMD instructions supported on current hardware platform"));
Environment.Exit(-1);
break;
default:
IntrinsicLevel = (IntrinsicLevel)statCode;
break;
}
IntrinsicMask = id;
}
[LibraryImport(Native.LibName, EntryPoint = "load_intrinsic_function")]
[UnmanagedCallConv(CallConvs =[ typeof(CallConvCdecl) ])]
private static partial int LoadIntrinsicFunction(IntrinsicContext* context);
}
internal enum IntrinsicLevel : sbyte
{
Non = 0,
SSE = 1,
AVX2 = 2,
AVX512 = 3,
NEON = -1
private static partial ulong LoadIntrinsicFunction(
IntrinsicContext* context
);
}
}
StgSharp.Common/Mathematics/Linear/Matrix.cs
+2 -5
View File
@@ -178,12 +178,9 @@ namespace StgSharp.Mathematics.Numeric
)
{
// Caller guarantees: indices in range, layout is dense square.
int k_c = y >> 2;
int k_r = x >> 2;
MatrixKernelEnumerator enumerator = new(_size.KernelColumnLength, _size.KernelRowLength,
k_c, k_r);
return Buffer + _boarder.GetIndexOffset(k_c, k_r, in enumerator);
x, y);
return Buffer + _boarder.GetIndexOffset(x, y, in enumerator);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
StgSharp.Common/Mathematics/Scalar/Random.cs
+4 -8
View File
@@ -1,4 +1,4 @@
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="Random"
// Project: StgSharp
@@ -88,7 +88,7 @@ namespace StgSharp.Mathematics
///
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static T Random<T>() where T: unmanaged, IConvertible, INumber<T>
public static T Random<T>() where T : unmanaged, IConvertible, INumber<T>
{
if (randomSeed == null)
@@ -97,13 +97,9 @@ namespace StgSharp.Mathematics
string originSeed =
($"{((Assembly.GetEntryAssembly() == null) ? (Assembly.GetAssembly(typeof(World))!) :
Assembly.GetEntryAssembly())}") + ($"{DateTime.UtcNow.ToString("o")}");
using (SHA256 sha = SHA256.Create()) {
randomSeed = sha.ComputeHash(Encoding.UTF8.GetBytes(originSeed));
}
}
using (SHA256 sha = SHA256.Create()) {
randomSeed = sha.ComputeHash(randomSeed);
randomSeed = SHA256.HashData(Encoding.UTF8.GetBytes(originSeed));
}
randomSeed = SHA256.HashData(randomSeed);
fixed (byte* bptr = randomSeed) {
return *(T*)bptr;
}
StgSharp.Mathematics/Numeric/MatrixCompute.Float.cs
+17 -6
View File
@@ -142,9 +142,13 @@ namespace StgSharp.Mathematics.Numeric
{
throw new IndexOutOfRangeException("Matrix dimensions must agree.");
}
if (left.Layout != MatrixLayout.DenseRectangle ||
right.Layout != MatrixLayout.DenseRectangle ||
ans.Layout != MatrixLayout.DenseRectangle) {
throw new NotSupportedException("Only dense rectangle matrices are supported.");
}
long count = (long)left.KernelColumnLength * left.KernelRowLength;
// int size = MatrixPanel.KernelSideCount<float>();
if (count <= 1024)
{
MatrixParallelThread[] pool = null;
@@ -155,15 +159,22 @@ namespace StgSharp.Mathematics.Numeric
{
for (int j = 0; j < ans.KernelColumnLength; j++)
{
MatrixKernel<float>* ansPanel = ans.GetKernelAddressUnsafe(i, j);
NativeIntrinsic.Context.mat[f32].clear_panel((MatrixPanel*)ansPanel);
MatrixKernel<float>* ansPanel = GetKernelAddressUnsafe(ans.Buffer,
ans.KernelColumnLength,
i, j);
// Console.WriteLine($"{i}{j}");
// NativeIntrinsic.Context.mat[f32].clear_panel((MatrixPanel*)ansPanel);
for (int k = 0; k < left.KernelColumnLength; k++)
{
NativeIntrinsic.Context
.mat[f32].kernel_fma((MatrixPanel*)left.GetKernelAddressUnsafe(i, k),
(MatrixPanel*)right.GetKernelAddressUnsafe(k, j),
.mat[f32].kernel_fma((MatrixPanel*)GetKernelAddressUnsafe(left.Buffer,
left.KernelColumnLength,
i,
k),
(MatrixPanel*)GetKernelAddressUnsafe(right.Buffer,
right.KernelColumnLength,
k,
j),
(MatrixPanel*)ansPanel);
}
}
StgSharp.Mathematics/Numeric/MatrixCompute.cs
+3 -6
View File
@@ -45,7 +45,6 @@ namespace StgSharp.Mathematics.Numeric
)
{
int f32 = (int)MatrixElementType.F32;
int size = MatrixPanel.KernelSideCount<float>();
MatrixKernel* leftBuffer = (MatrixKernel*)p->Left;
MatrixKernel* rightBuffer = (MatrixKernel*)p->Right;
MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer;
@@ -62,17 +61,15 @@ namespace StgSharp.Mathematics.Numeric
{
int i = (int)(c / ansWidth);
int j = (int)(c % ansWidth);
MatrixKernel<float>* ansPanel = GetKernelAddressUnsafe((MatrixKernel<float>*)ansBuffer,
ansHeight, i, j);
NativeIntrinsic.Context.mat[f32].clear_panel((MatrixPanel*)ansPanel);
for (int k = 0; k < commonK; k += size) {
for (int k = 0; k < commonK; k++) {
NativeIntrinsic.Context
.mat[f32].kernel_fma(
(MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel<float>*)leftBuffer,
ansHeight, k, j),
(MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel<float>*)rightBuffer,
commonK, j, k),
(MatrixPanel*)ansPanel);
(MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel<float>*)ansBuffer,
ansHeight, i, j));
}
}
}
StgSharp.Native/sn-src/cpuid.x86.c
+1 -1
View File
@@ -78,7 +78,7 @@ static int is_intel_native_avx512(int family, int model)
SN_API SIMDID SN_DECL sn_get_simd_level()
{
if (!global_simdid.mask) {
if (global_simdid.mask) {
return global_simdid;
}
StgSharp.Native/sn-src/m.fma.x86.c
+153 -54
View File
@@ -7,8 +7,8 @@
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
// ans = left * right + ans
// right is not transposed
// float
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
{
const __m128 a0 = _mm_load_ps((float *)&left->f32_x[0]);
@@ -57,56 +57,18 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
ans->f32_x[3] = c3;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
{
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
//group 0
register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0));
register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0));
c01 = _mm256_fmadd_ps(a01, b01_perm, c01);
c23 = _mm256_fmadd_ps(a23, b23_perm, c23);
//group 2
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2));
c01 = _mm256_fmadd_ps(a23, b01_perm, c01);
c23 = _mm256_fmadd_ps(a01, b23_perm, c23);
//group 1
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
register __m256 a03 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
c01 = _mm256_fmadd_ps(a12, b01_perm, c01);
c23 = _mm256_fmadd_ps(a03, b23_perm, c23);
//group 3
a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
a03 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
c01 = _mm256_fmadd_ps(a03, b01_perm, c01);
c23 = _mm256_fmadd_ps(a12, b23_perm, c23);
ans->f32_y[0] = c01;
ans->f32_y[1] = c23;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
{
static const __m256i b01_shift = { 0, 1, 2, 3, 5, 6, 7, 4 };
static const __m256i b23_shift = { 2, 3, 0, 1, 7, 4, 5, 6 };
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
b01 = _mm256_permutevar_ps(b01, b01_shift);
b23 = _mm256_permutevar_ps(b23, b23_shift);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
@@ -125,49 +87,186 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
//group 1
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
register __m256 a03 = _mm256_permute2f128_ps(a01, a23, 0x30);
register __m256 a30 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
c01 = _mm256_add_ps(_mm256_mul_ps(a12, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a03, b23_perm), c23);
c23 = _mm256_add_ps(_mm256_mul_ps(a30, b23_perm), c23);
//group 3
a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
a03 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
c01 = _mm256_add_ps(_mm256_mul_ps(a03, b01_perm), c01);
c01 = _mm256_add_ps(_mm256_mul_ps(a30, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a12, b23_perm), c23);
ans->f32_y[0] = c01;
ans->f32_y[1] = c23;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
{
static const __m256i b01_shift = { 0, 1, 2, 3, 5, 6, 7, 4 };
static const __m256i b23_shift = { 2, 3, 0, 1, 7, 4, 5, 6 };
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
b01 = _mm256_permutevar_ps(b01, b01_shift);
b23 = _mm256_permutevar_ps(b23, b23_shift);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
//group 0
register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0));
register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0));
c01 = _mm256_fmadd_ps(a01, b01_perm, c01);
c23 = _mm256_fmadd_ps(a23, b23_perm, c23);
//group 2
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2));
c01 = _mm256_fmadd_ps(a23, b01_perm, c01);
c23 = _mm256_fmadd_ps(a01, b23_perm, c23);
//group 1
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
register __m256 a30 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
c01 = _mm256_fmadd_ps(a12, b01_perm, c01);
c23 = _mm256_fmadd_ps(a30, b23_perm, c23);
//group 3
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
c01 = _mm256_fmadd_ps(a30, b01_perm, c01);
c23 = _mm256_fmadd_ps(a12, b23_perm, c23);
ans->f32_y[0] = c01;
ans->f32_y[1] = c23;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
{
static const __m512i b_shift = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9, 15, 12, 13, 14 };
register __m512 a = _mm512_load_ps((float *)left);
register __m512 b = _mm512_load_ps((float *)right);
register __m512 c = _mm512_load_ps((float *)ans);
b = _mm512_permutexvar_ps(b_shift, b);
register __m512 a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0));
register __m512 b_perm = _mm512_permute_ps(b, _FMA_SELECT(0));
c = _mm512_fmadd_ps(a, b_perm, c);
c = _mm512_fmadd_ps(a_perm, b_perm, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(1));
c = _mm512_fmadd_ps(a, b_perm, c);
c = _mm512_fmadd_ps(a_perm, b_perm, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(2));
c = _mm512_fmadd_ps(a, b_perm, c);
c = _mm512_fmadd_ps(a_perm, b_perm, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(3));
c = _mm512_fmadd_ps(a, b_perm, c);
c = _mm512_fmadd_ps(a_perm, b_perm, c);
_mm512_store_ps((float *)ans, c);
}
// double
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma)
{
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]);
__m256d b0 = _mm256_load_pd((double *)&right->f64_y[0]);
__m256d b1 = _mm256_load_pd((double *)&right->f64_y[1]);
__m256d b2 = _mm256_load_pd((double *)&right->f64_y[2]);
__m256d b3 = _mm256_load_pd((double *)&right->f64_y[3]);
register __m256d ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[0]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b0, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b0, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b0, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b0, _FMA_SELECT(3))));
ans->f64_y[0] = ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[1]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b1, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b1, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b1, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b1, _FMA_SELECT(3))));
ans->f64_y[0] = ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[2]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b2, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b2, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b2, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b2, _FMA_SELECT(3))));
ans->f64_y[0] = ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[3]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b3, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b3, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b3, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b3, _FMA_SELECT(3))));
ans->f64_y[0] = ccol;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma)
{
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]);
__m256d b0 = _mm256_load_pd((double *)&right->f64_y[0]);
__m256d b1 = _mm256_load_pd((double *)&right->f64_y[1]);
__m256d b2 = _mm256_load_pd((double *)&right->f64_y[2]);
__m256d b3 = _mm256_load_pd((double *)&right->f64_y[3]);
register __m256d ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[0]);
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b0, _FMA_SELECT(0)), ccol);
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b0, _FMA_SELECT(1)), ccol);
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b0, _FMA_SELECT(2)), ccol);
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b0, _FMA_SELECT(3)), ccol);
ans->f64_y[0] = ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[1]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b1, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b1, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b1, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b1, _FMA_SELECT(3))));
ans->f64_y[0] = ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[2]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b2, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b2, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b2, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b2, _FMA_SELECT(3))));
ans->f64_y[0] = ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[3]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b3, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b3, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b3, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b3, _FMA_SELECT(3))));
ans->f64_y[0] = ccol;
}
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
{
for (int col = 0; col < 4; ++col) {
StgSharpDebug/Program.cs
+7 -13
View File
@@ -66,19 +66,13 @@ namespace StgSharp
hlsf);
MatrixCompute.Fill(p, 0.0f);
/*
for (int i = 0; i < 10; i++)
/**/
for (int j = 0; j < 20; j++)
{
MatrixCompute.Add(n, m, n);
for (int j = 0; j < 5; j++)
{
for (int k = 0; k < 5; k++) {
Console.Write($"{n[j, k]:0.0E+00} \t");
}
Console.WriteLine();
for (int k = 0; k < 10; k++) {
Console.Write($"{m[j, k]:0.0E+00}\t");
}
Console.WriteLine($"{i}");
GC.Collect();
Console.WriteLine();
}
/**/
@@ -86,8 +80,8 @@ namespace StgSharp
MatrixCompute.Mul(n, m, p);
for (int i = 0; i < 20; i++)
{
for (int j = 0; j < 20; j++) {
Console.Write($"{p[i, j]:0.0E+00} \t");
for (int j = 0; j < 10; j++) {
Console.Write($"{p[i, j]:0.0E+00}\t");
}
Console.WriteLine();
}