Better FMA

This commit is contained in:
Youmu Liu
2026-03-05 18:04:34 +08:00
parent dfa6f4ba92
commit 83997f64d0
21 changed files with 893 additions and 1030 deletions
@@ -189,9 +189,9 @@ namespace StgSharp.Mathematics.Graphics
{
const int f32 = 0;
fixed (MatrixKernel<float>* source = &this.kernel, target = &transpose.kernel) {
MathematicsModule.GlobalContext.mat_sk[f32].Call(MatrixIntrinsicHandle.Transpose,
null, (MatrixKernel*)source,
(MatrixKernel*)target, 0);
NumericalModule.GlobalContext.mat_sk[f32].Call(MatrixIntrinsicHandle.Transpose,
null, (MatrixKernel*)source,
(MatrixKernel*)target, 0);
}
}
@@ -203,10 +203,10 @@ namespace StgSharp.Mathematics.Graphics
{
Matrix44 ret = new();
int f32 = MatrixElementType.F32.IntrinsicNode;
MathematicsModule.GlobalContext.mat_sk[f32].Call(MatrixIntrinsicHandle.Sub,
(MatrixKernel*)&left,
(MatrixKernel*)&right,
(MatrixKernel*)&ret, 1);
NumericalModule.GlobalContext.mat_sk[f32].Call(MatrixIntrinsicHandle.Sub,
(MatrixKernel*)&left,
(MatrixKernel*)&right,
(MatrixKernel*)&ret, 1);
return ret;
}
@@ -237,10 +237,10 @@ namespace StgSharp.Mathematics.Graphics
{
Matrix44 ret = new();
int f32 = MatrixElementType.F32.IntrinsicNode;
MathematicsModule.GlobalContext.mat_sk[f32].Call(MatrixIntrinsicHandle.Fma,
(MatrixKernel*)&left,
(MatrixKernel*)&right,
(MatrixKernel*)&ret, 0);
NumericalModule.GlobalContext.mat_sk[f32].Call(MatrixIntrinsicHandle.Fma,
(MatrixKernel*)&left,
(MatrixKernel*)&right,
(MatrixKernel*)&ret, 0);
return ret;
}
@@ -252,10 +252,10 @@ namespace StgSharp.Mathematics.Graphics
{
Matrix44 ret = new();
int f32 = MatrixElementType.F32.IntrinsicNode;
MathematicsModule.GlobalContext.mat_sk[f32].Call(MatrixIntrinsicHandle.Add,
(MatrixKernel*)&left,
(MatrixKernel*)&right,
(MatrixKernel*)&ret, 1);
NumericalModule.GlobalContext.mat_sk[f32].Call(MatrixIntrinsicHandle.Add,
(MatrixKernel*)&left,
(MatrixKernel*)&right,
(MatrixKernel*)&ret, 1);
return ret;
}
@@ -77,7 +77,7 @@ namespace StgSharp.Mathematics.Numeric
ulong* pivots = stackalloc ulong[4];
for (int kBlock = 0; kBlock < n; kBlock += 4)
{
MathematicsModule.GlobalContext
NumericalModule.GlobalContext
.mat_sk[(int)MatrixElementType.F32].pivot(matrix.Buffer,
(ulong)(kBlock >> 2),
(ulong)(matrix.KernelRowLength - (kBlock >> 2)),
@@ -49,7 +49,7 @@ namespace StgSharp.Mathematics.Numeric
Matrix<float> ans
)
{
int f32 = (int)MatrixElementType.F32;
MatrixElementType f32 = MatrixElementType.F32;
if (left.ColumnLength != right.ColumnLength ||
left.RowLength != right.RowLength ||
left.ColumnLength != ans.ColumnLength ||
@@ -62,8 +62,8 @@ namespace StgSharp.Mathematics.Numeric
MatrixKernel<float>* leftPtr = left.Buffer;
MatrixKernel<float>* rightPtr = right.Buffer;
MatrixKernel<float>* ansPtr = ans.Buffer;
MathematicsModule.GlobalContext.mat_sk[f32].buffer_add(leftPtr, rightPtr, ansPtr,
count);
NumericalModule.GlobalContext.mat_mk[f32].buffer_add(leftPtr, rightPtr, ansPtr,
count);
return;
} else if (count <= 1024)
@@ -73,21 +73,21 @@ namespace StgSharp.Mathematics.Numeric
MatrixKernel<float>* leftPtr = left.Buffer;
MatrixKernel<float>* rightPtr = right.Buffer;
MatrixKernel<float>* ansPtr = ans.Buffer;
MathematicsModule.GlobalContext.mat_sk[f32].buffer_add(leftPtr, rightPtr, ansPtr,
count);
NumericalModule.GlobalContext.mat_mk[f32].buffer_add(leftPtr, rightPtr, ansPtr,
count);
MatrixParallel.ReturnParallelResources(ref pool);
return;
}
MatrixParallelTask* package = MatrixParallelFactory.CreateBaseTask();
package->Left = (nint)left.Buffer;
package->Right = (nint)right.Buffer;
package->Answer = (nint)ans.Buffer;
package->Mat1 = (MatrixKernel*)left.Buffer;
package->Mat2 = (MatrixKernel*)right.Buffer;
package->Mat3 = (MatrixKernel*)ans.Buffer;
package->ElementSize = sizeof(float);
package->PrimCount = ans.KernelColumnLength;
package->SecCount = ans.KernelRowLength;
package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.LEFT_RIGHT_ANS_PARAM);
package->ComputeHandle = (IntPtr)MathematicsModule.GlobalContext.mat_sk[f32].buffer_add;
package->ComputeHandle = (IntPtr)NumericalModule.GlobalContext.mat_sk[f32].buffer_add;
MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package);
taskGroup.LaunchParallel(SleepMode.DeepSleep);
@@ -100,18 +100,18 @@ namespace StgSharp.Mathematics.Numeric
float value
)
{
int f32 = (int)MatrixElementType.F32;
MatrixElementType f32 = MatrixElementType.F32;
long count = (long)ans.KernelColumnLength * ans.KernelRowLength;
ScalarPacket* scalar = MatrixParallelFactory.CreateScalarPacket();
scalar->Data<float>(0) = value;
if (count <= 256)
{
MathematicsModule.GlobalContext.mat_sk[f32].buffer_fill(ans.Buffer, scalar, count);
NumericalModule.GlobalContext.mat_sk[f32].buffer_fill(ans.Buffer, scalar, count);
return;
} else if (count <= 1024)
{
MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1);
MathematicsModule.GlobalContext.mat_sk[f32].buffer_fill(ans.Buffer, scalar, count);
NumericalModule.GlobalContext.mat_sk[f32].buffer_fill(ans.Buffer, scalar, count);
MatrixParallel.ReturnParallelResources(ref pool);
return;
}
@@ -121,7 +121,7 @@ namespace StgSharp.Mathematics.Numeric
package->PrimCount = ans.KernelColumnLength;
package->SecCount = ans.KernelRowLength;
package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.ANS_SCALAR_PARAM);
package->ComputeHandle = (IntPtr)MathematicsModule.GlobalContext.mat_sk[f32].buffer_fill;
package->ComputeHandle = (IntPtr)NumericalModule.GlobalContext.mat_sk[f32].buffer_fill;
package->Scalar = scalar;
MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package);
taskGroup.LaunchParallel(SleepMode.DeepSleep);
@@ -135,7 +135,7 @@ namespace StgSharp.Mathematics.Numeric
Matrix<float> ans
)
{
int f32 = (int)MatrixElementType.F32;
MatrixElementType f32 = MatrixElementType.F32;
// For matrix multiplication (left: MxK) * (right: KxN) => ans: MxN
// ColumnLength = number of columns (N), RowLength = number of rows (M)
@@ -168,11 +168,11 @@ namespace StgSharp.Mathematics.Numeric
enumeration->Member<int>(0) = i;
enumeration->Member<int>(1) = j;
MathematicsModule.GlobalContext
NumericalModule.GlobalContext
.mat_sk[f32].kernel_tile_fma((MatrixKernel*)left.Buffer,
(MatrixKernel*)right.Buffer,
(MatrixKernel*)ans.Buffer,
enumeration);
(MatrixKernel*)right.Buffer,
(MatrixKernel*)ans.Buffer,
enumeration);
}
}
@@ -205,7 +205,7 @@ namespace StgSharp.Mathematics.Numeric
Matrix<float> ans
)
{
int f32 = (int)MatrixElementType.F32;
MatrixElementType f32 = MatrixElementType.F32;
if (left.ColumnLength != ans.ColumnLength || left.RowLength != ans.RowLength) {
throw new IndexOutOfRangeException("Matrix dimensions must agree.");
}
@@ -220,8 +220,8 @@ namespace StgSharp.Mathematics.Numeric
{
MatrixKernel<float>* leftPtr = left.Buffer;
MatrixKernel<float>* ansPtr = ans.Buffer;
MathematicsModule.GlobalContext.mat_sk[f32].buffer_scalar_mul(leftPtr, ansPtr, scalar,
count);
NumericalModule.GlobalContext.mat_sk[f32].buffer_scalar_mul(leftPtr, ansPtr, scalar,
count);
return;
} else if (count <= 1024)
@@ -230,8 +230,8 @@ namespace StgSharp.Mathematics.Numeric
MatrixKernel<float>* leftPtr = left.Buffer;
MatrixKernel<float>* ansPtr = ans.Buffer;
MathematicsModule.GlobalContext.mat_sk[f32].buffer_scalar_mul(leftPtr, ansPtr, scalar,
count);
NumericalModule.GlobalContext.mat_sk[f32].buffer_scalar_mul(leftPtr, ansPtr, scalar,
count);
MatrixParallel.ReturnParallelResources(ref pool);
return;
@@ -244,7 +244,7 @@ namespace StgSharp.Mathematics.Numeric
package->SecCount = left.KernelRowLength;
package->Scalar = scalar;
package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM);
package->ComputeHandle = (IntPtr)MathematicsModule.GlobalContext.mat_sk[f32].buffer_scalar_mul;
package->ComputeHandle = (IntPtr)NumericalModule.GlobalContext.mat_sk[f32].buffer_scalar_mul;
MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package);
taskGroup.LaunchParallel(SleepMode.DeepSleep);
@@ -259,7 +259,7 @@ namespace StgSharp.Mathematics.Numeric
Matrix<float> ans
)
{
int f32 = (int)MatrixElementType.F32;
MatrixElementType f32 = MatrixElementType.F32;
if (left.ColumnLength != right.ColumnLength ||
left.RowLength != right.RowLength ||
left.ColumnLength != ans.ColumnLength ||
@@ -277,8 +277,8 @@ namespace StgSharp.Mathematics.Numeric
MatrixKernel<float>* leftPtr = left.Buffer;
MatrixKernel<float>* rightPtr = right.Buffer;
MatrixKernel<float>* ansPtr = ans.Buffer;
MathematicsModule.GlobalContext.mat_sk[f32].buffer_sub(leftPtr, rightPtr, ansPtr,
count);
NumericalModule.GlobalContext.mat_sk[f32].buffer_sub(leftPtr, rightPtr, ansPtr,
count);
return;
} else if (count <= 1024)
@@ -288,8 +288,8 @@ namespace StgSharp.Mathematics.Numeric
MatrixKernel<float>* leftPtr = left.Buffer;
MatrixKernel<float>* rightPtr = right.Buffer;
MatrixKernel<float>* ansPtr = ans.Buffer;
MathematicsModule.GlobalContext.mat_sk[f32].buffer_sub(leftPtr, rightPtr, ansPtr,
count);
NumericalModule.GlobalContext.mat_sk[f32].buffer_sub(leftPtr, rightPtr, ansPtr,
count);
MatrixParallel.ReturnParallelResources(ref pool);
return;
@@ -302,7 +302,7 @@ namespace StgSharp.Mathematics.Numeric
package->PrimCount = left.KernelColumnLength;
package->SecCount = left.KernelRowLength;
package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.LEFT_RIGHT_ANS_PARAM);
package->ComputeHandle = (IntPtr)MathematicsModule.GlobalContext.mat_sk[f32].buffer_sub;
package->ComputeHandle = (IntPtr)NumericalModule.GlobalContext.mat_sk[f32].buffer_sub;
MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package);
taskGroup.LaunchParallel(SleepMode.DeepSleep);
@@ -58,7 +58,7 @@ namespace StgSharp.Mathematics.Numeric
MatrixKernel<double>* leftPtr = left.Buffer;
MatrixKernel<double>* rightPtr = right.Buffer;
MatrixKernel<double>* ansPtr = ans.Buffer;
MathematicsModule.GlobalContext.mat_sk[f64].buffer_add(leftPtr, rightPtr, ansPtr,
NumericalModule.GlobalContext.mat_sk[f64].buffer_add(leftPtr, rightPtr, ansPtr,
count);
return;
@@ -69,7 +69,7 @@ namespace StgSharp.Mathematics.Numeric
MatrixKernel<double>* leftPtr = left.Buffer;
MatrixKernel<double>* rightPtr = right.Buffer;
MatrixKernel<double>* ansPtr = ans.Buffer;
MathematicsModule.GlobalContext.mat_sk[f64].buffer_add(leftPtr, rightPtr, ansPtr,
NumericalModule.GlobalContext.mat_sk[f64].buffer_add(leftPtr, rightPtr, ansPtr,
count);
MatrixParallel.ReturnParallelResources(ref pool);
@@ -83,7 +83,7 @@ namespace StgSharp.Mathematics.Numeric
package->PrimCount = ans.KernelColumnLength;
package->SecCount = ans.KernelRowLength;
package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.LEFT_RIGHT_ANS_PARAM);
package->ComputeHandle = (IntPtr)MathematicsModule.GlobalContext.mat_sk[f64].buffer_add;
package->ComputeHandle = (IntPtr)NumericalModule.GlobalContext.mat_sk[f64].buffer_add;
MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package);
taskGroup.LaunchParallel(SleepMode.DeepSleep);
@@ -102,12 +102,12 @@ namespace StgSharp.Mathematics.Numeric
scalar->Data<double>(0) = value;
if (count <= 256)
{
MathematicsModule.GlobalContext.mat_sk[f64].buffer_fill(ans.Buffer, scalar, count);
NumericalModule.GlobalContext.mat_sk[f64].buffer_fill(ans.Buffer, scalar, count);
return;
} else if (count <= 1024)
{
MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1);
MathematicsModule.GlobalContext.mat_sk[f64].buffer_fill(ans.Buffer, scalar, count);
NumericalModule.GlobalContext.mat_sk[f64].buffer_fill(ans.Buffer, scalar, count);
MatrixParallel.ReturnParallelResources(ref pool);
return;
}
@@ -117,7 +117,7 @@ namespace StgSharp.Mathematics.Numeric
package->PrimCount = ans.KernelColumnLength;
package->SecCount = ans.KernelRowLength;
package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.ANS_SCALAR_PARAM);
package->ComputeHandle = (IntPtr)MathematicsModule.GlobalContext.mat_sk[f64].buffer_fill;
package->ComputeHandle = (IntPtr)NumericalModule.GlobalContext.mat_sk[f64].buffer_fill;
package->Scalar = scalar;
MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package);
taskGroup.LaunchParallel(SleepMode.DeepSleep);
@@ -158,7 +158,7 @@ namespace StgSharp.Mathematics.Numeric
enumeration->Member<int>(0) = i;
enumeration->Member<int>(1) = j;
MathematicsModule.GlobalContext
NumericalModule.GlobalContext
.mat_sk[f64].kernel_tile_fma((MatrixKernel*)left.Buffer,
(MatrixKernel*)right.Buffer,
(MatrixKernel*)ans.Buffer,
@@ -209,7 +209,7 @@ namespace StgSharp.Mathematics.Numeric
{
MatrixKernel<double>* leftPtr = left.Buffer;
MatrixKernel<double>* ansPtr = ans.Buffer;
MathematicsModule.GlobalContext.mat_sk[f64].buffer_scalar_mul(leftPtr, ansPtr, scalar,
NumericalModule.GlobalContext.mat_sk[f64].buffer_scalar_mul(leftPtr, ansPtr, scalar,
count);
return;
@@ -219,7 +219,7 @@ namespace StgSharp.Mathematics.Numeric
MatrixKernel<double>* leftPtr = left.Buffer;
MatrixKernel<double>* ansPtr = ans.Buffer;
MathematicsModule.GlobalContext.mat_sk[f64].buffer_scalar_mul(leftPtr, ansPtr, scalar,
NumericalModule.GlobalContext.mat_sk[f64].buffer_scalar_mul(leftPtr, ansPtr, scalar,
count);
MatrixParallel.ReturnParallelResources(ref pool);
@@ -233,7 +233,7 @@ namespace StgSharp.Mathematics.Numeric
package->SecCount = left.KernelRowLength;
package->Scalar = scalar;
package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM);
package->ComputeHandle = (IntPtr)MathematicsModule.GlobalContext.mat_sk[f64].buffer_scalar_mul;
package->ComputeHandle = (IntPtr)NumericalModule.GlobalContext.mat_sk[f64].buffer_scalar_mul;
MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package);
taskGroup.LaunchParallel(SleepMode.DeepSleep);
@@ -266,7 +266,7 @@ namespace StgSharp.Mathematics.Numeric
MatrixKernel<double>* leftPtr = left.Buffer;
MatrixKernel<double>* rightPtr = right.Buffer;
MatrixKernel<double>* ansPtr = ans.Buffer;
MathematicsModule.GlobalContext.mat_sk[f64].buffer_sub(leftPtr, rightPtr, ansPtr,
NumericalModule.GlobalContext.mat_sk[f64].buffer_sub(leftPtr, rightPtr, ansPtr,
count);
return;
@@ -277,7 +277,7 @@ namespace StgSharp.Mathematics.Numeric
MatrixKernel<double>* leftPtr = left.Buffer;
MatrixKernel<double>* rightPtr = right.Buffer;
MatrixKernel<double>* ansPtr = ans.Buffer;
MathematicsModule.GlobalContext.mat_sk[f64].buffer_sub(leftPtr, rightPtr, ansPtr,
NumericalModule.GlobalContext.mat_sk[f64].buffer_sub(leftPtr, rightPtr, ansPtr,
count);
MatrixParallel.ReturnParallelResources(ref pool);
@@ -291,7 +291,7 @@ namespace StgSharp.Mathematics.Numeric
package->PrimCount = left.KernelColumnLength;
package->SecCount = left.KernelRowLength;
package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.LEFT_RIGHT_ANS_PARAM);
package->ComputeHandle = (IntPtr)MathematicsModule.GlobalContext.mat_sk[f64].buffer_sub;
package->ComputeHandle = (IntPtr)NumericalModule.GlobalContext.mat_sk[f64].buffer_sub;
MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package);
taskGroup.LaunchParallel(SleepMode.DeepSleep);
@@ -1,4 +1,4 @@
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="NumericalModule"
// Project: StgSharp
@@ -25,9 +25,11 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Mathematics.Internal;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
@@ -36,12 +38,53 @@ namespace StgSharp.Mathematics.Numeric
public class NumericalModule : IStaticModule
{
public static unsafe int DefaultSIMDAlignment
{
get
{
SIMDID id = GlobalIntrinsicMask;
return (id.MaskByte[0] & 0b_00001111) switch
{
1 => id.MaskByte[1] switch
{
1 => 16,
2 => 32,
3 => 64,
4 => 64,
5 => 64,
_ => 8,
},
_ => 8,
};
}
}
public string ModuleName => "Math.Numerical";
internal static IntrinsicContext GlobalContext
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get;
} = new();
internal static SIMDID GlobalIntrinsicMask { get; set; }
public void InitializeModule()
{
throw new NotImplementedException();
}
public void InitializeModule(
IModuleInitializeProfile profile
)
{
MatrixParallel.Init();
}
public void UninitializeModule()
{
throw new NotImplementedException();
}
}
}
@@ -33,11 +33,12 @@ using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StgSharp.Mathematics.Numeric
{
public unsafe class MatrixParallelQueue
internal unsafe class MatrixParallelQueue
{
private static ConcurrentStack<MatrixParallelQueue> _recycle = [];
@@ -57,7 +58,7 @@ namespace StgSharp.Mathematics.Numeric
queue = new MatrixParallelQueue();
}
queue._package = pack;
queue._maxCount = (ulong)pack->PrimCount * (ulong)pack->SecCount;
queue._maxCount = (ulong)pack->CommonX * (ulong)pack->CommonY;
queue._cursor = 0;
return queue;
}
@@ -65,7 +65,7 @@ namespace StgSharp.Mathematics.Numeric
int j = (int)(c % ansHeight);
enumeration->Member<int>(0) = i;
enumeration->Member<int>(1) = j;
MathematicsModule.GlobalContext.mat_sk[f32].kernel_tile_fma(leftBuffer, rightBuffer,
NumericalModule.GlobalContext.mat_sk[f32].kernel_tile_fma(leftBuffer, rightBuffer,
ansBuffer, enumeration);
}
}
@@ -97,7 +97,7 @@ namespace StgSharp.Mathematics.Numeric
int j = (int)(c % ansHeight);
enumeration->Member<int>(0) = i;
enumeration->Member<int>(1) = j;
MathematicsModule.GlobalContext.mat_sk[f64].kernel_tile_fma(leftBuffer, rightBuffer,
NumericalModule.GlobalContext.mat_sk[f64].kernel_tile_fma(leftBuffer, rightBuffer,
ansBuffer, enumeration);
}
}
@@ -63,8 +63,6 @@ namespace StgSharp.Mathematics.Numeric
_count = threadArray.Length;
}
public MatrixParallelQueue TaskQueue { get; set; }
public ReadOnlySpan<MatrixParallelThread> Threads
{
get
@@ -77,6 +75,8 @@ namespace StgSharp.Mathematics.Numeric
public ThreadPriority AfterWork { get; private set; }
internal MatrixParallelQueue TaskQueue { get; set; }
private bool IsMultithread { get; init; }
public void Dispose()
@@ -54,7 +54,7 @@ namespace StgSharp.Mathematics
return factorialResult[n];
}
if (n < 30) {
return MathematicsModule.GlobalContext.factorial_simd(n);
return NumericalModule.GlobalContext.factorial_simd(n);
}
double
dn = n,