youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Daily update on 20260111-005714 by Nitload Youmu. Additional Message : Use optimized tile operation to accelerate matrix mul, still has problem

Browse Source 
Signed-off-by: Nitload <YoumuLiu_NSI@whv0t.onmicrosoft.com>
This commit is contained in:
Nitload
2026-01-11 00:57:14 +08:00
parent 6827259891
commit 4be5c1c7c9
9 changed files with 141 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/M128.cs
+6 -6
View File
@@ -1,9 +1,9 @@
//----------------------------------------------------------------------- //-----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="M128" // file="M128"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload // AuthorGroup: Nitload Space
// Copyright (c) Nitload. All rights reserved. // Copyright (c) Nitload Space. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -58,7 +58,7 @@ namespace StgSharp.HighPerformance
Vec = v; Vec = v;
} }
public ref T AsRef<T>() where T : unmanaged, INumber<T> public ref T AsRef<T>() where T: unmanaged, INumber<T>
{ {
return ref Unsafe.As<byte, T>(ref Buffer[0]); return ref Unsafe.As<byte, T>(ref Buffer[0]);
} }
@@ -69,7 +69,7 @@ namespace StgSharp.HighPerformance
return Vec.AsVector4(); return Vec.AsVector4();
} }
public void BroadCastFrom<T>(T value) where T : unmanaged, INumber<T> public void BroadCastFrom<T>(T value) where T: unmanaged, INumber<T>
{ {
Vec = Vector128.Create<T>(value).AsSingle(); Vec = Vector128.Create<T>(value).AsSingle();
} }
@@ -85,7 +85,7 @@ namespace StgSharp.HighPerformance
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public ref T Member<T>(int index) where T : unmanaged, INumber<T> public ref T Member<T>(int index) where T: unmanaged, INumber<T>
{ {
return ref Unsafe.Add(ref AsRef<T>(), index); return ref Unsafe.Add(ref AsRef<T>(), index);
} }
StgSharp.Common/HighPerformance/MatrixPanel.cs
+22 -30
View File
@@ -1,9 +1,9 @@
//----------------------------------------------------------------------- //-----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="MatrixPanel" // file="MatrixPanel"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload // AuthorGroup: Nitload Space
// Copyright (c) Nitload. All rights reserved. // Copyright (c) Nitload Space. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -41,41 +41,37 @@ namespace StgSharp.HighPerformance
private byte Head; private byte Head;
// Allocate aligned memory for one panel; caller must free via Destroy. // Allocate aligned memory for one panel; caller must free via Destroy.
public static unsafe MatrixPanel<T>* Create<T>() where T : unmanaged, INumber<T> public static unsafe MatrixPanel<T>* Create<T>() where T: unmanaged, INumber<T>
{ {
return (MatrixPanel<T>*)NativeMemory.AlignedAlloc(Size<T>(), AlignmentBytes); return (MatrixPanel<T>*)NativeMemory.AlignedAlloc(Size<T>(), AlignmentBytes);
} }
public static unsafe void Destroy<T>( public static unsafe void Destroy<T>(MatrixPanel<T>* panel) where T: unmanaged, INumber<T>
MatrixPanel<T>* panel
) where T : unmanaged, INumber<T>
{ {
NativeMemory.AlignedFree(panel); NativeMemory.AlignedFree(panel);
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static unsafe Span<T> Elements<T>( public static unsafe Span<T> Elements<T>(ref T head) where T: unmanaged, INumber<T>
ref T head
) where T : unmanaged, INumber<T>
{ {
void* p = Unsafe.AsPointer(ref head); void* p = Unsafe.AsPointer(ref head);
return new Span<T>(p, MaxElementCount<T>()); return new Span<T>(p, MaxElementCount<T>());
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int ElementSideLength<T>() where T : unmanaged, INumber<T> public static int ElementSideLength<T>() where T: unmanaged, INumber<T>
{ {
return PanelSideLength<T>(); return PanelSideLength<T>();
} }
public static int MaxElementCount<T>() where T : unmanaged, INumber<T> public static int MaxElementCount<T>() where T: unmanaged, INumber<T>
{ {
int side = ElementSideLength<T>(); int side = ElementSideLength<T>();
return side * side; return side * side;
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static nuint Size<T>() where T : unmanaged, INumber<T> public static nuint Size<T>() where T: unmanaged, INumber<T>
{ {
return (nuint)(MaxElementCount<T>() * sizeof(T)); return (nuint)(MaxElementCount<T>() * sizeof(T));
} }
@@ -84,7 +80,7 @@ namespace StgSharp.HighPerformance
/// Number of 4x4 kernels that fit in a panel for the given ISA and element T. /// Number of 4x4 kernels that fit in a panel for the given ISA and element T.
/// </summary> /// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static int KernelCapacity<T>() where T : unmanaged, INumber<T> internal static int KernelCapacity<T>() where T: unmanaged, INumber<T>
{ {
int perSide = KernelSideCount<T>(); int perSide = KernelSideCount<T>();
return perSide * perSide; return perSide * perSide;
@@ -94,7 +90,7 @@ namespace StgSharp.HighPerformance
/// How many 4x4 kernels fit along a single panel dimension for the given ISA. /// How many 4x4 kernels fit along a single panel dimension for the given ISA.
/// </summary> /// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static int KernelSideCount<T>() where T : unmanaged, INumber<T> internal static int KernelSideCount<T>() where T: unmanaged, INumber<T>
{ {
SIMDID id = NativeIntrinsic.IntrinsicMask; SIMDID id = NativeIntrinsic.IntrinsicMask;
switch (id.MaskByte[0] & 0b_00001111) switch (id.MaskByte[0] & 0b_00001111)
@@ -113,32 +109,31 @@ namespace StgSharp.HighPerformance
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe nuint Align64( private static unsafe nuint Align64(void* p)
void* p {
) => ((nuint)p + (AlignmentBytes - 1)) & ~(nuint)(AlignmentBytes - 1); return ((nuint)p + (AlignmentBytes - 1)) & ~(nuint)(AlignmentBytes - 1);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int KernelSideCountAvx2<T>() where T : unmanaged, INumber<T> private static int KernelSideCountAvx2<T>() where T: unmanaged, INumber<T>
{ {
return PanelSideFromBits<T>(256) / 4; return PanelSideFromBits<T>(256) / 4;
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int KernelSideCountAvx512<T>() where T : unmanaged, INumber<T> private static int KernelSideCountAvx512<T>() where T: unmanaged, INumber<T>
{ {
return PanelSideFromBits<T>(512) / 4; return PanelSideFromBits<T>(512) / 4;
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int KernelSideCountSse<T>() where T : unmanaged, INumber<T> private static int KernelSideCountSse<T>() where T: unmanaged, INumber<T>
{ {
return PanelSideFromBits<T>(128) / 4; return PanelSideFromBits<T>(128) / 4;
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int PanelSideFromBits<T>( private static int PanelSideFromBits<T>(int vectorBits) where T: unmanaged, INumber<T>
int vectorBits
) where T : unmanaged, INumber<T>
{ {
int elemBits = sizeof(T) * 8; int elemBits = sizeof(T) * 8;
@@ -150,7 +145,7 @@ namespace StgSharp.HighPerformance
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int PanelSideLength<T>() where T : unmanaged, INumber<T> private static int PanelSideLength<T>() where T: unmanaged, INumber<T>
{ {
SIMDID id = NativeIntrinsic.IntrinsicMask; SIMDID id = NativeIntrinsic.IntrinsicMask;
switch (id.MaskByte[0] & 0b_00001111) switch (id.MaskByte[0] & 0b_00001111)
@@ -171,15 +166,12 @@ namespace StgSharp.HighPerformance
} }
// Single POD-style panel: ZMM-compatible size. Alignment handled by allocator, not by the struct. // Single POD-style panel: ZMM-compatible size. Alignment handled by allocator, not by the struct.
public unsafe struct MatrixPanel<T> where T : unmanaged, INumber<T> public unsafe struct MatrixPanel<T> where T: unmanaged, INumber<T>
{ {
internal T Head; // first element of the panel buffer internal T Head; // first element of the panel buffer
public ref T this[ public ref T this[int column, int row]
int column,
int row
]
{ {
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
get => ref Unsafe.Add(ref Head, row * MatrixPanel.ElementSideLength<T>() + column); get => ref Unsafe.Add(ref Head, row * MatrixPanel.ElementSideLength<T>() + column);
StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs
+4 -4
View File
@@ -1,9 +1,9 @@
//----------------------------------------------------------------------- //-----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="IntrinsicContext" // file="IntrinsicContext"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload // AuthorGroup: Nitload Space
// Copyright (c) Nitload. All rights reserved. // Copyright (c) Nitload Space. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -46,7 +46,7 @@ namespace StgSharp.Internal.Intrinsic
public delegate* unmanaged[Cdecl]<MatrixPanel*, MatrixKernel*, int, int, int, int, void> build_panel; //f32_build_panel public delegate* unmanaged[Cdecl]<MatrixPanel*, MatrixKernel*, int, int, int, int, void> build_panel; //f32_build_panel
public delegate* unmanaged[Cdecl]<MatrixPanel*, void> clear_panel; //f32_clear_panel public delegate* unmanaged[Cdecl]<MatrixPanel*, void> clear_panel; //f32_clear_panel
public delegate* unmanaged[Cdecl]<void*, void*, void*, void> kernel_fma; //f32_panel_fma public delegate* unmanaged[Cdecl]<void*, void*, void*, void> kernel_fma; //f32_panel_fma
public delegate* unmanaged[Cdecl]<void*, void*, void*, M128*, void> kernel_tile_fma; //f32_panel_fma public delegate* unmanaged[Cdecl]<MatrixKernel*, MatrixKernel*, MatrixKernel*, M128*, void> kernel_tile_fma; //f32_panel_fma
public delegate* unmanaged[Cdecl]<void*, ulong, ulong, ulong, void> pivot; //f32_panel_mul public delegate* unmanaged[Cdecl]<void*, ulong, ulong, ulong, void> pivot; //f32_panel_mul
public delegate* unmanaged[Cdecl]<void*, void*, int, int, int, int, void> store_panel; //f32_store_panel public delegate* unmanaged[Cdecl]<void*, void*, int, int, int, int, void> store_panel; //f32_store_panel
StgSharp.Common/Internal/Intrinsic/SIMDID.cs
+1 -1
View File
@@ -1,4 +1,4 @@
//----------------------------------------------------------------------- //-----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="SIMDID" // file="SIMDID"
// Project: StgSharp // Project: StgSharp
StgSharp.Mathematics/Numeric/MatrixCompute.Float.cs
+28 -44
View File
@@ -1,9 +1,9 @@
//----------------------------------------------------------------------- //-----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="MatrixCompute.Float" // file="MatrixCompute.Float"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload // AuthorGroup: Nitload Space
// Copyright (c) Nitload. All rights reserved. // Copyright (c) Nitload Space. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -42,11 +42,7 @@ namespace StgSharp.Mathematics.Numeric
public static partial class MatrixCompute public static partial class MatrixCompute
{ {
public static unsafe void Add( public static unsafe void Add(Matrix<float> left, Matrix<float> right, Matrix<float> ans)
Matrix<float> left,
Matrix<float> right,
Matrix<float> ans
)
{ {
int f32 = (int)MatrixElementType.F32; int f32 = (int)MatrixElementType.F32;
if (left.ColumnLength != right.ColumnLength || if (left.ColumnLength != right.ColumnLength ||
@@ -92,10 +88,7 @@ namespace StgSharp.Mathematics.Numeric
return; return;
} }
public static unsafe void Fill( public static unsafe void Fill(Matrix<float> ans, float value)
Matrix<float> ans,
float value
)
{ {
int f32 = (int)MatrixElementType.F32; int f32 = (int)MatrixElementType.F32;
long count = (long)ans.KernelColumnLength * ans.KernelRowLength; long count = (long)ans.KernelColumnLength * ans.KernelRowLength;
@@ -126,11 +119,7 @@ namespace StgSharp.Mathematics.Numeric
taskGroup.Dispose(); taskGroup.Dispose();
} }
public static unsafe void Mul( public static unsafe void Mul(Matrix<float> left, Matrix<float> right, Matrix<float> ans)
Matrix<float> left,
Matrix<float> right,
Matrix<float> ans
)
{ {
int f32 = (int)MatrixElementType.F32; int f32 = (int)MatrixElementType.F32;
@@ -155,28 +144,32 @@ namespace StgSharp.Mathematics.Numeric
if (count > 256) { if (count > 256) {
pool = MatrixParallel.LeadParallel(1); pool = MatrixParallel.LeadParallel(1);
} }
M128* enumeration = stackalloc M128[4];
for (int i = 0; i < ans.KernelRowLength; i++) for (int i = 0; i < ans.KernelRowLength; i++)
{ {
for (int j = 0; j < ans.KernelColumnLength; j++) for (int j = 0; j < ans.KernelColumnLength; j++)
{ {
MatrixKernel<float>* ansPanel = GetKernelAddressUnsafe(ans.Buffer, enumeration->Member<int>(0) = i;
ans.KernelColumnLength, enumeration->Member<int>(1) = j;
i, j); enumeration->Member<int>(2) = ans.KernelRowLength;
enumeration->Member<int>(3) = left.KernelColumnLength;
// NativeIntrinsic.Context.mat[f32].clear_panel((MatrixPanel*)ansPanel); NativeIntrinsic.Context
for (int k = 0; k < left.KernelColumnLength; k++) .mat[f32].kernel_tile_fma((MatrixKernel*)left.Buffer,
{ (MatrixKernel*)right.Buffer,
(MatrixKernel*)ans.Buffer, enumeration);
/*
for (int k = 0; k < left.KernelColumnLength; k++) {
NativeIntrinsic.Context NativeIntrinsic.Context
.mat[f32].kernel_fma((MatrixPanel*)GetKernelAddressUnsafe(left.Buffer, .mat[f32].kernel_fma(GetKernelAddressUnsafe(left.Buffer,
left.KernelColumnLength, left.KernelColumnLength, i,
i, k),
k), GetKernelAddressUnsafe(right.Buffer,
(MatrixPanel*)GetKernelAddressUnsafe(right.Buffer, right.KernelColumnLength, k, j),
right.KernelColumnLength, GetKernelAddressUnsafe(ans.Buffer,
k, ans.KernelColumnLength, i, j));
j),
(MatrixPanel*)ansPanel);
} }
/**/
} }
} }
@@ -203,18 +196,13 @@ namespace StgSharp.Mathematics.Numeric
return; return;
} }
public static unsafe void Mul( public static unsafe void Mul(Matrix<float> left, float right, Matrix<float> ans)
Matrix<float> left,
float right,
Matrix<float> ans
)
{ {
int f32 = (int)MatrixElementType.F32; int f32 = (int)MatrixElementType.F32;
if (left.ColumnLength != ans.ColumnLength || left.RowLength != ans.RowLength) { if (left.ColumnLength != ans.ColumnLength || left.RowLength != ans.RowLength) {
throw new IndexOutOfRangeException("Matrix dimensions must agree."); throw new IndexOutOfRangeException("Matrix dimensions must agree.");
} }
if (left.Layout != MatrixLayout.DenseRectangle || if (left.Layout != MatrixLayout.DenseRectangle || ans.Layout != MatrixLayout.DenseRectangle) {
ans.Layout != MatrixLayout.DenseRectangle) {
throw new NotSupportedException("Only dense rectangle matrices are supported."); throw new NotSupportedException("Only dense rectangle matrices are supported.");
} }
long count = (long)left.KernelColumnLength * left.KernelRowLength; long count = (long)left.KernelColumnLength * left.KernelRowLength;
@@ -255,11 +243,7 @@ namespace StgSharp.Mathematics.Numeric
return; return;
} }
public static unsafe void Sub( public static unsafe void Sub(Matrix<float> left, Matrix<float> right, Matrix<float> ans)
Matrix<float> left,
Matrix<float> right,
Matrix<float> ans
)
{ {
int f32 = (int)MatrixElementType.F32; int f32 = (int)MatrixElementType.F32;
if (left.ColumnLength != right.ColumnLength || if (left.ColumnLength != right.ColumnLength ||
StgSharp.Mathematics/Numeric/MatrixCompute.cs
+27 -28
View File
@@ -1,9 +1,9 @@
//----------------------------------------------------------------------- //-----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="MatrixCompute" // file="MatrixCompute"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload // AuthorGroup: Nitload Space
// Copyright (c) Nitload. All rights reserved. // Copyright (c) Nitload Space. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -40,9 +40,7 @@ namespace StgSharp.Mathematics.Numeric
public partial class MatrixCompute public partial class MatrixCompute
{ {
internal static unsafe void ComputeFmaPanelF32( internal static unsafe void ComputeFmaPanelF32(MatrixParallelTaskPackage* p)
MatrixParallelTaskPackage* p
)
{ {
int f32 = (int)MatrixElementType.F32; int f32 = (int)MatrixElementType.F32;
MatrixKernel* leftBuffer = (MatrixKernel*)p->Left; MatrixKernel* leftBuffer = (MatrixKernel*)p->Left;
@@ -64,19 +62,14 @@ namespace StgSharp.Mathematics.Numeric
for (int k = 0; k < commonK; k++) { for (int k = 0; k < commonK; k++) {
NativeIntrinsic.Context NativeIntrinsic.Context
.mat[f32].kernel_fma( .mat[f32].kernel_fma(
(MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel<float>*)leftBuffer, GetKernelAddressUnsafe((MatrixKernel<float>*)leftBuffer, ansHeight, k, j),
ansHeight, k, j), GetKernelAddressUnsafe((MatrixKernel<float>*)rightBuffer, commonK, j, k),
(MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel<float>*)rightBuffer, GetKernelAddressUnsafe((MatrixKernel<float>*)ansBuffer, ansHeight, i, j));
commonK, j, k),
(MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel<float>*)ansBuffer,
ansHeight, i, j));
} }
} }
} }
internal static unsafe void ComputeFmaPanelF64( internal static unsafe void ComputeFmaPanelF64(MatrixParallelTaskPackage* p)
MatrixParallelTaskPackage* p
)
{ {
int f64 = (int)MatrixElementType.F64; int f64 = (int)MatrixElementType.F64;
@@ -90,33 +83,39 @@ namespace StgSharp.Mathematics.Numeric
int ansHeight = p->AnsSecOffset; int ansHeight = p->AnsSecOffset;
int commonK = p->LeftPrimOffset; int commonK = p->LeftPrimOffset;
M128* enumeration = stackalloc M128[1];
// in this occasion size must be 1 // in this occasion size must be 1
for (long c = offsetRef; c < offsetRef + lengthRef; c++) for (long c = offsetRef; c < offsetRef + lengthRef; c++)
{ {
int i = (int)(c / ansWidth); int i = (int)(c / ansWidth);
int j = (int)(c % ansWidth); int j = (int)(c % ansWidth);
enumeration->Member<int>(0) = i;
enumeration->Member<int>(1) = j;
enumeration->Member<int>(2) = commonK;
enumeration->Member<int>(3) = ansHeight;
NativeIntrinsic.Context.mat[f64].kernel_tile_fma(leftBuffer, rightBuffer, ansBuffer, enumeration);
/*
for (int k = 0; k < commonK; k += 1) { for (int k = 0; k < commonK; k += 1) {
NativeIntrinsic.Context NativeIntrinsic.Context
.mat[f64].kernel_fma( .mat[f64].kernel_fma(
(MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel<double>*)leftBuffer, GetKernelAddressUnsafe((MatrixKernel<double>*)leftBuffer, ansHeight, k, j),
ansHeight, k, j), GetKernelAddressUnsafe((MatrixKernel<double>*)rightBuffer, commonK, j, k),
(MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel<double>*)rightBuffer, GetKernelAddressUnsafe((MatrixKernel<double>*)ansBuffer, ansHeight, i, j));
commonK, j, k),
(MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel<double>*)ansBuffer,
ansHeight, i, j));
} }
/**/
} }
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static unsafe MatrixKernel<T>* GetKernelAddressUnsafe<T>( internal static unsafe MatrixKernel* GetKernelAddressUnsafe<T>(
MatrixKernel<T>* buffer, MatrixKernel<T>* buffer,
int colLength, int colLength,
int x, int x,
int y int y)
) where T : unmanaged, INumber<T> where T: unmanaged, INumber<T>
{ {
return buffer + (colLength * x + y); return (MatrixKernel*)(buffer + (colLength * x + y));
} }
} }
StgSharp.Mathematics/Numeric/MatrixCompute.double.cs
+35 -38
View File
@@ -1,9 +1,9 @@
//----------------------------------------------------------------------- //-----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="MatrixCompute.double" // file="MatrixCompute.double"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload // AuthorGroup: Nitload Space
// Copyright (c) Nitload. All rights reserved. // Copyright (c) Nitload Space. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -38,11 +38,7 @@ namespace StgSharp.Mathematics.Numeric
public partial class MatrixCompute public partial class MatrixCompute
{ {
public static unsafe void Add( public static unsafe void Add(Matrix<double> left, Matrix<double> right, Matrix<double> ans)
Matrix<double> left,
Matrix<double> right,
Matrix<double> ans
)
{ {
int f64 = (int)MatrixElementType.F64; int f64 = (int)MatrixElementType.F64;
if (left.ColumnLength != right.ColumnLength || if (left.ColumnLength != right.ColumnLength ||
@@ -88,10 +84,7 @@ namespace StgSharp.Mathematics.Numeric
return; return;
} }
public static unsafe void Fill( public static unsafe void Fill(Matrix<double> ans, double value)
Matrix<double> ans,
double value
)
{ {
int f64 = (int)MatrixElementType.F64; int f64 = (int)MatrixElementType.F64;
long count = (long)ans.KernelColumnLength * ans.KernelRowLength; long count = (long)ans.KernelColumnLength * ans.KernelRowLength;
@@ -122,11 +115,7 @@ namespace StgSharp.Mathematics.Numeric
taskGroup.Dispose(); taskGroup.Dispose();
} }
public static unsafe void Mul( public static unsafe void Mul(Matrix<double> left, Matrix<double> right, Matrix<double> ans)
Matrix<double> left,
Matrix<double> right,
Matrix<double> ans
)
{ {
int f64 = (int)MatrixElementType.F64; int f64 = (int)MatrixElementType.F64;
@@ -139,25 +128,42 @@ namespace StgSharp.Mathematics.Numeric
throw new IndexOutOfRangeException("Matrix dimensions must agree."); throw new IndexOutOfRangeException("Matrix dimensions must agree.");
} }
long count = (long)left.KernelColumnLength * left.KernelRowLength; long count = (long)left.KernelColumnLength * left.KernelRowLength;
int size = MatrixPanel.KernelSideCount<double>();
// int size = MatrixPanel.KernelSideCount<double>();
if (count <= 1024) if (count <= 1024)
{ {
MatrixParallelThread[] pool = null; MatrixParallelThread[] pool = null;
if (count > 256) { if (count > 256) {
pool = MatrixParallel.LeadParallel(1); pool = MatrixParallel.LeadParallel(1);
} }
for (int i = 0; i < ans.KernelRowLength; i += size) M128* enumeration = stackalloc M128[1];
for (int i = 0; i < ans.KernelRowLength; i++)
{ {
for (int j = 0; j < ans.KernelColumnLength; j += size) for (int j = 0; j < ans.KernelColumnLength; j += 1)
{ {
enumeration->Member<int>(0) = i;
enumeration->Member<int>(1) = j;
enumeration->Member<int>(2) = ans.KernelRowLength;
enumeration->Member<int>(3) = left.KernelColumnLength;
NativeIntrinsic.Context
.mat[f64].kernel_tile_fma((MatrixKernel*)left.Buffer,
(MatrixKernel*)right.Buffer,
(MatrixKernel*)ans.Buffer, enumeration);
/*
// Console.WriteLine($"{i}{j}"); // Console.WriteLine($"{i}{j}");
for (int k = 0; k < left.KernelColumnLength; k += size) for (int k = 0; k < left.KernelColumnLength; k++)
{ {
NativeIntrinsic.Context NativeIntrinsic.Context
.mat[f64].kernel_fma((MatrixPanel*)left.GetKernelAddressUnsafe(i, k), .mat[f64].kernel_fma(GetKernelAddressUnsafe(left.Buffer,
(MatrixPanel*)right.GetKernelAddressUnsafe(k, j), left.KernelColumnLength, i,
(MatrixPanel*)ans.GetKernelAddressUnsafe(i, j)); k),
GetKernelAddressUnsafe(right.Buffer,
right.KernelColumnLength, k, j),
GetKernelAddressUnsafe(ans.Buffer,
ans.KernelColumnLength, i, j));
} }
/**/
} }
} }
if (count > 256) { if (count > 256) {
@@ -172,8 +178,8 @@ namespace StgSharp.Mathematics.Numeric
package->AnsPrimOffset = ans.KernelRowLength; package->AnsPrimOffset = ans.KernelRowLength;
package->AnsSecOffset = ans.KernelColumnLength; package->AnsSecOffset = ans.KernelColumnLength;
package->LeftPrimOffset = left.KernelColumnLength; package->LeftPrimOffset = left.KernelColumnLength;
package->PrimCount = ans.KernelColumnLength / size; package->PrimCount = ans.KernelColumnLength;
package->SecCount = ans.KernelRowLength / size; package->SecCount = ans.KernelRowLength;
package->ElementSize = (int)MatrixElementType.F64; package->ElementSize = (int)MatrixElementType.F64;
package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationSpecial.SEQ_FMA); package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationSpecial.SEQ_FMA);
MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package);
@@ -183,18 +189,13 @@ namespace StgSharp.Mathematics.Numeric
return; return;
} }
public static unsafe void Mul( public static unsafe void Mul(Matrix<double> left, double right, Matrix<double> ans)
Matrix<double> left,
double right,
Matrix<double> ans
)
{ {
int f64 = (int)MatrixElementType.F64; int f64 = (int)MatrixElementType.F64;
if (left.ColumnLength != ans.ColumnLength || left.RowLength != ans.RowLength) { if (left.ColumnLength != ans.ColumnLength || left.RowLength != ans.RowLength) {
throw new IndexOutOfRangeException("Matrix dimensions must agree."); throw new IndexOutOfRangeException("Matrix dimensions must agree.");
} }
if (left.Layout != MatrixLayout.DenseRectangle || if (left.Layout != MatrixLayout.DenseRectangle || ans.Layout != MatrixLayout.DenseRectangle) {
ans.Layout != MatrixLayout.DenseRectangle) {
throw new NotSupportedException("Only dense rectangle matrices are supported."); throw new NotSupportedException("Only dense rectangle matrices are supported.");
} }
long count = (long)left.KernelColumnLength * left.KernelRowLength; long count = (long)left.KernelColumnLength * left.KernelRowLength;
@@ -235,11 +236,7 @@ namespace StgSharp.Mathematics.Numeric
return; return;
} }
public static unsafe void Sub( public static unsafe void Sub(Matrix<double> left, Matrix<double> right, Matrix<double> ans)
Matrix<double> left,
Matrix<double> right,
Matrix<double> ans
)
{ {
int f64 = (int)MatrixElementType.F64; int f64 = (int)MatrixElementType.F64;
if (left.ColumnLength != right.ColumnLength || if (left.ColumnLength != right.ColumnLength ||
StgSharp.Native/sn-src/m.fma_tile.x86.c
+10 -52
View File
@@ -1,4 +1,4 @@
#include "sn_target.h" #include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64) #if SN_IS_ARCH(SN_ARCH_X86_64)
@@ -293,7 +293,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
a_hi = _mm_load_pd(left_cur + (3 * 2 + 1)); \ a_hi = _mm_load_pd(left_cur + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \ b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b));\
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma) DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
{ {
@@ -316,10 +316,12 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
const *left_cur = left + GET_INDEX(ansHeight, k, j), const *left_cur = left + GET_INDEX(ansHeight, k, j),
const *right_cur = right + GET_INDEX(commonK, j, k); // GetKernelAddressUnsafe const *right_cur = right + GET_INDEX(commonK, j, k); // GetKernelAddressUnsafe
DOUBLE_KERNEL_FMA_CYCLE(0) {DOUBLE_KERNEL_FMA_CYCLE(0)}
DOUBLE_KERNEL_FMA_CYCLE(1) {DOUBLE_KERNEL_FMA_CYCLE(1)}
DOUBLE_KERNEL_FMA_CYCLE(2) {DOUBLE_KERNEL_FMA_CYCLE(2)}
DOUBLE_KERNEL_FMA_CYCLE(3) {
DOUBLE_KERNEL_FMA_CYCLE(3)
}
} }
_mm_store_pd(ans->f64_x + (0 * 2 + 0), c0_lo); _mm_store_pd(ans->f64_x + (0 * 2 + 0), c0_lo);
@@ -519,8 +521,8 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, 512, , fma)
c23 = _mm512_fmadd_pd(a01, b23_perm, c23); c23 = _mm512_fmadd_pd(a01, b23_perm, c23);
//group 1 //group 1
register __m512d a12 = _mm512_permute2f128_pd(a01, a23, 0x21); register __m512d a12 = _mm512_shuffle_f64x2(a01, a23, 0x21);
register __m512d a30 = _mm512_permute2f128_pd(a01, a23, 0x30); register __m512d a30 = _mm512_shuffle_f64x2(a01, a23, 0x30);
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(1)); b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(1));
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(1)); b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(1));
c01 = _mm512_fmadd_pd(a12, b01_perm, c01); c01 = _mm512_fmadd_pd(a12, b01_perm, c01);
@@ -534,50 +536,6 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, 512, , fma)
} }
_mm512_store_pd((double *)&ans_cur->f64_z[0], c01); _mm512_store_pd((double *)&ans_cur->f64_z[0], c01);
_mm512_store_pd((double *)&ans_cur->f64_z[1], c23); _mm512_store_pd((double *)&ans_cur->f64_z[1], c23);
/*
static const __m512i b01_shift = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m512i b23_shift = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
register __m512d a01 = _mm512_load_pd((double *)&left->f64_z[0]);
register __m512d a23 = _mm512_load_pd((double *)&left->f64_z[1]);
register __m512d b01 = _mm512_load_pd((double *)&right->f64_z[0]);
register __m512d b23 = _mm512_load_pd((double *)&right->f64_z[1]);
b01 = _mm512_permutexvar_pd(b01_shift, b01);
b23 = _mm512_permutexvar_pd(b23_shift, b23);
register __m512d c01 = _mm512_load_pd((float *)&ans->f64_z[0]);
register __m512d c23 = _mm512_load_pd((float *)&ans->f64_z[1]);
//group 0
register __m512d b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(0));
register __m512d b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(0));
c01 = _mm512_fmadd_pd(a01, b01_perm, c01);
c23 = _mm512_fmadd_pd(a23, b23_perm, c23);
//group 2
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(2));
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(2));
c01 = _mm512_fmadd_pd(a23, b01_perm, c01);
c23 = _mm512_fmadd_pd(a01, b23_perm, c23);
//group 1
register __m512d a12 = _mm512_shuffle_f64x2(a01, a23, _MM_SHUFFLE(1, 0, 3, 2));
register __m512d a30 = _mm512_shuffle_f64x2(a23, a01, _MM_SHUFFLE(1, 0, 3, 2));
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(1));
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(1));
c01 = _mm512_fmadd_pd(a12, b01_perm, c01);
c23 = _mm512_fmadd_pd(a30, b23_perm, c23);
//group 3
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(3));
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(3));
c01 = _mm512_fmadd_pd(a30, b01_perm, c01);
c23 = _mm512_fmadd_pd(a12, b23_perm, c23);
ans->f64_z[0] = c01;
ans->f64_z[1] = c23;
/**/
} }
#endif #endif
StgSharpDebug/Program.cs
+8 -16
View File
@@ -1,9 +1,9 @@
//----------------------------------------------------------------------- //-----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="Program" // file="Program"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload // AuthorGroup: Nitload Space
// Copyright (c) Nitload. All rights reserved. // Copyright (c) Nitload Space. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -39,16 +39,11 @@ namespace StgSharp
private static string testExpPath = @"D:\Programmer\stepcode\data\ap203\ap203.exp"; private static string testExpPath = @"D:\Programmer\stepcode\data\ap203\ap203.exp";
internal static unsafe void Main( internal static unsafe void Main(string[] args)
string[] args
)
{ {
try try
{ {
World.Initialize() World.Initialize().UseModule<NumericalModule>().UseModule<GraphicModule>().InitializeModule();
.UseModule<NumericalModule>()
.UseModule<GraphicModule>()
.InitializeModule();
// TestRender(); // TestRender();
/**/ /**/
@@ -56,14 +51,11 @@ namespace StgSharp
unchecked { unchecked {
hlsf = new((nuint)(64 * (ulong)10000 * 10000)); hlsf = new((nuint)(64 * (ulong)10000 * 10000));
} }
Matrix<double> m = Matrix<double>.Create(4000, 4000, MatrixLayout.DenseRectangle, Matrix<double> m = Matrix<double>.Create(40, 40, MatrixLayout.DenseRectangle, hlsf);
hlsf);
MatrixCompute.Fill(m, 1.0f); MatrixCompute.Fill(m, 1.0f);
Matrix<double> n = Matrix<double>.Create(4000, 4000, MatrixLayout.DenseRectangle, Matrix<double> n = Matrix<double>.Create(40, 40, MatrixLayout.DenseRectangle, hlsf);
hlsf);
MatrixCompute.Fill(n, 1.0f); MatrixCompute.Fill(n, 1.0f);
Matrix<double> p = Matrix<double>.Create(4000, 4000, MatrixLayout.DenseRectangle, Matrix<double> p = Matrix<double>.Create(40, 40, MatrixLayout.DenseRectangle, hlsf);
hlsf);
MatrixCompute.Fill(p, 0.0f); MatrixCompute.Fill(p, 0.0f);
/**/ /**/