youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

keep working on PLN facotrize

Browse Source
This commit is contained in:
Youmu Liu
2025-12-24 18:09:16 +08:00
parent b60ce56f1c
commit 3b3a90a2af
12 changed files with 428 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/Mathematics/Linear/Matrix.Create.cs
+5 -11
View File
@@ -65,7 +65,7 @@ namespace StgSharp.Mathematics.Numeric
long kernelCount = layout switch
{
MatrixLayout.DenseSquare => (long)kernelColumns * kernelRows,
MatrixLayout.DenseRectangle => (long)kernelColumns * kernelRows,
MatrixLayout.UpperTriangle => TriangularKernelCount(kernelColumns),
MatrixLayout.LowerTriangle => TriangularKernelCount(kernelRows),
_ => throw new ArgumentOutOfRangeException(nameof(layout))
@@ -79,16 +79,10 @@ namespace StgSharp.Mathematics.Numeric
HybridLayerSegregatedFitAllocationHandle handle = allocator.Alloc((uint)requiredBytes);
MatrixStorageHLSF<T> storage = new MatrixStorageHLSF<T>(allocator, handle);
Matrix<T> matrix = new Matrix<T>
{
Layout = layout,
KernelColumnLength = kernelColumns,
KernelRowLength = kernelRows,
};
matrix._buffer = storage;
matrix._boarder = MatrixKernelBoarder.Create(layout);
matrix._size = new MatrixSize(columnLength, rowLength, kernelColumns, kernelRows);
Matrix<T> matrix = new Matrix<T>(MatrixKernelBoarder.Create(layout), new MatrixSize(columnLength,
rowLength,
kernelColumns,
kernelRows), storage);
return matrix;
}
StgSharp.Common/Mathematics/Linear/Matrix.cs
+35 -11
View File
@@ -41,11 +41,21 @@ namespace StgSharp.Mathematics.Numeric
private protected static readonly int t_count = 16 / sizeof(T);
private protected static readonly int t_size = sizeof(T);
private MatrixKernelBoarder _boarder;
private MatrixSize _size;
private MatrixStorage<T> _buffer;
private readonly MatrixKernelBoarder _boarder;
private readonly MatrixSize _size;
private readonly MatrixStorage<T> _buffer;
private protected Matrix() { }
private protected Matrix(
MatrixKernelBoarder boarder,
MatrixSize size,
MatrixStorage<T> buffer
)
{
_boarder = boarder;
_size = size;
_buffer = buffer;
}
public T this[int x, int y]
{
@@ -122,19 +132,33 @@ namespace StgSharp.Mathematics.Numeric
get => (MatrixKernel<T>*)_buffer.BufferPointer;
}
protected internal int KernelColumnLength { get; init; }
protected internal int KernelColumnLength => _size.KernelColumnLength;
protected internal int KernelRowLength { get; init; }
protected internal int KernelRowLength => _size.KernelRowLength;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public unsafe ref MatrixKernel<T> GetMatrixKernel(int x, int y)
{
fixed (MatrixSize *msPtr = &_size)
{
MatrixKernelEnumerator* e = (MatrixKernelEnumerator*)msPtr;
return ref _boarder.IsInBuffer(x, y, e) ?
ref _buffer[_boarder.GetIndexOffset(x, y, e)] :
MatrixKernelEnumerator enumerator = new(_size.KernelColumnLength, _size.KernelRowLength,
x, y);
return ref _boarder.IsInBuffer(x, y, in enumerator) ?
ref _buffer[_boarder.GetIndexOffset(x, y, in enumerator)] :
ref Unsafe.NullRef<MatrixKernel<T>>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal unsafe ref T GetElementUnsafe(int x, int y)
{
// Caller guarantees: indices in range, layout is dense square.
int k_c = y >> 2;
int k_r = x >> 2;
int c = y & 3;
int r = x & 3;
MatrixKernelEnumerator enumerator = new(_size.KernelColumnLength, _size.KernelRowLength,
k_c, k_r);
ref MatrixKernel<T> kernel = ref _buffer[_boarder.GetIndexOffset(k_c, k_r, in enumerator)];
return ref kernel[c, r];
}
}
StgSharp.Common/Mathematics/Linear/MatrixMetadata.cs
+121 -117
View File
@@ -25,169 +25,159 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.HighPerformance;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Mathematics.Numeric
{
public enum MatrixLayout
{
DenseSquare,
DenseRectangle,
UpperTriangle,
LowerTriangle,
}
[StructLayout(LayoutKind.Sequential)]
public unsafe struct MatrixKernelBoarder
public abstract class MatrixKernelBoarder
{
public readonly delegate* unmanaged[Cdecl]
<int, int, MatrixKernelEnumerator*, long> GetIndexOffset;
public readonly delegate* unmanaged[Cdecl]<int, int, MatrixKernelEnumerator*, bool> IsInBuffer;
[Obsolete("Do not use constructor, use static method instead", true)]
public MatrixKernelBoarder() { }
public MatrixKernelBoarder(
delegate* unmanaged[Cdecl]
<int, int, MatrixKernelEnumerator*, long> getOffset,
delegate* unmanaged[Cdecl]<int, int, MatrixKernelEnumerator*, bool> inBuffer)
{
GetIndexOffset = getOffset;
IsInBuffer = inBuffer;
}
private static delegate* unmanaged[Cdecl]<int, int, MatrixKernelEnumerator*, long> GetIndexOffsetDenseSquarePtr { get; } =
(delegate* unmanaged[Cdecl]<int, int, MatrixKernelEnumerator*, long>)&GetIndexOffsetDenseSquare;
private static delegate* unmanaged[Cdecl]<int, int, MatrixKernelEnumerator*, long> GetIndexOffsetUpperTrianglePtr { get; } =
(delegate* unmanaged[Cdecl]<int, int, MatrixKernelEnumerator*, long>)&GetIndexOffsetUpperTriangle;
private static delegate* unmanaged[Cdecl]<int, int, MatrixKernelEnumerator*, long> GetIndexOffsetLowerTrianglePtr { get; } =
(delegate* unmanaged[Cdecl]<int, int, MatrixKernelEnumerator*, long>)&GetIndexOffsetLowerTriangle;
private static delegate* unmanaged[Cdecl]<int, int, MatrixKernelEnumerator*, bool> IsInBufferDenseSquarePtr { get; } =
(delegate* unmanaged[Cdecl]<int, int, MatrixKernelEnumerator*, bool>)&IsInBufferDenseSquare;
private static delegate* unmanaged[Cdecl]<int, int, MatrixKernelEnumerator*, bool> IsInBufferUpperTrianglePtr { get; } =
(delegate* unmanaged[Cdecl]<int, int, MatrixKernelEnumerator*, bool>)&IsInBufferUpperTriangle;
private static delegate* unmanaged[Cdecl]<int, int, MatrixKernelEnumerator*, bool> IsInBufferLowerTrianglePtr { get; } =
(delegate* unmanaged[Cdecl]<int, int, MatrixKernelEnumerator*, bool>)&IsInBufferLowerTriangle;
private static readonly MatrixKernelBoarder DenseSquareInstance = new DenseSquareKernelBoarder();
private static readonly MatrixKernelBoarder LowerTriangleInstance = new LowerTriangleKernelBoarder();
private static readonly MatrixKernelBoarder UpperTriangleInstance = new UpperTriangleKernelBoarder();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static MatrixKernelBoarder Create(MatrixLayout layout)
{
return layout switch
{
MatrixLayout.DenseSquare => new MatrixKernelBoarder(GetIndexOffsetDenseSquarePtr, IsInBufferDenseSquarePtr),
MatrixLayout.UpperTriangle => new MatrixKernelBoarder(GetIndexOffsetUpperTrianglePtr, IsInBufferUpperTrianglePtr),
MatrixLayout.LowerTriangle => new MatrixKernelBoarder(GetIndexOffsetLowerTrianglePtr, IsInBufferLowerTrianglePtr),
MatrixLayout.DenseRectangle => DenseSquareInstance,
MatrixLayout.UpperTriangle => UpperTriangleInstance,
MatrixLayout.LowerTriangle => LowerTriangleInstance,
_ => throw new ArgumentOutOfRangeException(nameof(layout)),
};
}
[UnmanagedCallersOnly(CallConvs =[ typeof(CallConvCdecl) ])]
private static long GetIndexOffsetDenseSquare(
int x_kernel,
int y_kernel,
MatrixKernelEnumerator* enumerator)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public abstract long GetIndexOffset(
int xKernel,
int yKernel,
in MatrixKernelEnumerator enumerator);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public abstract bool IsInBuffer(
int xKernel,
int yKernel,
in MatrixKernelEnumerator enumerator);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected static long GetTriangularOrder(in MatrixKernelEnumerator enumerator)
{
long x = x_kernel;
long y = y_kernel;
long rows = enumerator->KernelRowLength;
return Math.Min(enumerator.KernelColumnLength, enumerator.KernelRowLength);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected static bool IsWithinKernelDomain(
in MatrixKernelEnumerator enumerator,
int xKernel,
int yKernel)
{
return xKernel >= 0 &&
yKernel >= 0 &&
xKernel < enumerator.KernelColumnLength &&
yKernel < enumerator.KernelRowLength;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected static bool IsWithinTriangularDomain(
in MatrixKernelEnumerator enumerator,
int xKernel,
int yKernel)
{
long order = GetTriangularOrder(enumerator);
return xKernel >= 0 && yKernel >= 0 && xKernel < order && yKernel < order;
}
private sealed class DenseSquareKernelBoarder : MatrixKernelBoarder
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override long GetIndexOffset(
int xKernel,
int yKernel,
in MatrixKernelEnumerator enumerator)
{
long x = xKernel;
long y = yKernel;
long rows = enumerator.KernelRowLength;
return x * rows + y;
}
[UnmanagedCallersOnly(CallConvs =[ typeof(CallConvCdecl) ])]
private static long GetIndexOffsetLowerTriangle(
int x_kernel,
int y_kernel,
MatrixKernelEnumerator* enumerator)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override bool IsInBuffer(
int xKernel,
int yKernel,
in MatrixKernelEnumerator enumerator)
{
return IsWithinKernelDomain(enumerator, xKernel, yKernel);
}
}
private sealed class LowerTriangleKernelBoarder : MatrixKernelBoarder
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override long GetIndexOffset(
int xKernel,
int yKernel,
in MatrixKernelEnumerator enumerator)
{
long order = GetTriangularOrder(enumerator);
long x = x_kernel;
long y = y_kernel;
long x = xKernel;
long y = yKernel;
long prefixColumns = x * order - (x * (x - 1L)) / 2L;
return prefixColumns + (y - x);
}
[UnmanagedCallersOnly(CallConvs =[ typeof(CallConvCdecl) ])]
private static long GetIndexOffsetUpperTriangle(
int x_kernel,
int y_kernel,
MatrixKernelEnumerator* enumerator)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override bool IsInBuffer(
int xKernel,
int yKernel,
in MatrixKernelEnumerator enumerator)
{
long x = x_kernel;
long y = y_kernel;
return IsWithinTriangularDomain(enumerator, xKernel, yKernel) && yKernel >= xKernel;
}
}
private sealed unsafe class UpperTriangleKernelBoarder : MatrixKernelBoarder
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override long GetIndexOffset(
int xKernel,
int yKernel,
in MatrixKernelEnumerator enumerator)
{
long x = xKernel;
long y = yKernel;
long prefixColumns = (x * (x + 1L)) / 2L;
return prefixColumns + y;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static long GetTriangularOrder(MatrixKernelEnumerator* enumerator)
public override bool IsInBuffer(
int xKernel,
int yKernel,
in MatrixKernelEnumerator enumerator)
{
return Math.Min(enumerator->KernelColumnLength, enumerator->KernelRowLength);
return IsWithinTriangularDomain(enumerator, xKernel, yKernel) && yKernel <= xKernel;
}
[UnmanagedCallersOnly(CallConvs =[ typeof(CallConvCdecl) ])]
private static bool IsInBufferDenseSquare(
int x_kernel,
int y_kernel,
MatrixKernelEnumerator* enumerator)
{
return IsWithinKernelDomain(enumerator, x_kernel, y_kernel);
}
[UnmanagedCallersOnly(CallConvs =[ typeof(CallConvCdecl) ])]
private static bool IsInBufferLowerTriangle(
int x_kernel,
int y_kernel,
MatrixKernelEnumerator* enumerator)
{
return IsWithinTriangularDomain(enumerator, x_kernel, y_kernel) && y_kernel >= x_kernel;
}
[UnmanagedCallersOnly(CallConvs =[ typeof(CallConvCdecl) ])]
private static bool IsInBufferUpperTriangle(
int x_kernel,
int y_kernel,
MatrixKernelEnumerator* enumerator)
{
return IsWithinTriangularDomain(enumerator, x_kernel, y_kernel) && y_kernel <= x_kernel;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsWithinKernelDomain(
MatrixKernelEnumerator* enumerator,
int x_kernel,
int y_kernel)
{
return x_kernel >= 0 &&
y_kernel >= 0 &&
x_kernel < enumerator->KernelColumnLength &&
y_kernel < enumerator->KernelRowLength;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsWithinTriangularDomain(
MatrixKernelEnumerator* enumerator,
int x_kernel,
int y_kernel)
{
long order = GetTriangularOrder(enumerator);
return x_kernel >= 0 && y_kernel >= 0 && x_kernel < order && y_kernel < order;
}
}
@@ -202,6 +192,7 @@ namespace StgSharp.Mathematics.Numeric
[FieldOffset(8)] public readonly int KernelColumnLength;
[FieldOffset(12)] public readonly int KernelRowLength;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public MatrixSize(
int elementColumnLength,
int elementRowLength,
@@ -226,5 +217,18 @@ namespace StgSharp.Mathematics.Numeric
[FieldOffset(8)] public readonly int KernelColumnLength;
[FieldOffset(12)] public readonly int KernelRowLength;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public MatrixKernelEnumerator(
int kernelColumnLength,
int kernelRowLength,
int currentColumn = 0,
int currentRow = 0)
{
CurrentColumn = currentColumn;
CurrentRow = currentRow;
KernelColumnLength = kernelColumnLength;
KernelRowLength = kernelRowLength;
}
}
}
StgSharp.Mathematics/Numeric/MatrixCompute.Factorize.cs
+54 -2
View File
@@ -38,17 +38,69 @@ namespace StgSharp.Mathematics.Numeric
public static void FactorizePLU(
Matrix<float> matrix,
Span<float> p,
Span<float> pivot,
Matrix<float> lower,
Matrix<float> upper)
{
if (matrix.ColumnLength != matrix.RowLength) {
throw new InvalidOperationException("Matrix must be square.");
}
if (matrix.Layout != MatrixLayout.DenseRectangle) {
throw new NotSupportedException("Only dense square matrices are supported.");
}
int n = matrix.RowLength;
if (pivot.Length < n) {
throw new ArgumentException("Pivot span length is insufficient.", nameof(pivot));
}
if (lower.RowLength != n ||
lower.ColumnLength != n ||
lower.Layout != MatrixLayout.LowerTriangle) {
throw new ArgumentException("Lower matrix must be lower-triangular with same dimensions as input.", nameof(lower));
}
if (upper.RowLength != n ||
upper.ColumnLength != n ||
upper.Layout != MatrixLayout.UpperTriangle) {
throw new ArgumentException("Upper matrix must be upper-triangular with same dimensions as input.", nameof(upper));
}
int size = matrix.KernelColumnLength * matrix.KernelRowLength;
if (size < 256)
{
for (int i = 0; i < matrix.KernelColumnLength; i++) { }
// Naive Doolittle-style PLU without row swaps (for illustration).
for (int i = 0; i < n; i++)
{
pivot[i] = i; // identity pivot
lower[i, i] = 1.0f; // unit diagonal for L
}
for (int k = 0; k < n; k++)
{
// Compute U[k, j] for j >= k
for (int j = k; j < n; j++)
{
float sum = 0f;
for (int s = 0; s < k; s++) {
sum += lower[k, s] * upper[s, j];
}
upper[k, j] = matrix[k, j] - sum;
}
// Compute L[i, k] for i > k
for (int i = k + 1; i < n; i++)
{
float sum = 0f;
for (int s = 0; s < k; s++) {
sum += lower[i, s] * upper[s, k];
}
float u = upper[k, k];
if (u == 0f) {
throw new InvalidOperationException("Zero pivot encountered in demonstration PLU.");
}
lower[i, k] = (matrix.GetElementUnsafe(i, k) - sum) / u;
}
}
// TODO: For larger kernels, consider calling native pivot helper to extract 4 pivots at once.
} else if (size < 1024) { } else { }
}
StgSharp.Mathematics/Numeric/MatrixCompute.cs
+9
View File
@@ -200,6 +200,10 @@ namespace StgSharp.Mathematics.Numeric
if (left.ColumnLength != ans.ColumnLength || left.RowLength != ans.RowLength) {
throw new IndexOutOfRangeException("Matrix dimensions must agree.");
}
if (left.Layout != MatrixLayout.DenseRectangle ||
ans.Layout != MatrixLayout.DenseRectangle) {
throw new NotSupportedException("Only dense rectangle matrices are supported.");
}
long count = (long)left.KernelColumnLength * left.KernelRowLength;
ScalarPacket* scalar = MatrixParallelFactory.CreateScalarPacket();
scalar->Data<float>(0) = right;
@@ -246,6 +250,11 @@ namespace StgSharp.Mathematics.Numeric
left.RowLength != ans.RowLength) {
throw new IndexOutOfRangeException("Matrix dimensions must agree.");
}
if (left.Layout != MatrixLayout.DenseRectangle ||
right.Layout != MatrixLayout.DenseRectangle ||
ans.Layout != MatrixLayout.DenseRectangle) {
throw new NotSupportedException("Only dense rectangle matrices are supported.");
}
long count = (long)left.KernelColumnLength * left.KernelRowLength;
if (count <= 256)
{
StgSharp.Native/StgSharp.Native.vcxproj
+1 -1
View File
@@ -151,7 +151,7 @@
<ClCompile Include="sn-src\k_panel.c" />
<ClCompile Include="sn-src\k_basicop.c" />
<ClCompile Include="sn-src\dllmain.c" />
<ClCompile Include="sn-src\k_kerop.c" />
<ClCompile Include="sn-src\k_bufop.c" />
<ClCompile Include="sn-src\k_pivot.c" />
<ClCompile Include="sn-src\str_factorial.c" />
<ClCompile Include="sn-src\k_fma.c" />
StgSharp.Native/StgSharp.Native.vcxproj.filters
+1 -1
View File
@@ -48,7 +48,7 @@
<ClCompile Include="sn-src\load_intrinsic.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\k_kerop.c">
<ClCompile Include="sn-src\k_bufop.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\k_panel.c">
StgSharp.Native/sn-head/sn_intrinsic.h
+46 -27
View File
@@ -96,6 +96,40 @@ static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mis
_Static_assert(sizeof(MAT_TASK(float)) == 128, "matrix_parallel_task_package size mismatch");
#endif
#pragma region matrix buffer operation function type
typedef void(SN_DECL *BUF_PROC_AS)(mat_kernel *ans, scalar_pack *scalar, int count);
typedef void(SN_DECL *BUF_PROC_RAS)(mat_kernel *right, mat_kernel *ans, scalar_pack *scalar,
int count);
typedef void(SN_DECL *BUF_PROC_LRAS)(mat_kernel *left, mat_kernel *right, mat_kernel *ans,
scalar_pack *scalar, int count);
typedef void(SN_DECL *BUF_PROC_RA)(mat_kernel *right, mat_kernel *ans, int count);
typedef void(SN_DECL *BUF_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans,
int count);
#define BUF_PROC(T_type, T_arch, T_op_name) T_type##_buf_##T_op_name##_##T_arch
#define DECLARE_BUF_PROC_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, size_t count)
#define DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
scalar_pack const *scalar, int count)
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, size_t count)
#define DECLARE_BUF_PROC_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, size_t count)
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, size_t count)
#pragma endregion
#pragma region matrix panel function type
typedef void(SN_DECL *PANEL_PROC)(mat_panel *panel, mat_kernel *kernel, int col_length,
@@ -142,9 +176,6 @@ typedef void(SN_DECL *KER_PROC_LAS)(mat_kernel *matrix, mat_kernel *ans, scalar_
#define KER_PROC(T_type, T_arch, T_op_name) T_type##_kernel_##T_op_name##_##T_arch
#define DECLARE_KER_PROC_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar)
#define DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
@@ -185,25 +216,25 @@ typedef uint64_t(SN_DECL *FACTORIALROC)(int n);
INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, add);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, add);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, add);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, sub);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, sub);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, sub);
DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose);
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul);
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul);
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul);
DECLARE_KER_PROC_ANS_SCALAR(float, sse, fill);
DECLARE_KER_PROC_ANS_SCALAR(float, avx, fill);
DECLARE_KER_PROC_ANS_SCALAR(float, 512, fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill);
#pragma endregion
@@ -249,18 +280,6 @@ INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int lengt
#pragma endregion
typedef struct sn_mat_enumerator {
int CurrentColumn;
int CurrentRow;
int KernelColumnLength;
int KernelRowLength;
} sn_mat_enumerator;
typedef struct sn_mat_boarder {
int (*SN_DECL is_in_buffer)(int col, int row, sn_mat_enumerator *enumerator);
long (*SN_DECL get_index_offset)(int col, int row, sn_mat_enumerator *wnumerator);
} sn_mat_boarder;
// Mirrors C# IntrinsicContext field order so managed/unmanaged layouts stay in sync.
typedef struct sn_intrinsic {
HASH city_hash_simplify;
StgSharp.Native/sn-src/k_bufop.c
+134
View File
@@ -0,0 +1,134 @@
#include "sn_intrinsic.h"
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill)
{
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].xmm[0] = vec;
ans[i].xmm[1] = vec;
ans[i].xmm[2] = vec;
ans[i].xmm[3] = vec;
}
}
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
ans[i].ymm[0] = vec;
ans[i].ymm[1] = vec;
}
}
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].zmm[0] = vec;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, add)
{
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_add_ps(left[i].xmm[0], right[i].xmm[0]);
register __m128 c1 = _mm_add_ps(left[i].xmm[1], right[i].xmm[1]);
register __m128 c2 = _mm_add_ps(left[i].xmm[2], right[i].xmm[2]);
register __m128 c3 = _mm_add_ps(left[i].xmm[3], right[i].xmm[3]);
ans[i].xmm[0] = c0;
ans[i].xmm[1] = c1;
ans[i].xmm[2] = c2;
ans[i].xmm[3] = c3;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, add)
{
for (size_t i = 0; i < count; i++) {
size_t s = sizeof(MAT_KERNEL(float));
register __m256 c0 = _mm256_add_ps(left[i].ymm[0], right[i].ymm[0]);
register __m256 c1 = _mm256_add_ps(left[i].ymm[1], right[i].ymm[1]);
ans[i].ymm[0] = c0;
ans[i].ymm[1] = c1;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, add)
{
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_add_ps(left[i].zmm[0], right[i].zmm[0]);
ans[i].zmm[0] = c0;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, sub)
{
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_sub_ps(left[i].xmm[0], right[i].xmm[0]);
register __m128 c1 = _mm_sub_ps(left[i].xmm[1], right[i].xmm[1]);
register __m128 c2 = _mm_sub_ps(left[i].xmm[2], right[i].xmm[2]);
register __m128 c3 = _mm_sub_ps(left[i].xmm[3], right[i].xmm[3]);
ans[i].xmm[0] = c0;
ans[i].xmm[1] = c1;
ans[i].xmm[2] = c2;
ans[i].xmm[3] = c3;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, sub)
{
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].ymm[0], right[i].ymm[0]);
register __m256 c1 = _mm256_sub_ps(left[i].ymm[1], right[i].ymm[1]);
ans[i].ymm[0] = c0;
ans[i].ymm[1] = c1;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, sub)
{
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].zmm[0], right[i].zmm[0]);
ans[i].zmm[0] = c0;
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul)
{
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_mul_ps(right[i].xmm[0], vec);
register __m128 c1 = _mm_mul_ps(right[i].xmm[1], vec);
register __m128 c2 = _mm_mul_ps(right[i].xmm[2], vec);
register __m128 c3 = _mm_mul_ps(right[i].xmm[3], vec);
ans[i].xmm[0] = c0;
ans[i].xmm[1] = c1;
ans[i].xmm[2] = c2;
ans[i].xmm[3] = c3;
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_mul_ps(right[i].ymm[0], vec);
register __m256 c1 = _mm256_mul_ps(right[i].ymm[1], vec);
ans[i].ymm[0] = c0;
ans[i].ymm[1] = c1;
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_mul_ps(right[i].zmm[0], vec);
ans[i].zmm[0] = c0;
}
}
StgSharp.Native/sn-src/k_kerop.c
-113
View File
@@ -1,113 +0,0 @@
#include "sn_intrinsic.h"
DECLARE_KER_PROC_ANS_SCALAR(float, sse, fill)
{
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
ans->xmm[0] = vec;
ans->xmm[1] = vec;
ans->xmm[2] = vec;
ans->xmm[3] = vec;
}
DECLARE_KER_PROC_ANS_SCALAR(float, avx, fill)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
ans->ymm[0] = vec;
ans->ymm[1] = vec;
}
DECLARE_KER_PROC_ANS_SCALAR(float, 512, fill)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
ans[i].zmm[0] = vec;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, add)
{
register __m128 c0 = _mm_add_ps(left->xmm[0], right->xmm[0]);
register __m128 c1 = _mm_add_ps(left->xmm[1], right->xmm[1]);
register __m128 c2 = _mm_add_ps(left->xmm[2], right->xmm[2]);
register __m128 c3 = _mm_add_ps(left->xmm[3], right->xmm[3]);
ans->xmm[0] = c0;
ans->xmm[1] = c1;
ans->xmm[2] = c2;
ans->xmm[3] = c3;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, add)
{
register __m256 c0 = _mm256_add_ps(left->ymm[0], right->ymm[0]);
register __m256 c1 = _mm256_add_ps(left->ymm[1], right->ymm[1]);
ans->ymm[0] = c0;
ans->ymm[1] = c1;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, add)
{
register __m512 c0 = _mm512_add_ps(left->zmm[0], right->zmm[0]);
ans->zmm[0] = c0;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, sub)
{
register __m128 c0 = _mm_sub_ps(left->xmm[0], right->xmm[0]);
register __m128 c1 = _mm_sub_ps(left->xmm[1], right->xmm[1]);
register __m128 c2 = _mm_sub_ps(left->xmm[2], right->xmm[2]);
register __m128 c3 = _mm_sub_ps(left->xmm[3], right->xmm[3]);
ans->xmm[0] = c0;
ans->xmm[1] = c1;
ans->xmm[2] = c2;
ans->xmm[3] = c3;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, sub)
{
register __m256 c0 = _mm256_sub_ps(left->ymm[0], right->ymm[0]);
register __m256 c1 = _mm256_sub_ps(left->ymm[1], right->ymm[1]);
ans->ymm[0] = c0;
ans->ymm[1] = c1;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, sub)
{
register __m512 c0 = _mm512_sub_ps(left->zmm[0], right->zmm[0]);
ans->zmm[0] = c0;
}
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul)
{
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
register __m128 c0 = _mm_mul_ps(right->xmm[0], vec);
register __m128 c1 = _mm_mul_ps(right->xmm[1], vec);
register __m128 c2 = _mm_mul_ps(right->xmm[2], vec);
register __m128 c3 = _mm_mul_ps(right->xmm[3], vec);
ans->xmm[0] = c0;
ans->xmm[1] = c1;
ans->xmm[2] = c2;
ans->xmm[3] = c3;
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_mul_ps(right[i].ymm[0], vec);
register __m256 c1 = _mm256_mul_ps(right[i].ymm[1], vec);
ans[i].ymm[0] = c0;
ans[i].ymm[1] = c1;
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_mul_ps(right[i].zmm[0], vec);
ans[i].zmm[0] = c0;
}
}
StgSharpDebug/Program.cs
+3 -3
View File
@@ -58,11 +58,11 @@ namespace StgSharp
unchecked {
hlsf = new((nuint)(64 * (ulong)10000 * 10000));
}
Matrix<float> m = Matrix<float>.Create(4000, 4000, MatrixLayout.DenseSquare, hlsf);
Matrix<float> m = Matrix<float>.Create(4000, 4000, MatrixLayout.DenseRectangle, hlsf);
MatrixCompute.Fill(m, 1.0f);
Matrix<float> n = Matrix<float>.Create(4000, 4000, MatrixLayout.DenseSquare, hlsf);
Matrix<float> n = Matrix<float>.Create(4000, 4000, MatrixLayout.DenseRectangle, hlsf);
MatrixCompute.Fill(n, 1.0f);
Matrix<float> p = Matrix<float>.Create(4000, 4000, MatrixLayout.DenseSquare, hlsf);
Matrix<float> p = Matrix<float>.Create(4000, 4000, MatrixLayout.DenseRectangle, hlsf);
// MatrixCompute.Fill(p, 0.0f);
/*
StgSharpDevelop.sln
+14 -14
View File
@@ -46,8 +46,8 @@ Global
{3EAF2EE6-FA9E-4F57-951F-7E6F8125DFBF}.Debug|x64.Build.0 = Debug|x64
{3EAF2EE6-FA9E-4F57-951F-7E6F8125DFBF}.PerformanceDebug|ARM64.ActiveCfg = PerformanceDebug|ARM64
{3EAF2EE6-FA9E-4F57-951F-7E6F8125DFBF}.PerformanceDebug|ARM64.Build.0 = PerformanceDebug|ARM64
{3EAF2EE6-FA9E-4F57-951F-7E6F8125DFBF}.PerformanceDebug|x64.ActiveCfg = PerformanceDebug|x64
{3EAF2EE6-FA9E-4F57-951F-7E6F8125DFBF}.PerformanceDebug|x64.Build.0 = PerformanceDebug|x64
{3EAF2EE6-FA9E-4F57-951F-7E6F8125DFBF}.PerformanceDebug|x64.ActiveCfg = Release|x64
{3EAF2EE6-FA9E-4F57-951F-7E6F8125DFBF}.PerformanceDebug|x64.Build.0 = Release|x64
{3EAF2EE6-FA9E-4F57-951F-7E6F8125DFBF}.Release|ARM64.ActiveCfg = Release|ARM64
{3EAF2EE6-FA9E-4F57-951F-7E6F8125DFBF}.Release|ARM64.Build.0 = Release|ARM64
{3EAF2EE6-FA9E-4F57-951F-7E6F8125DFBF}.Release|x64.ActiveCfg = Release|x64
@@ -58,8 +58,8 @@ Global
{610B992B-BB34-49C3-9993-676C595FC12E}.Debug|x64.Build.0 = Debug|x64
{610B992B-BB34-49C3-9993-676C595FC12E}.PerformanceDebug|ARM64.ActiveCfg = PerformanceDebug|ARM64
{610B992B-BB34-49C3-9993-676C595FC12E}.PerformanceDebug|ARM64.Build.0 = PerformanceDebug|ARM64
{610B992B-BB34-49C3-9993-676C595FC12E}.PerformanceDebug|x64.ActiveCfg = PerformanceDebug|x64
{610B992B-BB34-49C3-9993-676C595FC12E}.PerformanceDebug|x64.Build.0 = PerformanceDebug|x64
{610B992B-BB34-49C3-9993-676C595FC12E}.PerformanceDebug|x64.ActiveCfg = Release|x64
{610B992B-BB34-49C3-9993-676C595FC12E}.PerformanceDebug|x64.Build.0 = Release|x64
{610B992B-BB34-49C3-9993-676C595FC12E}.Release|ARM64.ActiveCfg = Release|ARM64
{610B992B-BB34-49C3-9993-676C595FC12E}.Release|ARM64.Build.0 = Release|ARM64
{610B992B-BB34-49C3-9993-676C595FC12E}.Release|x64.ActiveCfg = Release|x64
@@ -69,8 +69,8 @@ Global
{20CBFF26-6746-402E-992C-7B643E9423EB}.Debug|x64.Build.0 = Debug|x64
{20CBFF26-6746-402E-992C-7B643E9423EB}.PerformanceDebug|ARM64.ActiveCfg = PerformanceDebug|x64
{20CBFF26-6746-402E-992C-7B643E9423EB}.PerformanceDebug|ARM64.Build.0 = PerformanceDebug|x64
{20CBFF26-6746-402E-992C-7B643E9423EB}.PerformanceDebug|x64.ActiveCfg = PerformanceDebug|x64
{20CBFF26-6746-402E-992C-7B643E9423EB}.PerformanceDebug|x64.Build.0 = PerformanceDebug|x64
{20CBFF26-6746-402E-992C-7B643E9423EB}.PerformanceDebug|x64.ActiveCfg = Release|x64
{20CBFF26-6746-402E-992C-7B643E9423EB}.PerformanceDebug|x64.Build.0 = Release|x64
{20CBFF26-6746-402E-992C-7B643E9423EB}.Release|ARM64.ActiveCfg = Release|ARM64
{20CBFF26-6746-402E-992C-7B643E9423EB}.Release|ARM64.Build.0 = Release|ARM64
{20CBFF26-6746-402E-992C-7B643E9423EB}.Release|x64.ActiveCfg = Release|x64
@@ -81,8 +81,8 @@ Global
{10B5819F-2A62-41B6-980A-EB9E76A0E95C}.Debug|x64.Build.0 = Debug|x64
{10B5819F-2A62-41B6-980A-EB9E76A0E95C}.PerformanceDebug|ARM64.ActiveCfg = PerformanceDebug|x64
{10B5819F-2A62-41B6-980A-EB9E76A0E95C}.PerformanceDebug|ARM64.Build.0 = PerformanceDebug|x64
{10B5819F-2A62-41B6-980A-EB9E76A0E95C}.PerformanceDebug|x64.ActiveCfg = PerformanceDebug|x64
{10B5819F-2A62-41B6-980A-EB9E76A0E95C}.PerformanceDebug|x64.Build.0 = PerformanceDebug|x64
{10B5819F-2A62-41B6-980A-EB9E76A0E95C}.PerformanceDebug|x64.ActiveCfg = Release|x64
{10B5819F-2A62-41B6-980A-EB9E76A0E95C}.PerformanceDebug|x64.Build.0 = Release|x64
{10B5819F-2A62-41B6-980A-EB9E76A0E95C}.Release|ARM64.ActiveCfg = Release|x64
{10B5819F-2A62-41B6-980A-EB9E76A0E95C}.Release|ARM64.Build.0 = Release|x64
{10B5819F-2A62-41B6-980A-EB9E76A0E95C}.Release|x64.ActiveCfg = Release|x64
@@ -93,8 +93,8 @@ Global
{5B51F5EA-138C-46B5-B36F-1D389968B6CF}.Debug|x64.Build.0 = Debug|x64
{5B51F5EA-138C-46B5-B36F-1D389968B6CF}.PerformanceDebug|ARM64.ActiveCfg = PerformanceDebug|x64
{5B51F5EA-138C-46B5-B36F-1D389968B6CF}.PerformanceDebug|ARM64.Build.0 = PerformanceDebug|x64
{5B51F5EA-138C-46B5-B36F-1D389968B6CF}.PerformanceDebug|x64.ActiveCfg = PerformanceDebug|x64
{5B51F5EA-138C-46B5-B36F-1D389968B6CF}.PerformanceDebug|x64.Build.0 = PerformanceDebug|x64
{5B51F5EA-138C-46B5-B36F-1D389968B6CF}.PerformanceDebug|x64.ActiveCfg = Release|x64
{5B51F5EA-138C-46B5-B36F-1D389968B6CF}.PerformanceDebug|x64.Build.0 = Release|x64
{5B51F5EA-138C-46B5-B36F-1D389968B6CF}.Release|ARM64.ActiveCfg = Release|x64
{5B51F5EA-138C-46B5-B36F-1D389968B6CF}.Release|ARM64.Build.0 = Release|x64
{5B51F5EA-138C-46B5-B36F-1D389968B6CF}.Release|x64.ActiveCfg = Release|x64
@@ -105,8 +105,8 @@ Global
{D9538DE0-EE4C-4183-86D6-7DFC08A7BDFB}.Debug|x64.Build.0 = Debug|x64
{D9538DE0-EE4C-4183-86D6-7DFC08A7BDFB}.PerformanceDebug|ARM64.ActiveCfg = PerformanceDebug|x64
{D9538DE0-EE4C-4183-86D6-7DFC08A7BDFB}.PerformanceDebug|ARM64.Build.0 = PerformanceDebug|x64
{D9538DE0-EE4C-4183-86D6-7DFC08A7BDFB}.PerformanceDebug|x64.ActiveCfg = PerformanceDebug|x64
{D9538DE0-EE4C-4183-86D6-7DFC08A7BDFB}.PerformanceDebug|x64.Build.0 = PerformanceDebug|x64
{D9538DE0-EE4C-4183-86D6-7DFC08A7BDFB}.PerformanceDebug|x64.ActiveCfg = Release|x64
{D9538DE0-EE4C-4183-86D6-7DFC08A7BDFB}.PerformanceDebug|x64.Build.0 = Release|x64
{D9538DE0-EE4C-4183-86D6-7DFC08A7BDFB}.Release|ARM64.ActiveCfg = Release|ARM64
{D9538DE0-EE4C-4183-86D6-7DFC08A7BDFB}.Release|ARM64.Build.0 = Release|ARM64
{D9538DE0-EE4C-4183-86D6-7DFC08A7BDFB}.Release|x64.ActiveCfg = Release|x64
@@ -117,8 +117,8 @@ Global
{9946599F-4D34-440F-840D-F9443D5AEFAA}.Debug|x64.Build.0 = Debug|x64
{9946599F-4D34-440F-840D-F9443D5AEFAA}.PerformanceDebug|ARM64.ActiveCfg = PerformanceDebug|ARM64
{9946599F-4D34-440F-840D-F9443D5AEFAA}.PerformanceDebug|ARM64.Build.0 = PerformanceDebug|ARM64
{9946599F-4D34-440F-840D-F9443D5AEFAA}.PerformanceDebug|x64.ActiveCfg = PerformanceDebug|x64
{9946599F-4D34-440F-840D-F9443D5AEFAA}.PerformanceDebug|x64.Build.0 = PerformanceDebug|x64
{9946599F-4D34-440F-840D-F9443D5AEFAA}.PerformanceDebug|x64.ActiveCfg = Release|x64
{9946599F-4D34-440F-840D-F9443D5AEFAA}.PerformanceDebug|x64.Build.0 = Release|x64
{9946599F-4D34-440F-840D-F9443D5AEFAA}.Release|ARM64.ActiveCfg = Release|ARM64
{9946599F-4D34-440F-840D-F9443D5AEFAA}.Release|ARM64.Build.0 = Release|ARM64
{9946599F-4D34-440F-840D-F9443D5AEFAA}.Release|x64.ActiveCfg = Release|x64