youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

small size buffer op ok

Browse Source
This commit is contained in:
Youmu Liu
2025-11-14 18:28:15 +08:00
parent 1417785726
commit a8c522ab23
19 changed files with 578 additions and 1048 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/Collections/CapacityFixedStack.cs
+1 -3
View File
@@ -99,9 +99,7 @@ namespace StgSharp.Collections
public void Push(T item)
{
_values[++_index] = _index < _values.Length - 1 ?
item :
throw new InvalidOperationException("Stack is full.");
_values[++_index] = _index < _values.Length ? item : throw new InvalidOperationException("Stack is full.");
}
public void PushRange(ReadOnlySpan<T> span)
StgSharp.Common/HighPerformance/Memory/SlabAllocator.ChunkedConcurrent.cs
+1 -1
View File
@@ -38,7 +38,7 @@ using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
internal sealed unsafe class ConcurrentChunkedSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
internal sealed unsafe class ConcurrentChunkedSlabAllocator<T> : SlabAllocator<T> where T : unmanaged
{
private readonly BufferExpansionLock _lock = new();
StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs
+5 -4
View File
@@ -30,6 +30,7 @@ using System;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.InteropServices.Marshalling;
namespace StgSharp.Internal.Intrinsic
{
@@ -38,11 +39,11 @@ namespace StgSharp.Internal.Intrinsic
{
public delegate* unmanaged[Cdecl]<char*, int, int> city_hash_simplify;
public delegate* unmanaged[Cdecl]<void*, void*, void*, void> f32_add;
public delegate* unmanaged[Cdecl]<void*, ScalarPacket*, void> f32_fill;
public delegate* unmanaged[Cdecl]<void*, void*, void*, long, void> f32_add;
public delegate* unmanaged[Cdecl]<void*, ScalarPacket*,long, void> f32_fill;
public delegate* unmanaged[Cdecl]<void*, void*, void*, void> f32_fma;
public delegate* unmanaged[Cdecl]<void*, void*, ScalarPacket*, void> f32_scalar_mul;
public delegate* unmanaged[Cdecl]<void*, void*, void*, void> f32_sub;
public delegate* unmanaged[Cdecl]<void*, void*, ScalarPacket*, long, void> f32_scalar_mul;
public delegate* unmanaged[Cdecl]<void*, void*, void*, long, void> f32_sub;
public delegate* unmanaged[Cdecl]<void*, void*, void> f32_transpose;
public delegate* unmanaged[Cdecl]<int, ulong> factorial_simd;
public delegate* unmanaged[Cdecl]<char*, int, int, int> index_pair;
StgSharp.Common/Mathematics/Graphic/Matrix44.cs
+2 -2
View File
@@ -212,7 +212,7 @@ namespace StgSharp.Mathematics.Graphic
public static unsafe Matrix44 operator -(Matrix44 left, Matrix44 right)
{
Matrix44 ret = new();
NativeIntrinsic.Intrinsic.f32_sub(&left, &right, &ret);
NativeIntrinsic.Intrinsic.f32_sub(&left, &right, &ret, 1);
return ret;
}
@@ -254,7 +254,7 @@ namespace StgSharp.Mathematics.Graphic
public static unsafe Matrix44 operator +(Matrix44 left, Matrix44 right)
{
Matrix44 ret = new();
NativeIntrinsic.Intrinsic.f32_add(&left, &right, &ret);
NativeIntrinsic.Intrinsic.f32_add(&left, &right, &ret, 1);
return ret;
}
StgSharp.Common/Mathematics/Linear/Matrix.GC.cs
+8 -3
View File
@@ -38,7 +38,7 @@ using System.Threading.Tasks;
namespace StgSharp.Mathematics.Numeric
{
internal unsafe partial class GCMatrix<T> : Matrix<T> where T: unmanaged, INumber<T>
internal unsafe partial class GCMatrix<T> : Matrix<T> where T : unmanaged, INumber<T>
{
private readonly MatrixKernel<T>* _buffer;
@@ -47,6 +47,8 @@ namespace StgSharp.Mathematics.Numeric
{
KernelColumnLength = (column + 3) / 4;
KernelRowLength = (row + 3) / t_count;
ColumnLength = column;
RowLength = row;
_buffer = (MatrixKernel<T>*)NativeMemory.AlignedAlloc((nuint)(column * row * sizeof(T)), (nuint)World.DefaultSIMDAlignment);
}
@@ -95,7 +97,8 @@ namespace StgSharp.Mathematics.Numeric
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override unsafe MatrixSegmentEnumeration<T> GetColumnEnumeration()
{
return new MatrixSegmentEnumeration<T>(_buffer, KernelColumnLength, KernelColumnLength * KernelRowLength, 1, KernelColumnLength);
return new MatrixSegmentEnumeration<T>(_buffer, KernelColumnLength, KernelColumnLength * KernelRowLength, 1,
KernelColumnLength);
}
public override unsafe MatrixSegmentEnumeration<T> GetRowEnumeration()
@@ -106,7 +109,9 @@ namespace StgSharp.Mathematics.Numeric
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override unsafe MatrixSegmentEnumeration<T> GetSequentialEnumeration()
{
return new MatrixSegmentEnumeration<T>(_buffer, KernelRowLength * KernelColumnLength, KernelColumnLength * KernelRowLength, 1, KernelRowLength * KernelColumnLength);
return new MatrixSegmentEnumeration<T>(_buffer, KernelRowLength * KernelColumnLength,
KernelColumnLength * KernelRowLength, 1,
KernelRowLength * KernelColumnLength);
}
#endregion
StgSharp.Common/Mathematics/Linear/Matrix.HLSF.cs
+10 -4
View File
@@ -39,7 +39,7 @@ using hlsfHandle = global::StgSharp.HighPerformance.Memory.HybridLayerSegregated
namespace StgSharp.Mathematics.Numeric
{
internal unsafe partial class HlsfMatrix<T> : Matrix<T> where T: unmanaged, INumber<T>
internal unsafe partial class HlsfMatrix<T> : Matrix<T> where T : unmanaged, INumber<T>
{
private readonly MatrixKernel<T>* k_buffer;
@@ -50,6 +50,8 @@ namespace StgSharp.Mathematics.Numeric
{
KernelColumnLength = (column + 3) / 4;
KernelRowLength = (row + 3) / t_count;
ColumnLength = column;
RowLength = row;
this.handle = allocator.Alloc((uint)(KernelColumnLength * KernelRowLength * sizeof(MatrixKernel<T>)));
k_buffer = (MatrixKernel<T>*)handle.Pointer;
}
@@ -104,18 +106,22 @@ namespace StgSharp.Mathematics.Numeric
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override unsafe MatrixSegmentEnumeration<T> GetColumnEnumeration()
{
return new MatrixSegmentEnumeration<T>(k_buffer, KernelColumnLength, KernelColumnLength * KernelRowLength, 1, KernelColumnLength);
return new MatrixSegmentEnumeration<T>(k_buffer, KernelColumnLength, KernelColumnLength * KernelRowLength,
1, KernelColumnLength);
}
public override unsafe MatrixSegmentEnumeration<T> GetRowEnumeration()
{
return new MatrixSegmentEnumeration<T>(k_buffer, 1, KernelColumnLength, KernelColumnLength, KernelRowLength);
return new MatrixSegmentEnumeration<T>(k_buffer, 1, KernelColumnLength, KernelColumnLength,
KernelRowLength);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override unsafe MatrixSegmentEnumeration<T> GetSequentialEnumeration()
{
return new MatrixSegmentEnumeration<T>(k_buffer, KernelRowLength * KernelColumnLength, KernelColumnLength * KernelRowLength, 1, KernelRowLength * KernelColumnLength);
return new MatrixSegmentEnumeration<T>(k_buffer, KernelRowLength * KernelColumnLength,
KernelColumnLength * KernelRowLength, 1,
KernelRowLength * KernelColumnLength);
}
#endregion
StgSharp.Common/Threading/NumaBuffer.cs
+47
View File
@@ -0,0 +1,47 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="NumaBuffer"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Common.Threading
{
public unsafe struct NumaBuffer<T> where T : unmanaged
{
public readonly T* Buffers;
public readonly int BufferColumnCapacity;
public readonly int BufferRowCapacity;
public int CurrentColumnCount;
public int CurrentRowCount;
}
}
StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs
+28 -30
View File
@@ -41,41 +41,41 @@ namespace StgSharp.Mathematics.Numeric
public static class MatrixCompute
{
public static unsafe Matrix<float> Add(Matrix<float> left, Matrix<float> right)
public static unsafe Matrix<float> Add(this Matrix<float> left, Matrix<float> right)
{
if (left.ColumnLength != right.ColumnLength || left.RowLength != right.RowLength) {
throw new IndexOutOfRangeException("Matrix dimensions must agree.");
}
Matrix<float> ans = Matrix<float>.FromDefault(left.ColumnLength, left.RowLength);
long count = (long)left.KernelColumnLength * left.KernelRowLength;
Matrix<float> ans = Matrix<float>.FromDefault(left.ColumnLength, left.RowLength);
if (count <= 256)
{
MatrixKernel<float>* leftPtr = left.Buffer;
MatrixKernel<float>* rightPtr = right.Buffer;
MatrixKernel<float>* ansPtr = ans.Buffer;
for (int i = 0; i < count; i++) {
NativeIntrinsic.Intrinsic.f32_add(leftPtr + i, rightPtr + i, ansPtr + i);
}
NativeIntrinsic.Intrinsic.f32_add(leftPtr, rightPtr, ansPtr, count);
return ans;
}
if (count <= 1024)
} else if (count <= 1024)
{
MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1);
MatrixKernel<float>* leftPtr = left.Buffer;
MatrixKernel<float>* rightPtr = right.Buffer;
MatrixKernel<float>* ansPtr = ans.Buffer;
for (int i = 0; i < count; i++) {
NativeIntrinsic.Intrinsic.f32_add(leftPtr + i, rightPtr + i, ansPtr + i);
}
NativeIntrinsic.Intrinsic.f32_add(leftPtr, rightPtr, ansPtr, count);
MatrixParallel.ReturnParallelResources(ref pool);
return ans;
}
MatrixParallelHandle taskGroup = MatrixParallel.PublicTask(left.GetColumnEnumeration(),
MatrixParallelHandle taskGroup = /**/ default
/*
MatrixParallel.PublicTask(left.GetColumnEnumeration(),
right.GetColumnEnumeration(),
ans.GetColumnEnumeration(),
(delegate*<MatrixKernel<float>*, MatrixKernel<float>*, MatrixKernel<float>*, void>)NativeIntrinsic.Intrinsic.f32_add);
(delegate*<MatrixKernel<float>*, MatrixKernel<float>*, MatrixKernel<float>*, void>)NativeIntrinsic.Intrinsic.f32_add)
/**/
;
MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep);
/*
* run parallel
@@ -84,35 +84,33 @@ namespace StgSharp.Mathematics.Numeric
return ans;
}
public static unsafe void Fill<T>(Matrix<float> left, T value) where T: unmanaged, INumber<T>
public static unsafe void Fill<T>(this Matrix<float> right, T value) where T : unmanaged, INumber<T>
{
long count = (long)left.KernelColumnLength * left.KernelRowLength;
long count = (long)right.KernelColumnLength * right.KernelRowLength;
ScalarPacket* scaler = MatrixParallelFactory.CreateScalarPacket();
scaler->BroadCast<T, M512>(0, value);
scaler->Data<T>(0) = value;
if (count <= 256)
{
MatrixKernel<float>* leftPtr = left.Buffer;
for (int i = 0; i < count; i++) {
NativeIntrinsic.Intrinsic.f32_fill(leftPtr + i, scaler);
}
}
if (count <= 1024)
NativeIntrinsic.Intrinsic.f32_fill(right.Buffer, scaler, count);
return;
} else if (count <= 1024)
{
MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1);
MatrixKernel<float>* leftPtr = left.Buffer;
for (int i = 0; i < count; i++) {
NativeIntrinsic.Intrinsic.f32_fill(leftPtr + i, scaler);
}
MatrixParallel.ReturnParallelResources(ref pool);
}
MatrixParallelHandle taskGroup = MatrixParallel.PublicNoAnswerTask(left.GetColumnEnumeration(), scaler,
(delegate* unmanaged[Cdecl]<MatrixKernel<float>*, ScalarPacket*, void>)NativeIntrinsic.Intrinsic.f32_add);
NativeIntrinsic.Intrinsic.f32_fill(right.Buffer, scaler, count);
MatrixParallel.ReturnParallelResources(ref pool);
return;
}
MatrixParallelHandle taskGroup =/**/ default
/*MatrixParallel.PublicNoAnswerTask(left.GetColumnEnumeration(), scaler,
(delegate* unmanaged[Cdecl]<MatrixKernel<float>*, ScalarPacket*, void>)NativeIntrinsic.Intrinsic.f32_add)
/**/;
MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep);
/*
* run parallel
*/
MatrixParallelFactory.Release(scaler);
taskGroup.Dispose();
}
StgSharp.Mathematics/Numeric/MatrixComputeModel.cs
+23
View File
@@ -34,6 +34,29 @@ namespace StgSharp.Mathematics.Numeric
internal static class MatrixComputeModel
{
internal static unsafe void BufferComputeBinary(MatrixParallelTaskPackageNonGeneric* p)
{
int size = p->ElementSize * 16;
int count = p->PrimCount * p->SecCount;
int offset = p->LeftPrimOffset * p->SecCount;
((delegate* unmanaged[Cdecl]<void*, void*, void*, int, void>)p->ComputeHandle)((void*)(p->Left + offset * size),
(void*)(p->Right + offset * size),
(void*)(p->Result + offset * size),
count);
}
internal static unsafe void BufferComputeUnaryScalar(MatrixParallelTaskPackageNonGeneric* p)
{
int size = p->ElementSize * 16;
int count = p->PrimCount * p->SecCount;
int offset = p->LeftPrimOffset * p->SecCount;
((delegate* unmanaged[Cdecl]<void*, void*, int, void>)p->ComputeHandle)((void*)(p->Right + offset * size),
(void*)(p->Result + offset * size),
count);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static unsafe void Compute_Binary(MatrixParallelTaskPackageNonGeneric* p)
{
StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs
+1 -794
View File
@@ -40,801 +40,8 @@ namespace StgSharp.Mathematics.Numeric
public static partial class MatrixParallel
{
public static unsafe MatrixParallelHandle PublicNoAnswerTask<T>(
MatrixSegmentEnumeration<T> source,
ScalarPacket* additional,
delegate* unmanaged[Cdecl]<MatrixKernel<T>*, ScalarPacket*, void> computeHandle)
where T: unmanaged, INumber<T>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Scalar = additional;
package->Right = source.Source;
package->RightPrimOffset = source.PrimOffset;
package->RightPrimStride = source.PrimStride;
package->RightSecOffset = source.SecondaryOffset;
package->RightSecStride = source.SecondaryStride;
package->PrimCount = source.PrimSize;
package->SecCount = source.SecondarySize;
package->ElementSize = sizeof(T);
package->ComputeHandle = (nint)computeHandle;
package->ComputeMode = 6; // no ans
return PublicTaskPrivate(package);
}
public static unsafe MatrixParallelHandle PublicTask<T>(
MatrixSegmentEnumeration<T> source,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation)
where T: unmanaged, INumber<T>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Right = source.Source;
package->Result = ans.Source;
package->RightPrimOffset = source.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->RightPrimStride = source.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->RightSecOffset = source.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->RightSecStride = source.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = source.PrimSize;
package->SecCount = source.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 1; // Unary
return PublicTaskPrivate<T>(package);
}
public static unsafe MatrixParallelHandle PublicTask<T>(
MatrixSegmentEnumeration<T> left,
MatrixSegmentEnumeration<T> right,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation)
where T: unmanaged, INumber<T>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Left = left.Source;
package->Right = right.Source;
package->Result = ans.Source;
package->LeftPrimOffset = left.PrimOffset;
package->RightPrimOffset = right.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->LeftPrimStride = left.PrimStride;
package->RightPrimStride = right.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->LeftSecOffset = left.SecondaryOffset;
package->RightSecOffset = right.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->LeftSecStride = left.SecondaryStride;
package->RightSecStride = right.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = left.PrimSize;
package->SecCount = left.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 3; // Binary
return PublicTaskPrivate<T>(package);
}
// Unary with 1..8 scalars (each scalar is independent generic type)
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0>(
MatrixSegmentEnumeration<T> source,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Right = source.Source;
package->Result = ans.Source;
package->RightPrimOffset = source.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->RightPrimStride = source.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->RightSecOffset = source.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->RightSecStride = source.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = source.PrimSize;
package->SecCount = source.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 2; // UnaryScalar
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0, TScalar1>(
MatrixSegmentEnumeration<T> source,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0,
TScalar1 s1)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
where TScalar1: unmanaged, INumber<TScalar1>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Right = source.Source;
package->Result = ans.Source;
package->RightPrimOffset = source.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->RightPrimStride = source.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->RightSecOffset = source.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->RightSecStride = source.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = source.PrimSize;
package->SecCount = source.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 2;
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
sp->Data<TScalar1>(1) = s1;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
// Binary with 1..8 scalars (each scalar is independent generic type)
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0>(
MatrixSegmentEnumeration<T> left,
MatrixSegmentEnumeration<T> right,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Left = left.Source;
package->Right = right.Source;
package->Result = ans.Source;
package->LeftPrimOffset = left.PrimOffset;
package->RightPrimOffset = right.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->LeftPrimStride = left.PrimStride;
package->RightPrimStride = right.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->LeftSecOffset = left.SecondaryOffset;
package->RightSecOffset = right.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->LeftSecStride = left.SecondaryStride;
package->RightSecStride = right.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = left.PrimSize;
package->SecCount = left.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 4; // BinaryScalar
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0, TScalar1, TScalar2>(
MatrixSegmentEnumeration<T> source,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0,
TScalar1 s1,
TScalar2 s2)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
where TScalar1: unmanaged, INumber<TScalar1>
where TScalar2: unmanaged, INumber<TScalar2>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Right = source.Source;
package->Result = ans.Source;
package->RightPrimOffset = source.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->RightPrimStride = source.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->RightSecOffset = source.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->RightSecStride = source.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = source.PrimSize;
package->SecCount = source.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 2;
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
sp->Data<TScalar1>(1) = s1;
sp->Data<TScalar2>(2) = s2;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0, TScalar1>(
MatrixSegmentEnumeration<T> left,
MatrixSegmentEnumeration<T> right,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0,
TScalar1 s1)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
where TScalar1: unmanaged, INumber<TScalar1>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Left = left.Source;
package->Right = right.Source;
package->Result = ans.Source;
package->LeftPrimOffset = left.PrimOffset;
package->RightPrimOffset = right.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->LeftPrimStride = left.PrimStride;
package->RightPrimStride = right.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->LeftSecOffset = left.SecondaryOffset;
package->RightSecOffset = right.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->LeftSecStride = left.SecondaryStride;
package->RightSecStride = right.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = left.PrimSize;
package->SecCount = left.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 4;
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
sp->Data<TScalar1>(1) = s1;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0, TScalar1, TScalar2, TScalar3>(
MatrixSegmentEnumeration<T> source,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0,
TScalar1 s1,
TScalar2 s2,
TScalar3 s3)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
where TScalar1: unmanaged, INumber<TScalar1>
where TScalar2: unmanaged, INumber<TScalar2>
where TScalar3: unmanaged, INumber<TScalar3>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Right = source.Source;
package->Result = ans.Source;
package->RightPrimOffset = source.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->RightPrimStride = source.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->RightSecOffset = source.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->RightSecStride = source.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = source.PrimSize;
package->SecCount = source.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 2;
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
sp->Data<TScalar1>(1) = s1;
sp->Data<TScalar2>(2) = s2;
sp->Data<TScalar3>(3) = s3;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0, TScalar1, TScalar2>(
MatrixSegmentEnumeration<T> left,
MatrixSegmentEnumeration<T> right,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0,
TScalar1 s1,
TScalar2 s2)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
where TScalar1: unmanaged, INumber<TScalar1>
where TScalar2: unmanaged, INumber<TScalar2>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Left = left.Source;
package->Right = right.Source;
package->Result = ans.Source;
package->LeftPrimOffset = left.PrimOffset;
package->RightPrimOffset = right.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->LeftPrimStride = left.PrimStride;
package->RightPrimStride = right.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->LeftSecOffset = left.SecondaryOffset;
package->RightSecOffset = right.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->LeftSecStride = left.SecondaryStride;
package->RightSecStride = right.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = left.PrimSize;
package->SecCount = left.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 4;
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
sp->Data<TScalar1>(1) = s1;
sp->Data<TScalar2>(2) = s2;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0, TScalar1, TScalar2, TScalar3, TScalar4>(
MatrixSegmentEnumeration<T> source,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0,
TScalar1 s1,
TScalar2 s2,
TScalar3 s3,
TScalar4 s4)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
where TScalar1: unmanaged, INumber<TScalar1>
where TScalar2: unmanaged, INumber<TScalar2>
where TScalar3: unmanaged, INumber<TScalar3>
where TScalar4: unmanaged, INumber<TScalar4>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Right = source.Source;
package->Result = ans.Source;
package->RightPrimOffset = source.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->RightPrimStride = source.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->RightSecOffset = source.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->RightSecStride = source.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = source.PrimSize;
package->SecCount = source.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 2;
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
sp->Data<TScalar1>(1) = s1;
sp->Data<TScalar2>(2) = s2;
sp->Data<TScalar3>(3) = s3;
sp->Data<TScalar4>(4) = s4;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0, TScalar1, TScalar2, TScalar3>(
MatrixSegmentEnumeration<T> left,
MatrixSegmentEnumeration<T> right,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0,
TScalar1 s1,
TScalar2 s2,
TScalar3 s3)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
where TScalar1: unmanaged, INumber<TScalar1>
where TScalar2: unmanaged, INumber<TScalar2>
where TScalar3: unmanaged, INumber<TScalar3>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Left = left.Source;
package->Right = right.Source;
package->Result = ans.Source;
package->LeftPrimOffset = left.PrimOffset;
package->RightPrimOffset = right.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->LeftPrimStride = left.PrimStride;
package->RightPrimStride = right.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->LeftSecOffset = left.SecondaryOffset;
package->RightSecOffset = right.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->LeftSecStride = left.SecondaryStride;
package->RightSecStride = right.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = left.PrimSize;
package->SecCount = left.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 4;
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
sp->Data<TScalar1>(1) = s1;
sp->Data<TScalar2>(2) = s2;
sp->Data<TScalar3>(3) = s3;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0, TScalar1, TScalar2, TScalar3, TScalar4, TScalar5>(
MatrixSegmentEnumeration<T> source,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0,
TScalar1 s1,
TScalar2 s2,
TScalar3 s3,
TScalar4 s4,
TScalar5 s5)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
where TScalar1: unmanaged, INumber<TScalar1>
where TScalar2: unmanaged, INumber<TScalar2>
where TScalar3: unmanaged, INumber<TScalar3>
where TScalar4: unmanaged, INumber<TScalar4>
where TScalar5: unmanaged, INumber<TScalar5>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Right = source.Source;
package->Result = ans.Source;
package->RightPrimOffset = source.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->RightPrimStride = source.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->RightSecOffset = source.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->RightSecStride = source.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = source.PrimSize;
package->SecCount = source.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 2;
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
sp->Data<TScalar1>(1) = s1;
sp->Data<TScalar2>(2) = s2;
sp->Data<TScalar3>(3) = s3;
sp->Data<TScalar4>(4) = s4;
sp->Data<TScalar5>(5) = s5;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0, TScalar1, TScalar2, TScalar3, TScalar4>(
MatrixSegmentEnumeration<T> left,
MatrixSegmentEnumeration<T> right,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0,
TScalar1 s1,
TScalar2 s2,
TScalar3 s3,
TScalar4 s4)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
where TScalar1: unmanaged, INumber<TScalar1>
where TScalar2: unmanaged, INumber<TScalar2>
where TScalar3: unmanaged, INumber<TScalar3>
where TScalar4: unmanaged, INumber<TScalar4>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Left = left.Source;
package->Right = right.Source;
package->Result = ans.Source;
package->LeftPrimOffset = left.PrimOffset;
package->RightPrimOffset = right.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->LeftPrimStride = left.PrimStride;
package->RightPrimStride = right.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->LeftSecOffset = left.SecondaryOffset;
package->RightSecOffset = right.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->LeftSecStride = left.SecondaryStride;
package->RightSecStride = right.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = left.PrimSize;
package->SecCount = left.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 4;
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
sp->Data<TScalar1>(1) = s1;
sp->Data<TScalar2>(2) = s2;
sp->Data<TScalar3>(3) = s3;
sp->Data<TScalar4>(4) = s4;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0, TScalar1, TScalar2, TScalar3, TScalar4, TScalar5, TScalar6>(
MatrixSegmentEnumeration<T> source,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0,
TScalar1 s1,
TScalar2 s2,
TScalar3 s3,
TScalar4 s4,
TScalar5 s5,
TScalar6 s6)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
where TScalar1: unmanaged, INumber<TScalar1>
where TScalar2: unmanaged, INumber<TScalar2>
where TScalar3: unmanaged, INumber<TScalar3>
where TScalar4: unmanaged, INumber<TScalar4>
where TScalar5: unmanaged, INumber<TScalar5>
where TScalar6: unmanaged, INumber<TScalar6>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Right = source.Source;
package->Result = ans.Source;
package->RightPrimOffset = source.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->RightPrimStride = source.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->RightSecOffset = source.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->RightSecStride = source.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = source.PrimSize;
package->SecCount = source.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 2;
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
sp->Data<TScalar1>(1) = s1;
sp->Data<TScalar2>(2) = s2;
sp->Data<TScalar3>(3) = s3;
sp->Data<TScalar4>(4) = s4;
sp->Data<TScalar5>(5) = s5;
sp->Data<TScalar6>(6) = s6;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0, TScalar1, TScalar2, TScalar3, TScalar4, TScalar5>(
MatrixSegmentEnumeration<T> left,
MatrixSegmentEnumeration<T> right,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0,
TScalar1 s1,
TScalar2 s2,
TScalar3 s3,
TScalar4 s4,
TScalar5 s5)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
where TScalar1: unmanaged, INumber<TScalar1>
where TScalar2: unmanaged, INumber<TScalar2>
where TScalar3: unmanaged, INumber<TScalar3>
where TScalar4: unmanaged, INumber<TScalar4>
where TScalar5: unmanaged, INumber<TScalar5>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Left = left.Source;
package->Right = right.Source;
package->Result = ans.Source;
package->LeftPrimOffset = left.PrimOffset;
package->RightPrimOffset = right.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->LeftPrimStride = left.PrimStride;
package->RightPrimStride = right.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->LeftSecOffset = left.SecondaryOffset;
package->RightSecOffset = right.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->LeftSecStride = left.SecondaryStride;
package->RightSecStride = right.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = left.PrimSize;
package->SecCount = left.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 4;
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
sp->Data<TScalar1>(1) = s1;
sp->Data<TScalar2>(2) = s2;
sp->Data<TScalar3>(3) = s3;
sp->Data<TScalar4>(4) = s4;
sp->Data<TScalar5>(5) = s5;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0, TScalar1, TScalar2, TScalar3, TScalar4, TScalar5, TScalar6, TScalar7>(
MatrixSegmentEnumeration<T> source,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0,
TScalar1 s1,
TScalar2 s2,
TScalar3 s3,
TScalar4 s4,
TScalar5 s5,
TScalar6 s6,
TScalar7 s7)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
where TScalar1: unmanaged, INumber<TScalar1>
where TScalar2: unmanaged, INumber<TScalar2>
where TScalar3: unmanaged, INumber<TScalar3>
where TScalar4: unmanaged, INumber<TScalar4>
where TScalar5: unmanaged, INumber<TScalar5>
where TScalar6: unmanaged, INumber<TScalar6>
where TScalar7: unmanaged, INumber<TScalar7>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Right = source.Source;
package->Result = ans.Source;
package->RightPrimOffset = source.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->RightPrimStride = source.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->RightSecOffset = source.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->RightSecStride = source.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = source.PrimSize;
package->SecCount = source.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 2;
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
sp->Data<TScalar1>(1) = s1;
sp->Data<TScalar2>(2) = s2;
sp->Data<TScalar3>(3) = s3;
sp->Data<TScalar4>(4) = s4;
sp->Data<TScalar5>(5) = s5;
sp->Data<TScalar6>(6) = s6;
sp->Data<TScalar7>(7) = s7;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0, TScalar1, TScalar2, TScalar3, TScalar4, TScalar5, TScalar6>(
MatrixSegmentEnumeration<T> left,
MatrixSegmentEnumeration<T> right,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0,
TScalar1 s1,
TScalar2 s2,
TScalar3 s3,
TScalar4 s4,
TScalar5 s5,
TScalar6 s6)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
where TScalar1: unmanaged, INumber<TScalar1>
where TScalar2: unmanaged, INumber<TScalar2>
where TScalar3: unmanaged, INumber<TScalar3>
where TScalar4: unmanaged, INumber<TScalar4>
where TScalar5: unmanaged, INumber<TScalar5>
where TScalar6: unmanaged, INumber<TScalar6>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Left = left.Source;
package->Right = right.Source;
package->Result = ans.Source;
package->LeftPrimOffset = left.PrimOffset;
package->RightPrimOffset = right.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->LeftPrimStride = left.PrimStride;
package->RightPrimStride = right.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->LeftSecOffset = left.SecondaryOffset;
package->RightSecOffset = right.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->LeftSecStride = left.SecondaryStride;
package->RightSecStride = right.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = left.PrimSize;
package->SecCount = left.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 4;
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
sp->Data<TScalar1>(1) = s1;
sp->Data<TScalar2>(2) = s2;
sp->Data<TScalar3>(3) = s3;
sp->Data<TScalar4>(4) = s4;
sp->Data<TScalar5>(5) = s5;
sp->Data<TScalar6>(6) = s6;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
public static unsafe MatrixParallelHandle PublicTask<T, TScalar0, TScalar1, TScalar2, TScalar3, TScalar4, TScalar5, TScalar6, TScalar7>(
MatrixSegmentEnumeration<T> left,
MatrixSegmentEnumeration<T> right,
MatrixSegmentEnumeration<T> ans,
delegate*<MatrixKernel<T>*, MatrixKernel<T>*, MatrixKernel<T>*, void> Operation,
TScalar0 s0,
TScalar1 s1,
TScalar2 s2,
TScalar3 s3,
TScalar4 s4,
TScalar5 s5,
TScalar6 s6,
TScalar7 s7)
where T: unmanaged, INumber<T>
where TScalar0: unmanaged, INumber<TScalar0>
where TScalar1: unmanaged, INumber<TScalar1>
where TScalar2: unmanaged, INumber<TScalar2>
where TScalar3: unmanaged, INumber<TScalar3>
where TScalar4: unmanaged, INumber<TScalar4>
where TScalar5: unmanaged, INumber<TScalar5>
where TScalar6: unmanaged, INumber<TScalar6>
where TScalar7: unmanaged, INumber<TScalar7>
{
MatrixParallelTaskPackage<T>* package = MatrixParallelFactory.CreateBaseTask<T>();
package->Left = left.Source;
package->Right = right.Source;
package->Result = ans.Source;
package->LeftPrimOffset = left.PrimOffset;
package->RightPrimOffset = right.PrimOffset;
package->ResultPrimOffset = ans.PrimOffset;
package->LeftPrimStride = left.PrimStride;
package->RightPrimStride = right.PrimStride;
package->ResultPrimStride = ans.PrimStride;
package->LeftSecOffset = left.SecondaryOffset;
package->RightSecOffset = right.SecondaryOffset;
package->ResultSecOffset = ans.SecondaryOffset;
package->LeftSecStride = left.SecondaryStride;
package->RightSecStride = right.SecondaryStride;
package->ResultSecStride = ans.SecondaryStride;
package->PrimCount = left.PrimSize;
package->SecCount = left.SecondarySize;
package->ComputeHandle = (nint)Operation;
package->ElementSize = sizeof(T);
package->ComputeMode = 4;
ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket();
sp->Data<TScalar0>(0) = s0;
sp->Data<TScalar1>(1) = s1;
sp->Data<TScalar2>(2) = s2;
sp->Data<TScalar3>(3) = s3;
sp->Data<TScalar4>(4) = s4;
sp->Data<TScalar5>(5) = s5;
sp->Data<TScalar6>(6) = s6;
sp->Data<TScalar7>(7) = s7;
package->Scalar = sp;
return PublicTaskPrivate<T>(package);
}
private static unsafe MatrixParallelHandle PublicTaskPrivate<T>(MatrixParallelTaskPackage<T>* package)
where T: unmanaged, INumber<T>
where T : unmanaged, INumber<T>
{
int primCount = package->PrimCount;
int secCount = package->SecCount;
StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs
+76 -29
View File
@@ -34,7 +34,54 @@ using System.Runtime.InteropServices;
namespace StgSharp.Mathematics.Numeric
{
/*
* #define PANEL_OP 1
* #define BUFFER_OP 2
* #define KERNEL_OP 3
*
* #define RIGHT_ANS_PARAM 1
* #define LEFT_RIGHT_ANS_PARAM 2
* #define RIGHT_ANS_SCALAR_PARAM 3
* #define LEFT_RIGHT_ANS_SCALAR_PARAM 4
* #define ANS_SCALAR_PARAM 5
*
* typedef struct matrix_op_mode {
* int16_t operation_style; // PANEL_OP, BUFFER_OP, KERNEL_OP
* int16_t parameter_style; // RIGHT_ANS_PARAM, LEFT_RIGHT_ANS_PARAM, e.g.
* }
* matrix_op_mode;
*/
public enum MatrixOperationStyle : short
{
PANEL_OP = 1,
BUFFER_OP = 2,
KERNEL_OP = 3,
}
public enum MatrixOperationParam : short
{
RIGHT_ANS_PARAM = 1,
LEFT_RIGHT_ANS_PARAM = 2,
RIGHT_ANS_SCALAR_PARAM = 3,
LEFT_RIGHT_ANS_SCALAR_PARAM = 4,
ANS_SCALAR_PARAM = 5,
}
[StructLayout(LayoutKind.Sequential)]
public struct MatrixOpMode
{
public MatrixOperationParam ParameterStyle;
public MatrixOperationStyle OperationStyle;
}
[StructLayout(LayoutKind.Explicit, Size = 128)]
internal unsafe struct MatrixParallelTaskPackageNonGeneric
{
@@ -50,69 +97,69 @@ namespace StgSharp.Mathematics.Numeric
#region source
// Group 1: Left/Right matrix kernel base address
public IntPtr Left; // 8
public IntPtr Right; // 8
[FieldOffset(0)]public IntPtr Left; // 8
[FieldOffset(8)] public IntPtr Right; // 8
#endregion
#region ans
// Group 2: Result kernel base address + reserved
public IntPtr Result; // 8
public int ElementSize;
public int ReservedPtr0; // 4 (extension/reserved)
[FieldOffset(16)] public IntPtr Result; // 8
[FieldOffset(24)]public int ElementSize;
[FieldOffset(28)] public int ReservedPtr0; // 4 (extension/reserved)
#endregion
#region left enum
public int LeftPrimOffset; // 4
public int LeftPrimStride; // 4
public int LeftSecOffset; // 4
public int LeftSecStride; // 4
[FieldOffset(32)] public int LeftPrimOffset; // 4
[FieldOffset(36)]public int LeftPrimStride; // 4
[FieldOffset(40)]public int LeftSecOffset; // 4
[FieldOffset(44)]public int LeftSecStride; // 4
#endregion
#region right enum
public int RightPrimOffset; // 4
public int RightPrimStride; // 4
public int RightSecOffset; // 4
public int RightSecStride; // 4
[FieldOffset(32)] public int RightPrimOffset; // 4
[FieldOffset(36)]public int RightPrimStride; // 4
[FieldOffset(40)]public int RightSecOffset; // 4
[FieldOffset(44)]public int RightSecStride; // 4
#endregion
#region ans enum
public int ResultPrimOffset; // 4
public int ResultPrimStride; // 4
public int ResultSecOffset; // 4
public int ResultSecStride; // 4
[FieldOffset(48)]public int ResultPrimOffset; // 4
[FieldOffset(52)]public int ResultPrimStride; // 4
[FieldOffset(56)]public int ResultSecOffset; // 4
[FieldOffset(60)] public int ResultSecStride; // 4
#endregion
#region scalar/compute
public ScalarPacket* Scalar; // 8
public IntPtr ComputeHandle; // 8
[FieldOffset(80)]public ScalarPacket* Scalar; // 8
[FieldOffset(88)]public IntPtr ComputeHandle; // 8
#endregion
#region global profile
public int PrimCount; // 4
public int ComputeMode; // 4
public int SecCount; // 4
private int ReservedInt0; // 4
[FieldOffset(96)]public int PrimCount; // 4
[FieldOffset(100)]public MatrixOpMode ComputeMode; // 4
[FieldOffset(104)]public int SecCount; // 4
[FieldOffset(108)]private int ReservedInt0; // 4
#endregion
#region count offset
public int PrimTileOffset; // 4
public int SecTileOffset; // 4
public int PrimCountInTile; // 4
public int SecCountInTile; // 4
[FieldOffset(112)]public int PrimTileOffset; // 4
[FieldOffset(116)]public int SecTileOffset; // 4
[FieldOffset(120)]public int PrimCountInTile; // 4
[FieldOffset(124)]public int SecCountInTile; // 4
#endregion
}
@@ -186,8 +233,8 @@ namespace StgSharp.Mathematics.Numeric
#region global profile
[FieldOffset(96)]public int PrimCount; // 4
[FieldOffset(100)]public int ComputeMode; // 4
[FieldOffset(96)] public int PrimCount; // 4
[FieldOffset(100)]public MatrixOpMode ComputeMode; // 4
[FieldOffset(104)]public int SecCount; // 4
[FieldOffset(108)] private readonly int ReservedInt0; // 4
StgSharp.Mathematics/Numeric/MatrixParallelThread.cs
+185 -10
View File
@@ -30,6 +30,7 @@ using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
@@ -38,12 +39,19 @@ namespace StgSharp.Mathematics.Numeric
public class MatrixParallelThread : IDisposable
{
// Affinity binding info for this thread
private AffinityBinding _affinity;
private bool _affinityApplied;
private bool disposedValue;
private ManualResetEventSlim _resetEvent;
private Thread _managedThread;
public MatrixParallelThread()
{
// Allocate a unique logical processor for this worker (mutually exclusive)
_affinity = AffinityAllocator.Allocate();
_managedThread = new Thread(MatrixParallelWorker);
_resetEvent = new ManualResetEventSlim(false);
}
@@ -71,19 +79,47 @@ namespace StgSharp.Mathematics.Numeric
while (taskQueue.TryDequeue(out nint handle))
{
MatrixParallelTaskPackageNonGeneric* p = (MatrixParallelTaskPackageNonGeneric*)handle;
switch (p->ComputeMode)
switch (p->ComputeMode.OperationStyle)
{
case 1:
MatrixComputeModel.Compute_Unary(p);
case MatrixOperationStyle.PANEL_OP:
break;
case 2:
MatrixComputeModel.Compute_UnaryScalar(p);
case MatrixOperationStyle.BUFFER_OP:
switch (p->ComputeMode.ParameterStyle)
{
case MatrixOperationParam.RIGHT_ANS_PARAM:
break;
case MatrixOperationParam.LEFT_RIGHT_ANS_PARAM:
MatrixComputeModel.BufferComputeBinary(p);
break;
case MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM:
MatrixComputeModel.BufferComputeUnaryScalar(p);
break;
case MatrixOperationParam.LEFT_RIGHT_ANS_SCALAR_PARAM:
break;
case MatrixOperationParam.ANS_SCALAR_PARAM:
break;
default:
break;
}
break;
case 3:
MatrixComputeModel.Compute_Binary(p);
break;
case 4:
MatrixComputeModel.Compute_BinaryScalar(p);
case MatrixOperationStyle.KERNEL_OP:
switch (p->ComputeMode.ParameterStyle)
{
case MatrixOperationParam.RIGHT_ANS_PARAM:
MatrixComputeModel.Compute_Unary(p);
break;
case MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM:
MatrixComputeModel.Compute_UnaryScalar(p);
break;
case MatrixOperationParam.LEFT_RIGHT_ANS_PARAM:
MatrixComputeModel.Compute_Binary(p);
break;
case MatrixOperationParam.LEFT_RIGHT_ANS_SCALAR_PARAM:
MatrixComputeModel.Compute_BinaryScalar(p);
break;
default:
break;
}
break;
default:
break;
@@ -111,12 +147,18 @@ namespace StgSharp.Mathematics.Numeric
_managedThread.Join();
}
// Release the reserved logical processor back to the pool
AffinityAllocator.Release(_affinity.LogicalProcessor);
disposedValue = true;
}
}
private unsafe void MatrixParallelWorker()
{
// Bind OS-thread affinity once when the worker starts running
ApplyAffinityIfNeeded();
// Thread self = Thread.CurrentThread;
_resetEvent.Wait();
_resetEvent.Reset();
@@ -141,6 +183,139 @@ namespace StgSharp.Mathematics.Numeric
_managedThread.Priority = AfterWork;
}
#region core affinity
/*
* data to record core affinity
* preparation for future NUMA affinity setting
*/
// Represents the binding information for a worker thread
internal readonly struct AffinityBinding
{
public readonly int LogicalProcessor;
public readonly nuint Mask;
public AffinityBinding(int logicalProcessor, nuint mask)
{
LogicalProcessor = logicalProcessor;
Mask = mask;
}
}
// Thread-safe allocator that hands out unique logical processors (mutually exclusive)
internal static class AffinityAllocator
{
// Bitset of reserved logical processors (0..63). One bit per CPU.
private static long s_reservedMask; // treat as unsigned bits
public static AffinityBinding Allocate()
{
int cpuCount = Environment.ProcessorCount;
if (cpuCount > 64)
{
cpuCount = 64; // SetThreadAffinityMask supports up to 64-bit mask
}
while (true)
{
long snapshot = Volatile.Read(ref s_reservedMask);
// Find first zero bit
int candidate = -1;
long bit = 0;
for (int i = 0; i < cpuCount; i++)
{
bit = 1L << i;
if ((snapshot & bit) == 0)
{
candidate = i;
break;
}
}
if (candidate < 0)
{
// No free CPU found: fall back to CPU 0 (shared)
return new AffinityBinding(0, (nuint)1);
}
long desired = snapshot | bit;
if (Interlocked.CompareExchange(ref s_reservedMask, desired, snapshot) == snapshot)
{
// Successfully reserved candidate
nuint mask = (nuint)(1UL << candidate);
return new AffinityBinding(candidate, mask);
}
// else: another thread raced us; retry
}
}
public static void Release(int logicalProcessor)
{
if ((uint)logicalProcessor >= 64u) {
return;
}
long bit = 1L << logicalProcessor;
while (true)
{
long snapshot = Volatile.Read(ref s_reservedMask);
if ((snapshot & bit) == 0)
{
return; // already free
}
long desired = snapshot & ~bit;
if (Interlocked.CompareExchange(ref s_reservedMask, desired, snapshot) == snapshot) {
return;
}
}
}
}
private void ApplyAffinityIfNeeded()
{
if (_affinityApplied) {
return;
}
#if WINDOWS
try
{
// On Windows pin the current native thread to the reserved logical processor
SetThreadAffinityMask(GetCurrentThread(), _affinity.Mask);
}
catch
{
// Ignore binding errors; continue without affinity
}
#elif LINUX
#endif
_affinityApplied = true;
}
#endregion
#if WINDOWS
[DllImport("kernel32.dll")]
private static extern IntPtr GetCurrentThread();
[DllImport("kernel32.dll")]
private static extern nuint SetThreadAffinityMask(IntPtr hThread, nuint dwThreadAffinityMask);
#elif LINUX
#endif
#region wrap belong
internal MatrixParallelWrap BindedWrap { get; set; } = null!;
StgSharp.Mathematics/Numeric/TaskPackageUsage.md
View File
StgSharp.Mathematics/StgSharp.Mathematics.csproj
+16
View File
@@ -13,4 +13,20 @@
<ProjectReference Include="..\StgSharp.Common\StgSharp.Common.csproj" />
</ItemGroup>
<PropertyGroup>
<IsWindows Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true'">true</IsWindows>
<IsOSX Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true'">true</IsOSX>
<IsLinux Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true'">true</IsLinux>
</PropertyGroup>
<PropertyGroup Condition="'$(IsWindows)'=='true'">
<DefineConstants>WINDOWS</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$(IsOSX)'=='true'">
<DefineConstants>OSX</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$(IsLinux)'=='true'">
<DefineConstants>LINUX</DefineConstants>
</PropertyGroup>
</Project>
StgSharp.Native/k_bufop.c
+104
View File
@@ -27,3 +27,107 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill)
ans[i].zmm[0] = vec;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, add)
{
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_add_ps(left[i].xmm[0], right[i].xmm[0]);
register __m128 c1 = _mm_add_ps(left[i].xmm[1], right[i].xmm[1]);
register __m128 c2 = _mm_add_ps(left[i].xmm[2], right[i].xmm[2]);
register __m128 c3 = _mm_add_ps(left[i].xmm[3], right[i].xmm[3]);
ans[i].xmm[0] = c0;
ans[i].xmm[1] = c1;
ans[i].xmm[2] = c2;
ans[i].xmm[3] = c3;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, add)
{
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_add_ps(left[i].ymm[0], right[i].ymm[0]);
register __m256 c1 = _mm256_add_ps(left[i].ymm[1], right[i].ymm[1]);
ans[i].ymm[0] = c0;
ans[i].ymm[1] = c1;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, add)
{
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_add_ps(left[i].zmm[0], right[i].zmm[0]);
ans[i].zmm[0] = c0;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, sub)
{
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_sub_ps(left[i].xmm[0], right[i].xmm[0]);
register __m128 c1 = _mm_sub_ps(left[i].xmm[1], right[i].xmm[1]);
register __m128 c2 = _mm_sub_ps(left[i].xmm[2], right[i].xmm[2]);
register __m128 c3 = _mm_sub_ps(left[i].xmm[3], right[i].xmm[3]);
ans[i].xmm[0] = c0;
ans[i].xmm[1] = c1;
ans[i].xmm[2] = c2;
ans[i].xmm[3] = c3;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, sub)
{
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].ymm[0], right[i].ymm[0]);
register __m256 c1 = _mm256_sub_ps(left[i].ymm[1], right[i].ymm[1]);
ans[i].ymm[0] = c0;
ans[i].ymm[1] = c1;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, sub)
{
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].zmm[0], right[i].zmm[0]);
ans[i].zmm[0] = c0;
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul)
{
register __m128 scalar_vec = ((__m128 *)(scalar->data))[0];
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_mul_ps(right[i].xmm[0], scalar_vec);
register __m128 c1 = _mm_mul_ps(right[i].xmm[1], scalar_vec);
register __m128 c2 = _mm_mul_ps(right[i].xmm[2], scalar_vec);
register __m128 c3 = _mm_mul_ps(right[i].xmm[3], scalar_vec);
ans[i].xmm[0] = c0;
ans[i].xmm[1] = c1;
ans[i].xmm[2] = c2;
ans[i].xmm[3] = c3;
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul)
{
register __m256 scalar_vec = ((__m256 *)(scalar->data))[0];
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_mul_ps(right[i].ymm[0], scalar_vec);
register __m256 c1 = _mm256_mul_ps(right[i].ymm[1], scalar_vec);
ans[i].ymm[0] = c0;
ans[i].ymm[1] = c1;
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul)
{
register __m512 scalar_vec = ((__m512 *)(scalar->data))[0];
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_mul_ps(right[i].zmm[0], scalar_vec);
ans[i].zmm[0] = c0;
}
}
StgSharp.Native/sn-head/sn_intrinsic.h
+45 -24
View File
@@ -39,6 +39,23 @@ typedef union scalar_pack {
uint64_t data[8];
} scalar_pack;
// define operation style:
#define PANEL_OP 1
#define BUFFER_OP 2
#define KERNEL_OP 3
#define RIGHT_ANS_PARAM 1
#define LEFT_RIGHT_ANS_PARAM 2
#define RIGHT_ANS_SCALAR_PARAM 3
#define LEFT_RIGHT_ANS_SCALAR_PARAM 4
#define ANS_SCALAR_PARAM 5
typedef struct matrix_op_mode {
int16_t operation_style; /* PANEL_OP, BUFFER_OP, KERNEL_OP */
int16_t parameter_style; /* RIGHT_ANS_PARAM, LEFT_RIGHT_ANS_PARAM, e.g. */
} matrix_op_mode;
/* Mirrors StgSharp.Mathematics.Numeric.MatrixParallelTaskPackage<T> (128 bytes). */
#define MAT_TASK(T) \
struct { \
@@ -62,7 +79,7 @@ typedef union scalar_pack {
scalar_pack *scalar; /* Offset 80 */ \
void *compute_handle; /* Offset 88 */ \
int32_t prim_count; /* Offset 96 */ \
int32_t compute_mode; /* Offset 100 */ \
matrix_op_mode compute_mode; /* Offset 100 */ \
int32_t sec_count; /* Offset 104 */ \
int32_t reserved_int0; /* Offset 108 */ \
int32_t prim_tile_offset; /* Offset 112 */ \
@@ -93,7 +110,7 @@ typedef void(SN_DECL *BUF_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_ker
#define DECLARE_BUF_PROC_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, int count)
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, size_t count)
#define DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
@@ -101,14 +118,14 @@ typedef void(SN_DECL *BUF_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_ker
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, int count)
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, size_t count)
#define DECLARE_BUF_PROC_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, int count)
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, size_t count)
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, int count)
MAT_KERNEL(T_type) *restrict ans, size_t count)
#pragma endregion
@@ -176,23 +193,25 @@ typedef uint64_t(SN_DECL *FACTORIALROC)(int n);
INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, add);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, add);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, add);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, sub);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, sub);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, sub);
DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose);
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul);
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul);
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul);
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill);
#pragma endregion
@@ -231,17 +250,19 @@ INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int lengt
#pragma endregion
// Reordered to match C# IntrinsicContext field order
// C# order: city_hash_simplify, f32_add, f32_fill, f32_fma, f32_scalar_mul, f32_sub, f32_transpose, factorial_simd, index_pair, normalize_3
typedef struct sn_intrinsic {
HASH city_hash_simplify;
FACTORIALROC factorial_simd;
INDEX_PAIR index_pair;
KER_PROC_LRA f32_kernel_add;
KER_PROC_AS f32_kernel_fill;
PNL_PROC_LRA f32_panel_fma;
KER_PROC_LRA f32_kernel_scalar_mul;
KER_PROC_LRA f32_kernel_sub;
KER_PROC_RA f32_kernel_transpose;
VECTORNORMALIZEPROC f32_normalize_3;
HASH city_hash_simplify; // city_hash_simplify
BUF_PROC_LRA f32_buffer_add; // f32_add
BUF_PROC_AS f32_buffer_fill; // f32_fill
PNL_PROC_LRA f32_panel_fma; // f32_fma (panel variant)
BUF_PROC_LRA f32_kernel_scalar_mul; // f32_scalar_mul (right+ans+scalar variant mapped here)
BUF_PROC_LRA f32_kernel_sub; // f32_sub
KER_PROC_RA f32_kernel_transpose; // f32_transpose
FACTORIALROC factorial_simd; // factorial_simd
INDEX_PAIR index_pair; // index_pair
VECTORNORMALIZEPROC f32_normalize_3; // normalize_3
} sn_intrinsic;
typedef enum most_advanced_instruction {
StgSharp.Native/sn-src/k_basicop.c
-117
View File
@@ -1,87 +1,5 @@
#include "sn_intrinsic.h"
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill)
{
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].xmm[0] = vec;
ans[i].xmm[1] = vec;
ans[i].xmm[2] = vec;
ans[i].xmm[3] = vec;
}
}
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
ans[i].ymm[0] = vec;
ans[i].ymm[1] = vec;
}
}
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].zmm[0] = vec;
}
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, add)
{
register __m128 c0 = _mm_add_ps(left->xmm[0], right->xmm[0]);
register __m128 c1 = _mm_add_ps(left->xmm[1], right->xmm[1]);
register __m128 c2 = _mm_add_ps(left->xmm[2], right->xmm[2]);
register __m128 c3 = _mm_add_ps(left->xmm[3], right->xmm[3]);
ans->xmm[0] = c0;
ans->xmm[1] = c1;
ans->xmm[2] = c2;
ans->xmm[3] = c3;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, add)
{
register __m256 c0 = _mm256_add_ps(left->ymm[0], right->ymm[0]);
register __m256 c1 = _mm256_add_ps(left->ymm[1], right->ymm[1]);
ans->ymm[0] = c0;
ans->ymm[1] = c1;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, add)
{
register __m512 c0 = _mm512_add_ps(left->zmm[0], right->zmm[0]);
ans->zmm[0] = c0;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, sub)
{
register __m128 c0 = _mm_sub_ps(left->xmm[0], right->xmm[0]);
register __m128 c1 = _mm_sub_ps(left->xmm[1], right->xmm[1]);
register __m128 c2 = _mm_sub_ps(left->xmm[2], right->xmm[2]);
register __m128 c3 = _mm_sub_ps(left->xmm[3], right->xmm[3]);
ans->xmm[0] = c0;
ans->xmm[1] = c1;
ans->xmm[2] = c2;
ans->xmm[3] = c3;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, sub)
{
register __m256 c0 = _mm256_sub_ps(left->ymm[0], right->ymm[0]);
register __m256 c1 = _mm256_sub_ps(left->ymm[1], right->ymm[1]);
ans->ymm[0] = c0;
ans->ymm[1] = c1;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, sub)
{
register __m512 c0 = _mm512_sub_ps(left->zmm[0], right->zmm[0]);
ans->zmm[0] = c0;
}
// Matrix transpose implementations
DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose)
{
@@ -124,38 +42,3 @@ DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose)
ans->zmm[0] = _mm512_permutexvar_ps(transpose_indices, matrix);
}
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul)
{
register __m128 scalar_vec = ((__m128 *)(scalar->data))[0];
register __m128 c0 = _mm_mul_ps(right->xmm[0], scalar_vec);
register __m128 c1 = _mm_mul_ps(right->xmm[1], scalar_vec);
register __m128 c2 = _mm_mul_ps(right->xmm[2], scalar_vec);
register __m128 c3 = _mm_mul_ps(right->xmm[3], scalar_vec);
ans->xmm[0] = c0;
ans->xmm[1] = c1;
ans->xmm[2] = c2;
ans->xmm[3] = c3;
}
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul)
{
register __m256 scalar_vec = ((__m256 *)(scalar->data))[0];
register __m256 c0 = _mm256_mul_ps(right->ymm[0], scalar_vec);
register __m256 c1 = _mm256_mul_ps(right->ymm[1], scalar_vec);
ans->ymm[0] = c0;
ans->ymm[1] = c1;
}
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul)
{
register __m512 scalar_vec = ((__m512 *)(scalar->data))[0];
register __m512 c0 = _mm512_mul_ps(right->zmm[0], scalar_vec);
ans->zmm[0] = c0;
}
StgSharp.Native/sn-src/load_intrinsic.c
+12 -9
View File
@@ -65,10 +65,11 @@ void load_intrinsic_sse(sn_intrinsic *context)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->f32_kernel_add = KER_PROC(float, sse, add);
context->f32_buffer_add = BUF_PROC(float, sse, add);
context->f32_panel_fma = KER_PROC(float, sse, fma);
context->f32_kernel_scalar_mul = KER_PROC(float, sse, scalar_mul);
context->f32_kernel_sub = KER_PROC(float, sse, sub);
context->f32_buffer_fill = BUF_PROC(float, sse, fill);
context->f32_kernel_scalar_mul = BUF_PROC(float, sse, scalar_mul);
context->f32_kernel_sub = BUF_PROC(float, sse, sub);
context->f32_kernel_transpose = KER_PROC(float, sse, transpose);
context->f32_normalize_3 = f32_normalize;
}
@@ -78,10 +79,11 @@ void load_intrinsic_avx(sn_intrinsic *context)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->f32_kernel_add = KER_PROC(float, avx, add);
context->f32_buffer_add = BUF_PROC(float, avx, add);
context->f32_panel_fma = KER_PROC(float, sse, fma);
context->f32_kernel_scalar_mul = KER_PROC(float, avx, scalar_mul);
context->f32_kernel_sub = KER_PROC(float, avx, sub);
context->f32_buffer_fill = BUF_PROC(float, avx, fill);
context->f32_kernel_scalar_mul = BUF_PROC(float, avx, scalar_mul);
context->f32_kernel_sub = BUF_PROC(float, avx, sub);
context->f32_kernel_transpose = KER_PROC(float, avx, transpose);
context->f32_normalize_3 = f32_normalize;
}
@@ -91,10 +93,11 @@ void load_intrinsic_512(sn_intrinsic *context)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->f32_kernel_add = KER_PROC(float, 512, add);
context->f32_buffer_add = BUF_PROC(float, 512, add);
context->f32_panel_fma = KER_PROC(float, sse, fma);
context->f32_kernel_scalar_mul = KER_PROC(float, 512, scalar_mul);
context->f32_kernel_sub = KER_PROC(float, 512, sub);
context->f32_buffer_fill = BUF_PROC(float, 512, fill);
context->f32_kernel_scalar_mul = BUF_PROC(float, 512, scalar_mul);
context->f32_kernel_sub = BUF_PROC(float, 512, sub);
context->f32_kernel_transpose = KER_PROC(float, 512, transpose);
context->f32_normalize_3 = f32_normalize;
}
StgSharpDebug/Program.cs
+14 -18
View File
@@ -31,6 +31,7 @@ using StgSharp.Geometries;
using StgSharp.Graphics;
using StgSharp.HighPerformance;
using StgSharp.HighPerformance.Memory;
using StgSharp.Internal;
using StgSharp.Mathematics;
using StgSharp.Mathematics.Graphic;
using StgSharp.Mathematics.Numeric;
@@ -60,9 +61,9 @@ namespace StgSharp
{
try
{
World.Init();
World.Initialize().UseModule<NumericalModule>().UseModule<GraphicModule>().InitializeModule();
/*
// New test with synchronized start:
// - 8 producer threads, each pushing 1000 unique integers
// - 8 consumer threads, each attempting to pop 1000 times
@@ -155,7 +156,7 @@ namespace StgSharp
}
Console.WriteLine($"Popped count: {popped.Count}");
/**/
/*
Console.WriteLine("Popped values:");
foreach (int v in popped)
{
@@ -163,7 +164,7 @@ namespace StgSharp
Console.Write('\t');
}
Console.WriteLine();
/**/
/*
Console.WriteLine($"Remaining count: {remaining.Count}");
Console.WriteLine("Remaining values:");
foreach (int v in remaining)
@@ -176,28 +177,23 @@ namespace StgSharp
// Sanity check
Console.WriteLine($"Produced: {producerThreads * perProducer}, Popped+Remaining: {popped.Count + remaining.Count}");
/*
// TestMatrixParallel();
/**/
// 原有的矩阵测试代码
HybridLayerSegregatedFitAllocator hlsf = new HybridLayerSegregatedFitAllocator(64 * 400 * 400);
Matrix<float> m = Matrix<float>.FromHlsf(400, 200, hlsf);
int i = 0, ii = 0;
foreach (MatrixSegment<float> lines in m.GetRowEnumeration())
Matrix<float> m = Matrix<float>.FromHlsf(20, 20, hlsf);
m.Fill(1.0f);
Matrix<float> n = m.Add(m);
for (int i = 0; i < 20; i++)
{
i = 0;
foreach (MatrixKernel<float>* item in lines)
{
i++;
Console.WriteLine(i);
for (int j = 0; j < 20; j++) {
Console.Write($"{n[i, j].ToString("0.000")}\t");
}
ii++;
Console.WriteLine($"============================{ii}");
Console.WriteLine();
}
/**/
/**/
}
catch (Exception ex)
{