diff --git a/StgSharp.Common/Collections/CapacityFixedStack.cs b/StgSharp.Common/Collections/CapacityFixedStack.cs index 6c3bc5a8..bdb29dd7 100644 --- a/StgSharp.Common/Collections/CapacityFixedStack.cs +++ b/StgSharp.Common/Collections/CapacityFixedStack.cs @@ -99,9 +99,7 @@ namespace StgSharp.Collections public void Push(T item) { - _values[++_index] = _index < _values.Length - 1 ? - item : - throw new InvalidOperationException("Stack is full."); + _values[++_index] = _index < _values.Length ? item : throw new InvalidOperationException("Stack is full."); } public void PushRange(ReadOnlySpan span) diff --git a/StgSharp.Common/HighPerformance/Memory/SlabAllocator.ChunkedConcurrent.cs b/StgSharp.Common/HighPerformance/Memory/SlabAllocator.ChunkedConcurrent.cs index 10ed094a..2191a256 100644 --- a/StgSharp.Common/HighPerformance/Memory/SlabAllocator.ChunkedConcurrent.cs +++ b/StgSharp.Common/HighPerformance/Memory/SlabAllocator.ChunkedConcurrent.cs @@ -38,7 +38,7 @@ using System.Threading.Tasks; namespace StgSharp.HighPerformance.Memory { - internal sealed unsafe class ConcurrentChunkedSlabAllocator : SlabAllocator where T: unmanaged + internal sealed unsafe class ConcurrentChunkedSlabAllocator : SlabAllocator where T : unmanaged { private readonly BufferExpansionLock _lock = new(); diff --git a/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs b/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs index ed08ad24..2ae48623 100644 --- a/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs +++ b/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs @@ -30,6 +30,7 @@ using System; using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.InteropServices.Marshalling; namespace StgSharp.Internal.Intrinsic { @@ -38,11 +39,11 @@ namespace StgSharp.Internal.Intrinsic { public delegate* unmanaged[Cdecl] city_hash_simplify; - public delegate* unmanaged[Cdecl] f32_add; - public delegate* unmanaged[Cdecl] f32_fill; + public delegate* unmanaged[Cdecl] f32_add; + public delegate* unmanaged[Cdecl] f32_fill; public delegate* unmanaged[Cdecl] f32_fma; - public delegate* unmanaged[Cdecl] f32_scalar_mul; - public delegate* unmanaged[Cdecl] f32_sub; + public delegate* unmanaged[Cdecl] f32_scalar_mul; + public delegate* unmanaged[Cdecl] f32_sub; public delegate* unmanaged[Cdecl] f32_transpose; public delegate* unmanaged[Cdecl] factorial_simd; public delegate* unmanaged[Cdecl] index_pair; diff --git a/StgSharp.Common/Mathematics/Graphic/Matrix44.cs b/StgSharp.Common/Mathematics/Graphic/Matrix44.cs index 7ff55d95..d0b21fdd 100644 --- a/StgSharp.Common/Mathematics/Graphic/Matrix44.cs +++ b/StgSharp.Common/Mathematics/Graphic/Matrix44.cs @@ -212,7 +212,7 @@ namespace StgSharp.Mathematics.Graphic public static unsafe Matrix44 operator -(Matrix44 left, Matrix44 right) { Matrix44 ret = new(); - NativeIntrinsic.Intrinsic.f32_sub(&left, &right, &ret); + NativeIntrinsic.Intrinsic.f32_sub(&left, &right, &ret, 1); return ret; } @@ -254,7 +254,7 @@ namespace StgSharp.Mathematics.Graphic public static unsafe Matrix44 operator +(Matrix44 left, Matrix44 right) { Matrix44 ret = new(); - NativeIntrinsic.Intrinsic.f32_add(&left, &right, &ret); + NativeIntrinsic.Intrinsic.f32_add(&left, &right, &ret, 1); return ret; } diff --git a/StgSharp.Common/Mathematics/Linear/Matrix.GC.cs b/StgSharp.Common/Mathematics/Linear/Matrix.GC.cs index aa9e4919..52daa4c7 100644 --- a/StgSharp.Common/Mathematics/Linear/Matrix.GC.cs +++ b/StgSharp.Common/Mathematics/Linear/Matrix.GC.cs @@ -38,7 +38,7 @@ using System.Threading.Tasks; namespace StgSharp.Mathematics.Numeric { - internal unsafe partial class GCMatrix : Matrix where T: unmanaged, INumber + internal unsafe partial class GCMatrix : Matrix where T : unmanaged, INumber { private readonly MatrixKernel* _buffer; @@ -47,6 +47,8 @@ namespace StgSharp.Mathematics.Numeric { KernelColumnLength = (column + 3) / 4; KernelRowLength = (row + 3) / t_count; + ColumnLength = column; + RowLength = row; _buffer = (MatrixKernel*)NativeMemory.AlignedAlloc((nuint)(column * row * sizeof(T)), (nuint)World.DefaultSIMDAlignment); } @@ -95,7 +97,8 @@ namespace StgSharp.Mathematics.Numeric [MethodImpl(MethodImplOptions.AggressiveInlining)] public override unsafe MatrixSegmentEnumeration GetColumnEnumeration() { - return new MatrixSegmentEnumeration(_buffer, KernelColumnLength, KernelColumnLength * KernelRowLength, 1, KernelColumnLength); + return new MatrixSegmentEnumeration(_buffer, KernelColumnLength, KernelColumnLength * KernelRowLength, 1, + KernelColumnLength); } public override unsafe MatrixSegmentEnumeration GetRowEnumeration() @@ -106,7 +109,9 @@ namespace StgSharp.Mathematics.Numeric [MethodImpl(MethodImplOptions.AggressiveInlining)] public override unsafe MatrixSegmentEnumeration GetSequentialEnumeration() { - return new MatrixSegmentEnumeration(_buffer, KernelRowLength * KernelColumnLength, KernelColumnLength * KernelRowLength, 1, KernelRowLength * KernelColumnLength); + return new MatrixSegmentEnumeration(_buffer, KernelRowLength * KernelColumnLength, + KernelColumnLength * KernelRowLength, 1, + KernelRowLength * KernelColumnLength); } #endregion diff --git a/StgSharp.Common/Mathematics/Linear/Matrix.HLSF.cs b/StgSharp.Common/Mathematics/Linear/Matrix.HLSF.cs index 073a22e7..8689a906 100644 --- a/StgSharp.Common/Mathematics/Linear/Matrix.HLSF.cs +++ b/StgSharp.Common/Mathematics/Linear/Matrix.HLSF.cs @@ -39,7 +39,7 @@ using hlsfHandle = global::StgSharp.HighPerformance.Memory.HybridLayerSegregated namespace StgSharp.Mathematics.Numeric { - internal unsafe partial class HlsfMatrix : Matrix where T: unmanaged, INumber + internal unsafe partial class HlsfMatrix : Matrix where T : unmanaged, INumber { private readonly MatrixKernel* k_buffer; @@ -50,6 +50,8 @@ namespace StgSharp.Mathematics.Numeric { KernelColumnLength = (column + 3) / 4; KernelRowLength = (row + 3) / t_count; + ColumnLength = column; + RowLength = row; this.handle = allocator.Alloc((uint)(KernelColumnLength * KernelRowLength * sizeof(MatrixKernel))); k_buffer = (MatrixKernel*)handle.Pointer; } @@ -104,18 +106,22 @@ namespace StgSharp.Mathematics.Numeric [MethodImpl(MethodImplOptions.AggressiveInlining)] public override unsafe MatrixSegmentEnumeration GetColumnEnumeration() { - return new MatrixSegmentEnumeration(k_buffer, KernelColumnLength, KernelColumnLength * KernelRowLength, 1, KernelColumnLength); + return new MatrixSegmentEnumeration(k_buffer, KernelColumnLength, KernelColumnLength * KernelRowLength, + 1, KernelColumnLength); } public override unsafe MatrixSegmentEnumeration GetRowEnumeration() { - return new MatrixSegmentEnumeration(k_buffer, 1, KernelColumnLength, KernelColumnLength, KernelRowLength); + return new MatrixSegmentEnumeration(k_buffer, 1, KernelColumnLength, KernelColumnLength, + KernelRowLength); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public override unsafe MatrixSegmentEnumeration GetSequentialEnumeration() { - return new MatrixSegmentEnumeration(k_buffer, KernelRowLength * KernelColumnLength, KernelColumnLength * KernelRowLength, 1, KernelRowLength * KernelColumnLength); + return new MatrixSegmentEnumeration(k_buffer, KernelRowLength * KernelColumnLength, + KernelColumnLength * KernelRowLength, 1, + KernelRowLength * KernelColumnLength); } #endregion diff --git a/StgSharp.Common/Threading/NumaBuffer.cs b/StgSharp.Common/Threading/NumaBuffer.cs new file mode 100644 index 00000000..7fff2701 --- /dev/null +++ b/StgSharp.Common/Threading/NumaBuffer.cs @@ -0,0 +1,47 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="NumaBuffer" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.Common.Threading +{ + public unsafe struct NumaBuffer where T : unmanaged + { + + public readonly T* Buffers; + public readonly int BufferColumnCapacity; + public readonly int BufferRowCapacity; + + public int CurrentColumnCount; + public int CurrentRowCount; + + } +} diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs index c09ab6e9..8c3c1d55 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs @@ -41,41 +41,41 @@ namespace StgSharp.Mathematics.Numeric public static class MatrixCompute { - public static unsafe Matrix Add(Matrix left, Matrix right) + public static unsafe Matrix Add(this Matrix left, Matrix right) { if (left.ColumnLength != right.ColumnLength || left.RowLength != right.RowLength) { throw new IndexOutOfRangeException("Matrix dimensions must agree."); } - Matrix ans = Matrix.FromDefault(left.ColumnLength, left.RowLength); long count = (long)left.KernelColumnLength * left.KernelRowLength; + Matrix ans = Matrix.FromDefault(left.ColumnLength, left.RowLength); if (count <= 256) { MatrixKernel* leftPtr = left.Buffer; MatrixKernel* rightPtr = right.Buffer; MatrixKernel* ansPtr = ans.Buffer; - for (int i = 0; i < count; i++) { - NativeIntrinsic.Intrinsic.f32_add(leftPtr + i, rightPtr + i, ansPtr + i); - } + NativeIntrinsic.Intrinsic.f32_add(leftPtr, rightPtr, ansPtr, count); + return ans; - } - if (count <= 1024) + } else if (count <= 1024) { MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1); MatrixKernel* leftPtr = left.Buffer; MatrixKernel* rightPtr = right.Buffer; MatrixKernel* ansPtr = ans.Buffer; - for (int i = 0; i < count; i++) { - NativeIntrinsic.Intrinsic.f32_add(leftPtr + i, rightPtr + i, ansPtr + i); - } + NativeIntrinsic.Intrinsic.f32_add(leftPtr, rightPtr, ansPtr, count); + MatrixParallel.ReturnParallelResources(ref pool); return ans; } - MatrixParallelHandle taskGroup = MatrixParallel.PublicTask(left.GetColumnEnumeration(), + MatrixParallelHandle taskGroup = /**/ default + /* + MatrixParallel.PublicTask(left.GetColumnEnumeration(), right.GetColumnEnumeration(), ans.GetColumnEnumeration(), - (delegate**, MatrixKernel*, MatrixKernel*, void>)NativeIntrinsic.Intrinsic.f32_add); - + (delegate**, MatrixKernel*, MatrixKernel*, void>)NativeIntrinsic.Intrinsic.f32_add) + /**/ + ; MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); /* * run parallel @@ -84,35 +84,33 @@ namespace StgSharp.Mathematics.Numeric return ans; } - public static unsafe void Fill(Matrix left, T value) where T: unmanaged, INumber + public static unsafe void Fill(this Matrix right, T value) where T : unmanaged, INumber { - long count = (long)left.KernelColumnLength * left.KernelRowLength; + long count = (long)right.KernelColumnLength * right.KernelRowLength; ScalarPacket* scaler = MatrixParallelFactory.CreateScalarPacket(); - scaler->BroadCast(0, value); + scaler->Data(0) = value; if (count <= 256) { - MatrixKernel* leftPtr = left.Buffer; - for (int i = 0; i < count; i++) { - NativeIntrinsic.Intrinsic.f32_fill(leftPtr + i, scaler); - } - } - if (count <= 1024) + NativeIntrinsic.Intrinsic.f32_fill(right.Buffer, scaler, count); + return; + } else if (count <= 1024) { MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1); - MatrixKernel* leftPtr = left.Buffer; - for (int i = 0; i < count; i++) { - NativeIntrinsic.Intrinsic.f32_fill(leftPtr + i, scaler); - } - MatrixParallel.ReturnParallelResources(ref pool); - } - MatrixParallelHandle taskGroup = MatrixParallel.PublicNoAnswerTask(left.GetColumnEnumeration(), scaler, - (delegate* unmanaged[Cdecl]*, ScalarPacket*, void>)NativeIntrinsic.Intrinsic.f32_add); + NativeIntrinsic.Intrinsic.f32_fill(right.Buffer, scaler, count); + MatrixParallel.ReturnParallelResources(ref pool); + return; + } + MatrixParallelHandle taskGroup =/**/ default + /*MatrixParallel.PublicNoAnswerTask(left.GetColumnEnumeration(), scaler, + (delegate* unmanaged[Cdecl]*, ScalarPacket*, void>)NativeIntrinsic.Intrinsic.f32_add) + /**/; MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); /* * run parallel */ + MatrixParallelFactory.Release(scaler); taskGroup.Dispose(); } diff --git a/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs b/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs index d16f2df1..d14dc082 100644 --- a/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs +++ b/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs @@ -34,6 +34,29 @@ namespace StgSharp.Mathematics.Numeric internal static class MatrixComputeModel { + internal static unsafe void BufferComputeBinary(MatrixParallelTaskPackageNonGeneric* p) + { + int size = p->ElementSize * 16; + int count = p->PrimCount * p->SecCount; + int offset = p->LeftPrimOffset * p->SecCount; + + ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Left + offset * size), + (void*)(p->Right + offset * size), + (void*)(p->Result + offset * size), + count); + } + + internal static unsafe void BufferComputeUnaryScalar(MatrixParallelTaskPackageNonGeneric* p) + { + int size = p->ElementSize * 16; + int count = p->PrimCount * p->SecCount; + int offset = p->LeftPrimOffset * p->SecCount; + + ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Right + offset * size), + (void*)(p->Result + offset * size), + count); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static unsafe void Compute_Binary(MatrixParallelTaskPackageNonGeneric* p) { diff --git a/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs b/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs index e98b5dd9..2a883b76 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs @@ -40,801 +40,8 @@ namespace StgSharp.Mathematics.Numeric public static partial class MatrixParallel { - public static unsafe MatrixParallelHandle PublicNoAnswerTask( - MatrixSegmentEnumeration source, - ScalarPacket* additional, - delegate* unmanaged[Cdecl]*, ScalarPacket*, void> computeHandle) - where T: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Scalar = additional; - package->Right = source.Source; - package->RightPrimOffset = source.PrimOffset; - package->RightPrimStride = source.PrimStride; - package->RightSecOffset = source.SecondaryOffset; - package->RightSecStride = source.SecondaryStride; - package->PrimCount = source.PrimSize; - package->SecCount = source.SecondarySize; - package->ElementSize = sizeof(T); - package->ComputeHandle = (nint)computeHandle; - package->ComputeMode = 6; // no ans - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration source, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation) - where T: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Right = source.Source; - package->Result = ans.Source; - package->RightPrimOffset = source.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->RightPrimStride = source.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->RightSecOffset = source.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->RightSecStride = source.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = source.PrimSize; - package->SecCount = source.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 1; // Unary - - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration left, - MatrixSegmentEnumeration right, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation) - where T: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Left = left.Source; - package->Right = right.Source; - package->Result = ans.Source; - package->LeftPrimOffset = left.PrimOffset; - package->RightPrimOffset = right.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->LeftPrimStride = left.PrimStride; - package->RightPrimStride = right.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->LeftSecOffset = left.SecondaryOffset; - package->RightSecOffset = right.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->LeftSecStride = left.SecondaryStride; - package->RightSecStride = right.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = left.PrimSize; - package->SecCount = left.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 3; // Binary - - return PublicTaskPrivate(package); - } - - // Unary with 1..8 scalars (each scalar is independent generic type) - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration source, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Right = source.Source; - package->Result = ans.Source; - package->RightPrimOffset = source.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->RightPrimStride = source.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->RightSecOffset = source.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->RightSecStride = source.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = source.PrimSize; - package->SecCount = source.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 2; // UnaryScalar - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration source, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0, - TScalar1 s1) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - where TScalar1: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Right = source.Source; - package->Result = ans.Source; - package->RightPrimOffset = source.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->RightPrimStride = source.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->RightSecOffset = source.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->RightSecStride = source.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = source.PrimSize; - package->SecCount = source.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 2; - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - sp->Data(1) = s1; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - - // Binary with 1..8 scalars (each scalar is independent generic type) - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration left, - MatrixSegmentEnumeration right, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Left = left.Source; - package->Right = right.Source; - package->Result = ans.Source; - package->LeftPrimOffset = left.PrimOffset; - package->RightPrimOffset = right.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->LeftPrimStride = left.PrimStride; - package->RightPrimStride = right.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->LeftSecOffset = left.SecondaryOffset; - package->RightSecOffset = right.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->LeftSecStride = left.SecondaryStride; - package->RightSecStride = right.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = left.PrimSize; - package->SecCount = left.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 4; // BinaryScalar - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration source, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0, - TScalar1 s1, - TScalar2 s2) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - where TScalar1: unmanaged, INumber - where TScalar2: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Right = source.Source; - package->Result = ans.Source; - package->RightPrimOffset = source.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->RightPrimStride = source.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->RightSecOffset = source.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->RightSecStride = source.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = source.PrimSize; - package->SecCount = source.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 2; - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - sp->Data(1) = s1; - sp->Data(2) = s2; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration left, - MatrixSegmentEnumeration right, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0, - TScalar1 s1) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - where TScalar1: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Left = left.Source; - package->Right = right.Source; - package->Result = ans.Source; - package->LeftPrimOffset = left.PrimOffset; - package->RightPrimOffset = right.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->LeftPrimStride = left.PrimStride; - package->RightPrimStride = right.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->LeftSecOffset = left.SecondaryOffset; - package->RightSecOffset = right.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->LeftSecStride = left.SecondaryStride; - package->RightSecStride = right.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = left.PrimSize; - package->SecCount = left.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 4; - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - sp->Data(1) = s1; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration source, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0, - TScalar1 s1, - TScalar2 s2, - TScalar3 s3) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - where TScalar1: unmanaged, INumber - where TScalar2: unmanaged, INumber - where TScalar3: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Right = source.Source; - package->Result = ans.Source; - package->RightPrimOffset = source.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->RightPrimStride = source.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->RightSecOffset = source.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->RightSecStride = source.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = source.PrimSize; - package->SecCount = source.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 2; - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - sp->Data(1) = s1; - sp->Data(2) = s2; - sp->Data(3) = s3; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration left, - MatrixSegmentEnumeration right, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0, - TScalar1 s1, - TScalar2 s2) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - where TScalar1: unmanaged, INumber - where TScalar2: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Left = left.Source; - package->Right = right.Source; - package->Result = ans.Source; - package->LeftPrimOffset = left.PrimOffset; - package->RightPrimOffset = right.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->LeftPrimStride = left.PrimStride; - package->RightPrimStride = right.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->LeftSecOffset = left.SecondaryOffset; - package->RightSecOffset = right.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->LeftSecStride = left.SecondaryStride; - package->RightSecStride = right.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = left.PrimSize; - package->SecCount = left.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 4; - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - sp->Data(1) = s1; - sp->Data(2) = s2; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration source, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0, - TScalar1 s1, - TScalar2 s2, - TScalar3 s3, - TScalar4 s4) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - where TScalar1: unmanaged, INumber - where TScalar2: unmanaged, INumber - where TScalar3: unmanaged, INumber - where TScalar4: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Right = source.Source; - package->Result = ans.Source; - package->RightPrimOffset = source.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->RightPrimStride = source.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->RightSecOffset = source.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->RightSecStride = source.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = source.PrimSize; - package->SecCount = source.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 2; - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - sp->Data(1) = s1; - sp->Data(2) = s2; - sp->Data(3) = s3; - sp->Data(4) = s4; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration left, - MatrixSegmentEnumeration right, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0, - TScalar1 s1, - TScalar2 s2, - TScalar3 s3) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - where TScalar1: unmanaged, INumber - where TScalar2: unmanaged, INumber - where TScalar3: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Left = left.Source; - package->Right = right.Source; - package->Result = ans.Source; - package->LeftPrimOffset = left.PrimOffset; - package->RightPrimOffset = right.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->LeftPrimStride = left.PrimStride; - package->RightPrimStride = right.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->LeftSecOffset = left.SecondaryOffset; - package->RightSecOffset = right.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->LeftSecStride = left.SecondaryStride; - package->RightSecStride = right.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = left.PrimSize; - package->SecCount = left.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 4; - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - sp->Data(1) = s1; - sp->Data(2) = s2; - sp->Data(3) = s3; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration source, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0, - TScalar1 s1, - TScalar2 s2, - TScalar3 s3, - TScalar4 s4, - TScalar5 s5) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - where TScalar1: unmanaged, INumber - where TScalar2: unmanaged, INumber - where TScalar3: unmanaged, INumber - where TScalar4: unmanaged, INumber - where TScalar5: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Right = source.Source; - package->Result = ans.Source; - package->RightPrimOffset = source.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->RightPrimStride = source.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->RightSecOffset = source.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->RightSecStride = source.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = source.PrimSize; - package->SecCount = source.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 2; - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - sp->Data(1) = s1; - sp->Data(2) = s2; - sp->Data(3) = s3; - sp->Data(4) = s4; - sp->Data(5) = s5; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration left, - MatrixSegmentEnumeration right, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0, - TScalar1 s1, - TScalar2 s2, - TScalar3 s3, - TScalar4 s4) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - where TScalar1: unmanaged, INumber - where TScalar2: unmanaged, INumber - where TScalar3: unmanaged, INumber - where TScalar4: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Left = left.Source; - package->Right = right.Source; - package->Result = ans.Source; - package->LeftPrimOffset = left.PrimOffset; - package->RightPrimOffset = right.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->LeftPrimStride = left.PrimStride; - package->RightPrimStride = right.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->LeftSecOffset = left.SecondaryOffset; - package->RightSecOffset = right.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->LeftSecStride = left.SecondaryStride; - package->RightSecStride = right.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = left.PrimSize; - package->SecCount = left.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 4; - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - sp->Data(1) = s1; - sp->Data(2) = s2; - sp->Data(3) = s3; - sp->Data(4) = s4; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration source, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0, - TScalar1 s1, - TScalar2 s2, - TScalar3 s3, - TScalar4 s4, - TScalar5 s5, - TScalar6 s6) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - where TScalar1: unmanaged, INumber - where TScalar2: unmanaged, INumber - where TScalar3: unmanaged, INumber - where TScalar4: unmanaged, INumber - where TScalar5: unmanaged, INumber - where TScalar6: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Right = source.Source; - package->Result = ans.Source; - package->RightPrimOffset = source.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->RightPrimStride = source.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->RightSecOffset = source.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->RightSecStride = source.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = source.PrimSize; - package->SecCount = source.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 2; - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - sp->Data(1) = s1; - sp->Data(2) = s2; - sp->Data(3) = s3; - sp->Data(4) = s4; - sp->Data(5) = s5; - sp->Data(6) = s6; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration left, - MatrixSegmentEnumeration right, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0, - TScalar1 s1, - TScalar2 s2, - TScalar3 s3, - TScalar4 s4, - TScalar5 s5) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - where TScalar1: unmanaged, INumber - where TScalar2: unmanaged, INumber - where TScalar3: unmanaged, INumber - where TScalar4: unmanaged, INumber - where TScalar5: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Left = left.Source; - package->Right = right.Source; - package->Result = ans.Source; - package->LeftPrimOffset = left.PrimOffset; - package->RightPrimOffset = right.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->LeftPrimStride = left.PrimStride; - package->RightPrimStride = right.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->LeftSecOffset = left.SecondaryOffset; - package->RightSecOffset = right.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->LeftSecStride = left.SecondaryStride; - package->RightSecStride = right.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = left.PrimSize; - package->SecCount = left.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 4; - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - sp->Data(1) = s1; - sp->Data(2) = s2; - sp->Data(3) = s3; - sp->Data(4) = s4; - sp->Data(5) = s5; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration source, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0, - TScalar1 s1, - TScalar2 s2, - TScalar3 s3, - TScalar4 s4, - TScalar5 s5, - TScalar6 s6, - TScalar7 s7) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - where TScalar1: unmanaged, INumber - where TScalar2: unmanaged, INumber - where TScalar3: unmanaged, INumber - where TScalar4: unmanaged, INumber - where TScalar5: unmanaged, INumber - where TScalar6: unmanaged, INumber - where TScalar7: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Right = source.Source; - package->Result = ans.Source; - package->RightPrimOffset = source.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->RightPrimStride = source.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->RightSecOffset = source.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->RightSecStride = source.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = source.PrimSize; - package->SecCount = source.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 2; - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - sp->Data(1) = s1; - sp->Data(2) = s2; - sp->Data(3) = s3; - sp->Data(4) = s4; - sp->Data(5) = s5; - sp->Data(6) = s6; - sp->Data(7) = s7; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration left, - MatrixSegmentEnumeration right, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0, - TScalar1 s1, - TScalar2 s2, - TScalar3 s3, - TScalar4 s4, - TScalar5 s5, - TScalar6 s6) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - where TScalar1: unmanaged, INumber - where TScalar2: unmanaged, INumber - where TScalar3: unmanaged, INumber - where TScalar4: unmanaged, INumber - where TScalar5: unmanaged, INumber - where TScalar6: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Left = left.Source; - package->Right = right.Source; - package->Result = ans.Source; - package->LeftPrimOffset = left.PrimOffset; - package->RightPrimOffset = right.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->LeftPrimStride = left.PrimStride; - package->RightPrimStride = right.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->LeftSecOffset = left.SecondaryOffset; - package->RightSecOffset = right.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->LeftSecStride = left.SecondaryStride; - package->RightSecStride = right.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = left.PrimSize; - package->SecCount = left.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 4; - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - sp->Data(1) = s1; - sp->Data(2) = s2; - sp->Data(3) = s3; - sp->Data(4) = s4; - sp->Data(5) = s5; - sp->Data(6) = s6; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - - public static unsafe MatrixParallelHandle PublicTask( - MatrixSegmentEnumeration left, - MatrixSegmentEnumeration right, - MatrixSegmentEnumeration ans, - delegate**, MatrixKernel*, MatrixKernel*, void> Operation, - TScalar0 s0, - TScalar1 s1, - TScalar2 s2, - TScalar3 s3, - TScalar4 s4, - TScalar5 s5, - TScalar6 s6, - TScalar7 s7) - where T: unmanaged, INumber - where TScalar0: unmanaged, INumber - where TScalar1: unmanaged, INumber - where TScalar2: unmanaged, INumber - where TScalar3: unmanaged, INumber - where TScalar4: unmanaged, INumber - where TScalar5: unmanaged, INumber - where TScalar6: unmanaged, INumber - where TScalar7: unmanaged, INumber - { - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Left = left.Source; - package->Right = right.Source; - package->Result = ans.Source; - package->LeftPrimOffset = left.PrimOffset; - package->RightPrimOffset = right.PrimOffset; - package->ResultPrimOffset = ans.PrimOffset; - package->LeftPrimStride = left.PrimStride; - package->RightPrimStride = right.PrimStride; - package->ResultPrimStride = ans.PrimStride; - package->LeftSecOffset = left.SecondaryOffset; - package->RightSecOffset = right.SecondaryOffset; - package->ResultSecOffset = ans.SecondaryOffset; - package->LeftSecStride = left.SecondaryStride; - package->RightSecStride = right.SecondaryStride; - package->ResultSecStride = ans.SecondaryStride; - package->PrimCount = left.PrimSize; - package->SecCount = left.SecondarySize; - package->ComputeHandle = (nint)Operation; - package->ElementSize = sizeof(T); - package->ComputeMode = 4; - ScalarPacket* sp = MatrixParallelFactory.CreateScalarPacket(); - - sp->Data(0) = s0; - sp->Data(1) = s1; - sp->Data(2) = s2; - sp->Data(3) = s3; - sp->Data(4) = s4; - sp->Data(5) = s5; - sp->Data(6) = s6; - sp->Data(7) = s7; - package->Scalar = sp; - return PublicTaskPrivate(package); - } - private static unsafe MatrixParallelHandle PublicTaskPrivate(MatrixParallelTaskPackage* package) - where T: unmanaged, INumber + where T : unmanaged, INumber { int primCount = package->PrimCount; int secCount = package->SecCount; diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs b/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs index c4d14874..df38b04d 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs @@ -34,7 +34,54 @@ using System.Runtime.InteropServices; namespace StgSharp.Mathematics.Numeric { + /* + * #define PANEL_OP 1 + * #define BUFFER_OP 2 + * #define KERNEL_OP 3 + * + * #define RIGHT_ANS_PARAM 1 + * #define LEFT_RIGHT_ANS_PARAM 2 + * #define RIGHT_ANS_SCALAR_PARAM 3 + * #define LEFT_RIGHT_ANS_SCALAR_PARAM 4 + * #define ANS_SCALAR_PARAM 5 + * + * typedef struct matrix_op_mode { + * int16_t operation_style; // PANEL_OP, BUFFER_OP, KERNEL_OP + * int16_t parameter_style; // RIGHT_ANS_PARAM, LEFT_RIGHT_ANS_PARAM, e.g. + * } + * matrix_op_mode; + */ + + public enum MatrixOperationStyle : short + { + + PANEL_OP = 1, + BUFFER_OP = 2, + KERNEL_OP = 3, + + } + + public enum MatrixOperationParam : short + { + + RIGHT_ANS_PARAM = 1, + LEFT_RIGHT_ANS_PARAM = 2, + RIGHT_ANS_SCALAR_PARAM = 3, + LEFT_RIGHT_ANS_SCALAR_PARAM = 4, + ANS_SCALAR_PARAM = 5, + + } + [StructLayout(LayoutKind.Sequential)] + public struct MatrixOpMode + { + + public MatrixOperationParam ParameterStyle; + public MatrixOperationStyle OperationStyle; + + } + + [StructLayout(LayoutKind.Explicit, Size = 128)] internal unsafe struct MatrixParallelTaskPackageNonGeneric { @@ -50,69 +97,69 @@ namespace StgSharp.Mathematics.Numeric #region source // Group 1: Left/Right matrix kernel base address - public IntPtr Left; // 8 - public IntPtr Right; // 8 + [FieldOffset(0)]public IntPtr Left; // 8 + [FieldOffset(8)] public IntPtr Right; // 8 #endregion #region ans // Group 2: Result kernel base address + reserved - public IntPtr Result; // 8 - public int ElementSize; - public int ReservedPtr0; // 4 (extension/reserved) + [FieldOffset(16)] public IntPtr Result; // 8 + [FieldOffset(24)]public int ElementSize; + [FieldOffset(28)] public int ReservedPtr0; // 4 (extension/reserved) #endregion #region left enum - public int LeftPrimOffset; // 4 - public int LeftPrimStride; // 4 - public int LeftSecOffset; // 4 - public int LeftSecStride; // 4 + [FieldOffset(32)] public int LeftPrimOffset; // 4 + [FieldOffset(36)]public int LeftPrimStride; // 4 + [FieldOffset(40)]public int LeftSecOffset; // 4 + [FieldOffset(44)]public int LeftSecStride; // 4 #endregion #region right enum - public int RightPrimOffset; // 4 - public int RightPrimStride; // 4 - public int RightSecOffset; // 4 - public int RightSecStride; // 4 + [FieldOffset(32)] public int RightPrimOffset; // 4 + [FieldOffset(36)]public int RightPrimStride; // 4 + [FieldOffset(40)]public int RightSecOffset; // 4 + [FieldOffset(44)]public int RightSecStride; // 4 #endregion #region ans enum - public int ResultPrimOffset; // 4 - public int ResultPrimStride; // 4 - public int ResultSecOffset; // 4 - public int ResultSecStride; // 4 + [FieldOffset(48)]public int ResultPrimOffset; // 4 + [FieldOffset(52)]public int ResultPrimStride; // 4 + [FieldOffset(56)]public int ResultSecOffset; // 4 + [FieldOffset(60)] public int ResultSecStride; // 4 #endregion #region scalar/compute - public ScalarPacket* Scalar; // 8 - public IntPtr ComputeHandle; // 8 + [FieldOffset(80)]public ScalarPacket* Scalar; // 8 + [FieldOffset(88)]public IntPtr ComputeHandle; // 8 #endregion #region global profile - public int PrimCount; // 4 - public int ComputeMode; // 4 - public int SecCount; // 4 - private int ReservedInt0; // 4 + [FieldOffset(96)]public int PrimCount; // 4 + [FieldOffset(100)]public MatrixOpMode ComputeMode; // 4 + [FieldOffset(104)]public int SecCount; // 4 + [FieldOffset(108)]private int ReservedInt0; // 4 #endregion #region count offset - public int PrimTileOffset; // 4 - public int SecTileOffset; // 4 - public int PrimCountInTile; // 4 - public int SecCountInTile; // 4 + [FieldOffset(112)]public int PrimTileOffset; // 4 + [FieldOffset(116)]public int SecTileOffset; // 4 + [FieldOffset(120)]public int PrimCountInTile; // 4 + [FieldOffset(124)]public int SecCountInTile; // 4 #endregion } @@ -186,8 +233,8 @@ namespace StgSharp.Mathematics.Numeric #region global profile - [FieldOffset(96)]public int PrimCount; // 4 - [FieldOffset(100)]public int ComputeMode; // 4 + [FieldOffset(96)] public int PrimCount; // 4 + [FieldOffset(100)]public MatrixOpMode ComputeMode; // 4 [FieldOffset(104)]public int SecCount; // 4 [FieldOffset(108)] private readonly int ReservedInt0; // 4 diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs b/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs index e123e41a..834430ce 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs @@ -30,6 +30,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Text; using System.Threading.Tasks; @@ -38,12 +39,19 @@ namespace StgSharp.Mathematics.Numeric public class MatrixParallelThread : IDisposable { + // Affinity binding info for this thread + private AffinityBinding _affinity; + private bool _affinityApplied; + private bool disposedValue; private ManualResetEventSlim _resetEvent; private Thread _managedThread; public MatrixParallelThread() { + // Allocate a unique logical processor for this worker (mutually exclusive) + _affinity = AffinityAllocator.Allocate(); + _managedThread = new Thread(MatrixParallelWorker); _resetEvent = new ManualResetEventSlim(false); } @@ -71,19 +79,47 @@ namespace StgSharp.Mathematics.Numeric while (taskQueue.TryDequeue(out nint handle)) { MatrixParallelTaskPackageNonGeneric* p = (MatrixParallelTaskPackageNonGeneric*)handle; - switch (p->ComputeMode) + switch (p->ComputeMode.OperationStyle) { - case 1: - MatrixComputeModel.Compute_Unary(p); + case MatrixOperationStyle.PANEL_OP: break; - case 2: - MatrixComputeModel.Compute_UnaryScalar(p); + case MatrixOperationStyle.BUFFER_OP: + switch (p->ComputeMode.ParameterStyle) + { + case MatrixOperationParam.RIGHT_ANS_PARAM: + break; + case MatrixOperationParam.LEFT_RIGHT_ANS_PARAM: + MatrixComputeModel.BufferComputeBinary(p); + break; + case MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM: + MatrixComputeModel.BufferComputeUnaryScalar(p); + break; + case MatrixOperationParam.LEFT_RIGHT_ANS_SCALAR_PARAM: + break; + case MatrixOperationParam.ANS_SCALAR_PARAM: + break; + default: + break; + } break; - case 3: - MatrixComputeModel.Compute_Binary(p); - break; - case 4: - MatrixComputeModel.Compute_BinaryScalar(p); + case MatrixOperationStyle.KERNEL_OP: + switch (p->ComputeMode.ParameterStyle) + { + case MatrixOperationParam.RIGHT_ANS_PARAM: + MatrixComputeModel.Compute_Unary(p); + break; + case MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM: + MatrixComputeModel.Compute_UnaryScalar(p); + break; + case MatrixOperationParam.LEFT_RIGHT_ANS_PARAM: + MatrixComputeModel.Compute_Binary(p); + break; + case MatrixOperationParam.LEFT_RIGHT_ANS_SCALAR_PARAM: + MatrixComputeModel.Compute_BinaryScalar(p); + break; + default: + break; + } break; default: break; @@ -111,12 +147,18 @@ namespace StgSharp.Mathematics.Numeric _managedThread.Join(); } + // Release the reserved logical processor back to the pool + AffinityAllocator.Release(_affinity.LogicalProcessor); + disposedValue = true; } } private unsafe void MatrixParallelWorker() { + // Bind OS-thread affinity once when the worker starts running + ApplyAffinityIfNeeded(); + // Thread self = Thread.CurrentThread; _resetEvent.Wait(); _resetEvent.Reset(); @@ -141,6 +183,139 @@ namespace StgSharp.Mathematics.Numeric _managedThread.Priority = AfterWork; } + #region core affinity + + /* + * data to record core affinity + * preparation for future NUMA affinity setting + */ + + // Represents the binding information for a worker thread + internal readonly struct AffinityBinding + { + + public readonly int LogicalProcessor; + public readonly nuint Mask; + + public AffinityBinding(int logicalProcessor, nuint mask) + { + LogicalProcessor = logicalProcessor; + Mask = mask; + } + + } + + // Thread-safe allocator that hands out unique logical processors (mutually exclusive) + internal static class AffinityAllocator + { + + // Bitset of reserved logical processors (0..63). One bit per CPU. + private static long s_reservedMask; // treat as unsigned bits + + public static AffinityBinding Allocate() + { + int cpuCount = Environment.ProcessorCount; + if (cpuCount > 64) + { + cpuCount = 64; // SetThreadAffinityMask supports up to 64-bit mask + } + + while (true) + { + long snapshot = Volatile.Read(ref s_reservedMask); + + // Find first zero bit + int candidate = -1; + long bit = 0; + for (int i = 0; i < cpuCount; i++) + { + bit = 1L << i; + if ((snapshot & bit) == 0) + { + candidate = i; + break; + } + } + + if (candidate < 0) + { + // No free CPU found: fall back to CPU 0 (shared) + return new AffinityBinding(0, (nuint)1); + } + + long desired = snapshot | bit; + if (Interlocked.CompareExchange(ref s_reservedMask, desired, snapshot) == snapshot) + { + // Successfully reserved candidate + nuint mask = (nuint)(1UL << candidate); + return new AffinityBinding(candidate, mask); + } + + // else: another thread raced us; retry + } + } + + public static void Release(int logicalProcessor) + { + if ((uint)logicalProcessor >= 64u) { + return; + } + + long bit = 1L << logicalProcessor; + while (true) + { + long snapshot = Volatile.Read(ref s_reservedMask); + if ((snapshot & bit) == 0) + { + return; // already free + } + + long desired = snapshot & ~bit; + if (Interlocked.CompareExchange(ref s_reservedMask, desired, snapshot) == snapshot) { + return; + } + } + } + + } + + private void ApplyAffinityIfNeeded() + { + if (_affinityApplied) { + return; + } + +#if WINDOWS + try + { + // On Windows pin the current native thread to the reserved logical processor + SetThreadAffinityMask(GetCurrentThread(), _affinity.Mask); + } + catch + { + // Ignore binding errors; continue without affinity + } +#elif LINUX + +#endif + + _affinityApplied = true; + } + + #endregion + + +#if WINDOWS + [DllImport("kernel32.dll")] + private static extern IntPtr GetCurrentThread(); + + [DllImport("kernel32.dll")] + private static extern nuint SetThreadAffinityMask(IntPtr hThread, nuint dwThreadAffinityMask); + +#elif LINUX + +#endif + #region wrap belong internal MatrixParallelWrap BindedWrap { get; set; } = null!; diff --git a/StgSharp.Mathematics/Numeric/TaskPackageUsage.md b/StgSharp.Mathematics/Numeric/TaskPackageUsage.md new file mode 100644 index 00000000..e69de29b diff --git a/StgSharp.Mathematics/StgSharp.Mathematics.csproj b/StgSharp.Mathematics/StgSharp.Mathematics.csproj index 0919652e..6f33de76 100644 --- a/StgSharp.Mathematics/StgSharp.Mathematics.csproj +++ b/StgSharp.Mathematics/StgSharp.Mathematics.csproj @@ -13,4 +13,20 @@ + + true + true + true + + + + WINDOWS + + + OSX + + + LINUX + + diff --git a/StgSharp.Native/k_bufop.c b/StgSharp.Native/k_bufop.c index e6976ec5..cfde2dc5 100644 --- a/StgSharp.Native/k_bufop.c +++ b/StgSharp.Native/k_bufop.c @@ -27,3 +27,107 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill) ans[i].zmm[0] = vec; } } + +DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, add) +{ + for (size_t i = 0; i < count; i++) { + register __m128 c0 = _mm_add_ps(left[i].xmm[0], right[i].xmm[0]); + register __m128 c1 = _mm_add_ps(left[i].xmm[1], right[i].xmm[1]); + register __m128 c2 = _mm_add_ps(left[i].xmm[2], right[i].xmm[2]); + register __m128 c3 = _mm_add_ps(left[i].xmm[3], right[i].xmm[3]); + + ans[i].xmm[0] = c0; + ans[i].xmm[1] = c1; + ans[i].xmm[2] = c2; + ans[i].xmm[3] = c3; + } +} + +DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, add) +{ + for (size_t i = 0; i < count; i++) { + register __m256 c0 = _mm256_add_ps(left[i].ymm[0], right[i].ymm[0]); + register __m256 c1 = _mm256_add_ps(left[i].ymm[1], right[i].ymm[1]); + ans[i].ymm[0] = c0; + ans[i].ymm[1] = c1; + } +} + +DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, add) +{ + for (size_t i = 0; i < count; i++) { + register __m512 c0 = _mm512_add_ps(left[i].zmm[0], right[i].zmm[0]); + ans[i].zmm[0] = c0; + } +} + +DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, sub) +{ + for (size_t i = 0; i < count; i++) { + register __m128 c0 = _mm_sub_ps(left[i].xmm[0], right[i].xmm[0]); + register __m128 c1 = _mm_sub_ps(left[i].xmm[1], right[i].xmm[1]); + register __m128 c2 = _mm_sub_ps(left[i].xmm[2], right[i].xmm[2]); + register __m128 c3 = _mm_sub_ps(left[i].xmm[3], right[i].xmm[3]); + + ans[i].xmm[0] = c0; + ans[i].xmm[1] = c1; + ans[i].xmm[2] = c2; + ans[i].xmm[3] = c3; + } +} + +DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, sub) +{ + for (size_t i = 0; i < count; i++) { + register __m256 c0 = _mm256_sub_ps(left[i].ymm[0], right[i].ymm[0]); + register __m256 c1 = _mm256_sub_ps(left[i].ymm[1], right[i].ymm[1]); + ans[i].ymm[0] = c0; + ans[i].ymm[1] = c1; + } +} + +DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, sub) +{ + for (size_t i = 0; i < count; i++) { + register __m512 c0 = _mm512_sub_ps(left[i].zmm[0], right[i].zmm[0]); + ans[i].zmm[0] = c0; + } +} + +DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul) +{ + register __m128 scalar_vec = ((__m128 *)(scalar->data))[0]; + for (size_t i = 0; i < count; i++) { + register __m128 c0 = _mm_mul_ps(right[i].xmm[0], scalar_vec); + register __m128 c1 = _mm_mul_ps(right[i].xmm[1], scalar_vec); + register __m128 c2 = _mm_mul_ps(right[i].xmm[2], scalar_vec); + register __m128 c3 = _mm_mul_ps(right[i].xmm[3], scalar_vec); + + ans[i].xmm[0] = c0; + ans[i].xmm[1] = c1; + ans[i].xmm[2] = c2; + ans[i].xmm[3] = c3; + } +} + +DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul) +{ + register __m256 scalar_vec = ((__m256 *)(scalar->data))[0]; + for (size_t i = 0; i < count; i++) { + register __m256 c0 = _mm256_mul_ps(right[i].ymm[0], scalar_vec); + register __m256 c1 = _mm256_mul_ps(right[i].ymm[1], scalar_vec); + + ans[i].ymm[0] = c0; + ans[i].ymm[1] = c1; + } +} + +DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul) +{ + register __m512 scalar_vec = ((__m512 *)(scalar->data))[0]; + for (size_t i = 0; i < count; i++) { + register __m512 c0 = _mm512_mul_ps(right[i].zmm[0], scalar_vec); + + ans[i].zmm[0] = c0; + } +} diff --git a/StgSharp.Native/sn-head/sn_intrinsic.h b/StgSharp.Native/sn-head/sn_intrinsic.h index 184ee8a8..996bc7a4 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.h @@ -39,6 +39,23 @@ typedef union scalar_pack { uint64_t data[8]; } scalar_pack; +// define operation style: + +#define PANEL_OP 1 +#define BUFFER_OP 2 +#define KERNEL_OP 3 + +#define RIGHT_ANS_PARAM 1 +#define LEFT_RIGHT_ANS_PARAM 2 +#define RIGHT_ANS_SCALAR_PARAM 3 +#define LEFT_RIGHT_ANS_SCALAR_PARAM 4 +#define ANS_SCALAR_PARAM 5 + +typedef struct matrix_op_mode { + int16_t operation_style; /* PANEL_OP, BUFFER_OP, KERNEL_OP */ + int16_t parameter_style; /* RIGHT_ANS_PARAM, LEFT_RIGHT_ANS_PARAM, e.g. */ +} matrix_op_mode; + /* Mirrors StgSharp.Mathematics.Numeric.MatrixParallelTaskPackage (128 bytes). */ #define MAT_TASK(T) \ struct { \ @@ -62,7 +79,7 @@ typedef union scalar_pack { scalar_pack *scalar; /* Offset 80 */ \ void *compute_handle; /* Offset 88 */ \ int32_t prim_count; /* Offset 96 */ \ - int32_t compute_mode; /* Offset 100 */ \ + matrix_op_mode compute_mode; /* Offset 100 */ \ int32_t sec_count; /* Offset 104 */ \ int32_t reserved_int0; /* Offset 108 */ \ int32_t prim_tile_offset; /* Offset 112 */ \ @@ -93,7 +110,7 @@ typedef void(SN_DECL *BUF_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_ker #define DECLARE_BUF_PROC_ANS_SCALAR(T_type, T_arch, T_op_name) \ INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \ - MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, int count) + MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, size_t count) #define DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \ MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \ @@ -101,14 +118,14 @@ typedef void(SN_DECL *BUF_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_ker #define DECLARE_BUF_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \ MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \ - MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, int count) + MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, size_t count) #define DECLARE_BUF_PROC_RIGHT_ANS(T_type, T_arch, T_op_name) \ INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \ - MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, int count) + MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, size_t count) #define DECLARE_BUF_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \ INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \ MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \ - MAT_KERNEL(T_type) *restrict ans, int count) + MAT_KERNEL(T_type) *restrict ans, size_t count) #pragma endregion @@ -176,23 +193,25 @@ typedef uint64_t(SN_DECL *FACTORIALROC)(int n); INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target); -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, add); -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, add); -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, add); +DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, add); +DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, add); +DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, add); -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, sub); -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, sub); -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, sub); +DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, sub); +DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, sub); +DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, sub); DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose); DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose); DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose); -DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul); -DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul); -DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul); +DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul); +DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul); +DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul); DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill); +DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill); +DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill); #pragma endregion @@ -231,17 +250,19 @@ INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int lengt #pragma endregion +// Reordered to match C# IntrinsicContext field order +// C# order: city_hash_simplify, f32_add, f32_fill, f32_fma, f32_scalar_mul, f32_sub, f32_transpose, factorial_simd, index_pair, normalize_3 typedef struct sn_intrinsic { - HASH city_hash_simplify; - FACTORIALROC factorial_simd; - INDEX_PAIR index_pair; - KER_PROC_LRA f32_kernel_add; - KER_PROC_AS f32_kernel_fill; - PNL_PROC_LRA f32_panel_fma; - KER_PROC_LRA f32_kernel_scalar_mul; - KER_PROC_LRA f32_kernel_sub; - KER_PROC_RA f32_kernel_transpose; - VECTORNORMALIZEPROC f32_normalize_3; + HASH city_hash_simplify; // city_hash_simplify + BUF_PROC_LRA f32_buffer_add; // f32_add + BUF_PROC_AS f32_buffer_fill; // f32_fill + PNL_PROC_LRA f32_panel_fma; // f32_fma (panel variant) + BUF_PROC_LRA f32_kernel_scalar_mul; // f32_scalar_mul (right+ans+scalar variant mapped here) + BUF_PROC_LRA f32_kernel_sub; // f32_sub + KER_PROC_RA f32_kernel_transpose; // f32_transpose + FACTORIALROC factorial_simd; // factorial_simd + INDEX_PAIR index_pair; // index_pair + VECTORNORMALIZEPROC f32_normalize_3; // normalize_3 } sn_intrinsic; typedef enum most_advanced_instruction { diff --git a/StgSharp.Native/sn-src/k_basicop.c b/StgSharp.Native/sn-src/k_basicop.c index d0e4f44b..b3ff326c 100644 --- a/StgSharp.Native/sn-src/k_basicop.c +++ b/StgSharp.Native/sn-src/k_basicop.c @@ -1,87 +1,5 @@ #include "sn_intrinsic.h" -DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill) -{ - register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]); - for (size_t i = 0; i < count; i++) { - ans[i].xmm[0] = vec; - ans[i].xmm[1] = vec; - ans[i].xmm[2] = vec; - ans[i].xmm[3] = vec; - } -} - -DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill) -{ - register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data))); - for (size_t i = 0; i < count; i++) { - ans[i].ymm[0] = vec; - ans[i].ymm[1] = vec; - } -} - -DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill) -{ - register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]); - for (size_t i = 0; i < count; i++) { - ans[i].zmm[0] = vec; - } -} - -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, add) -{ - register __m128 c0 = _mm_add_ps(left->xmm[0], right->xmm[0]); - register __m128 c1 = _mm_add_ps(left->xmm[1], right->xmm[1]); - register __m128 c2 = _mm_add_ps(left->xmm[2], right->xmm[2]); - register __m128 c3 = _mm_add_ps(left->xmm[3], right->xmm[3]); - - ans->xmm[0] = c0; - ans->xmm[1] = c1; - ans->xmm[2] = c2; - ans->xmm[3] = c3; -} - -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, add) -{ - register __m256 c0 = _mm256_add_ps(left->ymm[0], right->ymm[0]); - register __m256 c1 = _mm256_add_ps(left->ymm[1], right->ymm[1]); - ans->ymm[0] = c0; - ans->ymm[1] = c1; -} - -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, add) -{ - register __m512 c0 = _mm512_add_ps(left->zmm[0], right->zmm[0]); - ans->zmm[0] = c0; -} - -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, sub) -{ - register __m128 c0 = _mm_sub_ps(left->xmm[0], right->xmm[0]); - register __m128 c1 = _mm_sub_ps(left->xmm[1], right->xmm[1]); - register __m128 c2 = _mm_sub_ps(left->xmm[2], right->xmm[2]); - register __m128 c3 = _mm_sub_ps(left->xmm[3], right->xmm[3]); - - ans->xmm[0] = c0; - ans->xmm[1] = c1; - ans->xmm[2] = c2; - ans->xmm[3] = c3; -} - -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, sub) -{ - register __m256 c0 = _mm256_sub_ps(left->ymm[0], right->ymm[0]); - register __m256 c1 = _mm256_sub_ps(left->ymm[1], right->ymm[1]); - ans->ymm[0] = c0; - ans->ymm[1] = c1; -} - -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, sub) -{ - register __m512 c0 = _mm512_sub_ps(left->zmm[0], right->zmm[0]); - ans->zmm[0] = c0; -} - // Matrix transpose implementations DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose) { @@ -124,38 +42,3 @@ DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose) ans->zmm[0] = _mm512_permutexvar_ps(transpose_indices, matrix); } - -DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul) -{ - register __m128 scalar_vec = ((__m128 *)(scalar->data))[0]; - - register __m128 c0 = _mm_mul_ps(right->xmm[0], scalar_vec); - register __m128 c1 = _mm_mul_ps(right->xmm[1], scalar_vec); - register __m128 c2 = _mm_mul_ps(right->xmm[2], scalar_vec); - register __m128 c3 = _mm_mul_ps(right->xmm[3], scalar_vec); - - ans->xmm[0] = c0; - ans->xmm[1] = c1; - ans->xmm[2] = c2; - ans->xmm[3] = c3; -} - -DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul) -{ - register __m256 scalar_vec = ((__m256 *)(scalar->data))[0]; - - register __m256 c0 = _mm256_mul_ps(right->ymm[0], scalar_vec); - register __m256 c1 = _mm256_mul_ps(right->ymm[1], scalar_vec); - - ans->ymm[0] = c0; - ans->ymm[1] = c1; -} - -DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul) -{ - register __m512 scalar_vec = ((__m512 *)(scalar->data))[0]; - - register __m512 c0 = _mm512_mul_ps(right->zmm[0], scalar_vec); - - ans->zmm[0] = c0; -} diff --git a/StgSharp.Native/sn-src/load_intrinsic.c b/StgSharp.Native/sn-src/load_intrinsic.c index af5b8062..0e2b17d4 100644 --- a/StgSharp.Native/sn-src/load_intrinsic.c +++ b/StgSharp.Native/sn-src/load_intrinsic.c @@ -65,10 +65,11 @@ void load_intrinsic_sse(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; - context->f32_kernel_add = KER_PROC(float, sse, add); + context->f32_buffer_add = BUF_PROC(float, sse, add); context->f32_panel_fma = KER_PROC(float, sse, fma); - context->f32_kernel_scalar_mul = KER_PROC(float, sse, scalar_mul); - context->f32_kernel_sub = KER_PROC(float, sse, sub); + context->f32_buffer_fill = BUF_PROC(float, sse, fill); + context->f32_kernel_scalar_mul = BUF_PROC(float, sse, scalar_mul); + context->f32_kernel_sub = BUF_PROC(float, sse, sub); context->f32_kernel_transpose = KER_PROC(float, sse, transpose); context->f32_normalize_3 = f32_normalize; } @@ -78,10 +79,11 @@ void load_intrinsic_avx(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; - context->f32_kernel_add = KER_PROC(float, avx, add); + context->f32_buffer_add = BUF_PROC(float, avx, add); context->f32_panel_fma = KER_PROC(float, sse, fma); - context->f32_kernel_scalar_mul = KER_PROC(float, avx, scalar_mul); - context->f32_kernel_sub = KER_PROC(float, avx, sub); + context->f32_buffer_fill = BUF_PROC(float, avx, fill); + context->f32_kernel_scalar_mul = BUF_PROC(float, avx, scalar_mul); + context->f32_kernel_sub = BUF_PROC(float, avx, sub); context->f32_kernel_transpose = KER_PROC(float, avx, transpose); context->f32_normalize_3 = f32_normalize; } @@ -91,10 +93,11 @@ void load_intrinsic_512(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; - context->f32_kernel_add = KER_PROC(float, 512, add); + context->f32_buffer_add = BUF_PROC(float, 512, add); context->f32_panel_fma = KER_PROC(float, sse, fma); - context->f32_kernel_scalar_mul = KER_PROC(float, 512, scalar_mul); - context->f32_kernel_sub = KER_PROC(float, 512, sub); + context->f32_buffer_fill = BUF_PROC(float, 512, fill); + context->f32_kernel_scalar_mul = BUF_PROC(float, 512, scalar_mul); + context->f32_kernel_sub = BUF_PROC(float, 512, sub); context->f32_kernel_transpose = KER_PROC(float, 512, transpose); context->f32_normalize_3 = f32_normalize; } diff --git a/StgSharpDebug/Program.cs b/StgSharpDebug/Program.cs index cf61ef1f..f80a4c49 100644 --- a/StgSharpDebug/Program.cs +++ b/StgSharpDebug/Program.cs @@ -31,6 +31,7 @@ using StgSharp.Geometries; using StgSharp.Graphics; using StgSharp.HighPerformance; using StgSharp.HighPerformance.Memory; +using StgSharp.Internal; using StgSharp.Mathematics; using StgSharp.Mathematics.Graphic; using StgSharp.Mathematics.Numeric; @@ -60,9 +61,9 @@ namespace StgSharp { try { - World.Init(); - + World.Initialize().UseModule().UseModule().InitializeModule(); + /* // New test with synchronized start: // - 8 producer threads, each pushing 1000 unique integers // - 8 consumer threads, each attempting to pop 1000 times @@ -155,7 +156,7 @@ namespace StgSharp } Console.WriteLine($"Popped count: {popped.Count}"); - /**/ + /* Console.WriteLine("Popped values:"); foreach (int v in popped) { @@ -163,7 +164,7 @@ namespace StgSharp Console.Write('\t'); } Console.WriteLine(); - /**/ + /* Console.WriteLine($"Remaining count: {remaining.Count}"); Console.WriteLine("Remaining values:"); foreach (int v in remaining) @@ -176,28 +177,23 @@ namespace StgSharp // Sanity check Console.WriteLine($"Produced: {producerThreads * perProducer}, Popped+Remaining: {popped.Count + remaining.Count}"); - /* - // TestMatrixParallel(); + /**/ // 原有的矩阵测试代码 HybridLayerSegregatedFitAllocator hlsf = new HybridLayerSegregatedFitAllocator(64 * 400 * 400); - Matrix m = Matrix.FromHlsf(400, 200, hlsf); - - int i = 0, ii = 0; - - foreach (MatrixSegment lines in m.GetRowEnumeration()) + Matrix m = Matrix.FromHlsf(20, 20, hlsf); + m.Fill(1.0f); + Matrix n = m.Add(m); + for (int i = 0; i < 20; i++) { - i = 0; - foreach (MatrixKernel* item in lines) - { - i++; - Console.WriteLine(i); + for (int j = 0; j < 20; j++) { + Console.Write($"{n[i, j].ToString("0.000")}\t"); } - ii++; - Console.WriteLine($"============================{ii}"); + Console.WriteLine(); } /**/ + /**/ } catch (Exception ex) {