youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewrite of f32 fma, and other optmization of simd kernel ops

Browse Source
This commit is contained in:
Youmu Liu
2025-11-10 20:40:19 +08:00
parent 1dee467220
commit 678a846e54
18 changed files with 302 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs
+40 -9
View File
@@ -31,6 +31,7 @@ using StgSharp.Internal;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Security.AccessControl;
using System.Text;
using System.Threading.Tasks;
@@ -59,23 +60,21 @@ namespace StgSharp.Mathematics.Numeric
}
if (count <= 1024)
{
ReadOnlySpan<MatrixParallelThread> pool = MatrixParallel.LeadParallel(1);
if (pool.Length == 0)
{
/*
* wait
*/
}
MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1);
MatrixKernel<float>* leftPtr = left.Buffer;
MatrixKernel<float>* rightPtr = right.Buffer;
MatrixKernel<float>* ansPtr = ans.Buffer;
for (int i = 0; i < count; i++) {
NativeIntrinsic.Intrinsic.f32_add(leftPtr + i, rightPtr + i, ansPtr + i);
}
MatrixParallel.ReturnParallelResources(ref pool);
return ans;
}
MatrixParallelHandle taskGroup = MatrixParallel.PublicTask(left.GetColumnEnumeration(), right.GetColumnEnumeration(), ans.GetColumnEnumeration(),
(delegate*<MatrixKernel<float>*, MatrixKernel<float>*, MatrixKernel<float>*, void>)NativeIntrinsic.Intrinsic.f32_add);
MatrixParallelHandle taskGroup = MatrixParallel.PublicTask(left.GetColumnEnumeration(),
right.GetColumnEnumeration(),
ans.GetColumnEnumeration(),
(delegate*<MatrixKernel<float>*, MatrixKernel<float>*, MatrixKernel<float>*, void>)NativeIntrinsic.Intrinsic.f32_add);
MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep);
/*
@@ -85,5 +84,37 @@ namespace StgSharp.Mathematics.Numeric
return ans;
}
public static unsafe void Fill<T>(Matrix<float> left, T value) where T: unmanaged, INumber<T>
{
long count = (long)left.KernelColumnLength * left.KernelRowLength;
ScalarPacket* scaler = MatrixParallelFactory.CreateScalarPacket();
scaler->BroadCast<T, M512>(0, value);
if (count <= 256)
{
MatrixKernel<float>* leftPtr = left.Buffer;
for (int i = 0; i < count; i++) {
NativeIntrinsic.Intrinsic.f32_fill(leftPtr + i, scaler);
}
}
if (count <= 1024)
{
MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1);
MatrixKernel<float>* leftPtr = left.Buffer;
for (int i = 0; i < count; i++) {
NativeIntrinsic.Intrinsic.f32_fill(leftPtr + i, scaler);
}
MatrixParallel.ReturnParallelResources(ref pool);
}
MatrixParallelHandle taskGroup = MatrixParallel.PublicTask(left.GetColumnEnumeration(),
(delegate*<MatrixKernel<float>*, MatrixKernel<float>*, MatrixKernel<float>*, void>)NativeIntrinsic.Intrinsic.f32_add);
MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep);
/*
* run parallel
*/
taskGroup.Dispose();
}
}
}
StgSharp.Mathematics/Numeric/MatrixParallel.Await.cs
+23 -5
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="MatrixParallel.Await"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -56,7 +56,16 @@ namespace StgSharp.Mathematics.Numeric
min,
recommendCount
);
_awaitingLeaderHandle.Enqueue(handle);
do
{
if (Interlocked.Exchange(ref _sortState, 1) == 0)
{
_awaitingLeaderHandle.Enqueue(handle);
_ = Interlocked.Exchange(ref _sortState, 0);
Interlocked.Increment(ref _createGeneration);
break;
}
} while (true);
int cycleGeneration = _createGeneration;
do
{
@@ -66,7 +75,7 @@ namespace StgSharp.Mathematics.Numeric
{
if (Interlocked.Exchange(ref _sortState, 1) == 0)
{
if (_awaitingLeaderHandle.TryPopIfRefEqual(cycleGeneration, ref handle))
if (_awaitingLeaderHandle.TryPopIfRefEqual(cycleGeneration, in handle))
{
// Sort is implemented in TryPopIfRefEqual
ret = _threads.PopRange(handle.BestCount);
@@ -101,7 +110,16 @@ namespace StgSharp.Mathematics.Numeric
public static void ReturnParallelResources(ref MatrixParallelThread[] pool)
{
_threads.PushRange(pool);
while (true)
{
if (Interlocked.Exchange(ref _sortState, 1) == 0)
{
_threads.PushRange(pool);
Interlocked.Exchange(ref _sortState, 0);
break;
}
}
pool = null!;
}
StgSharp.Mathematics/Numeric/MatrixParallel.AwaitingQueue.cs
+12 -28
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="MatrixParallel.AwaitingQueue"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -47,7 +47,6 @@ namespace StgSharp.Mathematics.Numeric
private Element _quickElements = new();
private volatile int _count;
private volatile int _lastGeneration;
private readonly Queue<WaitHandle> _fallback = [];
public AwaitingQueue() { }
@@ -68,22 +67,16 @@ namespace StgSharp.Mathematics.Numeric
{
if (_count == 0)
{
_lastGeneration = generation;
handle = null!;
return false;
}
if (_lastGeneration == generation)
{
for (int i = 0; i < _count; i++) {
_quickElements[i].UpdatePriority(generation);
}
Element.Sort(_quickElements);
} else
{
_lastGeneration = generation;
for (int i = 0; i < _count; i++) {
_quickElements[i].UpdatePriority(generation);
}
Element.Sort(ref _quickElements);
handle = _quickElements[0];
_count--;
_quickElements[0] = default!;
if (_count > 7) {
_ = _fallback.TryDequeue(out _quickElements[0]!);
}
@@ -94,40 +87,31 @@ namespace StgSharp.Mathematics.Numeric
{
if (_count == 0)
{
_lastGeneration = generation;
handle = null!;
return false;
}
for (int i = 0; i < _count; i++) {
_quickElements[i].UpdatePriority(generation);
}
Element.Sort(_quickElements);
_lastGeneration = generation;
Element.Sort(ref _quickElements);
handle = _quickElements[0];
_count--;
if (_count > 7) {
_ = _fallback.TryDequeue(out _quickElements[0]!);
}
return true;
}
public bool TryPopIfRefEqual(int generation, ref WaitHandle handle)
public bool TryPopIfRefEqual(int generation, in WaitHandle handle)
{
if (_count == 0)
{
handle = null!;
if (_count == 0) {
return false;
}
for (int i = 0; i < _count; i++) {
_quickElements[i].UpdatePriority(generation);
}
Element.Sort(ref _quickElements);
_lastGeneration = generation;
handle = _quickElements[0];
_count--;
if (!ReferenceEquals(_quickElements[0], handle)) {
return false;
}
_count--;
_quickElements[0] = null!;
if (_count > 7) {
_ = _fallback.TryDequeue(out _quickElements[0]!);
@@ -147,7 +131,7 @@ namespace StgSharp.Mathematics.Numeric
{
int maxIdx = 0;
WaitHandle first = e[0];
int maxPr = first?.Priority ?? int.MaxValue;
int maxPr = first?.Priority ?? int.MinValue;
for (int i = 1; i < 8; i++)
{
StgSharp.Mathematics/Numeric/MatrixParallel.cs
+3 -3
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="MatrixParallel"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -40,7 +40,7 @@ namespace StgSharp.Mathematics.Numeric
private static bool _meaningful;
private static ConcurrentCapacityFixedStack<MatrixParallelThread> _threads;
private static CapacityFixedStack<MatrixParallelThread> _threads;
private static volatile int RemainThreadCount = 0;
private static readonly object _lock = new();
StgSharp.Mathematics/Numeric/MatrixParallelHandle.cs
+5 -27
View File
@@ -1,6 +1,6 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="MatrixParallelTask"
// file="MatrixParallelHandle"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
@@ -34,22 +34,6 @@ using System.Runtime.InteropServices;
namespace StgSharp.Mathematics.Numeric
{
public unsafe struct ScalarPacket
{
private fixed ulong data[8];
public ref T Data<T>(int index) where T: unmanaged,INumber<T>
{
if (index is < 0 or >= 8) {
throw new IndexOutOfRangeException();
}
return ref Unsafe.As<ulong, T>(ref data[index]);
}
}
[StructLayout(LayoutKind.Sequential)]
internal unsafe struct MatrixParallelTaskPackageNonGeneric
{
@@ -59,11 +43,8 @@ namespace StgSharp.Mathematics.Numeric
MatrixParallelTaskPackageNonGeneric* source,
MatrixParallelTaskPackageNonGeneric* target)
{
Buffer.MemoryCopy(
source,
target,
sizeof(MatrixParallelTaskPackageNonGeneric),
sizeof(MatrixParallelTaskPackageNonGeneric));
Buffer.MemoryCopy(source, target, sizeof(MatrixParallelTaskPackageNonGeneric),
sizeof(MatrixParallelTaskPackageNonGeneric));
}
#region source
@@ -148,11 +129,8 @@ namespace StgSharp.Mathematics.Numeric
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static unsafe void Copy(MatrixParallelTaskPackage<T>* source, MatrixParallelTaskPackage<T>* target)
{
Buffer.MemoryCopy(
source,
target,
sizeof(MatrixParallelTaskPackage<T>),
sizeof(MatrixParallelTaskPackage<T>));
Buffer.MemoryCopy(source, target, sizeof(MatrixParallelTaskPackage<T>),
sizeof(MatrixParallelTaskPackage<T>));
}
#region source