youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

matrix parallel start to use two different policy

Browse Source
This commit is contained in:
Youmu Liu
2026-04-27 20:08:10 +08:00
parent ecb1f04770
commit ae664cb541
16 changed files with 170 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/Collections/LinearBag.cs
+3 -2
View File
@@ -151,10 +151,11 @@ namespace StgSharp.Collections
/// Removes the first occurrence of value (unordered). Does nothing if not found.
/// </summary>
public void Remove(
T value
T value,
EqualityComparer<T> comparer = default
)
{
EqualityComparer<T> comparer = EqualityComparer<T>.Default;
comparer ??= EqualityComparer<T>.Default;
for (int i = 0; i < Count; i++)
{
if (comparer.Equals(_items[i], value))
StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs
+2 -2
View File
@@ -39,7 +39,7 @@ namespace StgSharp.HighPerformance.Memory
private struct PredictionArray
{
private short PredictionCount;
private int PredictionCount;
}
@@ -47,7 +47,7 @@ namespace StgSharp.HighPerformance.Memory
private struct MapArray
{
private short MapCount;
private int MapCount;
}
StgSharp.Common/HighPerformance/Memory/L4.Evict.cs
+25 -2
View File
@@ -25,6 +25,7 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
@@ -56,7 +57,29 @@ namespace StgSharp.HighPerformance.Memory
)
{
count = (count < 0) ? 0 : count;
return _intrinsic.ScanEvict((short*)_predictCount, (short*)_mapCount, count, this);
int* pPtr = (int*)_predictCount, mPtr = (int*)_mapCount;
int remain = count;
ConcurrentFlexibleArray<IL4Predict> predictors = _predictors;
predictors.RequestSpanOperation();
Span<IL4Predict> pSpan = predictors.AsSpan();
try
{
for (int i = 0; i < CacheLineCount; i++)
{
if (pPtr[i] > mPtr[i])
{
continue;
}
remain -= TryEvictAt(i, pSpan) ? 1 : 0;
}
}
finally
{
predictors.ReleaseSpanOperation();
}
return remain;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -85,7 +108,7 @@ namespace StgSharp.HighPerformance.Memory
predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan<byte>((byte*)head->Address, (int)head->Size));
RecycleLine(index);
(*_mapCount)[index] = 0;
(*_predictCount)[index] = short.MaxValue;
(*_predictCount)[index] = int.MaxValue;
return true;
}
return false;
StgSharp.Common/HighPerformance/Memory/L4.IntrinsicKernel.cs
-117
View File
@@ -1,117 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="L4.IntrinsicKernel"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class L4
{
private interface IIntrinsicKernel
{
static IIntrinsicKernel Instance { get; } = Create();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int ScanEvict(
short* pPtr,
short* mPtr,
int count,
L4 owner
);
private static IIntrinsicKernel Create()
{
if (Sse42.IsSupported)
{
return new SseKernel();
} else
{
throw new PlatformNotSupportedException("No supported intrinsic kernel found for current platform.");
}
}
}
private readonly struct SseKernel : IIntrinsicKernel
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public readonly int ScanEvict(
short* pPtr,
short* mPtr,
int count,
L4 owner
)
{
int remain = count;
ConcurrentFlexibleArray<IL4Predict> predictors = owner._predictors;
predictors.RequestSpanOperation();
Span<IL4Predict> pSpan = predictors.AsSpan();
try
{
for (int i = 0; i < CacheLineCount; i += 8)
{
Vector128<short> p = Sse2.LoadVector128(pPtr + i);
Vector128<short> m = Sse2.LoadVector128(mPtr + i);
Vector128<short> great = Sse2.CompareGreaterThan(p, m);
Vector128<short> le = Sse42.Xor(great, Vector128.Create<short>(-1));
int mask = Sse42.MoveMask(le.AsByte());
while ((mask != 0) && (remain > 0))
{
int bit = BitOperations.TrailingZeroCount(mask);
int lane = bit >> 1;
if (owner.TryEvictAt(i + lane, pSpan)) {
remain--;
}
mask &= mask - 1;
mask &= mask - 1;
}
}
}
finally
{
predictors.ReleaseSpanOperation();
}
return remain;
}
}
}
}
StgSharp.Common/HighPerformance/Memory/L4.cs
+1 -1
View File
@@ -190,7 +190,7 @@ namespace StgSharp.HighPerformance.Memory
(*_predictCount)[entry->Id] = 0;
_ = Interlocked.Increment(ref _activeCount);
_ = Interlocked.Decrement(ref _aheadCount);
_ = Interlocked.Decrement(ref entry->RemainingPredictionCount);
_ = Interlocked.Decrement(ref (*_mapCount)[entry->Id]);
return new L4CacheLine(entry->Address, ref _activeCount, ref entry->RefCount);
} else
{
StgSharp.Mathematics/Numeric/Runtime/MatrixEnumerator.cs → StgSharp.Mathematics/Numeric/Linear/MatrixCompute.Double.cs
+7 -59
View File
@@ -1,6 +1,6 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="MatrixEnumerator"
// file="MatrixCompute.Sequential"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
@@ -26,73 +26,21 @@
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.HighPerformance.Memory;
using StgSharp.Mathematics.Numeric.Runtime;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Mathematics.Numeric.Runtime
using unsafe MkProc = delegate* unmanaged[Cdecl]<StgSharp.Mathematics.Numeric.Runtime.MatrixParallelTask*, void>;
namespace StgSharp.Mathematics.Numeric
{
public enum PredictPolicy
public static unsafe partial class MatrixCompute
{
Transparent = 0,
BufferSequential = 1,
SquareRowMajor = 2,
SquareColMajor = 3,
}
public unsafe interface IMatrixCachePredictor<TSelf> : IL4Predict
where TSelf : class, IMatrixCachePredictor<TSelf>
{
public static abstract PredictPolicy Policy { get; set; }
}
public sealed unsafe class BufferSequentialPredictor : IMatrixCachePredictor<BufferSequentialPredictor>
{
private readonly MatrixParallelTask* origin;
private readonly int _kernelSize;
public static PredictPolicy Policy { get; set; } = PredictPolicy.SquareRowMajor;
public bool IsSamePolicy(
uint policy,
IL4Predict predict
)
{
throw new NotImplementedException();
}
public PredictResult Predict(
Span<CacheLineDescription> node,
out int count
)
{
throw new NotImplementedException();
}
public void Prefetch(
nuint origin,
uint policy,
Span<byte> cache
)
{
throw new NotImplementedException();
}
public void WriteBack(
nuint origin,
uint policy,
ReadOnlySpan<byte> cache
)
{
throw new NotImplementedException();
}
}
}
StgSharp.Mathematics/Numeric/Linear/MatrixCompute.Sequential.cs → StgSharp.Mathematics/Numeric/Linear/MatrixCompute.Single.cs
+2 -32
View File
@@ -1,6 +1,6 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="MatrixCompute.Sequential"
// file="MatrixCompute.Single"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
@@ -25,18 +25,16 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.HighPerformance.Memory;
using StgSharp.Mathematics.Numeric.Runtime;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using unsafe MkProc = delegate* unmanaged[Cdecl]<StgSharp.Mathematics.Numeric.Runtime.MatrixParallelTask*, void>;
namespace StgSharp.Mathematics.Numeric.Linear
namespace StgSharp.Mathematics.Numeric
{
public static unsafe partial class MatrixCompute
{
@@ -78,33 +76,5 @@ namespace StgSharp.Mathematics.Numeric.Linear
SequentialOperation(t, count);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SequentialOperation(
MatrixParallelTask* task,
long count
)
{
MatrixElementType elementType = task->ElementType;
MatrixParallelThread[] pool;
MkProc op = (MkProc)NumericalModule.GlobalContext.
mat_mk[elementType.IntrinsicNode].
Get(task->ComputeHandle);
if ((count <= 256) || !MatrixParallel.IsParallelMeaningful)
{
op(task);
return;
} else if (count <= 1024)
{
pool = MatrixParallel.LeadParallel(1);
op(task);
MatrixParallel.ReturnParallelResources(ref pool);
return;
}
pool = MatrixParallel.LeadParallel(count / 1024);
MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask<BufferSequentialPredictor>(pool, task);
taskGroup.LaunchParallel(SleepMode.DeepSleep);
taskGroup.Dispose();
}
}
}
StgSharp.Mathematics/Numeric/Linear/MatrixCompute.Float.cs.old → StgSharp.Mathematics/Numeric/Linear/MatrixCompute.Single.cs.old
View File
StgSharp.Mathematics/Numeric/Linear/MatrixCompute.cs
+28 -4
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="MatrixCompute"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -28,6 +28,7 @@
using StgSharp.HighPerformance.Memory;
using StgSharp.Internal;
using StgSharp.Mathematics;
using StgSharp.Mathematics.Numeric.Runtime;
using System;
using System.Collections.Generic;
using System.Linq;
@@ -36,9 +37,11 @@ using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using unsafe MkProc = delegate* unmanaged[Cdecl]<StgSharp.Mathematics.Numeric.Runtime.MatrixParallelTask*, void>;
namespace StgSharp.Mathematics.Numeric
{
public static partial class MatrixCompute
public static unsafe partial class MatrixCompute
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -49,7 +52,28 @@ namespace StgSharp.Mathematics.Numeric
int y
) where T : unmanaged, INumber<T>
{
return (MatrixKernel*)(buffer + (colLength * x + y));
return (MatrixKernel*)(buffer + ((colLength * x) + y));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SequentialOperation(
MatrixParallelTask* task,
long count
)
{
MatrixElementType elementType = task->ElementType;
MatrixParallelThread[] pool;
MkProc op = (MkProc)NumericalModule.GlobalContext.
mat_mk[elementType.IntrinsicNode].
Get(task->ComputeHandle);
if ((count <= 256) || !MatrixParallel.IsParallelMeaningful)
{
op(task);
return;
}
MatrixParallelWrap taskWrap = MatrixParallel.ScheduleTask<BufferSequentialQueue>(task);
taskWrap.LaunchParallel(SleepMode.DeepSleep);
taskWrap.Dispose();
}
}
StgSharp.Mathematics/Numeric/Runtime/MatrixOperatorPolicy.cs
+1
View File
@@ -50,6 +50,7 @@ namespace StgSharp.Mathematics.Numeric.Runtime
RowMajor = 4,
ColMajor = 5,
Sequential = 6,
#endregion
StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.Numa.cs
+2 -2
View File
@@ -42,8 +42,8 @@ namespace StgSharp.Mathematics.Numeric
private static NumaMeasurement[] _numaMeasurements;
private static int[] _numaPayload;
private static LinearBag<MatrixParallelThread>[] _numaSyncHeavy;
private static LinearBag<MatrixParallelThread>[] _numaWaveFront;
private static MatrixParallelThread[][] _numaSyncHeavy;
private static MatrixParallelThread[][] _numaWaveFront;
private static MatrixParallelThread[] _threads;
private static void UpdateThreadPayload()
StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.Publish.cs
+5 -4
View File
@@ -49,9 +49,9 @@ namespace StgSharp.Mathematics.Numeric
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static unsafe MatrixParallelWrap ScheduleTask<TTask>(
internal static unsafe MatrixParallelWrap ScheduleTask<TTaskQueue>(
MatrixParallelTask* package
) where TTask : IL4Predict
) where TTaskQueue : IMatrixParallelQueue<TTaskQueue>
{
/*
int primCount = package->PrimCount;
@@ -73,9 +73,10 @@ namespace StgSharp.Mathematics.Numeric
ulong total = (ulong)_numaWaveFront[i].Length;
NumaMeasurement measure = _numaMeasurements[i];
ulong numaPayload = measure.ActiveTasks;
uint numaActive = measure.ActiveThreadCount;
// uint numaActive = measure.ActiveThreadCount;
long curPayload = ((long)total) - (((long)(numaPayload + additionalPayload + 63)) / 64);
if (curPayload > payload)
if (curPayload < payload)
{
payload = curPayload;
payloadIndex = i;
StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.cs
+5 -6
View File
@@ -84,7 +84,7 @@ namespace StgSharp.Mathematics.Numeric
_retiredThreads = new(RESERVED_THREAD_COUNT);
// init thread on all logical cores
MatrixParallelThread[][] numaWaveOrigin = Numa.CreateForeachThread<MatrixParallelThread>(
_numaWaveFront = Numa.CreateForeachThread<MatrixParallelThread>(
core => {
MatrixParallelThread thread = new(core);
SpinWait.SpinUntil(() => thread.State == MatrixThreadState.Ready);
@@ -108,15 +108,15 @@ namespace StgSharp.Mathematics.Numeric
// retire threads
if (ContextCount == 1)
{
MatrixParallelThread[] threadArr = numaWaveOrigin[0];
MatrixParallelThread[] threadArr = _numaWaveFront[0];
int retireCount = Math.Min(RESERVED_THREAD_COUNT, Math.Max(0, threadArr.Length));
for (int i = 1; i <= retireCount; i++) {
RetireThread(threadArr[^i]);
}
_numaWaveFront[0] = [.. threadArr[0..^retireCount]];
_numaWaveFront[0] = threadArr[0..^retireCount];
} else
{
RetireWhenMultipleSIMDID(numaWaveOrigin);
RetireWhenMultipleSIMDID(_numaWaveFront);
}
_meaningful = _threads.Length > 1;
@@ -151,7 +151,6 @@ namespace StgSharp.Mathematics.Numeric
) => CompareSIMDID(left.Simd, right.Simd));
int retireCount = Math.Min(RESERVED_THREAD_COUNT, Math.Max(0, allThreads.Length));
// HashSet<MatrixParallelThread> retiredSet = [];
MatrixParallelThread[] retireArr = new MatrixParallelThread[retireCount];
for (int i = 0; i < retireCount; i++)
{
@@ -183,7 +182,7 @@ namespace StgSharp.Mathematics.Numeric
}
}
_numaWaveFront[numa] = [.. active];
_numaWaveFront[numa] = active;
}
_threads = allThreads[retireCount..];
StgSharp.Mathematics/Numeric/Runtime/MatrixParallelQueue.cs
+84 -6
View File
@@ -39,18 +39,96 @@ using System.Threading.Tasks;
namespace StgSharp.Mathematics.Numeric
{
internal unsafe class MatrixParallelQueue<TPredictor>
where TPredictor : class, IMatrixCachePredictor<TPredictor>
internal unsafe interface IMatrixParallelQueue<TSelf> where TSelf : IMatrixParallelQueue<TSelf>
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool Steal(
MatrixParallelTask* task
static abstract bool NeedNumaBuffer { get; }
static abstract TSelf Create(
MatrixParallelTask* baseTask,
L4 buffer
);
void Return(
MatrixParallelTask* task
);
bool Steal(
MatrixParallelTask* task
);
}
internal sealed unsafe class BufferSequentialQueue : IMatrixParallelQueue<BufferSequentialQueue>
{
private MatrixParallelTask* baseTask;
private BufferSequentialQueue() { }
private BufferSequentialQueue(
MatrixParallelTask* baseTask
)
{
this.baseTask = baseTask;
}
public static bool NeedNumaBuffer => throw new NotImplementedException();
public static unsafe BufferSequentialQueue Create(
MatrixParallelTask* baseTask,
L4 buffer
)
{
return new BufferSequentialQueue(baseTask);
}
public unsafe void Return(
MatrixParallelTask* task
) { }
public unsafe bool Steal(
MatrixParallelTask* task
)
{
// TODO 使用提取的内容填充task
throw new NotImplementedException();
}
private class Predictor : IL4Predict
{
public PredictResult Predict(
Span<CacheLineDescription> node,
out int count
)
{
throw new NotImplementedException();
}
public void Prefetch(
nuint origin,
uint policy,
Span<byte> cache
)
{
throw new NotImplementedException();
}
public void WriteBack(
nuint origin,
uint policy,
ReadOnlySpan<byte> cache
)
{
MatrixCachePolicy pEnum = (MatrixCachePolicy)policy;
if ((pEnum & (MatrixCachePolicy.Ans | MatrixCachePolicy.Sequential)) != 0)
{
byte* span = (byte*) origin;
cache.CopyTo(new Span<byte>(span, cache.Length));
}
}
}
}
}
StgSharp.Native/sn-head/StgSharpNative.h
+3
View File
@@ -23,6 +23,9 @@
#define cs_char uint16_t
#define sn_null_assert(ptr) assert(ptr != NULL)
#ifndef SN_
#endif
#define ALIGN(simdVec) _mm_loadu_ps(&simdVec)
#define ALIGN256(simdVec) _mm256_loadu_ps(&simdVec)
#define ALIGN512(simdVec) _mm512_loadu_ps(&simdVec)
StgSharp.Native/sn-head/sn_intrinsic.context.x86.h
+2 -2
View File
@@ -6,9 +6,9 @@
#include "sn_intrinsic.std.h"
#ifndef SN_X86_TARGET_ATTR_sse
#define SN_X86_TARGET_ATTR_sse SN_CLANG_TARGET_ATTR("sse2")
#define SN_X86_TARGET_ATTR_sse SN_CLANG_TARGET_ATTR("sse42")
#define SN_X86_TARGET_ATTR_avx SN_CLANG_TARGET_ATTR("avx")
#define SN_X86_TARGET_ATTR_avx_fma SN_CLANG_TARGET_ATTR("avx,fma")
#define SN_X86_TARGET_ATTR_avx_fma SN_CLANG_TARGET_ATTR("avx,avx2,fma")
#define SN_X86_TARGET_ATTR_512 SN_CLANG_TARGET_ATTR("avx512f,avx512vl")
#endif