youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

L4 completely add support for multi predictor, working on ver 5 mat para

Browse Source
This commit is contained in:
Youmu Liu
2026-04-10 18:05:40 +08:00
parent b5bee5c442
commit 8a66c1697a
16 changed files with 602 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs
+1 -1
View File
@@ -45,10 +45,10 @@ namespace StgSharp.HighPerformance.Memory
{
internal int _id;
internal int _predictor;
internal int _refCount;
internal nuint _address;
internal nuint _origin;
internal uint _predictor;
internal uint _profile;
}
StgSharp.Common/HighPerformance/Memory/L4.Evict.cs
+26 -17
View File
@@ -62,29 +62,38 @@ namespace StgSharp.HighPerformance.Memory
int remain = count;
Vector128<byte> three = Vector128.Create((byte)0x03);
byte* p = (byte*)_eviction;
for (int i = 0; (i < CacheLineCount) && (remain > 0); i += 16)
_predictors.RequestSpanOperation();
Span<IL4Predict> pSpan = _predictors.AsSpan();
try
{
int evictMask = _intrinsic.ScanEvict(p + i, three);
while ((evictMask != 0) && (remain > 0))
for (int i = 0; (i < CacheLineCount) && (remain > 0); i += 16)
{
int bitPos = BitOperations.TrailingZeroCount(evictMask);
int index = bitPos + i;
CacheLineHead* head = &_head[index];
if (Interlocked.CompareExchange(ref head->_refCount, int.MinValue, 0) == 0)
int evictMask = _intrinsic.ScanEvict(p + i, three);
while ((evictMask != 0) && (remain > 0))
{
// EvictLine(i + index);
_ = _map.TryRemove(head->_origin, out _);
int id = (int)head->_predictor;
IL4Predict predict = _predictions[id] ??
throw new L4PredictUnregisteredException(id);
predict.WriteBack(head->_origin, head->_profile, new ReadOnlySpan<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
RecycleLine(index);
remain--;
int bitPos = BitOperations.TrailingZeroCount(evictMask);
int index = bitPos + i;
CacheLineHead* head = &_head[index];
if (Interlocked.CompareExchange(ref head->_refCount, int.MinValue, 0) == 0)
{
// EvictLine(i + index);
_ = _map.TryRemove(head->_origin, out _);
int id = (int)head->_predictor;
IL4Predict predict = pSpan[id] ??
throw new L4PredictUnregisteredException(id);
predict.WriteBack(head->_origin, head->_profile, new ReadOnlySpan<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
RecycleLine(index);
remain--;
}
evictMask &= evictMask - 1;
}
evictMask &= evictMask - 1;
}
return remain;
}
finally
{
_predictors.ReleaseSpanOperation();
}
return remain;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
StgSharp.Common/HighPerformance/Memory/L4.Predict.cs
+17 -13
View File
@@ -28,6 +28,7 @@
using StgSharp.Common.Collections;
using StgSharp.Mathematics.Memory;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Text;
@@ -40,14 +41,16 @@ namespace StgSharp.HighPerformance.Memory
private ConcurrentBufferStack<int> _idReuse = new(32);
private ConcurrentFlexibleArray<IL4Predict> _predictions = new(32);
private ConcurrentFlexibleArray<IL4Predict> _predictors = new(32);
private int _largestId = 0;
private ConcurrentQueue<int> _predictionQueue = new();
private Queue<int> _predictionQueue = [];
private int _largestId ;
private IL4Predict CurrentPredict { get; set; }
private int CurrentPredictIndex { get; set; }
public int RegisterPredict(
IL4Predict predict
)
@@ -58,7 +61,7 @@ namespace StgSharp.HighPerformance.Memory
if (!_idReuse.TryPop(out int id)) {
id = _largestId++;
}
_predictions[id] = predict;
_predictors[id] = predict;
_predictionQueue.Enqueue(id);
return id;
}
@@ -70,29 +73,30 @@ namespace StgSharp.HighPerformance.Memory
if (id < 0) {
return null;
}
IL4Predict p = _predictions[id];
IL4Predict p = _predictors[id];
if (p == null) {
return p;
}
_predictions[id] = null!;
// TODO: scan and write back
_predictors[id] = null!;
_idReuse.Push(id);
return p;
}
private void UseNewPredict()
private bool TryGetNextPredict(
)
{
if (_predictionQueue.Count == 0) {
return;
if (_predictionQueue.IsEmpty) {
return false;
}
if (_predictionQueue.TryDequeue(out int id))
{
IL4Predict predict = _predictions[id] ??
IL4Predict predict = _predictors[id] ??
throw new L4PredictUnregisteredException(id);
CurrentPredict = predict;
CurrentPredictIndex = id;
return true;
}
return false;
}
}
StgSharp.Common/HighPerformance/Memory/L4.Predictor.cs
+6 -1
View File
@@ -45,6 +45,11 @@ namespace StgSharp.HighPerformance.Memory
public interface IL4Predict
{
bool IsSamePolicy(
uint policy,
IL4Predict predict
);
PredictResult Predict(
Span<CacheLinePrediction> node,
out int count
@@ -65,7 +70,7 @@ namespace StgSharp.HighPerformance.Memory
/// </param>
void Prefetch(
nuint origin,
ulong policy,
uint policy,
Span<byte> cache
);
StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs
+64 -41
View File
@@ -42,9 +42,13 @@ namespace StgSharp.HighPerformance.Memory
if (_activeCount > MaxActivateCacheLineCount) {
throw new OverflowException("Too many activated cache line in external reference.");
}
IL4Predict predict = CurrentPredict;
Span<CacheLinePrediction> span = new(_prediction, PredictionCount);
PredictResult result = PrefetchPredict.Predict(span, out int actualCount);
if ((result == PredictResult.Exhausted) && (actualCount <= 0))
PredictResult result = predict.Predict(span, out int actualCount);
if ((result & PredictResult.Exhausted) != 0) {
_exhausted = !TryGetNextPredict();
}
if (actualCount <= 0)
{
_exhausted = true;
return;
@@ -62,53 +66,72 @@ namespace StgSharp.HighPerformance.Memory
missIndex++;
}
}
// now prefetch the misses
for (int i = 0; i < missIndex; i++)
_predictors.RequestSpanOperation();
Span<IL4Predict> predictorSpan = _predictors.AsSpan();
try
{
ref CacheLinePrediction handle = ref span[i];
nuint address = handle.Address;
if (TestHit(address))
// now prefetch the misses
for (int i = 0; i < missIndex; i++)
{
continue;
}
CacheLineHead* head;
ref CacheLinePrediction pHandle = ref span[i];
nuint address = pHandle.Address;
int index;
if (_map.TryGet(address, out nuint handle))
{
head = (CacheLineHead*)handle;
index = head->_id;
(*_eviction)[index] = 0; // reset eviction counter
int retryCount = 0, index, remain = missIndex - i;
const int maxRetry = 1024;
while (!TryGetEmptyLine(out index))
{
remain = Evict(remain);
retryCount++;
if (retryCount > maxRetry) {
throw new OverflowException($"Failed to evict cache line after {maxRetry} retries.");
#region update predictor belonging
int predictId = head->_predictor;
IL4Predict belong = predictorSpan[predictId];
if (predict.IsSamePolicy(head->_profile, belong))
{
head->_predictor = predictId; // wrong line
} else
{
belong.WriteBack(head->_origin, head->_profile, new ReadOnlySpan<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
head->_profile = pHandle.MapPolicy;
predict.Prefetch(head->_origin, head->_profile, new Span<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
}
#endregion
continue;
}
int retryCount = 0, remain = missIndex - i;
const int maxRetry = 1024;
while (!TryGetEmptyLine(out index))
{
remain = Evict(remain);
retryCount++;
if (retryCount > maxRetry) {
throw new OverflowException($"Failed to evict cache line after {maxRetry} retries.");
}
if (retryCount > maxRetry / 2) {
Thread.Sleep(1);
}
}
head = &_head[index];
head->_id = index;
head->_origin = address;
head->_profile = pHandle.MapPolicy;
head->_refCount = 0;
head->_address = (nuint)(_data + index);
// pre fetch them here
predict.Prefetch(head->_origin, head->_profile, new Span<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
_ = _map.TryAddOrSet(address, (nuint)head, out _);
}
CacheLineHead* head = &_head[index];
head->_id = index;
head->_origin = address;
head->_profile = handle.MapPolicy;
head->_refCount = 0;
head->_address = (nuint)(_data + index);
// pre fetch them here
PrefetchPredict.Prefetch(head->_origin, head->_profile, new Span<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
_ = _map.TryAddOrSet(address, (nuint)head, out _);
_ = Interlocked.Add(ref _aheadCount, actualCount);
}
_ = Interlocked.Add(ref _aheadCount, actualCount);
}
private partial bool TestHit(
nuint origin
)
{
if (_map.TryGet(origin, out nuint handle))
finally
{
CacheLineHead* head = (CacheLineHead*)handle;
int index = head->_id;
(*_eviction)[index] = 0; // reset eviction counter
return true;
_predictors.ReleaseSpanOperation();
}
return false;
}
}
StgSharp.Common/HighPerformance/Memory/L4.cs
+20 -7
View File
@@ -26,6 +26,7 @@
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Collections;
using StgSharp.Common.HighPerformance.ProcessorAbstraction;
using System.Data;
using System.Runtime.InteropServices;
@@ -49,14 +50,20 @@ namespace StgSharp.HighPerformance.Memory
private int _activeCount;
private int _aheadCount;
private readonly int _bindedNuma;
private int _maxId;
private volatile int _prefetching;
private readonly nuint _baseAddress;
private readonly SwissTable _map;
private readonly UnmanagedStack<int> _idMux;
public L4()
public L4(
int bindedNuma
)
{
_bindedNuma = bindedNuma;
// init order is fallow by align requirement of each component.
nuint cacheSize = ((nuint)Unsafe.SizeOf<CacheLineData>()) * CacheLineCount; // SIMDID, no less than 16 byte bit
nuint entrySize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
@@ -67,7 +74,7 @@ namespace StgSharp.HighPerformance.Memory
// malloc all memory at once, and free at once in Dispose.
// This is to avoid fragmentation and improve performance.
_baseAddress = (nuint)NativeMemory.AlignedAlloc(cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize, 16);
_baseAddress = (nuint)Numa.Alloc(bindedNuma, cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize);
nuint ptr = _baseAddress;
// init all components with the buffer, and move the buffer pointer forward.
@@ -86,6 +93,8 @@ namespace StgSharp.HighPerformance.Memory
ptr += stackSize;
}
public static nuint AllocSize { get; }
// public IL4Predict PrefetchPredict { get; set; }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -114,7 +123,15 @@ namespace StgSharp.HighPerformance.Memory
_disposed = true;
_map.Dispose();
_idMux.Dispose();
NativeMemory.AlignedFree((void*)_baseAddress);
nuint cacheSize = ((nuint)Unsafe.SizeOf<CacheLineData>()) * CacheLineCount; // SIMDID, no less than 16 byte bit
nuint entrySize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLinePrediction>()) * PredictionCount; //16 byte align
nuint mapSize = (nuint)SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>(); // 16 byte align
nuint stackSize = ((nuint)sizeof(int)) * CacheLineCount; // int align
Numa.Free((void*)_baseAddress, _bindedNuma, cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize);
GC.SuppressFinalize(this);
}
@@ -164,10 +181,6 @@ namespace StgSharp.HighPerformance.Memory
private partial void Prefetch();
private partial bool TestHit(
nuint origin
);
private struct CacheLineData
{
StgSharp.Common/HighPerformance/ProcessorAbstraction/Numa.cs
+69
View File
@@ -25,8 +25,10 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.HighPerformance.Memory;
using System;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
@@ -43,6 +45,35 @@ namespace StgSharp.Common.HighPerformance.ProcessorAbstraction
public static ReadOnlySpan<NumaNode> AllNumaNode => _numaNodes;
public static unsafe void* Alloc(
int numaId,
nuint size
)
{
if (_numaNodes.Length == 0) {
throw new InvalidOperationException("Numa info has not been loaded.");
}
if ((numaId < 0) || (numaId >= _numaNodes.Length)) {
throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID.");
}
return NumaAllocNative(size, numaId);
}
public static unsafe void Free(
void* address,
int numaId,
nuint size
)
{
if (_numaNodes.Length == 0) {
throw new InvalidOperationException("Numa info has not been loaded.");
}
if ((numaId < 0) || (numaId >= _numaNodes.Length)) {
throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID.");
}
NumaFreeNative(address, size, numaId);
}
[LibraryImport(Native.LibName, EntryPoint = "sn_get_numa_each")]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
@@ -60,6 +91,20 @@ namespace StgSharp.Common.HighPerformance.ProcessorAbstraction
Span<int> coreCountInNuma
);
public static L4[] RequestBufferForEachNuma()
{
if (_numaNodes.Length == 0) {
throw new InvalidOperationException("Numa info has not been loaded.");
}
L4[] buffers = new L4[NumaNodeCount];
for (int i = 0; i < NumaNodeCount; i++)
{
L4 buffer = new L4(i);
buffers[i] = buffer;
}
return buffers;
}
internal static void Initialize()
{
Span<int> coreCountInNuma = stackalloc int[(Environment.ProcessorCount / 32) + 1];
@@ -73,6 +118,13 @@ namespace StgSharp.Common.HighPerformance.ProcessorAbstraction
}
}
[LibraryImport(Native.LibName, EntryPoint = "sn_set_thread_affinity")]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
internal static partial void SetThreadAffinity(
int coreId
);
[MethodImpl(MethodImplOptions.NoInlining)]
private static NumaNode InitNumaNode(
uint numaId,
@@ -84,5 +136,22 @@ namespace StgSharp.Common.HighPerformance.ProcessorAbstraction
return new(numaId, count, globalIndex);
}
[LibraryImport(Native.LibName, EntryPoint = "sn_numa_alloc")]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
private static unsafe partial void* NumaAllocNative(
nuint size,
int numaId
);
[LibraryImport(Native.LibName, EntryPoint = "sn_numa_free")]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
private static unsafe partial void NumaFreeNative(
void* ptr,
nuint size,
int numa_id
);
}
}