youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove id from L4 head, and other some useless types.

Browse Source
This commit is contained in:
Nitload
2026-06-02 00:35:08 +08:00
parent 9a09d32960
commit 07f8178431
6 changed files with 98 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs
+13 -29
View File
@@ -35,22 +35,6 @@ namespace StgSharp.HighPerformance.Memory
public partial class L4
{
[InlineArray(CacheLineCount)]
private struct PredictionArray
{
private int PredictionCount;
}
[InlineArray(CacheLineCount)]
private struct MapArray
{
private int MapCount;
}
internal enum AllocationState : byte
{
@@ -60,25 +44,32 @@ namespace StgSharp.HighPerformance.Memory
}
[StructLayout(LayoutKind.Explicit, Size = 64, Pack = 64)]
[StructLayout(LayoutKind.Explicit, Size = 32, Pack = 32)]
private unsafe struct CacheLineMetadata
{
#region reference counting and state information
[FieldOffset(24)] internal int RefCount;
// [FieldOffset(28)] internal readonly short Id;
#endregion
#region tlsf area
// TLSF area use only 24 bytes
internal const ulong Mask48Bits = 0x0000FFFFFFFFFFFF;
[FieldOffset(0)] private readonly M512 FillingMask;
[FieldOffset(0)] private readonly M256 FillingMask;
[FieldOffset(8)] private ulong PositionMask;
[FieldOffset(16)] private ulong SizeMask;
[FieldOffset(14)] internal AllocationState State;
// [FieldOffset(15)] internal sbyte Level;
[FieldOffset(0)] public short NextLevel;
[FieldOffset(2)] public short NextNear;
[FieldOffset(4)] public short PreviousLevel;
@@ -106,19 +97,12 @@ namespace StgSharp.HighPerformance.Memory
#endregion
#region reference counting and state information
[FieldOffset(24)] internal int RefCount;
[FieldOffset(28)] internal readonly short Id;
#endregion
#region cache line information
[FieldOffset(34)] internal ushort Profile;
[FieldOffset(36)] internal nuint Origin;
[FieldOffset(24)] internal nuint Origin;
#endregion
#endregion
}
[StructLayout(LayoutKind.Sequential)]
StgSharp.Common/HighPerformance/Memory/L4.Evict.cs
+13 -39
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="L4.Evict"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -52,63 +52,37 @@ namespace StgSharp.HighPerformance.Memory
/// Count of lines not evicted
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe int Evict(
int count = 1
)
private unsafe int Evict(int count = 1)
{
count = (count < 0) ? 0 : count;
int* pPtr = (int*)_predictCount, mPtr = (int*)_mapCount;
int* pPtr = _predictCount, mPtr = _mapCount;
int remain = count;
ConcurrentFlexibleArray<IL4Predict> predictors = _predictors;
predictors.RequestSpanOperation();
Span<IL4Predict> pSpan = predictors.AsSpan();
try
for (int i = 0; i < CacheLineCount; i++)
{
for (int i = 0; i < CacheLineCount; i++)
if (pPtr[i] > mPtr[i])
{
if (pPtr[i] > mPtr[i])
{
continue;
}
remain -= TryEvictAt(i, pSpan) ? 1 : 0;
continue;
}
}
finally
{
predictors.ReleaseSpanOperation();
remain -= TryEvictAt(i) ? 1 : 0;
}
return remain;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe void RecycleLine(
int id
)
{
_bufferAllocator.Free(id);
/**/
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe bool TryEvictAt(
int index,
Span<IL4Predict> predictorSpan
)
private unsafe bool TryEvictAt(int index)
{
CacheLineMetadata* head = &_head[index];
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
{
// EvictLine(i + index);
_ = _map.TryRemove(head->Origin, out _);
int id = head->Predictor;
IL4Predict predict = predictorSpan[id] ??
throw new L4PredictUnregisteredException(id);
IL4Predict predict = CurrentPredict;
predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan<byte>((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator));
RecycleLine(index);
(*_mapCount)[index] = 0;
(*_predictCount)[index] = int.MaxValue;
_bufferAllocator.Free(index);
_mapCount[index] = 0;
_predictCount[index] = int.MaxValue;
return true;
}
return false;
StgSharp.Common/HighPerformance/Memory/L4.Predict.cs
+7 -56
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="L4.Predict"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -39,66 +39,17 @@ namespace StgSharp.HighPerformance.Memory
public partial class L4
{
private ConcurrentBufferStack<int> _idReuse = new(32);
private ConcurrentFlexibleArray<IL4Predict> _predictors = new(32);
private ConcurrentQueue<int> _predictionQueue = new();
private int _largestId ;
private IL4Predict CurrentPredict { get; set; }
private int CurrentPredictIndex { get; set; }
public int RegisterPredict(
IL4Predict predict
)
public void ResetPredict(IL4Predict predict)
{
if (predict is null) {
return -1;
}
if (!_idReuse.TryPop(out int id)) {
id = _largestId++;
}
if (id > short.MaxValue) {
throw new OverflowException("Too many predictors in waiting");
}
_predictors[id] = predict;
_predictionQueue.Enqueue(id);
return id;
}
public IL4Predict? UnregisterPredict(
int id
)
{
if (id < 0) {
return null;
}
IL4Predict p = _predictors[id];
if (p == null) {
return p;
}
_predictors[id] = null!;
_idReuse.Push(id);
return p;
}
private bool TryGetNextPredict(
)
{
if (_predictionQueue.IsEmpty) {
return false;
}
if (_predictionQueue.TryDequeue(out int id))
if (CurrentPredict is null)
{
IL4Predict predict = _predictors[id] ?? throw new L4PredictUnregisteredException(id);
CurrentPredict = predict;
CurrentPredictIndex = id;
return true;
return;
}
return false;
// TODO: write back all and register new predictor
}
}
StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs
+52 -73
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="L4.Prefetch"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -39,12 +39,9 @@ namespace StgSharp.HighPerformance.Memory
return;
}
IL4Predict predict = CurrentPredict;
int predictIndex = CurrentPredictIndex;
Span<CacheLineDescription> span = new(_prediction, PredictionCount);
PredictResult result = predict.Predict(span, out int actualCount);
if ((result & PredictResult.Exhausted) != 0) {
_exhausted = !TryGetNextPredict();
}
_exhausted = (result & PredictResult.Exhausted) != 0;
if (actualCount <= 0)
{
_exhausted = true;
@@ -62,81 +59,63 @@ namespace StgSharp.HighPerformance.Memory
missIndex++;
}
}
_predictors.RequestSpanOperation();
Span<IL4Predict> predictorSpan = _predictors.AsSpan();
try
// now prefetch the misses
for (int i = 0; i < missIndex; i++)
{
// now prefetch the misses
for (int i = 0; i < missIndex; i++)
CacheLineMetadata* head;
ref CacheLineDescription pHandle = ref span[i];
nuint address = pHandle.Address;
int index;
if (_map.TryGet(address, out nuint handle))
{
CacheLineMetadata* head;
ref CacheLineDescription pHandle = ref span[i];
nuint address = pHandle.Address;
int index;
if (_map.TryGet(address, out nuint handle))
head = (CacheLineMetadata*)handle;
index = (int)(head - _head);
_predictCount[index] = 0; // reset eviction counter
#region update predictor belonging
if (_predictCount[index] > short.MaxValue / 2)
{
head = (CacheLineMetadata*)handle;
index = head->Id;
(*_predictCount)[index] = 0; // reset eviction counter
#region update predictor belonging
int predictId = head->Predictor;
IL4Predict belong = predictorSpan[predictId];
if (!ReferenceEquals(predict, belong))
{
throw new L4PredictConflictException(head->Origin, head->Profile, predictId);
} else
{
if ((*_predictCount)[index] > short.MaxValue / 2)
{
int predictCount = Volatile.Read(ref (*_predictCount)[index]);
int mapCount = Volatile.Read(ref (*_mapCount)[index]);
int decrement = (int.Min(predictCount, mapCount) + 1) / 2;
_ = Interlocked.Add(ref (*_predictCount)[index], -decrement);
_ = Interlocked.Add(ref (*_mapCount)[index], -decrement);
}
(*_predictCount)[index]++;
}
#endregion
continue;
int predictCount = Volatile.Read(ref _predictCount[index]);
int mapCount = Volatile.Read(ref _mapCount[index]);
int decrement = (int.Min(predictCount, mapCount) + 1) / 2;
_ = Interlocked.Add(ref _predictCount[index], -decrement);
_ = Interlocked.Add(ref _mapCount[index], -decrement);
}
_predictCount[index]++;
int retryCount = 0, remain = missIndex - i;
const int maxRetry = 1024;
while (!_bufferAllocator.TryAlloc(pHandle.Size, out index))
{
remain = Evict(remain);
retryCount++;
if (retryCount > maxRetry) {
throw new OverflowException($"Failed to evict cache line after {maxRetry} retries.");
}
if (retryCount > maxRetry / 2) {
Thread.Sleep(1);
}
}
head = _head + index;
head->Origin = address;
head->Profile = pHandle.MapPolicy;
head->RefCount = 0;
head->Predictor = (short)predictIndex;
#endregion
(*_predictCount)[index] = 1;
(*_mapCount)[index] = 1;
// pre fetch them here
predict.Prefetch(head->Origin, head->Profile, new Span<byte>((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator));
_ = _map.TryAddOrSet(address, (nuint)head, out _);
continue;
}
_ = Interlocked.Add(ref _aheadCount, actualCount);
}
finally
{
_predictors.ReleaseSpanOperation();
int retryCount = 0, remain = missIndex - i;
const int maxRetry = 1024;
while (!_bufferAllocator.TryAlloc(pHandle.Size, out index))
{
remain = Evict(remain);
retryCount++;
if (retryCount > maxRetry) {
throw new OverflowException($"Failed to evict cache line after {maxRetry} retries.");
}
if (retryCount > maxRetry / 2) {
Thread.Sleep(1);
}
}
head = _head + index;
head->Origin = address;
head->Profile = pHandle.MapPolicy;
head->RefCount = 0;
_predictCount[index] = 1;
_mapCount[index] = 1;
// pre fetch them here
predict.Prefetch(head->Origin, head->Profile, new Span<byte>((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator));
_ = _map.TryAddOrSet(address, (nuint)head, out _);
}
_ = Interlocked.Add(ref _aheadCount, actualCount);
}
}
StgSharp.Common/HighPerformance/Memory/L4.Tlsf.cs
+2 -2
View File
@@ -64,7 +64,7 @@ namespace StgSharp.HighPerformance.Memory
_maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, CacheLineMetadata.Mask48Bits)));
_maxLevel = int.MaxValue;
_maxLevel = GetLevel(_maxAllocSize);
_bucket = new int[GetLevel(_maxAllocSize) + 2];
_bucket = new short[GetLevel(_maxAllocSize) + 2];
_spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize);
_metadataEnd = 0;
@@ -163,7 +163,7 @@ namespace StgSharp.HighPerformance.Memory
private void RemoveFromBucketHead(int level, CacheLineMetadata* node, int nodeIdx)
{
int currentLevel = level;
int currentNextLevelIdx = node->NextLevel;
short currentNextLevelIdx = node->NextLevel;
// int currentPrevLevelIdx = 0;
StgSharp.Common/HighPerformance/Memory/L4.cs
+11 -20
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="L4"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -48,11 +48,11 @@ namespace StgSharp.HighPerformance.Memory
/// <summary>
/// Count of cache lines mapped for each prediction, used for eviction policy.
/// </summary>
private readonly MapArray* _mapCount;
private readonly int* _mapCount;
/// <summary>
/// Count of times each cache line is predicted, used for eviction policy.
/// </summary>
private readonly PredictionArray* _predictCount;
private readonly int* _predictCount;
/// <summary>
/// Original memory view for prediction span.
/// </summary>
@@ -69,12 +69,7 @@ namespace StgSharp.HighPerformance.Memory
private readonly TLSF _bufferAllocator;
public L4(
int bindedNuma,
nuint bufferSize = 8192 * 8192,
int align = 16,
nuint maxCacheLineSize = 8192
)
public L4(int bindedNuma, nuint bufferSize = 8192 * 8192, int align = 16, nuint maxCacheLineSize = 8192)
{
_bindedNuma = bindedNuma;
@@ -82,7 +77,7 @@ namespace StgSharp.HighPerformance.Memory
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineMetadata>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint predictionCountSize = (nuint)Unsafe.SizeOf<PredictionArray>(); // 16 byte align
nuint predictionCountSize = sizeof(int) * CacheLineCount; // 16 byte align
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
ByteCapacity = normalizedBufferSize;
@@ -103,9 +98,9 @@ namespace StgSharp.HighPerformance.Memory
ptr += predictionSize;
_map = SwissTable.Create(ptr, _ => { });
ptr += mapCountSize;
_predictCount = (PredictionArray*)ptr;
_predictCount = (int*)ptr;
ptr += predictionCountSize;
_mapCount = (MapArray*)ptr;
_mapCount = (int*)ptr;
ptr += predictionCountSize;
M512* mPtr = (M512*)_head;
@@ -121,9 +116,7 @@ namespace StgSharp.HighPerformance.Memory
// public IL4Predict PrefetchPredict { get; set; }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ContinueOnce(
Handle line
)
public void ContinueOnce(Handle line)
{
if (line.Address > 0) {
_ = Interlocked.Decrement(ref _aheadCount);
@@ -156,9 +149,7 @@ namespace StgSharp.HighPerformance.Memory
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Handle Map(
ref CacheLineDescription prediction
)
public Handle Map(ref CacheLineDescription prediction)
{
if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
{
@@ -186,7 +177,7 @@ namespace StgSharp.HighPerformance.Memory
CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle;
if (Interlocked.Increment(ref entry->RefCount) > 0)
{
(*_predictCount)[entry->Id] = 0;
_predictCount[entry - _head] = 0;
_ = Interlocked.Decrement(ref _aheadCount);
_ = Interlocked.Decrement(ref (*_mapCount)[entry->Id]);
return new Handle((nuint)entry->Position, ref entry->RefCount);