youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove id from L4 head, and other some useless types.

Browse Source
This commit is contained in:
Nitload
2026-06-02 00:35:08 +08:00
parent 9a09d32960
commit 07f8178431
6 changed files with 98 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs
+12 -28
View File
@@ -35,22 +35,6 @@ namespace StgSharp.HighPerformance.Memory
public partial class L4 public partial class L4
{ {
[InlineArray(CacheLineCount)]
private struct PredictionArray
{
private int PredictionCount;
}
[InlineArray(CacheLineCount)]
private struct MapArray
{
private int MapCount;
}
internal enum AllocationState : byte internal enum AllocationState : byte
{ {
@@ -60,25 +44,32 @@ namespace StgSharp.HighPerformance.Memory
} }
[StructLayout(LayoutKind.Explicit, Size = 64, Pack = 64)] [StructLayout(LayoutKind.Explicit, Size = 32, Pack = 32)]
private unsafe struct CacheLineMetadata private unsafe struct CacheLineMetadata
{ {
#region reference counting and state information
[FieldOffset(24)] internal int RefCount;
// [FieldOffset(28)] internal readonly short Id;
#endregion
#region tlsf area #region tlsf area
// TLSF area use only 24 bytes // TLSF area use only 24 bytes
internal const ulong Mask48Bits = 0x0000FFFFFFFFFFFF; internal const ulong Mask48Bits = 0x0000FFFFFFFFFFFF;
[FieldOffset(0)] private readonly M512 FillingMask; [FieldOffset(0)] private readonly M256 FillingMask;
[FieldOffset(8)] private ulong PositionMask; [FieldOffset(8)] private ulong PositionMask;
[FieldOffset(16)] private ulong SizeMask; [FieldOffset(16)] private ulong SizeMask;
[FieldOffset(14)] internal AllocationState State; [FieldOffset(14)] internal AllocationState State;
// [FieldOffset(15)] internal sbyte Level;
[FieldOffset(0)] public short NextLevel; [FieldOffset(0)] public short NextLevel;
[FieldOffset(2)] public short NextNear; [FieldOffset(2)] public short NextNear;
[FieldOffset(4)] public short PreviousLevel; [FieldOffset(4)] public short PreviousLevel;
@@ -106,17 +97,10 @@ namespace StgSharp.HighPerformance.Memory
#endregion #endregion
#region reference counting and state information
[FieldOffset(24)] internal int RefCount;
[FieldOffset(28)] internal readonly short Id;
#endregion
#region cache line information #region cache line information
[FieldOffset(34)] internal ushort Profile; [FieldOffset(34)] internal ushort Profile;
[FieldOffset(36)] internal nuint Origin; [FieldOffset(24)] internal nuint Origin;
#endregion #endregion
} }
StgSharp.Common/HighPerformance/Memory/L4.Evict.cs
+10 -36
View File
@@ -2,8 +2,8 @@
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="L4.Evict" // file="L4.Evict"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload // AuthorGroup: Nitload Space
// Copyright (c) Nitload. All rights reserved. // Copyright (c) Nitload Space. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -52,63 +52,37 @@ namespace StgSharp.HighPerformance.Memory
/// Count of lines not evicted /// Count of lines not evicted
/// </returns> /// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe int Evict( private unsafe int Evict(int count = 1)
int count = 1
)
{ {
count = (count < 0) ? 0 : count; count = (count < 0) ? 0 : count;
int* pPtr = (int*)_predictCount, mPtr = (int*)_mapCount; int* pPtr = _predictCount, mPtr = _mapCount;
int remain = count; int remain = count;
ConcurrentFlexibleArray<IL4Predict> predictors = _predictors;
predictors.RequestSpanOperation();
Span<IL4Predict> pSpan = predictors.AsSpan();
try
{
for (int i = 0; i < CacheLineCount; i++) for (int i = 0; i < CacheLineCount; i++)
{ {
if (pPtr[i] > mPtr[i]) if (pPtr[i] > mPtr[i])
{ {
continue; continue;
} }
remain -= TryEvictAt(i, pSpan) ? 1 : 0; remain -= TryEvictAt(i) ? 1 : 0;
}
}
finally
{
predictors.ReleaseSpanOperation();
} }
return remain; return remain;
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe void RecycleLine( private unsafe bool TryEvictAt(int index)
int id
)
{
_bufferAllocator.Free(id);
/**/
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe bool TryEvictAt(
int index,
Span<IL4Predict> predictorSpan
)
{ {
CacheLineMetadata* head = &_head[index]; CacheLineMetadata* head = &_head[index];
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0) if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
{ {
// EvictLine(i + index); // EvictLine(i + index);
_ = _map.TryRemove(head->Origin, out _); _ = _map.TryRemove(head->Origin, out _);
int id = head->Predictor; IL4Predict predict = CurrentPredict;
IL4Predict predict = predictorSpan[id] ??
throw new L4PredictUnregisteredException(id);
predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan<byte>((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator)); predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan<byte>((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator));
RecycleLine(index); _bufferAllocator.Free(index);
(*_mapCount)[index] = 0; _mapCount[index] = 0;
(*_predictCount)[index] = int.MaxValue; _predictCount[index] = int.MaxValue;
return true; return true;
} }
return false; return false;
StgSharp.Common/HighPerformance/Memory/L4.Predict.cs
+7 -56
View File
@@ -2,8 +2,8 @@
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="L4.Predict" // file="L4.Predict"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload // AuthorGroup: Nitload Space
// Copyright (c) Nitload. All rights reserved. // Copyright (c) Nitload Space. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -39,66 +39,17 @@ namespace StgSharp.HighPerformance.Memory
public partial class L4 public partial class L4
{ {
private ConcurrentBufferStack<int> _idReuse = new(32);
private ConcurrentFlexibleArray<IL4Predict> _predictors = new(32);
private ConcurrentQueue<int> _predictionQueue = new();
private int _largestId ;
private IL4Predict CurrentPredict { get; set; } private IL4Predict CurrentPredict { get; set; }
private int CurrentPredictIndex { get; set; } public void ResetPredict(IL4Predict predict)
public int RegisterPredict(
IL4Predict predict
)
{ {
if (predict is null) { if (CurrentPredict is null)
return -1;
}
if (!_idReuse.TryPop(out int id)) {
id = _largestId++;
}
if (id > short.MaxValue) {
throw new OverflowException("Too many predictors in waiting");
}
_predictors[id] = predict;
_predictionQueue.Enqueue(id);
return id;
}
public IL4Predict? UnregisterPredict(
int id
)
{ {
if (id < 0) {
return null;
}
IL4Predict p = _predictors[id];
if (p == null) {
return p;
}
_predictors[id] = null!;
_idReuse.Push(id);
return p;
}
private bool TryGetNextPredict(
)
{
if (_predictionQueue.IsEmpty) {
return false;
}
if (_predictionQueue.TryDequeue(out int id))
{
IL4Predict predict = _predictors[id] ?? throw new L4PredictUnregisteredException(id);
CurrentPredict = predict; CurrentPredict = predict;
CurrentPredictIndex = id; return;
return true;
} }
return false;
// TODO: write back all and register new predictor
} }
} }
StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs
+14 -35
View File
@@ -2,8 +2,8 @@
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="L4.Prefetch" // file="L4.Prefetch"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload // AuthorGroup: Nitload Space
// Copyright (c) Nitload. All rights reserved. // Copyright (c) Nitload Space. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -39,12 +39,9 @@ namespace StgSharp.HighPerformance.Memory
return; return;
} }
IL4Predict predict = CurrentPredict; IL4Predict predict = CurrentPredict;
int predictIndex = CurrentPredictIndex;
Span<CacheLineDescription> span = new(_prediction, PredictionCount); Span<CacheLineDescription> span = new(_prediction, PredictionCount);
PredictResult result = predict.Predict(span, out int actualCount); PredictResult result = predict.Predict(span, out int actualCount);
if ((result & PredictResult.Exhausted) != 0) { _exhausted = (result & PredictResult.Exhausted) != 0;
_exhausted = !TryGetNextPredict();
}
if (actualCount <= 0) if (actualCount <= 0)
{ {
_exhausted = true; _exhausted = true;
@@ -62,10 +59,7 @@ namespace StgSharp.HighPerformance.Memory
missIndex++; missIndex++;
} }
} }
_predictors.RequestSpanOperation();
Span<IL4Predict> predictorSpan = _predictors.AsSpan();
try
{
// now prefetch the misses // now prefetch the misses
for (int i = 0; i < missIndex; i++) for (int i = 0; i < missIndex; i++)
{ {
@@ -76,29 +70,20 @@ namespace StgSharp.HighPerformance.Memory
if (_map.TryGet(address, out nuint handle)) if (_map.TryGet(address, out nuint handle))
{ {
head = (CacheLineMetadata*)handle; head = (CacheLineMetadata*)handle;
index = head->Id; index = (int)(head - _head);
(*_predictCount)[index] = 0; // reset eviction counter _predictCount[index] = 0; // reset eviction counter
#region update predictor belonging #region update predictor belonging
int predictId = head->Predictor; if (_predictCount[index] > short.MaxValue / 2)
IL4Predict belong = predictorSpan[predictId];
if (!ReferenceEquals(predict, belong))
{ {
throw new L4PredictConflictException(head->Origin, head->Profile, predictId); int predictCount = Volatile.Read(ref _predictCount[index]);
} else int mapCount = Volatile.Read(ref _mapCount[index]);
{
if ((*_predictCount)[index] > short.MaxValue / 2)
{
int predictCount = Volatile.Read(ref (*_predictCount)[index]);
int mapCount = Volatile.Read(ref (*_mapCount)[index]);
int decrement = (int.Min(predictCount, mapCount) + 1) / 2; int decrement = (int.Min(predictCount, mapCount) + 1) / 2;
_ = Interlocked.Add(ref (*_predictCount)[index], -decrement); _ = Interlocked.Add(ref _predictCount[index], -decrement);
_ = Interlocked.Add(ref (*_mapCount)[index], -decrement); _ = Interlocked.Add(ref _mapCount[index], -decrement);
}
(*_predictCount)[index]++;
} }
_predictCount[index]++;
#endregion #endregion
@@ -122,10 +107,9 @@ namespace StgSharp.HighPerformance.Memory
head->Origin = address; head->Origin = address;
head->Profile = pHandle.MapPolicy; head->Profile = pHandle.MapPolicy;
head->RefCount = 0; head->RefCount = 0;
head->Predictor = (short)predictIndex;
(*_predictCount)[index] = 1; _predictCount[index] = 1;
(*_mapCount)[index] = 1; _mapCount[index] = 1;
// pre fetch them here // pre fetch them here
predict.Prefetch(head->Origin, head->Profile, new Span<byte>((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator)); predict.Prefetch(head->Origin, head->Profile, new Span<byte>((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator));
@@ -133,11 +117,6 @@ namespace StgSharp.HighPerformance.Memory
} }
_ = Interlocked.Add(ref _aheadCount, actualCount); _ = Interlocked.Add(ref _aheadCount, actualCount);
} }
finally
{
_predictors.ReleaseSpanOperation();
}
}
} }
} }
StgSharp.Common/HighPerformance/Memory/L4.Tlsf.cs
+2 -2
View File
@@ -64,7 +64,7 @@ namespace StgSharp.HighPerformance.Memory
_maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, CacheLineMetadata.Mask48Bits))); _maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, CacheLineMetadata.Mask48Bits)));
_maxLevel = int.MaxValue; _maxLevel = int.MaxValue;
_maxLevel = GetLevel(_maxAllocSize); _maxLevel = GetLevel(_maxAllocSize);
_bucket = new int[GetLevel(_maxAllocSize) + 2]; _bucket = new short[GetLevel(_maxAllocSize) + 2];
_spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize); _spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize);
_metadataEnd = 0; _metadataEnd = 0;
@@ -163,7 +163,7 @@ namespace StgSharp.HighPerformance.Memory
private void RemoveFromBucketHead(int level, CacheLineMetadata* node, int nodeIdx) private void RemoveFromBucketHead(int level, CacheLineMetadata* node, int nodeIdx)
{ {
int currentLevel = level; int currentLevel = level;
int currentNextLevelIdx = node->NextLevel; short currentNextLevelIdx = node->NextLevel;
// int currentPrevLevelIdx = 0; // int currentPrevLevelIdx = 0;
StgSharp.Common/HighPerformance/Memory/L4.cs
+11 -20
View File
@@ -2,8 +2,8 @@
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="L4" // file="L4"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload // AuthorGroup: Nitload Space
// Copyright (c) Nitload. All rights reserved. // Copyright (c) Nitload Space. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -48,11 +48,11 @@ namespace StgSharp.HighPerformance.Memory
/// <summary> /// <summary>
/// Count of cache lines mapped for each prediction, used for eviction policy. /// Count of cache lines mapped for each prediction, used for eviction policy.
/// </summary> /// </summary>
private readonly MapArray* _mapCount; private readonly int* _mapCount;
/// <summary> /// <summary>
/// Count of times each cache line is predicted, used for eviction policy. /// Count of times each cache line is predicted, used for eviction policy.
/// </summary> /// </summary>
private readonly PredictionArray* _predictCount; private readonly int* _predictCount;
/// <summary> /// <summary>
/// Original memory view for prediction span. /// Original memory view for prediction span.
/// </summary> /// </summary>
@@ -69,12 +69,7 @@ namespace StgSharp.HighPerformance.Memory
private readonly TLSF _bufferAllocator; private readonly TLSF _bufferAllocator;
public L4( public L4(int bindedNuma, nuint bufferSize = 8192 * 8192, int align = 16, nuint maxCacheLineSize = 8192)
int bindedNuma,
nuint bufferSize = 8192 * 8192,
int align = 16,
nuint maxCacheLineSize = 8192
)
{ {
_bindedNuma = bindedNuma; _bindedNuma = bindedNuma;
@@ -82,7 +77,7 @@ namespace StgSharp.HighPerformance.Memory
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineMetadata>()) * CacheLineCount; // 16 byte align nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineMetadata>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint predictionCountSize = (nuint)Unsafe.SizeOf<PredictionArray>(); // 16 byte align nuint predictionCountSize = sizeof(int) * CacheLineCount; // 16 byte align
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
ByteCapacity = normalizedBufferSize; ByteCapacity = normalizedBufferSize;
@@ -103,9 +98,9 @@ namespace StgSharp.HighPerformance.Memory
ptr += predictionSize; ptr += predictionSize;
_map = SwissTable.Create(ptr, _ => { }); _map = SwissTable.Create(ptr, _ => { });
ptr += mapCountSize; ptr += mapCountSize;
_predictCount = (PredictionArray*)ptr; _predictCount = (int*)ptr;
ptr += predictionCountSize; ptr += predictionCountSize;
_mapCount = (MapArray*)ptr; _mapCount = (int*)ptr;
ptr += predictionCountSize; ptr += predictionCountSize;
M512* mPtr = (M512*)_head; M512* mPtr = (M512*)_head;
@@ -121,9 +116,7 @@ namespace StgSharp.HighPerformance.Memory
// public IL4Predict PrefetchPredict { get; set; } // public IL4Predict PrefetchPredict { get; set; }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ContinueOnce( public void ContinueOnce(Handle line)
Handle line
)
{ {
if (line.Address > 0) { if (line.Address > 0) {
_ = Interlocked.Decrement(ref _aheadCount); _ = Interlocked.Decrement(ref _aheadCount);
@@ -156,9 +149,7 @@ namespace StgSharp.HighPerformance.Memory
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public Handle Map( public Handle Map(ref CacheLineDescription prediction)
ref CacheLineDescription prediction
)
{ {
if ((_aheadCount < PredictionLeastAhead) && !_exhausted) if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
{ {
@@ -186,7 +177,7 @@ namespace StgSharp.HighPerformance.Memory
CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle; CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle;
if (Interlocked.Increment(ref entry->RefCount) > 0) if (Interlocked.Increment(ref entry->RefCount) > 0)
{ {
(*_predictCount)[entry->Id] = 0; _predictCount[entry - _head] = 0;
_ = Interlocked.Decrement(ref _aheadCount); _ = Interlocked.Decrement(ref _aheadCount);
_ = Interlocked.Decrement(ref (*_mapCount)[entry->Id]); _ = Interlocked.Decrement(ref (*_mapCount)[entry->Id]);
return new Handle((nuint)entry->Position, ref entry->RefCount); return new Handle((nuint)entry->Position, ref entry->RefCount);