youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

swiss x86 is briefly ok, ready for test

Browse Source
This commit is contained in:
Youmu Liu
2026-10-09 18:17:03 +08:00
parent 2e7c3040f4
commit 3a2b123478
15 changed files with 457 additions and 1317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Nitload.Common/HighPerformance/Memory/L4.cs
+26 -133
View File
@@ -5,49 +5,20 @@
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using Nitload.Common.HighPerformance.Memory;
using Nitload.Common.HighPerformance.ProcessorAbstraction;
using System.Data;
using System.Runtime.InteropServices;
namespace Nitload.Common.HighPerformance.Memory
{
public sealed unsafe partial class L4 : IDisposable
{
private const int CacheLineCount = 8192;
private const nuint PageSize = 4096;
private const int PredictionCount = 2048;
private const int PredictionLeastAhead = 1024;
/// <summary>
/// Array of metadata for each cache line, also used as metadata for tlsf allocator.
/// </summary>
private readonly CacheLineMetadata* _head;
/// <summary>
/// Count of cache lines mapped for each prediction, used for eviction policy.
/// </summary>
private readonly int* _mapCount;
/// <summary>
/// Count of times each cache line is predicted, used for eviction policy.
/// </summary>
private readonly int* _predictCount;
/// <summary>
/// Original memory view for prediction span.
/// </summary>
private readonly CacheLineDescription* _prediction;
private bool _disposed;
private bool _exhausted;
private int _aheadCount;
private readonly int _bindedNuma;
private volatile int _prefetching;
private readonly nuint _baseAddress;
private readonly SwissTable _map;
private readonly TLSF _bufferAllocator;
private SwissTable _addressMap;
private TLSF _bufferAllocator;
public L4(
int bindedNuma,
@@ -59,130 +30,52 @@ namespace Nitload.Common.HighPerformance.Memory
ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize);
ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize);
_bindedNuma = bindedNuma;
// init order is fallow by align requirement of each component.
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineMetadata>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint predictionCountSize = sizeof(int) * CacheLineCount; // 16 byte align
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1));
ByteCapacity = normalizedBufferSize;
AllocSize = headSize + mapCountSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize;
// malloc all memory at once, and free at once in Dispose.
// This is to avoid fragmentation and improve performance.
nuint swissBufferOffset = normalizedBufferSize + TLSF.RequestedAllocation;
AllocSize = swissBufferOffset + (nuint)BufferSize;
_baseAddress = (nuint)Numa.Alloc(bindedNuma, AllocSize);
nuint ptr = _baseAddress;
// init all components with the buffer, and move the buffer pointer forward.
nuint _buffer = _baseAddress;
_bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress);
_buffer += normalizedBufferSize;
_head = (CacheLineMetadata*)ptr;
ptr += headSize;
_prediction = (CacheLineDescription*)ptr;
ptr += predictionSize;
_map = SwissTable.Create(ptr, _ => { });
ptr += mapCountSize;
_predictCount = (int*)ptr;
ptr += predictionCountSize;
_mapCount = (int*)ptr;
ptr += predictionCountSize;
M512* mPtr = (M512*)_head;
for (uint i = 1; i < CacheLineCount; i++) {
mPtr->BroadCastFrom<ulong>(0);
}
_addressMap = SwissTable.Create((byte*)(_baseAddress + swissBufferOffset));
}
public nuint ByteCapacity { get; init; }
public nuint AllocSize { get; init; }
// public IL4Predict PrefetchPredict { get; set; }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ContinueOnce(
Handle line
)
{
if (line.Address > 0) {
_ = Interlocked.Decrement(ref _aheadCount);
}
}
[System.Diagnostics.CodeAnalysis.SuppressMessage(
"Design",
"CA1063:Implement IDisposable correctly",
Justification = """
Only owns native memory via Numa.Free();
no managed IDisposable fields.
Dispose(bool) pattern is unnecessary.
""")]
public void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
_map.Dispose();
#pragma warning disable CA1816
GC.SuppressFinalize(_bufferAllocator);
#pragma warning restore CA1816
// init order is fallow by align requirement of each component.
_addressMap.Dispose();
Numa.Free((void*)_baseAddress, _bindedNuma, AllocSize);
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Handle Map(
ref CacheLineDescription prediction
)
internal void Collect()
{
if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
{
if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0)
{
try
{
if (_aheadCount < PredictionLeastAhead)
{
// do prefetch in work load now, sync
Prefetch();
}
}
finally
{
_prefetching = 0;
}
}
}
while (true)
{
if (SwissTable.Reader.TryGet(_map, prediction.GetHashCodeLong(), out nuint entryHandle))
{
// cache hit
CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle;
if (Interlocked.Increment(ref entry->RefCount) > 0)
{
_predictCount[entry - _head] = 0;
_ = Interlocked.Decrement(ref _aheadCount);
_ = Interlocked.Decrement(ref _mapCount[entry - _head]);
nuint* hPtr = stackalloc nuint[2] { entry->PositionMask + _baseAddress, (nuint)entry };
return *(Handle*)hPtr;
} else
{
_ = Interlocked.Decrement(ref entry->RefCount);
}
}
Thread.Sleep(0);
}
ObjectDisposedException.ThrowIf(_disposed, this);
_bufferAllocator.Collect();
}
private partial void Prefetch();
internal void Free(
int handle
)
{
ObjectDisposedException.ThrowIf(_disposed, this);
_bufferAllocator.Free(handle);
}
internal bool TryAllocate(
uint size,
out int handle
)
{
ObjectDisposedException.ThrowIf(_disposed, this);
return _bufferAllocator.TryAlloc(size, out handle);
}
}
}