diff --git a/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs b/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs index 05c9a15b..a47d96bb 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs @@ -35,22 +35,6 @@ namespace StgSharp.HighPerformance.Memory public partial class L4 { - [InlineArray(CacheLineCount)] - private struct PredictionArray - { - - private int PredictionCount; - - } - - [InlineArray(CacheLineCount)] - private struct MapArray - { - - private int MapCount; - - } - internal enum AllocationState : byte { @@ -60,25 +44,32 @@ namespace StgSharp.HighPerformance.Memory } - [StructLayout(LayoutKind.Explicit, Size = 64, Pack = 64)] + [StructLayout(LayoutKind.Explicit, Size = 32, Pack = 32)] private unsafe struct CacheLineMetadata { + #region reference counting and state information + + [FieldOffset(24)] internal int RefCount; + + // [FieldOffset(28)] internal readonly short Id; + + #endregion + + #region tlsf area // TLSF area use only 24 bytes internal const ulong Mask48Bits = 0x0000FFFFFFFFFFFF; - [FieldOffset(0)] private readonly M512 FillingMask; + [FieldOffset(0)] private readonly M256 FillingMask; [FieldOffset(8)] private ulong PositionMask; [FieldOffset(16)] private ulong SizeMask; [FieldOffset(14)] internal AllocationState State; - // [FieldOffset(15)] internal sbyte Level; - [FieldOffset(0)] public short NextLevel; [FieldOffset(2)] public short NextNear; [FieldOffset(4)] public short PreviousLevel; @@ -106,19 +97,12 @@ namespace StgSharp.HighPerformance.Memory #endregion - #region reference counting and state information - - [FieldOffset(24)] internal int RefCount; - [FieldOffset(28)] internal readonly short Id; - - #endregion - #region cache line information [FieldOffset(34)] internal ushort Profile; - [FieldOffset(36)] internal nuint Origin; + [FieldOffset(24)] internal nuint Origin; - #endregion + #endregion } [StructLayout(LayoutKind.Sequential)] diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs b/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs index b4646aea..3833925d 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="L4.Evict" // Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. +// AuthorGroup: Nitload Space +// Copyright (c) Nitload Space. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -52,63 +52,37 @@ namespace StgSharp.HighPerformance.Memory /// Count of lines not evicted /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private unsafe int Evict( - int count = 1 - ) + private unsafe int Evict(int count = 1) { count = (count < 0) ? 0 : count; - int* pPtr = (int*)_predictCount, mPtr = (int*)_mapCount; + int* pPtr = _predictCount, mPtr = _mapCount; int remain = count; - ConcurrentFlexibleArray predictors = _predictors; - predictors.RequestSpanOperation(); - Span pSpan = predictors.AsSpan(); - try + for (int i = 0; i < CacheLineCount; i++) { - for (int i = 0; i < CacheLineCount; i++) + if (pPtr[i] > mPtr[i]) { - if (pPtr[i] > mPtr[i]) - { - continue; - } - remain -= TryEvictAt(i, pSpan) ? 1 : 0; + continue; } - } - finally - { - predictors.ReleaseSpanOperation(); + remain -= TryEvictAt(i) ? 1 : 0; } return remain; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private unsafe void RecycleLine( - int id - ) - { - _bufferAllocator.Free(id); - /**/ - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private unsafe bool TryEvictAt( - int index, - Span predictorSpan - ) + private unsafe bool TryEvictAt(int index) { CacheLineMetadata* head = &_head[index]; if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0) { // EvictLine(i + index); _ = _map.TryRemove(head->Origin, out _); - int id = head->Predictor; - IL4Predict predict = predictorSpan[id] ?? - throw new L4PredictUnregisteredException(id); + IL4Predict predict = CurrentPredict; predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator)); - RecycleLine(index); - (*_mapCount)[index] = 0; - (*_predictCount)[index] = int.MaxValue; + _bufferAllocator.Free(index); + _mapCount[index] = 0; + _predictCount[index] = int.MaxValue; return true; } return false; diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Predict.cs b/StgSharp.Common/HighPerformance/Memory/L4.Predict.cs index 2662b5b4..db34c96b 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Predict.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Predict.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="L4.Predict" // Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. +// AuthorGroup: Nitload Space +// Copyright (c) Nitload Space. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -39,66 +39,17 @@ namespace StgSharp.HighPerformance.Memory public partial class L4 { - private ConcurrentBufferStack _idReuse = new(32); - - private ConcurrentFlexibleArray _predictors = new(32); - - private ConcurrentQueue _predictionQueue = new(); - - private int _largestId ; - private IL4Predict CurrentPredict { get; set; } - private int CurrentPredictIndex { get; set; } - - public int RegisterPredict( - IL4Predict predict - ) + public void ResetPredict(IL4Predict predict) { - if (predict is null) { - return -1; - } - if (!_idReuse.TryPop(out int id)) { - id = _largestId++; - } - if (id > short.MaxValue) { - throw new OverflowException("Too many predictors in waiting"); - } - _predictors[id] = predict; - _predictionQueue.Enqueue(id); - return id; - } - - public IL4Predict? UnregisterPredict( - int id - ) - { - if (id < 0) { - return null; - } - IL4Predict p = _predictors[id]; - if (p == null) { - return p; - } - _predictors[id] = null!; - _idReuse.Push(id); - return p; - } - - private bool TryGetNextPredict( - ) - { - if (_predictionQueue.IsEmpty) { - return false; - } - if (_predictionQueue.TryDequeue(out int id)) + if (CurrentPredict is null) { - IL4Predict predict = _predictors[id] ?? throw new L4PredictUnregisteredException(id); CurrentPredict = predict; - CurrentPredictIndex = id; - return true; + return; } - return false; + + // TODO: write back all and register new predictor } } diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs b/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs index b218708a..94e83a8c 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="L4.Prefetch" // Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. +// AuthorGroup: Nitload Space +// Copyright (c) Nitload Space. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -39,12 +39,9 @@ namespace StgSharp.HighPerformance.Memory return; } IL4Predict predict = CurrentPredict; - int predictIndex = CurrentPredictIndex; Span span = new(_prediction, PredictionCount); PredictResult result = predict.Predict(span, out int actualCount); - if ((result & PredictResult.Exhausted) != 0) { - _exhausted = !TryGetNextPredict(); - } + _exhausted = (result & PredictResult.Exhausted) != 0; if (actualCount <= 0) { _exhausted = true; @@ -62,81 +59,63 @@ namespace StgSharp.HighPerformance.Memory missIndex++; } } - _predictors.RequestSpanOperation(); - Span predictorSpan = _predictors.AsSpan(); - try + + // now prefetch the misses + for (int i = 0; i < missIndex; i++) { - // now prefetch the misses - for (int i = 0; i < missIndex; i++) + CacheLineMetadata* head; + ref CacheLineDescription pHandle = ref span[i]; + nuint address = pHandle.Address; + int index; + if (_map.TryGet(address, out nuint handle)) { - CacheLineMetadata* head; - ref CacheLineDescription pHandle = ref span[i]; - nuint address = pHandle.Address; - int index; - if (_map.TryGet(address, out nuint handle)) + head = (CacheLineMetadata*)handle; + index = (int)(head - _head); + _predictCount[index] = 0; // reset eviction counter + + #region update predictor belonging + + if (_predictCount[index] > short.MaxValue / 2) { - head = (CacheLineMetadata*)handle; - index = head->Id; - (*_predictCount)[index] = 0; // reset eviction counter - - #region update predictor belonging - - int predictId = head->Predictor; - IL4Predict belong = predictorSpan[predictId]; - - if (!ReferenceEquals(predict, belong)) - { - throw new L4PredictConflictException(head->Origin, head->Profile, predictId); - } else - { - if ((*_predictCount)[index] > short.MaxValue / 2) - { - int predictCount = Volatile.Read(ref (*_predictCount)[index]); - int mapCount = Volatile.Read(ref (*_mapCount)[index]); - int decrement = (int.Min(predictCount, mapCount) + 1) / 2; - _ = Interlocked.Add(ref (*_predictCount)[index], -decrement); - _ = Interlocked.Add(ref (*_mapCount)[index], -decrement); - } - (*_predictCount)[index]++; - } - - #endregion - - continue; + int predictCount = Volatile.Read(ref _predictCount[index]); + int mapCount = Volatile.Read(ref _mapCount[index]); + int decrement = (int.Min(predictCount, mapCount) + 1) / 2; + _ = Interlocked.Add(ref _predictCount[index], -decrement); + _ = Interlocked.Add(ref _mapCount[index], -decrement); } + _predictCount[index]++; - int retryCount = 0, remain = missIndex - i; - const int maxRetry = 1024; - while (!_bufferAllocator.TryAlloc(pHandle.Size, out index)) - { - remain = Evict(remain); - retryCount++; - if (retryCount > maxRetry) { - throw new OverflowException($"Failed to evict cache line after {maxRetry} retries."); - } - if (retryCount > maxRetry / 2) { - Thread.Sleep(1); - } - } - head = _head + index; - head->Origin = address; - head->Profile = pHandle.MapPolicy; - head->RefCount = 0; - head->Predictor = (short)predictIndex; + #endregion - (*_predictCount)[index] = 1; - (*_mapCount)[index] = 1; - - // pre fetch them here - predict.Prefetch(head->Origin, head->Profile, new Span((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator)); - _ = _map.TryAddOrSet(address, (nuint)head, out _); + continue; } - _ = Interlocked.Add(ref _aheadCount, actualCount); - } - finally - { - _predictors.ReleaseSpanOperation(); + + int retryCount = 0, remain = missIndex - i; + const int maxRetry = 1024; + while (!_bufferAllocator.TryAlloc(pHandle.Size, out index)) + { + remain = Evict(remain); + retryCount++; + if (retryCount > maxRetry) { + throw new OverflowException($"Failed to evict cache line after {maxRetry} retries."); + } + if (retryCount > maxRetry / 2) { + Thread.Sleep(1); + } + } + head = _head + index; + head->Origin = address; + head->Profile = pHandle.MapPolicy; + head->RefCount = 0; + + _predictCount[index] = 1; + _mapCount[index] = 1; + + // pre fetch them here + predict.Prefetch(head->Origin, head->Profile, new Span((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator)); + _ = _map.TryAddOrSet(address, (nuint)head, out _); } + _ = Interlocked.Add(ref _aheadCount, actualCount); } } diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Tlsf.cs b/StgSharp.Common/HighPerformance/Memory/L4.Tlsf.cs index 65e89ea7..dac33310 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Tlsf.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Tlsf.cs @@ -64,7 +64,7 @@ namespace StgSharp.HighPerformance.Memory _maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, CacheLineMetadata.Mask48Bits))); _maxLevel = int.MaxValue; _maxLevel = GetLevel(_maxAllocSize); - _bucket = new int[GetLevel(_maxAllocSize) + 2]; + _bucket = new short[GetLevel(_maxAllocSize) + 2]; _spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize); _metadataEnd = 0; @@ -163,7 +163,7 @@ namespace StgSharp.HighPerformance.Memory private void RemoveFromBucketHead(int level, CacheLineMetadata* node, int nodeIdx) { int currentLevel = level; - int currentNextLevelIdx = node->NextLevel; + short currentNextLevelIdx = node->NextLevel; // int currentPrevLevelIdx = 0; diff --git a/StgSharp.Common/HighPerformance/Memory/L4.cs b/StgSharp.Common/HighPerformance/Memory/L4.cs index 3c93616b..44b8ed4a 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="L4" // Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. +// AuthorGroup: Nitload Space +// Copyright (c) Nitload Space. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -48,11 +48,11 @@ namespace StgSharp.HighPerformance.Memory /// /// Count of cache lines mapped for each prediction, used for eviction policy. /// - private readonly MapArray* _mapCount; + private readonly int* _mapCount; /// /// Count of times each cache line is predicted, used for eviction policy. /// - private readonly PredictionArray* _predictCount; + private readonly int* _predictCount; /// /// Original memory view for prediction span. /// @@ -69,12 +69,7 @@ namespace StgSharp.HighPerformance.Memory private readonly TLSF _bufferAllocator; - public L4( - int bindedNuma, - nuint bufferSize = 8192 * 8192, - int align = 16, - nuint maxCacheLineSize = 8192 - ) + public L4(int bindedNuma, nuint bufferSize = 8192 * 8192, int align = 16, nuint maxCacheLineSize = 8192) { _bindedNuma = bindedNuma; @@ -82,7 +77,7 @@ namespace StgSharp.HighPerformance.Memory nuint headSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // 16 byte align nuint predictionSize = ((nuint)Unsafe.SizeOf()) * PredictionCount; //16 byte align nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align - nuint predictionCountSize = (nuint)Unsafe.SizeOf(); // 16 byte align + nuint predictionCountSize = sizeof(int) * CacheLineCount; // 16 byte align nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size ByteCapacity = normalizedBufferSize; @@ -103,9 +98,9 @@ namespace StgSharp.HighPerformance.Memory ptr += predictionSize; _map = SwissTable.Create(ptr, _ => { }); ptr += mapCountSize; - _predictCount = (PredictionArray*)ptr; + _predictCount = (int*)ptr; ptr += predictionCountSize; - _mapCount = (MapArray*)ptr; + _mapCount = (int*)ptr; ptr += predictionCountSize; M512* mPtr = (M512*)_head; @@ -121,9 +116,7 @@ namespace StgSharp.HighPerformance.Memory // public IL4Predict PrefetchPredict { get; set; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ContinueOnce( - Handle line - ) + public void ContinueOnce(Handle line) { if (line.Address > 0) { _ = Interlocked.Decrement(ref _aheadCount); @@ -156,9 +149,7 @@ namespace StgSharp.HighPerformance.Memory } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Handle Map( - ref CacheLineDescription prediction - ) + public Handle Map(ref CacheLineDescription prediction) { if ((_aheadCount < PredictionLeastAhead) && !_exhausted) { @@ -186,7 +177,7 @@ namespace StgSharp.HighPerformance.Memory CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle; if (Interlocked.Increment(ref entry->RefCount) > 0) { - (*_predictCount)[entry->Id] = 0; + _predictCount[entry - _head] = 0; _ = Interlocked.Decrement(ref _aheadCount); _ = Interlocked.Decrement(ref (*_mapCount)[entry->Id]); return new Handle((nuint)entry->Position, ref entry->RefCount);