diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Allocate.cs b/StgSharp.Common/HighPerformance/Memory/L4.Allocate.cs deleted file mode 100644 index 1cda29f9..00000000 --- a/StgSharp.Common/HighPerformance/Memory/L4.Allocate.cs +++ /dev/null @@ -1,53 +0,0 @@ -//----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -// file="L4.Allocate.cs" -// Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person -// obtaining a copy of this software and associated documentation -// files (the “Software”), to deal in the Software without restriction, -// including without limitation the rights to use, copy, modify, merge, -// publish, distribute, sublicense, and/or sell copies of the Software, -// and to permit persons to whom the Software is furnished to do so, -// subject to the following conditions: -// -// The above copyright notice and -// this permission notice shall be included in all copies -// or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED “AS IS”, -// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, -// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH -// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// -// ----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -namespace StgSharp.HighPerformance.Memory -{ - public partial class L4 - { - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void RecycleLine(int id) - { - _idMux.Push(id); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TryGetEmptyLine(out int index) - { - if (_idMux.TryPop(out index)) { - return true; - } - index = _maxId; - return Interlocked.Increment(ref _maxId) >= CacheLineCount; - } - - } -} diff --git a/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs b/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs index c6a2f468..5894bf40 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs @@ -40,21 +40,6 @@ namespace StgSharp.HighPerformance.Memory public partial class L4 { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private unsafe void AgeAll() - { - Vector128 one = Vector128.Create((byte)0x01); - Vector128 max = Vector128.Create((byte)0x03); - byte* p = (byte*)_eviction; - for (int i = 0; i < CacheLineCount; i += 16) - { - Vector128 v = Sse2.LoadVector128(p + i); - v = Sse2.AddSaturate(v, one); - v = Sse2.Min(v, max); - Sse2.Store(p + i, v); - } - } - [InlineArray(CacheLineCount)] private struct EvictionArray { diff --git a/StgSharp.Common/HighPerformance/Memory/L4.InternalOp.cs b/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs similarity index 73% rename from StgSharp.Common/HighPerformance/Memory/L4.InternalOp.cs rename to StgSharp.Common/HighPerformance/Memory/L4.Evict.cs index 55f703f5..e005fe8c 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.InternalOp.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs @@ -28,6 +28,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Numerics; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; using System.Text; @@ -69,13 +70,40 @@ namespace StgSharp.HighPerformance.Memory int evictMask = Sse2.MoveMask(mask); while (evictMask != 0 && remain > 0) { - int index = BitOperations.TrailingZeroCount(evictMask); - evictMask &= (evictMask - 1); - EvictLine(i + index); - remain--; + int bitPos = BitOperations.TrailingZeroCount(evictMask); + int index = bitPos + i; + CacheLineHead* head = &_head[index]; + if (Interlocked.CompareExchange(ref head->_refCount, -1, 0) == 0) + { + // EvictLine(i + index); + _ = _map.TryRemove(head->_origin, out _); + RecycleLine(index); + remain--; + } + evictMask &= evictMask - 1; } } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void RecycleLine( + int id + ) + { + _idMux.Push(id); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool TryGetEmptyLine( + out int index + ) + { + if (_idMux.TryPop(out index)) { + return true; + } + index = _maxId; + return Interlocked.Increment(ref _maxId) >= CacheLineCount; + } + } } diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs b/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs index decb6bcc..0eb325fc 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="L4.Prefetch" // Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -63,15 +63,19 @@ namespace StgSharp.HighPerformance.Memory { ref CacheLinePrediction handle = ref span[i]; nuint address = handle.Address; - nuint mapPtr = _cacheLineAllocator.Allocate(); - CacheLineHead* head = (CacheLineHead*)_entryAllocator.Allocate(); + if (!TryGetEmptyLine(out int index)) { + throw new OverflowException("L4 has no empty cache line."); + } + CacheLineHead* head = &_head[index]; // pre fetch them here } } - private partial bool TestHit(nuint origin) + private partial bool TestHit( + nuint origin + ) { if (_map.TryGet(origin, out nuint handle)) { diff --git a/StgSharp.Common/HighPerformance/Memory/L4.cs b/StgSharp.Common/HighPerformance/Memory/L4.cs index c3a817f6..ea2a32d0 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="L4" // Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -26,18 +26,12 @@ // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- using StgSharp.Collections; -using StgSharp.Mathematics.Memory; -using System; -using System.Collections.Generic; -using System.Linq; +using System.Data; using System.Runtime.InteropServices; -using System.Text; -using System.Threading.Tasks; -using System.Xml.Serialization; namespace StgSharp.HighPerformance.Memory { - public unsafe partial class L4 : IDisposable + public sealed unsafe partial class L4 : IDisposable { private const int CacheLineCount = 8192; @@ -55,25 +49,40 @@ namespace StgSharp.HighPerformance.Memory private int _aheadCount; private int _maxId; + private volatile int _prefetching; private readonly nuint _baseAddress; private readonly SwissTable _map; private readonly UnmanagedStack _idMux; public L4() { - int cacheSize = Unsafe.SizeOf() * CacheLineCount; - int entrySize = Unsafe.SizeOf() * CacheLineCount; - int stackSize = sizeof(int) * CacheLineCount; - int mapSize = SwissTable.RequestedNativeMemorySize; - int predictionSize = Unsafe.SizeOf() * PredictionCount; - int rrpvSize = Unsafe.SizeOf(); - _baseAddress = (nuint)NativeMemory.AlignedAlloc((nuint)(cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize), 16); + // init order is fallow by align requirement of each component. + nuint cacheSize = (nuint)Unsafe.SizeOf() * CacheLineCount; // SIMDID, no less than 16 byte bit + nuint entrySize = (nuint)Unsafe.SizeOf() * CacheLineCount; // 16 byte align + nuint predictionSize = (nuint)Unsafe.SizeOf() * PredictionCount; //16 byte align + nuint mapSize = (nuint)SwissTable.RequestedNativeMemorySize; // 16 byte align + nuint rrpvSize = (nuint)Unsafe.SizeOf(); // 16 byte align + nuint stackSize = (nuint)sizeof(int) * CacheLineCount; // int align + + // malloc all memory at once, and free at once in Dispose. + // This is to avoid fragmentation and improve performance. + _baseAddress = (nuint)NativeMemory.AlignedAlloc(cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize, 16); + nuint ptr = _baseAddress; + + // init all components with the buffer, and move the buffer pointer forward. nuint _buffer = _baseAddress; _data = (CacheLineData*)_buffer; - _head = (CacheLineHead*)(_buffer + (nuint)cacheSize); - _prediction = (CacheLinePrediction*)(_buffer + (nuint)(cacheSize + entrySize)); - _map = SwissTable.Create(_baseAddress + (nuint)(cacheSize + entrySize), _ => { }); - _eviction = (EvictionArray*)(_baseAddress + (nuint)(cacheSize + entrySize + mapSize)); + ptr += cacheSize; + _head = (CacheLineHead*)ptr; + ptr += entrySize; + _prediction = (CacheLinePrediction*)ptr; + ptr += predictionSize; + _map = SwissTable.Create(_baseAddress + cacheSize + entrySize, _ => { }); + _prediction = (CacheLinePrediction*)ptr; + _eviction = (EvictionArray*)ptr; + ptr += rrpvSize; + _idMux = new UnmanagedStack((int*)ptr, CacheLineCount, _ => { }); + ptr += stackSize; } public IPrefetchPredict PrefetchPredict { get; set; } @@ -93,30 +102,58 @@ namespace StgSharp.HighPerformance.Memory } _disposed = true; _map.Dispose(); + _idMux.Dispose(); NativeMemory.AlignedFree((void*)_baseAddress); GC.SuppressFinalize(this); } - public CacheLine Map(nuint origin) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public CacheLine Map( + nuint origin + ) { - if (_map.TryGet(origin, out nuint entryHandle)) + if (_aheadCount < PredictionLeastAhead) { - // cache hit - CacheLineHead* entry = (CacheLineHead*)entryHandle; - return new CacheLine(entry->_address, ref _activeCount, ref entry->_refCount); + while (true) + { + if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0) + { + if (_aheadCount < PredictionLeastAhead) + { + // do prefetch in work load now, sync + Prefetch(); + break; + } else + { + _prefetching = 0; + } + } + } + } + while (true) + { + if (_map.TryGet(origin, out nuint entryHandle)) + { + // cache hit + CacheLineHead* entry = (CacheLineHead*)entryHandle; + if (Interlocked.Increment(ref entry->_refCount) < 0) + { + (*_eviction)[entry->_id] = 0; + return new CacheLine(entry->_address, ref _activeCount, ref entry->_refCount); + } else + { + _ = Interlocked.Decrement(ref entry->_refCount); + } + } + Thread.Sleep(0); } - - // cache miss - /* - * post command - * then wait for completion - */ - return /*origin/**/default; //error now } private partial void Prefetch(); - private partial bool TestHit(nuint origin); + private partial bool TestHit( + nuint origin + ); public ref struct CacheLine : IDisposable { @@ -125,7 +162,11 @@ namespace StgSharp.HighPerformance.Memory private ref int _refCount; private nuint _handle; - internal CacheLine(nuint handle, ref int activeCount, ref int cacheRefCount) + internal CacheLine( + nuint handle, + ref int activeCount, + ref int cacheRefCount + ) { _handle = handle; _refCount = ref cacheRefCount; @@ -134,6 +175,9 @@ namespace StgSharp.HighPerformance.Memory _ = Interlocked.Increment(ref _refCount); } + [Obsolete("You should not create a CacheLineHandle outside a L4", true)] + public CacheLine() { } + public void Dispose() { if (_handle == 0) {