diff --git a/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs b/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs index e2b41e61..3a4ad883 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs @@ -51,12 +51,57 @@ namespace StgSharp.HighPerformance.Memory } - [StructLayout(LayoutKind.Explicit, Size = 64, Pack = 64)] - private unsafe struct CacheLineHead + internal enum AllocationState : byte { - [FieldOffset(0)] internal Tlsf.Handle AllocatorEntry; - [FieldOffset(0)] internal readonly M512 Buffer; + SpareOrCollect = 0, + Free = 1, + Alloc = 2, + + } + + [StructLayout(LayoutKind.Explicit, Size = 64, Pack = 64)] + private unsafe struct CacheLineMetadata + { + + #region lsf area + + internal const ulong Mask48Bits = 0x0000FFFFFFFFFFFF; + + [FieldOffset(0)] private readonly M512 FillingMask; + + [FieldOffset(16)] private ulong PositionMask; + [FieldOffset(24)] private ulong SizeMask; + + [FieldOffset(30)] internal AllocationState State; + [FieldOffset(31)] internal sbyte Level; + + [FieldOffset(0)] public int NextLevel; + [FieldOffset(4)] public int NextNear; + [FieldOffset(8)] public int PreviousLevel; + [FieldOffset(12)] public int PreviousNear; + + public Ptr64 Position + { + readonly get => PositionMask; + set => PositionMask = value; + } + + public Ptr64 SizeForAllocator + { + readonly get => (Ptr64)(SizeMask & Mask48Bits); + set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits); + } + + public Ptr64 SizeForSpare + { + readonly get => SizeMask; + set => SizeMask = value; + } + + public static uint SizeOf { get; } = (uint)Unsafe.SizeOf(); + + #endregion #region reference counting and state information @@ -65,21 +110,13 @@ namespace StgSharp.HighPerformance.Memory #endregion - #region hlsf entry alias - - public readonly Ptr64 Address => AllocatorEntry.Address; - - public readonly Ptr64 Size => AllocatorEntry.Size; - - #endregion - #region cache line information [FieldOffset(32)] internal short Predictor; [FieldOffset(34)] internal ushort Profile; [FieldOffset(36)] internal nuint Origin; - #endregion + #endregion } } diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs b/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs index d6ea54b8..b4646aea 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs @@ -87,10 +87,7 @@ namespace StgSharp.HighPerformance.Memory int id ) { - throw new NotImplementedException(); - /* - Tlsf* entry = (Tlsf*)(_entryManager._basePtr + id); - _bufferAllocator.Free(entry); + _bufferAllocator.Free(id); /**/ } @@ -100,7 +97,7 @@ namespace StgSharp.HighPerformance.Memory Span predictorSpan ) { - CacheLineHead* head = &_head[index]; + CacheLineMetadata* head = &_head[index]; if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0) { // EvictLine(i + index); @@ -108,7 +105,7 @@ namespace StgSharp.HighPerformance.Memory int id = head->Predictor; IL4Predict predict = predictorSpan[id] ?? throw new L4PredictUnregisteredException(id); - predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan((byte*)(ulong)head->Address, (int)(ulong)head->Size)); + predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator)); RecycleLine(index); (*_mapCount)[index] = 0; (*_predictCount)[index] = int.MaxValue; diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs b/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs index d81b850f..26851100 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs @@ -74,13 +74,13 @@ namespace StgSharp.HighPerformance.Memory // now prefetch the misses for (int i = 0; i < missIndex; i++) { - CacheLineHead* head; + CacheLineMetadata* head; ref CacheLineDescription pHandle = ref span[i]; nuint address = pHandle.Address; int index; if (_map.TryGet(address, out nuint handle)) { - head = (CacheLineHead*)handle; + head = (CacheLineMetadata*)handle; index = head->Id; (*_predictCount)[index] = 0; // reset eviction counter @@ -146,7 +146,7 @@ namespace StgSharp.HighPerformance.Memory (*_mapCount)[index] = 1; // pre fetch them here - predict.Prefetch(head->Origin, head->Profile, new Span((byte*)(ulong)head->Address, (int)(ulong)head->Size)); + predict.Prefetch(head->Origin, head->Profile, new Span((byte*)(ulong)head->Address, (int)(ulong)head->SizeForAllocator)); _ = _map.TryAddOrSet(address, (nuint)head, out _); } _ = Interlocked.Add(ref _aheadCount, actualCount); diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Tlsf.cs b/StgSharp.Common/HighPerformance/Memory/L4.Tlsf.cs index db9da375..78edfe72 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Tlsf.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Tlsf.cs @@ -25,6 +25,7 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- +using StgSharp.HighPerformance.ProcessorAbstraction; using System; using System.Collections.Generic; using System.Linq; @@ -41,13 +42,12 @@ namespace StgSharp.HighPerformance.Memory private unsafe class TLSF { - private const ulong Mask48Bits = 0x0000FFFFFFFFFFFF; public const int MaxCacheLineCount = 8192; - public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * Metadata.SizeOf; + public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * CacheLineMetadata.SizeOf; private readonly int[] _bucket; - private readonly Metadata* _spareMemory; + private readonly CacheLineMetadata* _spareMemory; private readonly int _align; private readonly int _maxLevel; @@ -65,17 +65,17 @@ namespace StgSharp.HighPerformance.Memory _totalSize = byteSize; _basePtr = (Ptr64)(ulong)externalPointer; _align = 4096; // align of a memory page - _maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, Mask48Bits))); + _maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, CacheLineMetadata.Mask48Bits))); _maxLevel = int.MaxValue; _maxLevel = GetLevel(_maxAllocSize); _bucket = new int[GetLevel(_maxAllocSize) + 2]; - _spareMemory = (Metadata*)(ulong)(_basePtr + _totalSize); + _spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize); _metadataEnd = 0; // init spare memory node _spareMemory->Position = _basePtr; - _spareMemory->SizeForSpare = _totalSize - Metadata.SizeOf; + _spareMemory->SizeForSpare = _totalSize - CacheLineMetadata.SizeOf; _spareMemory->NextLevel = 0; _spareMemory->NextNear = 0; _spareMemory->PreviousLevel = 0; @@ -98,11 +98,11 @@ namespace StgSharp.HighPerformance.Memory /// /// private void RecycleNode( - Metadata* node + CacheLineMetadata* node ) { int nodeIdx = (int)(_spareMemory - node); - int level = GetLevel(node->Size); + int level = GetLevel(node->SizeForAllocator); int nextLevelIdx = node->NextLevel; int prevLevelIdx = node->PreviousLevel; RemoveFromBucket(level, node, nodeIdx); @@ -110,11 +110,11 @@ namespace StgSharp.HighPerformance.Memory // remove from position chain int prevNearIdx = node->PreviousNear; int nextNearIdx = node->NextNear; - Metadata* prevNear = &_spareMemory[prevNearIdx]; + CacheLineMetadata* prevNear = &_spareMemory[prevNearIdx]; prevNear->NextNear = nextNearIdx; if (nextNearIdx != 0) { - Metadata* nextNear = &_spareMemory[nextNearIdx]; + CacheLineMetadata* nextNear = &_spareMemory[nextNearIdx]; nextNear->PreviousNear = prevNearIdx; } else { @@ -131,7 +131,7 @@ namespace StgSharp.HighPerformance.Memory int nextIdx = _spareMemory->NextLevel; if (nextIdx != 0) { - Metadata* next = &_spareMemory[nextIdx]; + CacheLineMetadata* next = &_spareMemory[nextIdx]; next->PreviousLevel = nodeIdx; } node->NextLevel = nextIdx; @@ -140,16 +140,16 @@ namespace StgSharp.HighPerformance.Memory [MethodImpl(MethodImplOptions.AggressiveInlining)] private void RemoveFromBucket( - Metadata* node, + CacheLineMetadata* node, int nodeIdx ) { - RemoveFromBucket(GetLevel(node->Size), node, nodeIdx); + RemoveFromBucket(GetLevel(node->SizeForAllocator), node, nodeIdx); } private void RemoveFromBucket( int level, - Metadata* node, + CacheLineMetadata* node, int nodeIdx ) { @@ -162,12 +162,12 @@ namespace StgSharp.HighPerformance.Memory _bucket[currentLevel] = currentNextLevelIdx; } else if (currentPrevLevelIdx != 0) { - Metadata* prevLevel = &_spareMemory[currentPrevLevelIdx]; + CacheLineMetadata* prevLevel = &_spareMemory[currentPrevLevelIdx]; prevLevel->NextLevel = currentNextLevelIdx; } if (currentNextLevelIdx != 0) { - Metadata* nextLevel = &_spareMemory[currentNextLevelIdx]; + CacheLineMetadata* nextLevel = &_spareMemory[currentNextLevelIdx]; nextLevel->PreviousLevel = currentPrevLevelIdx; } node->PreviousLevel = 0; @@ -177,7 +177,7 @@ namespace StgSharp.HighPerformance.Memory // Assuming node to remove is head f bucket private void RemoveFromBucketHead( int level, - Metadata* node, + CacheLineMetadata* node, int nodeIdx ) { @@ -199,7 +199,7 @@ namespace StgSharp.HighPerformance.Memory /**/ if (currentNextLevelIdx != 0) { - Metadata* nextLevel = &_spareMemory[currentNextLevelIdx]; + CacheLineMetadata* nextLevel = &_spareMemory[currentNextLevelIdx]; nextLevel->PreviousLevel = 0; } node->PreviousLevel = 0; @@ -208,11 +208,11 @@ namespace StgSharp.HighPerformance.Memory private bool TryAllocNewChunk( ulong size, - out Metadata* meta + out CacheLineMetadata* meta ) { int idx = _spareMemory->NextLevel; - ulong spareSize = _spareMemory->Size; + ulong spareSize = _spareMemory->SizeForAllocator; if (idx != 0) { if (spareSize < size) @@ -224,7 +224,7 @@ namespace StgSharp.HighPerformance.Memory int nextSpareIdx = meta->NextLevel; if (nextSpareIdx != 0) { - Metadata* nextSpare = &_spareMemory[nextSpareIdx]; + CacheLineMetadata* nextSpare = &_spareMemory[nextSpareIdx]; nextSpare->PreviousLevel = 0; } _spareMemory->NextLevel = nextSpareIdx; @@ -233,17 +233,17 @@ namespace StgSharp.HighPerformance.Memory return false; } spareSize -= size; - meta->Size = size; + meta->SizeForAllocator = size; meta->Position = _spareMemory->Position; meta->State = AllocationState.Free; meta->PreviousLevel = 0; meta->NextLevel = 0; - _spareMemory->Size = spareSize; + _spareMemory->SizeForAllocator = spareSize; _spareMemory->Position += size; // insert the new chunk to position chain int nearIdx = _spareMemory->PreviousNear; - Metadata* near = &_spareMemory[nearIdx]; + CacheLineMetadata* near = &_spareMemory[nearIdx]; near->NextNear = idx; meta->PreviousNear = nearIdx; _spareMemory->PreviousNear = idx; @@ -252,10 +252,10 @@ namespace StgSharp.HighPerformance.Memory } private bool TryGetNewMetadata( - out Metadata* meta + out CacheLineMetadata* meta ) { - ulong spareSize = _spareMemory->Size; + ulong spareSize = _spareMemory->SizeForAllocator; if (_metadataEnd >= MaxCacheLineCount) { meta = null; @@ -268,7 +268,7 @@ namespace StgSharp.HighPerformance.Memory private bool TryTakeBucketEntry( ulong minimumSize, - out Metadata* entry + out CacheLineMetadata* entry ) { int level = GetLevel(minimumSize); @@ -278,8 +278,8 @@ namespace StgSharp.HighPerformance.Memory int idx = _bucket[level]; while (idx != 0) { - Metadata* candidate = &_spareMemory[idx]; - if (candidate->Size >= minimumSize) + CacheLineMetadata* candidate = &_spareMemory[idx]; + if (candidate->SizeForAllocator >= minimumSize) { RemoveFromBucket(level, candidate, idx); entry = candidate; @@ -293,7 +293,7 @@ namespace StgSharp.HighPerformance.Memory int idx = _bucket[i]; if (idx != 0) { - Metadata* candidate = &_spareMemory[idx]; + CacheLineMetadata* candidate = &_spareMemory[idx]; RemoveFromBucketHead(i, candidate, idx); entry = candidate; return true; @@ -304,52 +304,6 @@ namespace StgSharp.HighPerformance.Memory return false; } - internal enum AllocationState : byte - { - - SpareOrCollect = 0, - Free = 1, - Alloc = 2, - - } - - [StructLayout(LayoutKind.Explicit, Size = 32)] - internal struct Metadata - { - - [FieldOffset(16)] private ulong PositionMask; - [FieldOffset(24)] private ulong SizeMask; - - [FieldOffset(30)] internal AllocationState State; - [FieldOffset(31)] internal sbyte Level; - - [FieldOffset(0)] public int NextLevel; - [FieldOffset(4)] public int NextNear; - [FieldOffset(8)] public int PreviousLevel; - [FieldOffset(12)] public int PreviousNear; - - public Ptr64 Position - { - readonly get => PositionMask; - set => PositionMask = value; - } - - public Ptr64 Size - { - readonly get => (Ptr64)(SizeMask & Mask48Bits); - set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits); - } - - public Ptr64 SizeForSpare - { - readonly get => SizeMask; - set => SizeMask = value; - } - - public static uint SizeOf { get; } = (uint)Unsafe.SizeOf(); - - } - #region alloc and free public int Alloc( @@ -360,12 +314,12 @@ namespace StgSharp.HighPerformance.Memory if (allocSize > _maxAllocSize) { throw new OverflowException("Requested allocation size exceeds the maximum block size."); } - if (!TryTakeBucketEntry(allocSize, out Metadata* entry) && + if (!TryTakeBucketEntry(allocSize, out CacheLineMetadata* entry) && !TryAllocNewChunk(allocSize, out entry)) { throw new OverflowException("No available "); } int entryIdx = (int)(_spareMemory - entry); - ulong entrySize = entry->Size; + ulong entrySize = entry->SizeForAllocator; if (entrySize < allocSize) { throw new OverflowException("Bucket returned an undersized block."); } @@ -373,18 +327,18 @@ namespace StgSharp.HighPerformance.Memory // slice tail if (remain > ulong.Min(4096UL, entrySize >> 7) && - TryGetNewMetadata(out Metadata* newEntry)) + TryGetNewMetadata(out CacheLineMetadata* newEntry)) { int newEntryIdx = (int)(_spareMemory - newEntry); newEntry->Position = entry->Position + allocSize; - newEntry->Size = remain; + newEntry->SizeForAllocator = remain; newEntry->State = AllocationState.Free; int newLevel = GetLevel(remain); int bucketHeadIdx = _bucket[newLevel]; if (bucketHeadIdx != 0) { // if there is already an entry in the bucket, link it to the new entry - Metadata* bucketHead = &_spareMemory[bucketHeadIdx]; + CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx]; bucketHead->PreviousLevel = newEntryIdx; } newEntry->NextLevel = bucketHeadIdx; @@ -394,7 +348,7 @@ namespace StgSharp.HighPerformance.Memory // push to position chain int afterIdx = entry->NextNear; - Metadata* after = &_spareMemory[afterIdx]; + CacheLineMetadata* after = &_spareMemory[afterIdx]; newEntry->NextNear = afterIdx; after->PreviousNear = newEntryIdx; entry->NextNear = newEntryIdx; @@ -416,7 +370,7 @@ namespace StgSharp.HighPerformance.Memory { break; } - Metadata* current = &_spareMemory[currentIdx]; + CacheLineMetadata* current = &_spareMemory[currentIdx]; if (current->State == AllocationState.Free) { // scan next @@ -424,24 +378,24 @@ namespace StgSharp.HighPerformance.Memory int nextIdx = current->NextNear; while (nextIdx != 0) { - Metadata* next = &_spareMemory[nextIdx]; + CacheLineMetadata* next = &_spareMemory[nextIdx]; if (next->State != AllocationState.Free) { break; } - size += next->Size; + size += next->SizeForAllocator; RecycleNode(next); nextIdx = current->NextNear; } if (size > 0) { - current->Size += size; + current->SizeForAllocator += size; RemoveFromBucket(current, currentIdx); - int newLevel = GetLevel(current->Size); + int newLevel = GetLevel(current->SizeForAllocator); int bucketHeadIdx = _bucket[newLevel]; if (bucketHeadIdx != 0) { - Metadata* bucketHead = &_spareMemory[bucketHeadIdx]; + CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx]; bucketHead->PreviousLevel = currentIdx; } current->NextLevel = bucketHeadIdx; @@ -458,9 +412,9 @@ namespace StgSharp.HighPerformance.Memory int collectScanWindow = 4 ) { - Metadata* head = &_spareMemory[handle]; + CacheLineMetadata* head = &_spareMemory[handle]; int headIdx = (int)(_spareMemory - head); - Ptr64 size = head->Size; + Ptr64 size = head->SizeForAllocator; for (int i = 0; i < collectScanWindow; i++) { // forward scan, previous direction @@ -469,12 +423,12 @@ namespace StgSharp.HighPerformance.Memory { break; } - Metadata* prev = &_spareMemory[forwardIdx]; + CacheLineMetadata* prev = &_spareMemory[forwardIdx]; if (prev->State != AllocationState.Free) { break; } - size += prev->Size; + size += prev->SizeForAllocator; head->Position = prev->Position; RecycleNode(prev); } @@ -486,25 +440,25 @@ namespace StgSharp.HighPerformance.Memory { break; } - Metadata* prev = &_spareMemory[nextIdx]; + CacheLineMetadata* prev = &_spareMemory[nextIdx]; if (prev->State != AllocationState.Free) { break; } - size += prev->Size; + size += prev->SizeForAllocator; RecycleNode(prev); } // recycle to level; - head->Size = size; + head->SizeForAllocator = size; head->State = AllocationState.Free; head->PreviousLevel = 0; - int level = GetLevel(head->Size); + int level = GetLevel(head->SizeForAllocator); int bucketHeadIdx = _bucket[level]; head->NextLevel = 0; if (bucketHeadIdx != 0) { - Metadata* bucketHead = &_spareMemory[bucketHeadIdx]; + CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx]; bucketHead->PreviousLevel = headIdx; head->NextLevel = bucketHeadIdx; } @@ -523,14 +477,14 @@ namespace StgSharp.HighPerformance.Memory h = -1; return false; } - if (!TryTakeBucketEntry(allocSize, out Metadata* entry) && + if (!TryTakeBucketEntry(allocSize, out CacheLineMetadata* entry) && !TryAllocNewChunk(allocSize, out entry)) { h = -1; return false; } int entryIdx = (int)(_spareMemory - entry); - ulong entrySize = entry->Size; + ulong entrySize = entry->SizeForAllocator; if (entrySize < allocSize) { h = -1; @@ -539,18 +493,18 @@ namespace StgSharp.HighPerformance.Memory ulong remain = entrySize - allocSize; // slice tail - if (remain > (entrySize >> 7) && TryGetNewMetadata(out Metadata* newEntry)) + if (remain > (entrySize >> 7) && TryGetNewMetadata(out CacheLineMetadata* newEntry)) { int newEntryIdx = (int)(_spareMemory - newEntry); newEntry->Position = entry->Position + allocSize; - newEntry->Size = remain; + newEntry->SizeForAllocator = remain; newEntry->State = AllocationState.Free; int newLevel = GetLevel(remain); int bucketHeadIdx = _bucket[newLevel]; if (bucketHeadIdx != 0) { // if there is already an entry in the bucket, link it to the new entry - Metadata* bucketHead = &_spareMemory[bucketHeadIdx]; + CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx]; newEntry->NextLevel = bucketHeadIdx; bucketHead->PreviousLevel = newEntryIdx; } else @@ -562,7 +516,7 @@ namespace StgSharp.HighPerformance.Memory // push to position chain int afterIdx = entry->NextNear; - Metadata* after = &_spareMemory[afterIdx]; + CacheLineMetadata* after = &_spareMemory[afterIdx]; newEntry->NextNear = afterIdx; after->PreviousNear = newEntryIdx; entry->NextNear = newEntryIdx; diff --git a/StgSharp.Common/HighPerformance/Memory/L4.cs b/StgSharp.Common/HighPerformance/Memory/L4.cs index 41b1978f..e3d80e28 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.cs @@ -42,7 +42,7 @@ namespace StgSharp.HighPerformance.Memory private const int PredictionLeastAhead = 1024; // private static readonly IIntrinsicKernel _intrinsic = IIntrinsicKernel.Instance; - private readonly CacheLineHead* _head; + private readonly CacheLineMetadata* _head; private readonly MapArray* _mapCount; private readonly PredictionArray* _predictCount; private readonly CacheLineDescription* _prediction; @@ -69,31 +69,30 @@ namespace StgSharp.HighPerformance.Memory _bindedNuma = bindedNuma; // init order is fallow by align requirement of each component. - nuint entrySize = TLSF.RequestedAllocation; // SIMDID, no less than 16 byte bit - nuint headSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // 16 byte align + nuint headSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // 16 byte align nuint predictionSize = ((nuint)Unsafe.SizeOf()) * PredictionCount; //16 byte align - nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align + nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align nuint predictionCountSize = (nuint)Unsafe.SizeOf(); // 16 byte align nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size ByteCapacity = normalizedBufferSize; + AllocSize = headSize + mapCountSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize; + // malloc all memory at once, and free at once in Dispose. // This is to avoid fragmentation and improve performance. - _baseAddress = (nuint)Numa.Alloc(bindedNuma, entrySize + headSize + mapSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize); + _baseAddress = (nuint)Numa.Alloc(bindedNuma, AllocSize); nuint ptr = _baseAddress; - AllocSize = entrySize; - // init all components with the buffer, and move the buffer pointer forward. - nuint _buffer = _baseAddress + normalizedBufferSize; + nuint _buffer = _baseAddress; _bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress); - ptr += entrySize; - _head = (CacheLineHead*)ptr; + _buffer += normalizedBufferSize; + _head = (CacheLineMetadata*)ptr; ptr += headSize; _prediction = (CacheLineDescription*)ptr; ptr += predictionSize; _map = SwissTable.Create(ptr, _ => { }); - ptr += mapSize; + ptr += mapCountSize; _predictCount = (PredictionArray*)ptr; ptr += predictionCountSize; _mapCount = (MapArray*)ptr; @@ -101,7 +100,7 @@ namespace StgSharp.HighPerformance.Memory M512* mPtr = (M512*)_head; for (uint i = 0; i < CacheLineCount; i++) { - mPtr->BroadCastFrom(i); + mPtr->BroadCastFrom(0); } } @@ -141,7 +140,7 @@ namespace StgSharp.HighPerformance.Memory // init order is fallow by align requirement of each component. nuint entrySize = TLSF.RequestedAllocation; // SIMDID, no less than 16 byte bit - nuint headSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // 16 byte align + nuint headSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // 16 byte align nuint predictionSize = ((nuint)Unsafe.SizeOf()) * PredictionCount; //16 byte align nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align nuint predictionCountSize = (nuint)Unsafe.SizeOf(); // 16 byte align @@ -178,7 +177,7 @@ namespace StgSharp.HighPerformance.Memory if (_map.TryGet(prediction.GetHashCodeLong(), out nuint entryHandle)) { // cache hit - CacheLineHead* entry = (CacheLineHead*)entryHandle; + CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle; if (Interlocked.Increment(ref entry->RefCount) > 0) { (*_predictCount)[entry->Id] = 0;