youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compress L4 metadata into 32 bytes

Browse Source
This commit is contained in:
Youmu Liu
2026-06-02 18:27:58 +08:00
parent 07f8178431
commit 9f3fc3b345
7 changed files with 156 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs
+26 -47
View File
@@ -2,8 +2,8 @@
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="L4.CacheLine" // file="L4.CacheLine"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload Space // AuthorGroup: Nitload
// Copyright (c) Nitload Space. All rights reserved. // Copyright (c) Nitload. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -28,8 +28,6 @@
using StgSharp.HighPerformance.ProcessorAbstraction; using StgSharp.HighPerformance.ProcessorAbstraction;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using Tlsf = StgSharp.HighPerformance.Memory.TwoLayerSegregatedFitAllocator;
namespace StgSharp.HighPerformance.Memory namespace StgSharp.HighPerformance.Memory
{ {
public partial class L4 public partial class L4
@@ -48,71 +46,52 @@ namespace StgSharp.HighPerformance.Memory
private unsafe struct CacheLineMetadata private unsafe struct CacheLineMetadata
{ {
#region reference counting and state information internal const uint MaxBufferSize = 0xFFFFFFFF - 64U;
[FieldOffset(24)] internal int RefCount;
// [FieldOffset(28)] internal readonly short Id;
#endregion
#region tlsf area
// TLSF area use only 24 bytes
internal const ulong Mask48Bits = 0x0000FFFFFFFFFFFF;
[FieldOffset(0)] private readonly M256 FillingMask; [FieldOffset(0)] private readonly M256 FillingMask;
[FieldOffset(8)] private ulong PositionMask; [FieldOffset(24)] internal nuint Origin;
[FieldOffset(16)] private ulong SizeMask;
[FieldOffset(14)] internal AllocationState State; public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
#region tlsf linked list
[FieldOffset(0)] public short NextLevel; [FieldOffset(0)] public short NextLevel;
[FieldOffset(2)] public short NextNear; [FieldOffset(2)] public short NextNear;
[FieldOffset(4)] public short PreviousLevel; [FieldOffset(4)] public short PreviousLevel;
[FieldOffset(6)] public short PreviousNear; [FieldOffset(6)] public short PreviousNear;
public Ptr64 Position #endregion
{
readonly get => PositionMask;
set => PositionMask = value;
}
public Ptr64 SizeForAllocator #region buffer
{
readonly get => (Ptr64)(SizeMask & Mask48Bits);
set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
}
public Ptr64 SizeForSpare [FieldOffset(8)] public uint PositionMask;
{ [FieldOffset(12)] public uint SizeMask;
readonly get => SizeMask;
set => SizeMask = value;
}
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
#endregion #endregion
#region cache line information #region mask
[FieldOffset(34)] internal ushort Profile; [FieldOffset(16)] internal AllocationState State;
[FieldOffset(24)] internal nuint Origin; [FieldOffset(17)] internal byte ReversedProfile;
[FieldOffset(18)] internal ushort CustomizeProfile;
[FieldOffset(20)] internal int RefCount;
#endregion #endregion
} }
[StructLayout(LayoutKind.Sequential)] #pragma warning disable IDE0032
public unsafe ref struct Handle : IDisposable [StructLayout(LayoutKind.Explicit)]
public unsafe struct Handle : IDisposable
{ {
private CacheLineMetadata* _head; [FieldOffset(8)] private readonly CacheLineMetadata* _head;
private nuint _handle; [FieldOffset(0)] private nuint _handle;
private Handle(nuint handle, CacheLineMetadata* cacheRefCount) private Handle(
nuint handle,
CacheLineMetadata* cacheRefCount
)
{ {
_handle = handle; _handle = handle;
_head = cacheRefCount; _head = cacheRefCount;
@@ -133,6 +112,6 @@ namespace StgSharp.HighPerformance.Memory
} }
} }
#pragma warning restore IDE0032
} }
} }
StgSharp.Common/HighPerformance/Memory/L4.Evict.cs
+9 -17
View File
@@ -2,8 +2,8 @@
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="L4.Evict" // file="L4.Evict"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload Space // AuthorGroup: Nitload
// Copyright (c) Nitload Space. All rights reserved. // Copyright (c) Nitload. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -25,18 +25,6 @@
// //
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
using StgSharp.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
using System.Text;
using System.Threading.Tasks;
using Tlsf = StgSharp.HighPerformance.Memory.TwoLayerSegregatedFitAllocator;
namespace StgSharp.HighPerformance.Memory namespace StgSharp.HighPerformance.Memory
{ {
public partial class L4 public partial class L4
@@ -52,7 +40,9 @@ namespace StgSharp.HighPerformance.Memory
/// Count of lines not evicted /// Count of lines not evicted
/// </returns> /// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe int Evict(int count = 1) private unsafe int Evict(
int count = 1
)
{ {
count = (count < 0) ? 0 : count; count = (count < 0) ? 0 : count;
int* pPtr = _predictCount, mPtr = _mapCount; int* pPtr = _predictCount, mPtr = _mapCount;
@@ -71,7 +61,9 @@ namespace StgSharp.HighPerformance.Memory
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe bool TryEvictAt(int index) private unsafe bool TryEvictAt(
int index
)
{ {
CacheLineMetadata* head = &_head[index]; CacheLineMetadata* head = &_head[index];
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0) if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
@@ -79,7 +71,7 @@ namespace StgSharp.HighPerformance.Memory
// EvictLine(i + index); // EvictLine(i + index);
_ = _map.TryRemove(head->Origin, out _); _ = _map.TryRemove(head->Origin, out _);
IL4Predict predict = CurrentPredict; IL4Predict predict = CurrentPredict;
predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan<byte>((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator)); predict.WriteBack(head->Origin, head->CustomizeProfile, new ReadOnlySpan<byte>((byte*)(ulong)(head->PositionMask + _baseAddress), (int)(ulong)head->SizeMask));
_bufferAllocator.Free(index); _bufferAllocator.Free(index);
_mapCount[index] = 0; _mapCount[index] = 0;
_predictCount[index] = int.MaxValue; _predictCount[index] = int.MaxValue;
StgSharp.Common/HighPerformance/Memory/L4.Exceptions.cs
-6
View File
@@ -25,12 +25,6 @@
// //
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory namespace StgSharp.HighPerformance.Memory
{ {
public sealed class L4PredictUnregisteredException : Exception public sealed class L4PredictUnregisteredException : Exception
StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs
+4 -4
View File
@@ -2,8 +2,8 @@
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="L4.Prefetch" // file="L4.Prefetch"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload Space // AuthorGroup: Nitload
// Copyright (c) Nitload Space. All rights reserved. // Copyright (c) Nitload. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -105,14 +105,14 @@ namespace StgSharp.HighPerformance.Memory
} }
head = _head + index; head = _head + index;
head->Origin = address; head->Origin = address;
head->Profile = pHandle.MapPolicy; head->CustomizeProfile = pHandle.MapPolicy;
head->RefCount = 0; head->RefCount = 0;
_predictCount[index] = 1; _predictCount[index] = 1;
_mapCount[index] = 1; _mapCount[index] = 1;
// pre fetch them here // pre fetch them here
predict.Prefetch(head->Origin, head->Profile, new Span<byte>((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator)); predict.Prefetch(head->Origin, head->CustomizeProfile, new Span<byte>((byte*)(head->PositionMask + _baseAddress), (int)head->SizeMask));
_ = _map.TryAddOrSet(address, (nuint)head, out _); _ = _map.TryAddOrSet(address, (nuint)head, out _);
} }
_ = Interlocked.Add(ref _aheadCount, actualCount); _ = Interlocked.Add(ref _aheadCount, actualCount);
StgSharp.Common/HighPerformance/Memory/L4.Tlsf.cs
+90 -54
View File
@@ -2,8 +2,8 @@
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="L4.Tlsf" // file="L4.Tlsf"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload Space // AuthorGroup: Nitload
// Copyright (c) Nitload Space. All rights reserved. // Copyright (c) Nitload. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -53,15 +53,19 @@ namespace StgSharp.HighPerformance.Memory
private int _metadataEnd; private int _metadataEnd;
private readonly Ptr64 _basePtr; private readonly Ptr64 _basePtr;
private readonly ulong _maxAllocSize; private readonly uint _maxAllocSize;
private readonly ulong _totalSize; private readonly uint _totalSize;
internal TLSF(nuint byteSize, ulong maxBlockSize, void* externalPointer) internal TLSF(
uint byteSize,
uint maxBlockSize,
void* externalPointer
)
{ {
_totalSize = byteSize; _totalSize = byteSize;
_basePtr = (Ptr64)(ulong)externalPointer; _basePtr = (Ptr64)(ulong)externalPointer;
_align = 4096; // align of a memory page _align = 4096; // align of a memory page
_maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, CacheLineMetadata.Mask48Bits))); _maxAllocSize = uint.Max(64, (~64U) & (uint.Min(maxBlockSize, CacheLineMetadata.MaxBufferSize) + 63U));
_maxLevel = int.MaxValue; _maxLevel = int.MaxValue;
_maxLevel = GetLevel(_maxAllocSize); _maxLevel = GetLevel(_maxAllocSize);
_bucket = new short[GetLevel(_maxAllocSize) + 2]; _bucket = new short[GetLevel(_maxAllocSize) + 2];
@@ -70,8 +74,8 @@ namespace StgSharp.HighPerformance.Memory
// init spare memory node // init spare memory node
_spareMemory->Position = _basePtr; _spareMemory->PositionMask = 0;
_spareMemory->SizeForSpare = _totalSize - CacheLineMetadata.SizeOf; _spareMemory->SizeMask = _totalSize;
_spareMemory->NextLevel = 0; _spareMemory->NextLevel = 0;
_spareMemory->NextNear = 0; _spareMemory->NextNear = 0;
_spareMemory->PreviousLevel = 0; _spareMemory->PreviousLevel = 0;
@@ -79,9 +83,11 @@ namespace StgSharp.HighPerformance.Memory
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private int GetLevel(ulong size) private int GetLevel(
uint size
)
{ {
int origin = 57 - BitOperations.LeadingZeroCount(size); int origin = 25 - BitOperations.LeadingZeroCount(size);
return int.Min(origin, _maxLevel); return int.Min(origin, _maxLevel);
} }
@@ -91,10 +97,12 @@ namespace StgSharp.HighPerformance.Memory
/// <param name="node"> /// <param name="node">
/// ///
/// </param> /// </param>
private void RecycleNode(CacheLineMetadata* node) private void RecycleNode(
CacheLineMetadata* node
)
{ {
short nodeIdx = (short)(_spareMemory - node); short nodeIdx = (short)(_spareMemory - node);
int level = GetLevel(node->SizeForAllocator); int level = GetLevel(node->SizeMask);
short nextLevelIdx = node->NextLevel; short nextLevelIdx = node->NextLevel;
short prevLevelIdx = node->PreviousLevel; short prevLevelIdx = node->PreviousLevel;
RemoveFromBucket(level, node, nodeIdx); RemoveFromBucket(level, node, nodeIdx);
@@ -115,8 +123,8 @@ namespace StgSharp.HighPerformance.Memory
node->PreviousNear = 0; node->PreviousNear = 0;
node->NextNear = 0; node->NextNear = 0;
node->Position = 0; node->PositionMask = 0;
node->SizeForSpare = 0; node->SizeMask = 0;
node->State = AllocationState.SpareOrCollect; node->State = AllocationState.SpareOrCollect;
node->PreviousLevel = 0; node->PreviousLevel = 0;
@@ -131,12 +139,19 @@ namespace StgSharp.HighPerformance.Memory
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private void RemoveFromBucket(CacheLineMetadata* node, int nodeIdx) private void RemoveFromBucket(
CacheLineMetadata* node,
int nodeIdx
)
{ {
RemoveFromBucket(GetLevel(node->SizeForAllocator), node, nodeIdx); RemoveFromBucket(GetLevel(node->SizeMask), node, nodeIdx);
} }
private void RemoveFromBucket(int level, CacheLineMetadata* node, int nodeIdx) private void RemoveFromBucket(
int level,
CacheLineMetadata* node,
int nodeIdx
)
{ {
int currentLevel = level; int currentLevel = level;
short currentNextLevelIdx = node->NextLevel; short currentNextLevelIdx = node->NextLevel;
@@ -160,7 +175,11 @@ namespace StgSharp.HighPerformance.Memory
} }
// Assuming node to remove is head f bucket // Assuming node to remove is head f bucket
private void RemoveFromBucketHead(int level, CacheLineMetadata* node, int nodeIdx) private void RemoveFromBucketHead(
int level,
CacheLineMetadata* node,
int nodeIdx
)
{ {
int currentLevel = level; int currentLevel = level;
short currentNextLevelIdx = node->NextLevel; short currentNextLevelIdx = node->NextLevel;
@@ -187,10 +206,13 @@ namespace StgSharp.HighPerformance.Memory
node->NextLevel = 0; node->NextLevel = 0;
} }
private bool TryAllocNewChunk(ulong size, out CacheLineMetadata* meta) private bool TryAllocNewChunk(
uint size,
out CacheLineMetadata* meta
)
{ {
short idx = _spareMemory->NextLevel; short idx = _spareMemory->NextLevel;
ulong spareSize = _spareMemory->SizeForAllocator; uint spareSize = _spareMemory->SizeMask;
if (idx != 0) if (idx != 0)
{ {
if (spareSize < size) if (spareSize < size)
@@ -211,13 +233,13 @@ namespace StgSharp.HighPerformance.Memory
return false; return false;
} }
spareSize -= size; spareSize -= size;
meta->SizeForAllocator = size; meta->SizeMask = size;
meta->Position = _spareMemory->Position; meta->PositionMask = _spareMemory->PositionMask;
meta->State = AllocationState.Free; meta->State = AllocationState.Free;
meta->PreviousLevel = 0; meta->PreviousLevel = 0;
meta->NextLevel = 0; meta->NextLevel = 0;
_spareMemory->SizeForAllocator = spareSize; _spareMemory->SizeMask = spareSize;
_spareMemory->Position += size; _spareMemory->PositionMask += size;
// insert the new chunk to position chain // insert the new chunk to position chain
short nearIdx = _spareMemory->PreviousNear; short nearIdx = _spareMemory->PreviousNear;
@@ -229,9 +251,11 @@ namespace StgSharp.HighPerformance.Memory
return true; return true;
} }
private bool TryGetNewMetadata(out CacheLineMetadata* meta) private bool TryGetNewMetadata(
out CacheLineMetadata* meta
)
{ {
ulong spareSize = _spareMemory->SizeForAllocator; ulong spareSize = _spareMemory->SizeMask;
if (_metadataEnd >= MaxCacheLineCount) if (_metadataEnd >= MaxCacheLineCount)
{ {
meta = null; meta = null;
@@ -242,7 +266,10 @@ namespace StgSharp.HighPerformance.Memory
return true; return true;
} }
private bool TryTakeBucketEntry(ulong minimumSize, out CacheLineMetadata* entry) private bool TryTakeBucketEntry(
uint minimumSize,
out CacheLineMetadata* entry
)
{ {
int level = GetLevel(minimumSize); int level = GetLevel(minimumSize);
@@ -252,7 +279,7 @@ namespace StgSharp.HighPerformance.Memory
while (idx != 0) while (idx != 0)
{ {
CacheLineMetadata* candidate = &_spareMemory[idx]; CacheLineMetadata* candidate = &_spareMemory[idx];
if (candidate->SizeForAllocator >= minimumSize) if (candidate->SizeMask >= minimumSize)
{ {
RemoveFromBucket(level, candidate, idx); RemoveFromBucket(level, candidate, idx);
entry = candidate; entry = candidate;
@@ -279,9 +306,11 @@ namespace StgSharp.HighPerformance.Memory
#region alloc and free #region alloc and free
public int Alloc(nuint size) public int Alloc(
uint size
)
{ {
ulong allocSize = ulong.Max(64, (~63UL) & ((ulong)size + 63)); uint allocSize = uint.Max(64, (~63U) & (size + 63));
if (allocSize > _maxAllocSize) { if (allocSize > _maxAllocSize) {
throw new OverflowException("Requested allocation size exceeds the maximum block size."); throw new OverflowException("Requested allocation size exceeds the maximum block size.");
} }
@@ -290,18 +319,19 @@ namespace StgSharp.HighPerformance.Memory
throw new OverflowException("No available "); throw new OverflowException("No available ");
} }
short entryIdx = (short)(_spareMemory - entry); short entryIdx = (short)(_spareMemory - entry);
ulong entrySize = entry->SizeForAllocator; uint entrySize = entry->SizeMask;
if (entrySize < allocSize) { if (entrySize < allocSize) {
throw new OverflowException("Bucket returned an undersized block."); throw new OverflowException("Bucket returned an undersized block.");
} }
ulong remain = entrySize - allocSize; uint remain = entrySize - allocSize;
// slice tail // slice tail
if (remain > ulong.Min(4096UL, entrySize >> 7) && TryGetNewMetadata(out CacheLineMetadata* newEntry)) if (remain > ulong.Min(4096UL, entrySize >> 7) &&
TryGetNewMetadata(out CacheLineMetadata* newEntry))
{ {
short newEntryIdx = (short)(_spareMemory - newEntry); short newEntryIdx = (short)(_spareMemory - newEntry);
newEntry->Position = entry->Position + allocSize; newEntry->PositionMask = entry->PositionMask + allocSize;
newEntry->SizeForAllocator = remain; newEntry->SizeMask = remain;
newEntry->State = AllocationState.Free; newEntry->State = AllocationState.Free;
int newLevel = GetLevel(remain); int newLevel = GetLevel(remain);
short bucketHeadIdx = _bucket[newLevel]; short bucketHeadIdx = _bucket[newLevel];
@@ -325,7 +355,7 @@ namespace StgSharp.HighPerformance.Memory
newEntry->PreviousNear = entryIdx; newEntry->PreviousNear = entryIdx;
} }
entry->State = AllocationState.Alloc; entry->State = AllocationState.Alloc;
if (entry->Position - (ulong)_basePtr > _totalSize) { if (entry->PositionMask - (ulong)_basePtr > _totalSize) {
throw new OverflowException("Allocated memory address exceeds the maximum block size."); throw new OverflowException("Allocated memory address exceeds the maximum block size.");
} }
return entryIdx; return entryIdx;
@@ -344,7 +374,7 @@ namespace StgSharp.HighPerformance.Memory
if (current->State == AllocationState.Free) if (current->State == AllocationState.Free)
{ {
// scan next // scan next
Ptr64 size = 0; uint size = 0;
int nextIdx = current->NextNear; int nextIdx = current->NextNear;
while (nextIdx != 0) while (nextIdx != 0)
{ {
@@ -353,15 +383,15 @@ namespace StgSharp.HighPerformance.Memory
{ {
break; break;
} }
size += next->SizeForAllocator; size += next->SizeMask;
RecycleNode(next); RecycleNode(next);
nextIdx = current->NextNear; nextIdx = current->NextNear;
} }
if (size > 0) if (size > 0)
{ {
current->SizeForAllocator += size; current->SizeMask += size;
RemoveFromBucket(current, currentIdx); RemoveFromBucket(current, currentIdx);
int newLevel = GetLevel(current->SizeForAllocator); int newLevel = GetLevel(current->SizeMask);
short bucketHeadIdx = _bucket[newLevel]; short bucketHeadIdx = _bucket[newLevel];
if (bucketHeadIdx != 0) if (bucketHeadIdx != 0)
{ {
@@ -377,11 +407,14 @@ namespace StgSharp.HighPerformance.Memory
} while (true); } while (true);
} }
public void Free(int handle, int collectScanWindow = 4) public void Free(
int handle,
int collectScanWindow = 4
)
{ {
CacheLineMetadata* head = &_spareMemory[handle]; CacheLineMetadata* head = &_spareMemory[handle];
short headIdx = (short)(_spareMemory - head); short headIdx = (short)(_spareMemory - head);
Ptr64 size = head->SizeForAllocator; uint size = head->SizeMask;
for (int i = 0; i < collectScanWindow; i++) for (int i = 0; i < collectScanWindow; i++)
{ {
// forward scan, previous direction // forward scan, previous direction
@@ -395,8 +428,8 @@ namespace StgSharp.HighPerformance.Memory
{ {
break; break;
} }
size += prev->SizeForAllocator; size += prev->SizeMask;
head->Position = prev->Position; head->PositionMask = prev->PositionMask;
RecycleNode(prev); RecycleNode(prev);
} }
for (int i = 0; i < collectScanWindow; i++) for (int i = 0; i < collectScanWindow; i++)
@@ -412,15 +445,15 @@ namespace StgSharp.HighPerformance.Memory
{ {
break; break;
} }
size += prev->SizeForAllocator; size += prev->SizeMask;
RecycleNode(prev); RecycleNode(prev);
} }
// recycle to level; // recycle to level;
head->SizeForAllocator = size; head->SizeMask = size;
head->State = AllocationState.Free; head->State = AllocationState.Free;
head->PreviousLevel = 0; head->PreviousLevel = 0;
int level = GetLevel(head->SizeForAllocator); int level = GetLevel(head->SizeMask);
short bucketHeadIdx = _bucket[level]; short bucketHeadIdx = _bucket[level];
head->NextLevel = 0; head->NextLevel = 0;
if (bucketHeadIdx != 0) if (bucketHeadIdx != 0)
@@ -433,9 +466,12 @@ namespace StgSharp.HighPerformance.Memory
return; return;
} }
public bool TryAlloc(nuint size, out int h) public bool TryAlloc(
uint size,
out int h
)
{ {
ulong allocSize = ulong.Max(64, (~63UL) & ((ulong)size + 63)); uint allocSize = uint.Max(64, (~63U) & (size + 63));
if (allocSize > _maxAllocSize) if (allocSize > _maxAllocSize)
{ {
h = -1; h = -1;
@@ -448,20 +484,20 @@ namespace StgSharp.HighPerformance.Memory
return false; return false;
} }
short entryIdx = (short)(_spareMemory - entry); short entryIdx = (short)(_spareMemory - entry);
ulong entrySize = entry->SizeForAllocator; uint entrySize = entry->SizeMask;
if (entrySize < allocSize) if (entrySize < allocSize)
{ {
h = -1; h = -1;
return false; return false;
} }
ulong remain = entrySize - allocSize; uint remain = entrySize - allocSize;
// slice tail // slice tail
if (remain > (entrySize >> 7) && TryGetNewMetadata(out CacheLineMetadata* newEntry)) if (remain > (entrySize >> 7) && TryGetNewMetadata(out CacheLineMetadata* newEntry))
{ {
short newEntryIdx = (short)(_spareMemory - newEntry); short newEntryIdx = (short)(_spareMemory - newEntry);
newEntry->Position = entry->Position + allocSize; newEntry->PositionMask = entry->PositionMask + allocSize;
newEntry->SizeForAllocator = remain; newEntry->SizeMask = remain;
newEntry->State = AllocationState.Free; newEntry->State = AllocationState.Free;
int newLevel = GetLevel(remain); int newLevel = GetLevel(remain);
short bucketHeadIdx = _bucket[newLevel]; short bucketHeadIdx = _bucket[newLevel];
@@ -487,7 +523,7 @@ namespace StgSharp.HighPerformance.Memory
newEntry->PreviousNear = entryIdx; newEntry->PreviousNear = entryIdx;
} }
entry->State = AllocationState.Alloc; entry->State = AllocationState.Alloc;
if (entry->Position - (ulong)_basePtr > _totalSize) if (entry->PositionMask - (ulong)_basePtr > _totalSize)
{ {
h = default; h = default;
throw new OverflowException("Allocated memory address exceeds the maximum block size."); throw new OverflowException("Allocated memory address exceeds the maximum block size.");
StgSharp.Common/HighPerformance/Memory/L4.cs
+19 -7
View File
@@ -2,8 +2,8 @@
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="L4" // file="L4"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload Space // AuthorGroup: Nitload
// Copyright (c) Nitload Space. All rights reserved. // Copyright (c) Nitload. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -69,8 +69,15 @@ namespace StgSharp.HighPerformance.Memory
private readonly TLSF _bufferAllocator; private readonly TLSF _bufferAllocator;
public L4(int bindedNuma, nuint bufferSize = 8192 * 8192, int align = 16, nuint maxCacheLineSize = 8192) public L4(
int bindedNuma,
uint bufferSize = 8192 * 8192,
int align = 16,
uint maxCacheLineSize = 8192
)
{ {
ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize);
ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize);
_bindedNuma = bindedNuma; _bindedNuma = bindedNuma;
// init order is fallow by align requirement of each component. // init order is fallow by align requirement of each component.
@@ -116,7 +123,9 @@ namespace StgSharp.HighPerformance.Memory
// public IL4Predict PrefetchPredict { get; set; } // public IL4Predict PrefetchPredict { get; set; }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ContinueOnce(Handle line) public void ContinueOnce(
Handle line
)
{ {
if (line.Address > 0) { if (line.Address > 0) {
_ = Interlocked.Decrement(ref _aheadCount); _ = Interlocked.Decrement(ref _aheadCount);
@@ -149,7 +158,9 @@ namespace StgSharp.HighPerformance.Memory
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public Handle Map(ref CacheLineDescription prediction) public Handle Map(
ref CacheLineDescription prediction
)
{ {
if ((_aheadCount < PredictionLeastAhead) && !_exhausted) if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
{ {
@@ -179,8 +190,9 @@ namespace StgSharp.HighPerformance.Memory
{ {
_predictCount[entry - _head] = 0; _predictCount[entry - _head] = 0;
_ = Interlocked.Decrement(ref _aheadCount); _ = Interlocked.Decrement(ref _aheadCount);
_ = Interlocked.Decrement(ref (*_mapCount)[entry->Id]); _ = Interlocked.Decrement(ref _mapCount[entry - _head]);
return new Handle((nuint)entry->Position, ref entry->RefCount); nuint* hPtr = stackalloc nuint[2] { entry->PositionMask + _baseAddress, (nuint)entry };
return *(Handle*)hPtr;
} else } else
{ {
_ = Interlocked.Decrement(ref entry->RefCount); _ = Interlocked.Decrement(ref entry->RefCount);
StgSharp.Native/sn-head/sn_intrinsic.h
+8
View File
@@ -47,6 +47,14 @@ typedef union scalar_pack {
uint64_t data[8]; uint64_t data[8];
} scalar_pack; } scalar_pack;
typedef struct sn_L4_cache_head {
uint64_t reverse_mask_0;
uint32_t position_mask;
uint32_t size;
uint64_t reverse_mask_1;
uint64_t reverse_mask_2;
} sn_L4_cache_head;
typedef struct matrix_task { typedef struct matrix_task {
void *mat_0; void *mat_0;
void *mat_1; void *mat_1;