308 lines
12 KiB
Plaintext
308 lines
12 KiB
Plaintext
//-----------------------------------------------------------------------
|
|
// -----------------------------------------------------------------------
|
|
// file="HLSFAllocator.SegmentEntry.cs"
|
|
// Project: StgSharp
|
|
// AuthorGroup: Nitload Space
|
|
// Copyright (c) Nitload Space. All rights reserved.
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person
|
|
// obtaining a copy of this software and associated documentation
|
|
// files (the “Software”), to deal in the Software without restriction,
|
|
// including without limitation the rights to use, copy, modify, merge,
|
|
// publish, distribute, sublicense, and/or sell copies of the Software,
|
|
// and to permit persons to whom the Software is furnished to do so,
|
|
// subject to the following conditions:
|
|
//
|
|
// The above copyright notice and
|
|
// this permission notice shall be included in all copies
|
|
// or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED “AS IS”,
|
|
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
|
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
|
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
|
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
|
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
|
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
//
|
|
// -----------------------------------------------------------------------
|
|
// -----------------------------------------------------------------------
|
|
using System;
|
|
using System.Runtime.CompilerServices;
|
|
using System.Runtime.InteropServices;
|
|
using System.Runtime.Serialization;
|
|
using System.Threading;
|
|
|
|
namespace StgSharp.HighPerformance.Memory
|
|
{
|
|
public unsafe partial class HybridLayerSegregatedFitAllocator
|
|
{
|
|
|
|
private static void AcquireEntrySpinLock(Entry* entry)
|
|
{
|
|
int thread = Environment.CurrentManagedThreadId;
|
|
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
|
|
{
|
|
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
|
|
Thread.Sleep(0);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Determine segment index based on size and level
|
|
/// </summary>
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
private int DetermineSegmentIndex(uint size, int level)
|
|
{
|
|
if (level < 0 || level >= levelSize.Length) {
|
|
return 0;
|
|
}
|
|
|
|
int segmentIndex = (int)size / levelSize[level];
|
|
return Math.Min(2, segmentIndex); // Ensure not exceeding maximum segment index
|
|
}
|
|
|
|
private void MergeEntry(Entry* remain, Entry* other)
|
|
{
|
|
if (other == null) {
|
|
return;
|
|
}
|
|
|
|
// Determine merged starting position and size
|
|
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
|
uint newSize = remain->BlockSize + other->BlockSize;
|
|
|
|
// Update current Entry
|
|
remain->Position = newPosition;
|
|
remain->BlockSize = newSize;
|
|
}
|
|
|
|
private void RecycleEntry(Entry* entry)
|
|
{
|
|
if (entry is null) {
|
|
return;
|
|
}
|
|
entry-> PreviousNear = EmptyHandle;
|
|
entry-> Position = EmptyHandle;
|
|
_nodes.Free((nuint)entry);
|
|
}
|
|
|
|
private static void ReleaseEntrySpinLock(Entry* entry)
|
|
{
|
|
int thread = Environment.CurrentManagedThreadId;
|
|
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
|
|
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Remove node from position linked list
|
|
/// </summary>
|
|
private void RemoveFromPositionChain(Entry* entry)
|
|
{
|
|
Entry* prev = (Entry*)entry->PreviousNear;
|
|
Entry* next = (Entry*)entry->NextNear;
|
|
|
|
if (prev != null) {
|
|
prev->NextNear = entry->NextNear;
|
|
}
|
|
if (next != null) {
|
|
next->PreviousNear = entry->PreviousNear;
|
|
}
|
|
|
|
// Clear node pointers
|
|
entry->NextNear = EmptyHandle;
|
|
entry->PreviousNear = EmptyHandle;
|
|
}
|
|
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
|
{
|
|
int thread = Environment.CurrentManagedThreadId;
|
|
return entry == null ||
|
|
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
|
|
0) == 0 ||
|
|
Volatile.Read(ref entry->SpinLock) == thread);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks
|
|
/// </summary>
|
|
private Entry* TryMergeAndRemoveFromBuckets(Entry* entry)
|
|
{
|
|
if (entry == null) {
|
|
return null;
|
|
}
|
|
|
|
Entry* result = entry;
|
|
uint originalSize = entry->BlockSize;
|
|
int originalLevel = GetLevelFromSize(originalSize);
|
|
int originalSegment = DetermineSegmentIndex(originalSize, originalLevel);
|
|
|
|
// Forward merge: check predecessor nodes in position linked list
|
|
Entry* prev = (Entry*)entry->PreviousNear;
|
|
while (prev != null && prev != _spareMemory && prev->State == EntryState.Empty && result->IsAdjacentTo(prev))
|
|
{
|
|
// Try acquire spin lock for prev entry
|
|
if (!TryAcquireEntrySpinLock(prev))
|
|
{
|
|
break; // Cannot acquire lock, might be operated by other thread
|
|
}
|
|
|
|
try
|
|
{
|
|
// Double-check state after acquiring lock
|
|
if (prev->State != EntryState.Empty || !result->IsAdjacentTo(prev))
|
|
{
|
|
break;
|
|
}
|
|
|
|
// Calculate prev's bucket position
|
|
uint prevSize = prev->BlockSize;
|
|
int prevLevel = GetLevelFromSize(prevSize);
|
|
int prevSegment = DetermineSegmentIndex(prevSize, prevLevel);
|
|
|
|
// Remove prev node from bucket (O(1) operation!)
|
|
BucketNode* prevBucketNode = ValidateOrRepairBucketNode(prev);
|
|
if (prevBucketNode != null && RemoveFromBucket(prevLevel, prevSegment, prevBucketNode))
|
|
{
|
|
// Merge into result
|
|
result->MergeWith(prev);
|
|
|
|
// Remove prev from position linked list
|
|
RemoveFromPositionChain(prev);
|
|
|
|
// Free prev's Entry node
|
|
_nodes.Free((nuint)prev);
|
|
|
|
// Continue searching forward
|
|
prev = (Entry*)result->PreviousNear;
|
|
continue;
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
ReleaseEntrySpinLock(prev);
|
|
}
|
|
|
|
break; // Cannot remove, might be operated by other thread
|
|
}
|
|
|
|
// Backward merge: check successor nodes in position linked list
|
|
Entry* next = (Entry*)entry->NextNear;
|
|
while (next != null && next->State == EntryState.Empty && result->IsAdjacentTo(next))
|
|
{
|
|
// Try acquire spin lock for next entry
|
|
if (!TryAcquireEntrySpinLock(next))
|
|
{
|
|
break; // Cannot acquire lock, might be operated by other thread
|
|
}
|
|
|
|
try
|
|
{
|
|
// Double-check state after acquiring lock
|
|
if (next->State != EntryState.Empty || !result->IsAdjacentTo(next))
|
|
{
|
|
break;
|
|
}
|
|
|
|
// Calculate next's bucket position
|
|
uint nextSize = next->BlockSize;
|
|
int nextLevel = GetLevelFromSize(nextSize);
|
|
int nextSegment = DetermineSegmentIndex(nextSize, nextLevel);
|
|
|
|
// Remove next node from bucket (O(1) operation!)
|
|
BucketNode* nextBucketNode = ValidateOrRepairBucketNode(next);
|
|
if (nextBucketNode != null && RemoveFromBucket(nextLevel, nextSegment, nextBucketNode))
|
|
{
|
|
// Merge into result
|
|
result->MergeWith(next);
|
|
|
|
// Remove next from position linked list
|
|
RemoveFromPositionChain(next);
|
|
|
|
// Free next's Entry node
|
|
_nodes.Free((nuint)next);
|
|
|
|
// Continue searching backward
|
|
next = (Entry*)result->NextNear;
|
|
continue;
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
ReleaseEntrySpinLock(next);
|
|
}
|
|
|
|
break; // Cannot remove, might be operated by other thread
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Entry structure - streamlined version (only position and state information) Stored in
|
|
/// external SLAB
|
|
/// </summary>
|
|
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
|
internal struct Entry
|
|
{
|
|
|
|
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
|
[FieldOffset(28)] internal int State; // Node state
|
|
[FieldOffset(32)] internal uint BlockSize; // Block size information
|
|
[FieldOffset(36)] internal uint Reverse;
|
|
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
|
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
|
[FieldOffset(0)] public nuint Position; // Memory block position
|
|
|
|
/// <summary>
|
|
/// Check if two Entries are adjacent in memory
|
|
/// </summary>
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public readonly bool IsAdjacentTo(Entry* other)
|
|
{
|
|
if (other == null) {
|
|
return false;
|
|
}
|
|
|
|
// Check if this immediately follows other
|
|
return other->Position + other->BlockSize == this.Position ||
|
|
this.Position + this.BlockSize == other->Position;
|
|
}
|
|
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static bool IsNullOrEmptyIndex(ulong index)
|
|
{
|
|
return index == EmptyHandle;
|
|
}
|
|
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static bool IsNullOrEmptyIndex(Entry* entry)
|
|
{
|
|
return entry == null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
|
/// state)
|
|
/// </summary>
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public void MergeWith(Entry* other) { }
|
|
|
|
}
|
|
|
|
private static class EntryState
|
|
{
|
|
|
|
public const int Allocated = 1; // Already allocated to user
|
|
public const int Empty = 0; // Free, can be allocated
|
|
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
|
|
|
}
|
|
|
|
}
|
|
}
|