Abort hlsf, goes back to tlsf, may use 'Arena-ed tlsf'

StgSharp.Benchmark/HlsfAllocatorBench.cs

@@ -53,12 +53,12 @@ namespace StgSharp.Benchmark
             1024,
             4096
         ];
-        private HybridLayerSegregatedFitAllocator _hlsfAllocator = null!;
+        private readonly int[] _freeOrderByBatch = new int[TotalOperationsPerInvoke];
+        private readonly uint[] _requestSizesByBatch = new uint[TotalOperationsPerInvoke];
         private readonly List<HlsfHandle> _hlsfLocalList = new();
         private readonly List<nint> _libcLocalList = new();
-        private readonly uint[] _requestSizesByBatch = new uint[TotalOperationsPerInvoke];
-        private readonly int[] _freeOrderByBatch = new int[TotalOperationsPerInvoke];
         private Random _random = new(42);
+        private HybridLayerSegregatedFitAllocator _hlsfAllocator = null!;
 
         [GlobalCleanup]
         public void Cleanup()
@@ -114,6 +114,21 @@ namespace StgSharp.Benchmark
             return (size + mask) & ~mask;
         }
 
+        private void BuildIterationWorkload()
+        {
+            for (int batch = 0; batch < BatchCount; batch++)
+            {
+                int batchOffset = batch * AllocationCount;
+                for (int i = 0; i < AllocationCount; i++)
+                {
+                    _requestSizesByBatch[batchOffset + i] = AlignSize((uint)AllocSizes[_random.Next(AllocSizes.Length)]);
+                    _freeOrderByBatch[batchOffset + i] = i;
+                }
+
+                ShuffleFreeOrder(batchOffset, AllocationCount);
+            }
+        }
+
         private static nuint GetArenaBytes()
         {
             const int defaultMb = 512;
@@ -152,7 +167,7 @@ namespace StgSharp.Benchmark
         }
 
         private unsafe void RunLibc(
-                     int batch
+                            int batch
         )
         {
             int batchOffset = batch * AllocationCount;
@@ -173,21 +188,6 @@ namespace StgSharp.Benchmark
             }
         }
 
-        private void BuildIterationWorkload()
-        {
-            for (int batch = 0; batch < BatchCount; batch++)
-            {
-                int batchOffset = batch * AllocationCount;
-                for (int i = 0; i < AllocationCount; i++)
-                {
-                    _requestSizesByBatch[batchOffset + i] = AlignSize((uint)AllocSizes[_random.Next(AllocSizes.Length)]);
-                    _freeOrderByBatch[batchOffset + i] = i;
-                }
-
-                ShuffleFreeOrder(batchOffset, AllocationCount);
-            }
-        }
-
         private void ShuffleFreeOrder(
                      int offset,
                      int count

StgSharp.Common/HighPerformance/Memory/Ptr64.cs

@@ -0,0 +1,384 @@
+//-----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+// file="Ptr64"
+// Project: StgSharp
+// AuthorGroup: Nitload
+// Copyright (c) Nitload. All rights reserved.
+//     
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//     
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Globalization;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StgSharp.HighPerformance.Memory
+{
+#pragma warning disable CA2225
+
+    public readonly struct Ptr64 : IEquatable<Ptr64>, IComparable<Ptr64>, IFormattable
+    {
+
+        public Ptr64(
+               ulong value
+        )
+        {
+            Value = value;
+        }
+
+        public ulong Value { get; }
+
+        public int CompareTo(
+                   Ptr64 other
+        )
+        {
+            return Value.CompareTo(other.Value);
+        }
+
+        public bool Equals(
+                    Ptr64 other
+        )
+        {
+            return Value == other.Value;
+        }
+
+        public override bool Equals(
+                             object? obj
+        )
+        {
+            return obj is Ptr64 other && Equals(other);
+        }
+
+        public override int GetHashCode()
+        {
+            return Value.GetHashCode();
+        }
+
+        public override string ToString()
+        {
+            return Value.ToString();
+        }
+
+        public string ToString(
+                      string? format,
+                      IFormatProvider? formatProvider
+        )
+        {
+            return Value.ToString(format, formatProvider);
+        }
+
+        public static Ptr64 operator -(
+                                     Ptr64 left,
+                                     ulong right
+        )
+        {
+            return new(left.Value - right);
+        }
+
+        public static ulong operator -(
+                                     Ptr64 left,
+                                     Ptr64 right
+        )
+        {
+            return left.Value - right.Value;
+        }
+
+        public static bool operator !=(
+                                    Ptr64 left,
+                                    Ptr64 right
+        )
+        {
+            return left.Value != right.Value;
+        }
+
+        public static Ptr64 operator &(
+                                     Ptr64 left,
+                                     ulong mask
+        )
+        {
+            return new(left.Value & mask);
+        }
+
+        public static Ptr64 operator |(
+                                     Ptr64 left,
+                                     ulong mask
+        )
+        {
+            return new(left.Value | mask);
+        }
+
+        public static Ptr64 operator +(
+                                     Ptr64 left,
+                                     ulong right
+        )
+        {
+            return new(left.Value + right);
+        }
+
+        public static bool operator <(
+                                    Ptr64 left,
+                                    Ptr64 right
+        )
+        {
+            return left.Value < right.Value;
+        }
+
+        public static Ptr64 operator <<(
+                                     Ptr64 left,
+                                     int shift
+        )
+        {
+            return new Ptr64(left.Value << shift);
+        }
+
+        public static bool operator <=(
+                                    Ptr64 left,
+                                    Ptr64 right
+        )
+        {
+            return left.Value <= right.Value;
+        }
+
+        public static bool operator ==(
+                                    Ptr64 left,
+                                    Ptr64 right
+        )
+        {
+            return left.Value == right.Value;
+        }
+
+        public static bool operator >(
+                                    Ptr64 left,
+                                    Ptr64 right
+        )
+        {
+            return left.Value > right.Value;
+        }
+
+        public static bool operator >=(
+                                    Ptr64 left,
+                                    Ptr64 right
+        )
+        {
+            return left.Value >= right.Value;
+        }
+
+        public static Ptr64 operator >>(
+                                     Ptr64 left,
+                                     int shift
+        )
+        {
+            return new Ptr64(left.Value >> shift);
+        }
+
+        public static implicit operator Ptr64(
+                                        ulong value
+        )
+        {
+            return new(value);
+        }
+
+        public static implicit operator ulong(
+                                        Ptr64 value
+        )
+        {
+            return value.Value;
+        }
+
+    }
+
+    public readonly struct SPtr64 : IEquatable<SPtr64>, IComparable<SPtr64>, IFormattable
+    {
+
+        public SPtr64(
+               long value
+        )
+        {
+            Value = value;
+        }
+
+        public long Value { get; }
+
+        public int CompareTo(
+                   SPtr64 other
+        )
+        {
+            return Value.CompareTo(other.Value);
+        }
+
+        public bool Equals(
+                    SPtr64 other
+        )
+        {
+            return Value == other.Value;
+        }
+
+        public override bool Equals(
+                             object? obj
+        )
+        {
+            return obj is SPtr64 other && Equals(other);
+        }
+
+        public override int GetHashCode()
+        {
+            return Value.GetHashCode();
+        }
+
+        public override string ToString()
+        {
+            return Value.ToString();
+        }
+
+        public string ToString(
+                      string? format,
+                      IFormatProvider? formatProvider
+        )
+        {
+            return Value.ToString(format, formatProvider);
+        }
+
+        public static SPtr64 operator -(
+                                      SPtr64 left,
+                                      long right
+        )
+        {
+            return new(left.Value - right);
+        }
+
+        public static long operator -(
+                                    SPtr64 left,
+                                    SPtr64 right
+        )
+        {
+            return left.Value - right.Value;
+        }
+
+        public static bool operator !=(
+                                    SPtr64 left,
+                                    SPtr64 right
+        )
+        {
+            return left.Value != right.Value;
+        }
+
+        public static SPtr64 operator &(
+                                      SPtr64 left,
+                                      long mask
+        )
+        {
+            return new(left.Value & mask);
+        }
+
+        public static SPtr64 operator |(
+                                      SPtr64 left,
+                                      long mask
+        )
+        {
+            return new(left.Value | mask);
+        }
+
+        public static SPtr64 operator +(
+                                      SPtr64 left,
+                                      long right
+        )
+        {
+            return new(left.Value + right);
+        }
+
+        public static bool operator <(
+                                    SPtr64 left,
+                                    SPtr64 right
+        )
+        {
+            return left.Value < right.Value;
+        }
+
+        public static SPtr64 operator <<(
+                                      SPtr64 left,
+                                      int shift
+        )
+        {
+            return new SPtr64(left.Value << shift);
+        }
+
+        public static bool operator <=(
+                                    SPtr64 left,
+                                    SPtr64 right
+        )
+        {
+            return left.Value <= right.Value;
+        }
+
+        public static bool operator ==(
+                                    SPtr64 left,
+                                    SPtr64 right
+        )
+        {
+            return left.Value == right.Value;
+        }
+
+        public static bool operator >(
+                                    SPtr64 left,
+                                    SPtr64 right
+        )
+        {
+            return left.Value > right.Value;
+        }
+
+        public static bool operator >=(
+                                    SPtr64 left,
+                                    SPtr64 right
+        )
+        {
+            return left.Value >= right.Value;
+        }
+
+        public static SPtr64 operator >>(
+                                      SPtr64 left,
+                                      int shift
+        )
+        {
+            return new SPtr64(left.Value >> shift);
+        }
+
+        public static implicit operator long(
+                                        SPtr64 value
+        )
+        {
+            return value.Value;
+        }
+
+        public static implicit operator SPtr64(
+                                        long value
+        )
+        {
+            return new(value);
+        }
+
+    }
+
+#pragma warning disable CA2225
+}

StgSharp.Common/HighPerformance/Memory/TwoLayerSegregatedFitAllocator.cs

@@ -0,0 +1,215 @@
+//-----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+// file="TwoLayerSegregatedFitAllocator"
+// Project: StgSharp
+// AuthorGroup: Nitload
+// Copyright (c) Nitload. All rights reserved.
+//     
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//     
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+using StgSharp.HighPerformance.Memory;
+using StgSharp.HighPerformance.ProcessorAbstraction;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Numerics;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Threading.Tasks;
+using static StgSharp.HighPerformance.Memory.TwoLayerSegregatedFitAllocator;
+
+namespace StgSharp.HighPerformance.Memory
+{
+    public unsafe class TwoLayerSegregatedFitAllocator : IDisposable
+    {
+
+        private const ulong Mask48Bits = 0x0000FFFFFFFFFFFF;
+
+        private Metadata* _allocEnd;
+        private readonly Metadata*[] _bucket;
+        private readonly Metadata* _spareMemory;
+        private readonly Action<nuint> _freeHandle;
+        private bool disposedValue;
+        private readonly int _align;
+        private readonly Ptr64 _basePtr;
+        private readonly ulong _maxAllocSize;
+        private readonly ulong _totalSize;
+
+        internal TwoLayerSegregatedFitAllocator(
+                 nuint byteSize,
+                 int align,
+                 ulong maxBlockSize,
+                 Func<nuint, int, nuint> allocator,
+                 Action<nuint> freeHandle
+        )
+            : this(byteSize, align, maxBlockSize, (void*)allocator(byteSize, align), freeHandle) { }
+
+        internal TwoLayerSegregatedFitAllocator(
+                 nuint byteSize,
+                 int align,
+                 ulong maxBlockSize,
+                 void* externalPointer,
+                 Action<nuint> freeHandle
+        )
+        {
+            _totalSize = byteSize;
+            _basePtr = (Ptr64)(ulong)externalPointer;
+            _align = align;
+            _maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, Mask48Bits)));
+            _bucket = new Metadata*[58 - BitOperations.LeadingZeroCount(_maxAllocSize)];
+            _spareMemory = &((Metadata*)(ulong)(_basePtr + _totalSize))[-1];
+            _freeHandle = freeHandle ?? (static_ => { });
+            _allocEnd = _spareMemory;
+
+            // init spare memory node
+
+            _spareMemory->Position = 0UL;
+            _spareMemory->Size = _totalSize - Metadata.SizeOf;
+            _spareMemory->NextLevel = 0;
+            _spareMemory->NextNear = 0;
+            _spareMemory->PreviousLevel = 0;
+            _spareMemory->PreviousNear = 0;
+        }
+
+        public TwoLayerSegregatedFitAllocator(
+               nuint byteSize
+        )
+            : this(byteSize, 16, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16),
+                   ptr => NativeMemory.AlignedFree((void*)ptr)) { }
+
+        public TwoLayerSegregatedFitAllocator(
+               nuint byteSize,
+               uint maxBlockSize
+        )
+            : this(byteSize, 16, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16),
+                   ptr => NativeMemory.AlignedFree((void*)ptr)) { }
+
+        public TwoLayerSegregatedFitAllocator(
+               nuint byteSize,
+               int align
+        )
+            : this(byteSize, align, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16),
+                   ptr => NativeMemory.AlignedFree((void*)ptr)) { }
+
+        public TwoLayerSegregatedFitAllocator(
+               nuint byteSize,
+               int align,
+               nuint maxBlockSize
+        )
+            : this(byteSize, align, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16),
+                   ptr => NativeMemory.AlignedFree((void*)ptr)) { }
+
+        public Handle Alloc(
+                      nuint size
+        )
+        {
+            ulong allocSize = nuint.Max(64, (~64U) & (size + 63));
+            if (allocSize > _maxAllocSize) {
+                throw new OverflowException("Requested allocation size exceeds the maximum block size.");
+            }
+            int level = 57 - BitOperations.LeadingZeroCount(allocSize);
+            Metadata* entry = null;
+            for (int i = level; i < _bucket.Length; i++)
+            {
+                entry = _bucket[i];
+                if (entry is not null)
+                {
+                    _bucket[i] = entry->NextLevel != 0 ? &_spareMemory[-entry->NextLevel] : null;
+                    break;
+                }
+            }
+            if (entry is null)
+            {
+                // alloc new from spare
+            }
+            ulong remain = entry->Size - allocSize;
+            throw new NotImplementedException();
+        }
+
+        public void Dispose()
+        {
+            Dispose(disposing:true);
+            GC.SuppressFinalize(this);
+        }
+
+        public void Free(
+                    Handle handle
+        )
+        {
+            throw new NotImplementedException();
+        }
+
+        protected virtual void Dispose(
+                               bool disposing
+        )
+        {
+            if (!disposedValue)
+            {
+                if (disposing) {
+                    _freeHandle((nuint)_basePtr);
+                }
+
+                disposedValue = true;
+            }
+        }
+
+        [StructLayout(LayoutKind.Sequential)]
+        public struct Handle
+        {
+
+            internal readonly Metadata* Meta;
+            public readonly void* Pointer;
+
+        }
+
+        [StructLayout(LayoutKind.Explicit, Size = 32)]
+        internal struct Metadata
+        {
+
+            [FieldOffset(24)] private ulong PositionMask;
+            [FieldOffset(16)] private ulong SizeMask;
+
+            [FieldOffset(0)] public int NextLevel;
+            [FieldOffset(4)] public int NextNear;
+            [FieldOffset(8)] public int PreviousLevel;
+            [FieldOffset(12)] public int PreviousNear;
+
+            [FieldOffset(22)] public ushort Mask1;
+            [FieldOffset(30)] public ushort Mask2;
+
+            public Ptr64 Position
+            {
+                readonly get => (Ptr64)(PositionMask & Mask48Bits);
+                set => PositionMask = (PositionMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
+            }
+
+            public Ptr64 Size
+            {
+                readonly get => (Ptr64)(SizeMask & Mask48Bits);
+                set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
+            }
+
+            public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<Metadata>();
+
+        }
+
+    }
+}

StgSharp.Common/HighPerformance/Memory/old/HLSFAllocator.SegmentEntry.cs

@@ -183,13 +183,25 @@ namespace StgSharp.HighPerformance.Memory
         internal struct Entry
         {
 
-            [FieldOffset(0)]  internal Entry* NextNear;      // Position linked list: next pointer
+            [FieldOffset(0)] internal Entry* NextNear;      // Position linked list: next pointer
             [FieldOffset(8)] internal Entry* PreviousNear;  // Position linked list: previous pointer
 
             [FieldOffset(32)] internal int Level;           // Level of the block in the allocator
             [FieldOffset(16)] internal nuint Size;           // Block size information
             [FieldOffset(36)] public EntryState State;
-            [FieldOffset(24)]  public nuint Position;        // Memory block position
+            [FieldOffset(24)] public nuint Position;        // Memory block position
+
+        }
+
+        [StructLayout(LayoutKind.Explicit, Size = 32)]
+        internal struct Metadata
+        {
+
+            [FieldOffset(0)] internal int NextLevel;
+            [FieldOffset(4)] internal int NextNear;
+            [FieldOffset(8)] internal int PreviousLevel;
+            [FieldOffset(12)] internal int PreviousNear;
+            [FieldOffset(16)] internal nuint Size;
 
         }
 

StgSharp.Common/HighPerformance/Memory/old/HLSFAllocator.cs

@@ -38,6 +38,7 @@ namespace StgSharp.HighPerformance.Memory
 
         // Bucket system field - directly in main allocator class
         private readonly BucketNode*[] _bucketHeads;
+        private Metadata* _metadata;
 
         // _spareMemory is also the beginning position of allocator
         private Entry* _spareMemory;
@@ -46,9 +47,12 @@ namespace StgSharp.HighPerformance.Memory
         private readonly Action<nuint> _freeHandle;
         private bool disposedValue;
         private readonly int _align;
+        private readonly int _maxAllocCount;
         private readonly int _maxLevel;
+        private int unusedMetaData;
         private readonly nuint _maxAllocSize;
         private readonly nuint _size;
+        private Ptr64 _basePtr;
         private readonly SlabAllocator<Entry> _entries;
 
         internal HybridLayerSegregatedFitAllocator(
@@ -78,6 +82,8 @@ namespace StgSharp.HighPerformance.Memory
             _entries = entryManager ??
                 SlabAllocator.Create<Entry>(64, SlabBufferLayout.Chunked, false);
             m_Buffer = (byte*)externalPointer;
+            _basePtr = (Ptr64)(nuint)externalPointer;
+            _metadata = (Metadata*)((nuint)externalPointer + byteSize);
             _align = align;
             _size = byteSize;
 

StgSharp.Common/HighPerformance/ProcessorAbstraction/M64.cs

@@ -71,6 +71,13 @@ namespace StgSharp.HighPerformance.ProcessorAbstraction
             return value.GetHashCode();
         }
 
+        public ulong GetLowerBits(
+                     int bitCount
+        )
+        {
+            return value & ((1UL << bitCount) - 1);
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public ref T Member<T>(
                      int index
@@ -79,6 +86,14 @@ namespace StgSharp.HighPerformance.ProcessorAbstraction
             return ref Unsafe.As<byte, T>(ref buffer[index * sizeof(T)]);
         }
 
+        public void SetLowerBits(
+                    int bitCount,
+                    ulong bits
+        )
+        {
+            value = (value & (~(1UL << bitCount))) | (bits & (1UL << bitCount);
+        }
+
         public static bool operator !=(
                                     M64 left,
                                     M64 right