From 5fc8e62f28d878de8d5f351058fe1087297acd82 Mon Sep 17 00:00:00 2001 From: Nitload Date: Sun, 19 Apr 2026 02:30:35 +0800 Subject: [PATCH] solution to length unfixed cacheline in l4 --- StgSharp.Benchmark/HlsfAllocatorBench.cs | 70 +++---- StgSharp.Common/Collections/SwissTable.cs | 106 +++++----- .../Memory/HLSFAllocationHandle.cs | 47 ++--- .../Memory/HLSFAllocator.Alloc.cs | 64 +++--- .../Memory/HLSFAllocator.Free.cs | 50 ++--- .../HighPerformance/Memory/HLSFAllocator.cs | 182 +++++++----------- StgSharp.Common/HighPerformance/Memory/L4.cs | 87 ++++----- .../Memory/SlabAllocator.Fixed.cs | 55 +++--- .../HighPerformance/Memory/SlabAllocator.cs | 118 ++++++------ .../Numeric/Linear/Matrix.Create.cs | 41 ++-- .../Numeric/Linear/MatrixStorage.cs | 2 +- .../Numeric/Runtime/MatrixEnumerator.cs | 68 +++---- StgSharpDebug/HlsfStressTest.cs | 61 +++--- StgSharpDebug/hlsf-test.cs | 2 +- 14 files changed, 421 insertions(+), 532 deletions(-) diff --git a/StgSharp.Benchmark/HlsfAllocatorBench.cs b/StgSharp.Benchmark/HlsfAllocatorBench.cs index cf3d09b8..dc830d94 100644 --- a/StgSharp.Benchmark/HlsfAllocatorBench.cs +++ b/StgSharp.Benchmark/HlsfAllocatorBench.cs @@ -1,27 +1,30 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="HlsfAllocatorBench" +// file="HlsfAllocatorBench.cs" // Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. +// AuthorGroup: Nitload Space +// Copyright (c) Nitload Space. All rights reserved. // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the “Software”), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and +// this permission notice shall be included in all copies +// or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED “AS IS”, +// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH +// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- @@ -42,17 +45,9 @@ namespace StgSharp.Benchmark private const int AllocationCount = 2048; - private static readonly int[] AllocSizes = [ - 64, - 96, - 150, - 200, - 400, - 1024, - 4096 - ]; + private static readonly int[] AllocSizes = [ 64, 96, 150, 200, 400, 1024, 4096 ]; private HybridLayerSegregatedFitAllocator _hlsfAllocator = null!; - private readonly List _hlsfLocalList = new(); + private readonly List _hlsfLocalList = new(); private readonly List _libcLocalList = new(); private Random _random = new(42); @@ -106,9 +101,7 @@ namespace StgSharp.Benchmark _hlsfAllocator = new HybridLayerSegregatedFitAllocator(GetArenaBytes(), align:(int)Alignment); } - private static uint AlignSize( - uint size - ) + private static uint AlignSize(uint size) { uint mask = Alignment - 1; return (size + mask) & ~mask; @@ -121,9 +114,7 @@ namespace StgSharp.Benchmark int mb = defaultMb; string? env = Environment.GetEnvironmentVariable("HLSF_BENCH_ARENA_MB"); - if (!string.IsNullOrWhiteSpace(env) && - int.TryParse(env, out int parsed) && - parsed >= minMb) { + if (!string.IsNullOrWhiteSpace(env) && int.TryParse(env, out int parsed) && parsed >= minMb) { mb = parsed; } @@ -132,17 +123,14 @@ namespace StgSharp.Benchmark private uint GetNextRandomSize() => (uint)AllocSizes[_random.Next(AllocSizes.Length)]; - private void RunHlsf( - HybridLayerSegregatedFitAllocator allocator, - FreePolicy policy - ) + private void RunHlsf(HybridLayerSegregatedFitAllocator allocator, FreePolicy policy) { _hlsfLocalList.Clear(); for (int i = 0; i < AllocationCount; i++) { uint size = AlignSize(GetNextRandomSize()); - HybridLayerSegregatedFitAllocationHandle allocate = allocator.Alloc(size); + hlsfHandle allocate = allocator.Alloc(size); _hlsfLocalList.Add(allocate); } diff --git a/StgSharp.Common/Collections/SwissTable.cs b/StgSharp.Common/Collections/SwissTable.cs index c5e40ee7..19dec5d1 100644 --- a/StgSharp.Common/Collections/SwissTable.cs +++ b/StgSharp.Common/Collections/SwissTable.cs @@ -1,27 +1,30 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="SwissTable" +// file="SwissTable.cs" // Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. +// AuthorGroup: Nitload Space +// Copyright (c) Nitload Space. All rights reserved. // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the “Software”), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and +// this permission notice shall be included in all copies +// or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED “AS IS”, +// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH +// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- @@ -46,16 +49,13 @@ namespace StgSharp.Collections private const int EntryMaskWidth = 10; // 1024 entries total, 16 per bucket public const int MaxCapacity = BucketCount * BucketSize; // 256 entries per bucket, 1024 buckets - public static readonly int RequestedNativeMemorySize = Unsafe.SizeOf(); + public static readonly nuint RequestedNativeMemorySize = (nuint)Unsafe.SizeOf(); private readonly Entry* _entries; private readonly Action _freeHandle; private bool _disposed; - private SwissTable( - Entry* buffer, - Action freeHandle - ) + private SwissTable(Entry* buffer, Action freeHandle) { _entries = buffer; _freeHandle = freeHandle; @@ -72,6 +72,17 @@ namespace StgSharp.Collections return new SwissTable(buffer, handle => NativeMemory.AlignedFree((void*)handle)); } + public static SwissTable Create(Func allocator, Action freeHandle) + { + Entry* buffer = (Entry*)allocator(RequestedNativeMemorySize); + for (int i = 0; i < BucketCount; i++) + { + // Set all control bytes to 0xFF (empty) + Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize); + } + return new SwissTable(buffer, freeHandle); + } + public void Dispose() { if (_disposed) { @@ -82,11 +93,7 @@ namespace StgSharp.Collections GC.SuppressFinalize(this); } - public bool TryAddOrSet( - nuint key, - nuint value, - out bool isNewKey - ) + public bool TryAddOrSet(nuint key, nuint value, out bool isNewKey) { uint hash = BitOperations.Crc32C(CrcMagic, (ulong)key); ushort entry = (ushort)(hash & (BucketCount - 1)); @@ -94,10 +101,7 @@ namespace StgSharp.Collections return IBucketScanner.Current.TrySet(key, hash, bucket, value, out isNewKey); } - public bool TryGet( - nuint key, - out nuint value - ) + public bool TryGet(nuint key, out nuint value) { uint hash = BitOperations.Crc32C(CrcMagic, key); ushort entry = (ushort)(hash & (BucketCount - 1)); @@ -112,21 +116,15 @@ namespace StgSharp.Collections */ } - public bool TryRemove( - nuint key, - out nuint value - ) + public bool TryRemove(nuint key, out nuint value) { - uint hash = BitOperations.Crc32C(CrcMagic, (ulong)key); + uint hash = BitOperations.Crc32C(CrcMagic, key); ushort entry = (ushort)(hash & (BucketCount - 1)); Bucket* bucket = &(*_entries)[entry]; return IBucketScanner.Current.TryRemove(key, hash, bucket, out value); } - internal static SwissTable Create( - nuint bufferPtr, - Action FreeHandle - ) + internal static SwissTable Create(nuint bufferPtr, Action FreeHandle) { Entry* buffer = (Entry*)bufferPtr; for (int i = 0; i < BucketCount; i++) @@ -164,27 +162,11 @@ namespace StgSharp.Collections static IBucketScanner Current { get; } = Create(); - bool TryFind( - nuint key, - uint Hash, - Bucket* bucket, - out nuint value - ); + bool TryFind(nuint key, uint Hash, Bucket* bucket, out nuint value); - unsafe bool TryRemove( - nuint key, - uint hash, - Bucket* bucket, - out nuint value - ); + unsafe bool TryRemove(nuint key, uint hash, Bucket* bucket, out nuint value); - bool TrySet( - nuint key, - uint hash, - Bucket* bucket, - nuint value, - out bool isNewKey - ); + bool TrySet(nuint key, uint hash, Bucket* bucket, nuint value, out bool isNewKey); private static IBucketScanner Create() { diff --git a/StgSharp.Common/HighPerformance/Memory/HLSFAllocationHandle.cs b/StgSharp.Common/HighPerformance/Memory/HLSFAllocationHandle.cs index 4ebe849a..7de31a8a 100644 --- a/StgSharp.Common/HighPerformance/Memory/HLSFAllocationHandle.cs +++ b/StgSharp.Common/HighPerformance/Memory/HLSFAllocationHandle.cs @@ -1,27 +1,30 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="HLSFAllocationHandle" +// file="HLSFAllocationHandle.cs" // Project: StgSharp // AuthorGroup: Nitload Space // Copyright (c) Nitload Space. All rights reserved. // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the “Software”), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and +// this permission notice shall be included in all copies +// or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED “AS IS”, +// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH +// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- @@ -36,13 +39,13 @@ using Allocator = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocator; namespace StgSharp.Mathematics.Memory { - public readonly unsafe struct HybridLayerSegregatedFitAllocationHandle + public readonly unsafe struct hlsfHandle { internal readonly Allocator.Entry* EntryHandle; public readonly long AllocSize; - internal HybridLayerSegregatedFitAllocationHandle(Allocator.Entry* handle, long s) + internal hlsfHandle(Allocator.Entry* handle, long s) { EntryHandle = handle; Pointer = (byte*)handle->Position; @@ -77,11 +80,11 @@ namespace StgSharp.Mathematics.Memory public ref struct Enumerator : IEnumerator { - private HybridLayerSegregatedFitAllocationHandle _handle; + private hlsfHandle _handle; private uint _index = ~0u; - internal Enumerator(HybridLayerSegregatedFitAllocationHandle span) + internal Enumerator(hlsfHandle span) { _handle = span; Span = span.BufferHandle; diff --git a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs index cbce7c6f..d6afeab5 100644 --- a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs +++ b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs @@ -1,42 +1,43 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="HLSFAllocator.Alloc" +// file="HLSFAllocator.Alloc.cs" // Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. +// AuthorGroup: Nitload Space +// Copyright (c) Nitload Space. All rights reserved. // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the “Software”), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and +// this permission notice shall be included in all copies +// or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED “AS IS”, +// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH +// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- using System; using System.Numerics; -using hlsfHandle = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocationHandle; +using hlsfHandle = StgSharp.Mathematics.Memory.hlsfHandle; namespace StgSharp.Mathematics.Memory { public unsafe partial class HybridLayerSegregatedFitAllocator { - public partial hlsfHandle Alloc( - long size - ) + public partial hlsfHandle Alloc(long size) { if (size > _maxSize) { throw new InvalidOperationException("Requested size exceeds maximum allocatable size."); @@ -93,10 +94,7 @@ namespace StgSharp.Mathematics.Memory /// /// /// - private void SliceMemory( - Entry* e, - long sizeToSlice - ) + private void SliceMemory(Entry* e, long sizeToSlice) { Entry* next = e; long remain = sizeToSlice; // remained size to slice @@ -194,9 +192,7 @@ namespace StgSharp.Mathematics.Memory /// /// True if allocation succeeded /// - private bool TryAllocateNew32MBBlock( - out Entry* handle - ) + private bool TryAllocateNew32MBBlock(out Entry* handle) { nuint pos = _spareMemory->Position; long remain = _spareMemory->Size; @@ -226,11 +222,7 @@ namespace StgSharp.Mathematics.Memory /// /// Helper method for allocation - converts BucketNode to Entry /// - private bool TryPopLevelForAllocation( - int levelIndex, - int sizeIndex, - out Entry* entry - ) + private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry) { if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode)) { diff --git a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs index 85733fd4..1a648965 100644 --- a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs +++ b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs @@ -1,27 +1,30 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="HLSFAllocator.Free" +// file="HLSFAllocator.Free.cs" // Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. +// AuthorGroup: Nitload Space +// Copyright (c) Nitload Space. All rights reserved. // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the “Software”), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and +// this permission notice shall be included in all copies +// or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED “AS IS”, +// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH +// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- @@ -35,7 +38,7 @@ using System.Runtime.InteropServices; using System.Text; using System.Threading.Tasks; -using hlsfHandle = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocationHandle; +using hlsfHandle = StgSharp.Mathematics.Memory.hlsfHandle; namespace StgSharp.Mathematics.Memory { @@ -90,10 +93,7 @@ namespace StgSharp.Mathematics.Memory /// Handle to the allocated memory block /// /// - public void Free( - hlsfHandle handle, - FreePolicy collectPolicy = FreePolicy.NoCollect - ) + public void Free(hlsfHandle handle, FreePolicy collectPolicy = FreePolicy.NoCollect) { Entry* e = handle.EntryHandle; if (e == null) { diff --git a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs index ff0bfdc8..9423bf06 100644 --- a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs +++ b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs @@ -1,27 +1,30 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="HLSFAllocator" +// file="HLSFAllocator.cs" // Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. +// AuthorGroup: Nitload Space +// Copyright (c) Nitload Space. All rights reserved. // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the “Software”), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and +// this permission notice shall be included in all copies +// or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED “AS IS”, +// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH +// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- @@ -30,7 +33,7 @@ using System.Buffers; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; -using hlsfHandle = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocationHandle; +using hlsfHandle = StgSharp.Mathematics.Memory.hlsfHandle; namespace StgSharp.Mathematics.Memory { @@ -43,47 +46,41 @@ namespace StgSharp.Mathematics.Memory // _spareMemory is also the beginning position of allocator private Entry *_spareMemory; private readonly byte* m_Buffer; + + private readonly Action _freeHandle; private bool disposedValue; private readonly int _align; private readonly long _maxSize; private readonly nuint _size; private readonly SlabAllocator _entries; - public HybridLayerSegregatedFitAllocator( - nuint byteSize - ) + internal HybridLayerSegregatedFitAllocator( + nuint byteSize, + int align, + long maxBlockSize, + Func allocator, + Action freeHandle, + SlabAllocator? entryManager) + : this(byteSize, align, maxBlockSize, (void*)allocator(byteSize, align), freeHandle, entryManager) { } + + internal HybridLayerSegregatedFitAllocator( + nuint byteSize, + int align, + long maxBlockSize, + void* externalPointer, + Action freeHandle, + SlabAllocator? entryManager = null) { - if (byteSize < 64) { - throw new InvalidOperationException(); + if (align < 16 || (align & (align - 1)) != 0) { + throw new ArgumentException( + "Alignment must be a power of two and at least 16 bytes."); } - _entries = SlabAllocator.Create(64, SlabBufferLayout.Chunked, false); - _size = byteSize; - m_Buffer = (byte *)NativeMemory.AlignedAlloc(byteSize, 16); - _align = 16; - - _bucketHeads = new BucketNode*[_levelSizeArray.Length * 3 + 1]; - - _spareMemory = (Entry *)_entries.Allocate(); - _spareMemory->State = EntryState.Spare; - _spareMemory->PreviousNear = _spareMemory; - _spareMemory->NextNear = _spareMemory; - _spareMemory->Position = (nuint)m_Buffer; - _spareMemory->Size = (long)byteSize; - - _maxSize = 32U * 1024U * 1024U; - } - - public HybridLayerSegregatedFitAllocator( - nuint byteSize, - uint maxBlockSize - ) - { - _entries = SlabAllocator.Create(64, SlabBufferLayout.Chunked, false); - m_Buffer = (byte *)NativeMemory.AlignedAlloc(byteSize, 16); - _align = 16; + _entries = entryManager ?? SlabAllocator.Create(64, SlabBufferLayout.Chunked, false); + m_Buffer = (byte*)externalPointer; + _align = align; _size = byteSize; - _spareMemory = (Entry *)_entries.Allocate(); + _spareMemory = (Entry*)_entries.Allocate(); _spareMemory->State = EntryState.Empty; _spareMemory->PreviousNear = _spareMemory; _spareMemory->NextNear = _spareMemory; @@ -93,7 +90,7 @@ namespace StgSharp.Mathematics.Memory int levelCountOrigin = _levelSizeArray.Length; int levelCount = GetLevelFromSize(maxBlockSize) + 1; Array.Resize(ref _levelSizeArray, levelCount); - _bucketHeads = new BucketNode *[(levelCount * 3) + 1]; + _bucketHeads = new BucketNode*[(levelCount * 3) + 1]; int size = _levelSizeArray[^1]; for (int i = levelCountOrigin; i < levelCount; i++) { @@ -101,72 +98,25 @@ namespace StgSharp.Mathematics.Memory _levelSizeArray[i] = size; } _maxSize = maxBlockSize; + _freeHandle = freeHandle ?? (static _ => { }); } - public HybridLayerSegregatedFitAllocator( - nuint byteSize, - int align - ) - { - if (align < 16 || (align & (align - 1)) != 0) { - throw new ArgumentException( - "Alignment must be a power of two and at least 16 bytes."); - } - _entries = SlabAllocator.Create(64, SlabBufferLayout.Chunked, false); - m_Buffer = (byte *)NativeMemory.AlignedAlloc(byteSize, (nuint)align); - _align = align; - _size = byteSize; + public HybridLayerSegregatedFitAllocator(nuint byteSize) + : this( + byteSize, 16, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16), ptr => NativeMemory.AlignedFree((void*)ptr)) { } - _bucketHeads = new BucketNode *[_levelSizeArray.Length * 3 + 1]; + public HybridLayerSegregatedFitAllocator(nuint byteSize, uint maxBlockSize) + : this( + byteSize, 16, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16), ptr => NativeMemory.AlignedFree((void*)ptr)) { } - _spareMemory = (Entry *)_entries.Allocate(); - _spareMemory->State = EntryState.Empty; - _spareMemory->PreviousNear = _spareMemory; - _spareMemory->NextNear = _spareMemory; - _spareMemory->Position = (nuint)m_Buffer; - _spareMemory->Size = (long)byteSize; + public HybridLayerSegregatedFitAllocator(nuint byteSize, int align) + : this( + byteSize, align, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16), ptr => NativeMemory.AlignedFree((void*)ptr)) { } - _maxSize = 32U * 1024U * 1024U; - } + public HybridLayerSegregatedFitAllocator(nuint byteSize, int align, long maxBlockSize) + : this(byteSize, align, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16), ptr => NativeMemory.AlignedFree((void*)ptr)) { } - public HybridLayerSegregatedFitAllocator( - nuint byteSize, - int align, - long maxBlockSize - ) - { - if (align < 16 || (align & (align - 1)) != 0) { - throw new ArgumentException( - "Alignment must be a power of two and at least 16 bytes."); - } - _entries = SlabAllocator.Create(64, SlabBufferLayout.Chunked, false); - m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align); - _align = align; - _size = byteSize; - - _spareMemory = (Entry *)_entries.Allocate(); - _spareMemory->State = EntryState.Empty; - _spareMemory->PreviousNear = _spareMemory; - _spareMemory->NextNear = _spareMemory; - _spareMemory->Position = (nuint)m_Buffer; - _spareMemory->Size = (long)byteSize; - - int levelCountOrigin = _levelSizeArray.Length; - int levelCount = GetLevelFromSize(maxBlockSize) + 1; - Array.Resize(ref _levelSizeArray, levelCount); - _bucketHeads = new BucketNode *[levelCount * 3 + 1]; - int size = _levelSizeArray[^1]; - for (int i = levelCountOrigin; i < levelCount; i++) - { - size *= 4; - _levelSizeArray[i] = size; - } - _maxSize = maxBlockSize; - } - - public partial hlsfHandle Alloc( - long size - ); + public partial hlsfHandle Alloc(long size); public partial void Collect(); @@ -176,16 +126,14 @@ namespace StgSharp.Mathematics.Memory GC.SuppressFinalize(this); } - protected virtual void Dispose( - bool disposing - ) + protected virtual void Dispose(bool disposing) { if (!disposedValue) { if (disposing) { _entries.Dispose(); } - NativeMemory.AlignedFree(m_Buffer); + _freeHandle((nuint)m_Buffer); disposedValue = true; } } diff --git a/StgSharp.Common/HighPerformance/Memory/L4.cs b/StgSharp.Common/HighPerformance/Memory/L4.cs index 279cbb23..13dfe5fe 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.cs @@ -1,35 +1,41 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="L4" +// file="L4.cs" // Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. +// AuthorGroup: Nitload Space +// Copyright (c) Nitload Space. All rights reserved. // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the “Software”), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and +// this permission notice shall be included in all copies +// or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED “AS IS”, +// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH +// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- using StgSharp.Collections; using StgSharp.Common.HighPerformance.ProcessorAbstraction; +using StgSharp.Mathematics.Memory; using System.Data; using System.Runtime.InteropServices; +using HLSF = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocator; + namespace StgSharp.HighPerformance.Memory { public sealed unsafe partial class L4 : IDisposable @@ -40,13 +46,13 @@ namespace StgSharp.HighPerformance.Memory private const int PredictionLeastAhead = 1024; private static readonly IIntrinsicKernel _intrinsic = IIntrinsicKernel.Instance; - - private readonly CacheLineData* _data; private readonly EvictionArray* _eviction; private readonly CacheLineHead* _head; private readonly CacheLinePrediction* _prediction; private bool _disposed; private bool _exhausted; + + private readonly HLSF _bufferAllocator; private int _activeCount; private int _aheadCount; @@ -55,20 +61,19 @@ namespace StgSharp.HighPerformance.Memory private int _maxId; private volatile int _prefetching; private readonly nuint _baseAddress; + private readonly SlabAllocator _entryManager; private readonly SwissTable _map; private readonly UnmanagedStack _idMux; - public L4( - int bindedNuma - ) + public L4(int bindedNuma, nuint bufferSize = 8192 * 8192, int align = 16, int maxCacheLineSize = 8192) { _bindedNuma = bindedNuma; // init order is fallow by align requirement of each component. - nuint cacheSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // SIMDID, no less than 16 byte bit + nuint cacheSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // SIMDID, no less than 16 byte bit nuint entrySize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // 16 byte align nuint predictionSize = ((nuint)Unsafe.SizeOf()) * PredictionCount; //16 byte align - nuint mapSize = (nuint)SwissTable.RequestedNativeMemorySize; // 16 byte align + nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align nuint rrpvSize = (nuint)Unsafe.SizeOf(); // 16 byte align nuint stackSize = ((nuint)sizeof(int)) * CacheLineCount; // int align @@ -78,8 +83,10 @@ namespace StgSharp.HighPerformance.Memory nuint ptr = _baseAddress; // init all components with the buffer, and move the buffer pointer forward. + _entryManager = SlabAllocator.FromBuffer(); + + _bufferAllocator = new HLSF(cacheSize, align, maxCacheLineSize, (void*)_baseAddress, static _ => { }); nuint _buffer = _baseAddress; - _data = (CacheLineData*)_buffer; ptr += cacheSize; _head = (CacheLineHead*)ptr; ptr += entrySize; @@ -98,9 +105,7 @@ namespace StgSharp.HighPerformance.Memory // public IL4Predict PrefetchPredict { get; set; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ContinueOnce( - L4CacheLine line - ) + public void ContinueOnce(L4CacheLine line) { if (line.Address > 0) { _ = Interlocked.Decrement(ref _aheadCount); @@ -123,12 +128,13 @@ namespace StgSharp.HighPerformance.Memory _disposed = true; _map.Dispose(); _idMux.Dispose(); + _bufferAllocator.Dispose(); nuint cacheSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // SIMDID, no less than 16 byte bit nuint entrySize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // 16 byte align nuint predictionSize = ((nuint)Unsafe.SizeOf()) * PredictionCount; //16 byte align - nuint mapSize = (nuint)SwissTable.RequestedNativeMemorySize; // 16 byte align + nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align nuint rrpvSize = (nuint)Unsafe.SizeOf(); // 16 byte align nuint stackSize = ((nuint)sizeof(int)) * CacheLineCount; // int align Numa.Free((void*)_baseAddress, _bindedNuma, cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize); @@ -136,9 +142,7 @@ namespace StgSharp.HighPerformance.Memory } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public L4CacheLine Map( - nuint origin - ) + public L4CacheLine Map(nuint origin) { if ((_aheadCount < PredictionLeastAhead) && !_exhausted) { @@ -181,13 +185,6 @@ namespace StgSharp.HighPerformance.Memory private partial void Prefetch(); - private struct CacheLineData - { - - private fixed byte Data[8192]; - - } - } public ref struct L4CacheLine : IDisposable @@ -197,11 +194,7 @@ namespace StgSharp.HighPerformance.Memory private ref int _refCount; private nuint _handle; - internal L4CacheLine( - nuint handle, - ref int activeCount, - ref int cacheRefCount - ) + internal L4CacheLine(nuint handle, ref int activeCount, ref int cacheRefCount) { _handle = handle; _refCount = ref cacheRefCount; diff --git a/StgSharp.Common/HighPerformance/Memory/SlabAllocator.Fixed.cs b/StgSharp.Common/HighPerformance/Memory/SlabAllocator.Fixed.cs index fb3a6d94..95e656b7 100644 --- a/StgSharp.Common/HighPerformance/Memory/SlabAllocator.Fixed.cs +++ b/StgSharp.Common/HighPerformance/Memory/SlabAllocator.Fixed.cs @@ -1,27 +1,30 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="SlabAllocator.Fixed" +// file="SlabAllocator.Fixed.cs" // Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. +// AuthorGroup: Nitload Space +// Copyright (c) Nitload Space. All rights reserved. // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the “Software”), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and +// this permission notice shall be included in all copies +// or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED “AS IS”, +// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH +// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- @@ -37,7 +40,7 @@ using System.Threading.Tasks; namespace StgSharp.HighPerformance.Memory { - internal unsafe class FixedSlabAllocator : SlabAllocator where T : unmanaged + internal unsafe class FixedSlabAllocator : SlabAllocator where T: unmanaged { private Action _freeHandle; @@ -50,11 +53,7 @@ namespace StgSharp.HighPerformance.Memory private nuint _Capacity, _currentIndex; - public FixedSlabAllocator( - nuint buffer, - nuint count, - Action freeHandle - ) + internal FixedSlabAllocator(nuint buffer, nuint count, Action freeHandle) { _freeHandle = freeHandle; _buffer = buffer; @@ -109,9 +108,7 @@ namespace StgSharp.HighPerformance.Memory public override void ExitBufferReading() { } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override void Free( - nuint index - ) + public override void Free(nuint index) { _recycle.Push(index); } diff --git a/StgSharp.Common/HighPerformance/Memory/SlabAllocator.cs b/StgSharp.Common/HighPerformance/Memory/SlabAllocator.cs index 0032794e..6df80ea8 100644 --- a/StgSharp.Common/HighPerformance/Memory/SlabAllocator.cs +++ b/StgSharp.Common/HighPerformance/Memory/SlabAllocator.cs @@ -1,27 +1,30 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="SlabAllocator" +// file="SlabAllocator.cs" // Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. +// AuthorGroup: Nitload Space +// Copyright (c) Nitload Space. All rights reserved. // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the “Software”), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and +// this permission notice shall be included in all copies +// or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED “AS IS”, +// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH +// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- @@ -56,62 +59,53 @@ namespace StgSharp.Mathematics.Memory /// /// /// Important Threading Considerations: While SLAB - /// /// allocators themselves may be designed for single-threaded use, allocated objects - /// are /// frequently shared across multiple threads in real-world applications. Due to - /// the /// special implementation of expandable concurrent sequential SLAB allocators, - /// they rely /// on internal expansion locks to ensure that memory buffer expansions - /// (which involve /// copying the entire buffer to a new location) do not occur while - /// other threads are /// reading from allocated regions. - /// Sequential Layout Limitations: ///
Sequential layout SLAB allocators - /// cannot provide a single-threaded /// version because:They use contiguous memory /// layout that requires - /// buffer expansion when capacity is exceeded - /// ///Buffer expansion involves copying all /// - /// existing data to a new memory location /// - /// Without proper synchronization, concurrent access to allocated objects during /// - /// expansion would cause memory corruptionThe - /// /// expansion lock mechanism is essential even if allocation itself is single- /// - /// threaded Chunked Layout: + /// /// /// allocators themselves may be designed for single-threaded use, allocated + /// objects /// are /// frequently shared across multiple threads in real-world + /// applications. Due to /// the /// special implementation of expandable concurrent + /// sequential SLAB allocators, /// they rely /// on internal expansion locks to + /// ensure that memory buffer expansions /// (which involve /// copying the entire + /// buffer to a new location) do not occur while /// other threads are /// reading + /// from allocated regions. /// Sequential Layout Limitations: + /// ///
Sequential layout SLAB allocators /// cannot provide a single- + /// threaded /// version because:They use + /// contiguous memory /// layout that requires /// buffer expansion when capacity is + /// exceeded /// ///Buffer expansion involves + /// copying all /// /// existing data to a new memory + /// location /// /// Without proper + /// synchronization, concurrent access to allocated objects during /// /// expansion + /// would cause memory corruptionThe /// /// + /// expansion lock mechanism is essential even if allocation itself is single- /// /// + /// threaded Chunked Layout: /// /// ///
Chunked layout allocators can optionally provide single-threaded - /// /// versions since they don't require buffer expansion and copying. Each chunk is - /// /// allocated independently, avoiding the memory relocation issues present in - /// sequential /// layouts. + /// /// /// versions since they don't require buffer expansion and copying. Each + /// chunk is /// /// allocated independently, avoiding the memory relocation issues + /// present in /// sequential /// layouts. /// - public static SlabAllocator Create( - nuint count, - SlabBufferLayout layout, - bool concurrentSupport = true - ) where T : unmanaged + public static SlabAllocator Create(nuint count, SlabBufferLayout layout, bool concurrentSupport = true) + where T: unmanaged { - if (concurrentSupport) { - return layout switch - { - SlabBufferLayout.Sequential => new ConcurrentSequentialSlabAllocator(count), - SlabBufferLayout.Chunked => new ConcurrentChunkedSlabAllocator(count), - _ => throw new ArgumentOutOfRangeException(nameof(layout), layout, null) - }; - } return layout switch { - SlabBufferLayout.Sequential => throw new NotSupportedException("Non-concurrent version of sequential layout SLAB is not supported due to expansion lock requirements for memory safety during buffer expansion."), - SlabBufferLayout.Chunked => new ChunkedSlabAllocator(count), + SlabBufferLayout.Sequential => concurrentSupport ? + new ConcurrentSequentialSlabAllocator(count) : + throw new NotSupportedException("Non-concurrent version of sequential layout SLAB is not supported due to expansion lock requirements for memory safety during buffer expansion."), + SlabBufferLayout.Chunked => concurrentSupport ? + new ConcurrentChunkedSlabAllocator(count) : + new ChunkedSlabAllocator(count), _ => throw new ArgumentOutOfRangeException(nameof(layout), layout, null) }; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static SlabAllocator FromBuffer( - nuint buffer, - nuint count, - Action freeHandle - ) where T : unmanaged + public static SlabAllocator FromBuffer(nuint buffer, nuint count, Action freeHandle) + where T: unmanaged { return new FixedSlabAllocator(buffer, count, freeHandle); } } - public abstract class SlabAllocator : IDisposable where T : unmanaged + public abstract class SlabAllocator : IDisposable where T: unmanaged { public abstract nuint Allocate(); @@ -122,9 +116,7 @@ namespace StgSharp.Mathematics.Memory public abstract void ExitBufferReading(); - public abstract void Free( - nuint index - ); + public abstract void Free(nuint index); public abstract SlabAllocationHandle ReadAllocation(); diff --git a/StgSharp.Mathematics/Numeric/Linear/Matrix.Create.cs b/StgSharp.Mathematics/Numeric/Linear/Matrix.Create.cs index b6b3ff7b..d5a38130 100644 --- a/StgSharp.Mathematics/Numeric/Linear/Matrix.Create.cs +++ b/StgSharp.Mathematics/Numeric/Linear/Matrix.Create.cs @@ -1,27 +1,30 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="Matrix.Create" +// file="Matrix.Create.cs" // Project: StgSharp // AuthorGroup: Nitload Space // Copyright (c) Nitload Space. All rights reserved. // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the “Software”), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and +// this permission notice shall be included in all copies +// or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED “AS IS”, +// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH +// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- @@ -74,7 +77,7 @@ namespace StgSharp.Mathematics.Numeric long requiredBytes = kernelCount * MatrixKernel.Size; ArgumentOutOfRangeException.ThrowIfNegativeOrZero(requiredBytes, nameof(requiredBytes)); - HybridLayerSegregatedFitAllocationHandle handle = allocator.Alloc((uint)requiredBytes); + hlsfHandle handle = allocator.Alloc((uint)requiredBytes); MatrixStorageHLSF storage = new MatrixStorageHLSF(allocator, handle); Matrix matrix = new Matrix(MatrixKernelBoarder.Create(layout), new MatrixSize(columnLength, rowLength, diff --git a/StgSharp.Mathematics/Numeric/Linear/MatrixStorage.cs b/StgSharp.Mathematics/Numeric/Linear/MatrixStorage.cs index 9efec916..928ac7bb 100644 --- a/StgSharp.Mathematics/Numeric/Linear/MatrixStorage.cs +++ b/StgSharp.Mathematics/Numeric/Linear/MatrixStorage.cs @@ -36,7 +36,7 @@ using System.Text; using System.Threading.Tasks; using HLSFAllocator = global::StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocator; -using HLSFHandle = global::StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocationHandle; +using HLSFHandle = global::StgSharp.Mathematics.Memory.hlsfHandle; namespace StgSharp.Mathematics.Numeric { diff --git a/StgSharp.Mathematics/Numeric/Runtime/MatrixEnumerator.cs b/StgSharp.Mathematics/Numeric/Runtime/MatrixEnumerator.cs index c711851f..675aec5b 100644 --- a/StgSharp.Mathematics/Numeric/Runtime/MatrixEnumerator.cs +++ b/StgSharp.Mathematics/Numeric/Runtime/MatrixEnumerator.cs @@ -1,27 +1,30 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="MatrixEnumerator" +// file="MatrixEnumerator.cs" // Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. +// AuthorGroup: Nitload Space +// Copyright (c) Nitload Space. All rights reserved. // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the “Software”), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and +// this permission notice shall be included in all copies +// or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED “AS IS”, +// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH +// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- @@ -44,8 +47,7 @@ namespace StgSharp.Mathematics.Numeric.Runtime } - public unsafe interface IMatrixCachePredictor : IL4Predict - where TSelf : class, IMatrixCachePredictor + public unsafe interface IMatrixCachePredictor : IL4Predict where TSelf: class, IMatrixCachePredictor { public static abstract PredictPolicy Policy { get; set; } @@ -60,36 +62,22 @@ namespace StgSharp.Mathematics.Numeric.Runtime public static PredictPolicy Policy { get; set; } = PredictPolicy.SquareRowMajor; - public bool IsSamePolicy( - uint policy, - IL4Predict predict - ) + public bool IsSamePolicy(uint policy, IL4Predict predict) { throw new NotImplementedException(); } - public PredictResult Predict( - Span node, - out int count - ) + public PredictResult Predict(Span node, out int count) { throw new NotImplementedException(); } - public void Prefetch( - nuint origin, - uint policy, - Span cache - ) + public void Prefetch(nuint origin, uint policy, Span cache) { throw new NotImplementedException(); } - public void WriteBack( - nuint origin, - uint policy, - ReadOnlySpan cache - ) + public void WriteBack(nuint origin, uint policy, ReadOnlySpan cache) { throw new NotImplementedException(); } diff --git a/StgSharpDebug/HlsfStressTest.cs b/StgSharpDebug/HlsfStressTest.cs index 92adc2ea..e1535c5e 100644 --- a/StgSharpDebug/HlsfStressTest.cs +++ b/StgSharpDebug/HlsfStressTest.cs @@ -1,27 +1,30 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="HlsfStressTest" +// file="HlsfStressTest.cs" // Project: StgSharp // AuthorGroup: Nitload Space // Copyright (c) Nitload Space. All rights reserved. // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the “Software”), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and +// this permission notice shall be included in all copies +// or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED “AS IS”, +// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH +// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- @@ -149,7 +152,7 @@ namespace StgSharp Stopwatch sw = Stopwatch.StartNew(); for (int i = 0; i < pairs; i++) { - HybridLayerSegregatedFitAllocationHandle h = allocator.Alloc(blockSize); + hlsfHandle h = allocator.Alloc(blockSize); allocator.Free(h); } sw.Stop(); @@ -175,11 +178,11 @@ namespace StgSharp return; } - List list = new List((int)ops); + List list = new List((int)ops); Stopwatch sw = Stopwatch.StartNew(); for (long i = 0; i < ops; i++) { - HybridLayerSegregatedFitAllocationHandle h = allocator.Alloc(blockSize); + hlsfHandle h = allocator.Alloc(blockSize); if (touch) { Span span = h.BufferHandle; @@ -193,7 +196,7 @@ namespace StgSharp } sw.Stop(); double allocOpsPerSec = ops / sw.Elapsed.TotalSeconds; - foreach (HybridLayerSegregatedFitAllocationHandle h in list) { + foreach (hlsfHandle h in list) { allocator.Free(h); } @@ -205,13 +208,13 @@ namespace StgSharp { int targetBytes = targetMiB * 1024 * 1024; Random rng = new Random(123); - List live = new List(128 * 1024); + List live = new List(128 * 1024); long allocated = 0; Stopwatch swA = Stopwatch.StartNew(); while (allocated < targetBytes) { uint sz = PickSize(rng); - HybridLayerSegregatedFitAllocationHandle h = allocator.Alloc(sz); + hlsfHandle h = allocator.Alloc(sz); live.Add(h); allocated += sz; } @@ -235,7 +238,7 @@ namespace StgSharp while (allocated < targetBytes / 2) { uint sz = PickSize(rng); - HybridLayerSegregatedFitAllocationHandle h = allocator.Alloc(sz); + hlsfHandle h = allocator.Alloc(sz); live.Add(h); allocated += sz; reCount++; @@ -244,7 +247,7 @@ namespace StgSharp Console.WriteLine($" phaseC re-alloc {reCount} in {swC.Elapsed.TotalSeconds:0.00}s ~{allocated / (1024.0 * 1024.0):0}MB"); Stopwatch swD = Stopwatch.StartNew(); - foreach (HybridLayerSegregatedFitAllocationHandle h in live) { + foreach (hlsfHandle h in live) { allocator.Free(h); } @@ -254,7 +257,7 @@ namespace StgSharp private static void BenchFreeThroughput(HybridLayerSegregatedFitAllocator allocator, int count, uint blockSize) { - List list = new List(count); + List list = new List(count); for (int i = 0; i < count; i++) { list.Add(allocator.Alloc(blockSize)); } @@ -292,12 +295,12 @@ namespace StgSharp private static void Warmup(HybridLayerSegregatedFitAllocator allocator) { - List tmp = new List(4096); + List tmp = new List(4096); for (int i = 0; i < 4096; i++) { tmp.Add(allocator.Alloc(256)); } - foreach (HybridLayerSegregatedFitAllocationHandle h in tmp) { + foreach (hlsfHandle h in tmp) { allocator.Free(h); } } diff --git a/StgSharpDebug/hlsf-test.cs b/StgSharpDebug/hlsf-test.cs index 528c1097..823b67f8 100644 --- a/StgSharpDebug/hlsf-test.cs +++ b/StgSharpDebug/hlsf-test.cs @@ -59,7 +59,7 @@ namespace StgSharp for (int round = 1; round <= maxRounds; round++) { // 100 次分配 - List batch = new List(100); + List batch = new List(100); for (int i = 0; i < 100; i++) { uint sz = sizes[rnd.Next(sizes.Length)];