From 588fa867af4ee4d54d62691947a25c463189eeae Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Wed, 3 Dec 2025 18:14:45 +0800 Subject: [PATCH] working on panel building, preparing for panel compute --- StgSharp.Common/HighPerformance/M128.cs | 67 +------- .../HighPerformance/MatrixPanel.cs | 157 +++++++++++++++--- .../Memory/HLSFAllocator.Alloc.cs | 3 - .../Memory/HLSFAllocator.SegmentEntry.cs | 73 +------- .../Memory/InroductionToHLSF.md | 107 ++++++++---- .../Mathematics/Graphic/Matrix44.cs | 22 ++- .../Numeric/MatrixCompute.Basic.cs | 27 ++- StgSharp.Native/StgSharp.Native.vcxproj | 1 + .../StgSharp.Native.vcxproj.filters | 3 + StgSharp.Native/sn-head/sn_intrinsic.h | 3 + StgSharp.Native/sn-head/sn_matkernel.h | 23 +++ StgSharp.Native/sn-src/dllmain.c | 18 +- StgSharp.Native/sn-src/k_build_panel.c | 42 +++++ StgSharpDebug/Program.cs | 4 +- 14 files changed, 338 insertions(+), 212 deletions(-) create mode 100644 StgSharp.Native/sn-src/k_build_panel.c diff --git a/StgSharp.Common/HighPerformance/M128.cs b/StgSharp.Common/HighPerformance/M128.cs index 61066ab5..8bff2079 100644 --- a/StgSharp.Common/HighPerformance/M128.cs +++ b/StgSharp.Common/HighPerformance/M128.cs @@ -48,6 +48,7 @@ namespace StgSharp.HighPerformance internal M128(Vector4 v) { Unsafe.SkipInit(out this); + Unsafe.SkipInit(out Vec); Vec = v.AsVector128(); } @@ -57,7 +58,7 @@ namespace StgSharp.HighPerformance Vec = v; } - public ref T AsRef() where T: unmanaged, INumber + public ref T AsRef() where T : unmanaged, INumber { return ref Unsafe.As(ref Buffer[0]); } @@ -68,7 +69,7 @@ namespace StgSharp.HighPerformance return Vec.AsVector4(); } - public void BroadCastFrom(T value) where T: unmanaged, INumber + public void BroadCastFrom(T value) where T : unmanaged, INumber { Vec = Vector128.Create(value).AsSingle(); } @@ -84,7 +85,7 @@ namespace StgSharp.HighPerformance } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ref T Member(int index) where T: unmanaged, INumber + public ref T Member(int index) where T : unmanaged, INumber { return ref Unsafe.Add(ref AsRef(), index); } @@ -115,65 +116,5 @@ namespace StgSharp.HighPerformance return left.Vec == right.Vec; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static explicit operator Vector128(M128 m) - { - return m.Vec.As(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static explicit operator Vector128(M128 m) - { - return m.Vec.As(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static explicit operator Vector128(M128 m) - { - return m.Vec.As(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static explicit operator Vector128(M128 m) - { - return m.Vec.As(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static explicit operator Vector128(M128 m) - { - return m.Vec.As(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static explicit operator Vector128(M128 m) - { - return m.Vec.As(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static explicit operator Vector128(M128 m) - { - return m.Vec.As(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static explicit operator Vector128(M128 m) - { - return m.Vec.As(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static explicit operator Vector128(M128 m) - { - return m.Vec.As(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static explicit operator Vector128(M128 m) - { - return m.Vec.As(); - } - } } diff --git a/StgSharp.Common/HighPerformance/MatrixPanel.cs b/StgSharp.Common/HighPerformance/MatrixPanel.cs index b2543a55..c400c8c9 100644 --- a/StgSharp.Common/HighPerformance/MatrixPanel.cs +++ b/StgSharp.Common/HighPerformance/MatrixPanel.cs @@ -32,11 +32,17 @@ using System.Runtime.InteropServices; namespace StgSharp.HighPerformance { - public interface IMatrixPanel : IDisposable where T: unmanaged,INumber + #pragma warning disable CA1000 + + public interface IMatrixPanel : IDisposable where T : unmanaged,INumber { + ref T this[int column, int row] { get; } + static abstract int MaxElementCount { get; } + static abstract int ElementSideLength { get; } + static abstract nuint Size { get; } void Dispose(); @@ -50,26 +56,39 @@ namespace StgSharp.HighPerformance /// /// Type of matrix element. /// - [InlineArray(16)] - public unsafe struct MatrixPanelSSE : IMatrixPanel where T: unmanaged, INumber + public unsafe struct MatrixPanelSSE : IMatrixPanel where T : unmanaged, INumber { - private M128 vector; + private T element; [Obsolete("Compute kernel should not be created from new(), use Create instead", true)] public MatrixPanelSSE() { } - public static int MaxElementCount + public ref T this[int column, int row] { - [MethodImpl(MethodImplOptions.AggressiveInlining)] get { - int size = sizeof(T); - return 16 * 16 / size; + int index = row * ElementSideLength + column; + return ref Unsafe.Add(ref element, index); } } - public static unsafe nuint Size { get; } = 16 * 16; + public static int ElementSideLength + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get; + } = int.Max(4, 16 / sizeof(T)); + + public static int MaxElementCount + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get; + } = ElementSideLength * ElementSideLength; + + public static unsafe nuint Size { get; } = (nuint)(MaxElementCount * sizeof(T)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int ChunkCount() => (ElementSideLength + LanesPerVector() - 1) / LanesPerVector(); [MethodImpl(MethodImplOptions.AggressiveInlining)] public static MatrixPanelSSE* Create() @@ -91,6 +110,25 @@ namespace StgSharp.HighPerformance } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ref M128 GetVectorChunk(int column, int chunk) + { + int lanes = LanesPerVector(); + int rowStart = chunk * lanes; + ref T colBase = ref this[column, 0]; + return ref Unsafe.As(ref Unsafe.Add(ref colBase, rowStart)); + } + + // SIMD access helpers (column-major) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int LanesPerVector() => 16 / sizeof(T); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void SetVectorChunk(int column, int chunk, in M128 value) + { + GetVectorChunk(column, chunk) = value; + } + } /// @@ -100,31 +138,44 @@ namespace StgSharp.HighPerformance /// /// Type of matrix element. /// - [InlineArray(16)] - public unsafe struct MatrixPanelAVX : IMatrixPanel where T: unmanaged, INumber + public unsafe struct MatrixPanelAVX : IMatrixPanel where T : unmanaged, INumber { - private M256 _buffer; + private T element; [Obsolete("Compute kernel should not be created from new(), use Create instead", true)] public MatrixPanelAVX() { } - public static unsafe int MaxElementCount + public ref T this[int column, int row] { - [MethodImpl(MethodImplOptions.AggressiveInlining)] get { - int size = sizeof(T); - return size < 4 ? 128 / size : 16; + int index = row * ElementSideLength + column; + return ref Unsafe.Add(ref element, index); } } - public static unsafe nuint Size { get; } = 32 * 16; + public static int ElementSideLength + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get; + } = int.Max(8, 32 / sizeof(T)); + + public static unsafe int MaxElementCount + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get; + } = ElementSideLength * ElementSideLength; + + public static unsafe nuint Size { get; } = (nuint)(MaxElementCount * sizeof(T)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int ChunkCount() => (ElementSideLength + LanesPerVector() - 1) / LanesPerVector(); [MethodImpl(MethodImplOptions.AggressiveInlining)] public static MatrixPanelAVX* Create() { - MatrixPanelAVX* kernel = (MatrixPanelAVX*)NativeMemory.AlignedAlloc(Size, 16); + MatrixPanelAVX* kernel = (MatrixPanelAVX*)NativeMemory.AlignedAlloc(Size, 32); return kernel; } @@ -141,6 +192,25 @@ namespace StgSharp.HighPerformance } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ref M256 GetVectorChunk(int column, int chunk) + { + int lanes = LanesPerVector(); + int rowStart = chunk * lanes; + ref T colBase = ref this[column, 0]; + return ref Unsafe.As(ref Unsafe.Add(ref colBase, rowStart)); + } + + // SIMD access helpers for AVX (column-major) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int LanesPerVector() => 32 / sizeof(T); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void SetVectorChunk(int column, int chunk, in M256 value) + { + GetVectorChunk(column, chunk) = value; + } + } /// @@ -150,31 +220,45 @@ namespace StgSharp.HighPerformance /// /// Type of matrix element. /// - [InlineArray(16)] - public unsafe struct MatrixComputeKernelAVX512 : IMatrixPanel where T: unmanaged, INumber + public unsafe struct MatrixComputeKernelAVX512 : IMatrixPanel + where T : unmanaged, INumber { - private M512 _buffer; + private T element; [Obsolete("Compute kernel should not be created from new(), use Create instead", true)] public MatrixComputeKernelAVX512() { } - public static unsafe int MaxElementCount + public ref T this[int column, int row] { - [MethodImpl(MethodImplOptions.AggressiveInlining)] get { - int size = sizeof(T); - return size < 4 ? 128 / size : 16; + int index = row * ElementSideLength + column; + return ref Unsafe.Add(ref element, index); } } - public static unsafe nuint Size { get; } = 32 * 16; + public static int ElementSideLength + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get; + } = int.Max(8, 64 / sizeof(T)); + + public static unsafe int MaxElementCount + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get; + } = ElementSideLength * ElementSideLength; + + public static unsafe nuint Size { get; } = (nuint)(MaxElementCount * sizeof(T)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int ChunkCount() => (ElementSideLength + LanesPerVector() - 1) / LanesPerVector(); [MethodImpl(MethodImplOptions.AggressiveInlining)] public static MatrixComputeKernelAVX512* Create() { - MatrixComputeKernelAVX512* kernel = (MatrixComputeKernelAVX512*)NativeMemory.AlignedAlloc(Size, 16); + MatrixComputeKernelAVX512* kernel = (MatrixComputeKernelAVX512*)NativeMemory.AlignedAlloc(Size, 64); return kernel; } @@ -191,5 +275,24 @@ namespace StgSharp.HighPerformance } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ref M512 GetVectorChunk(int column, int chunk) + { + int lanes = LanesPerVector(); + int rowStart = chunk * lanes; + ref T colBase = ref this[column, 0]; + return ref Unsafe.As(ref Unsafe.Add(ref colBase, rowStart)); + } + + // SIMD access helpers for AVX-512 (column-major) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int LanesPerVector() => 64 / sizeof(T); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void SetVectorChunk(int column, int chunk, in M512 value) + { + GetVectorChunk(column, chunk) = value; + } +#pragma warning restore CA1000 } } \ No newline at end of file diff --git a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs index a3e7c05a..e7bef0d1 100644 --- a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs +++ b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs @@ -86,9 +86,6 @@ namespace StgSharp.HighPerformance.Memory SliceMemory(handle->NextNear, remainder); } } - if (handle->PreviousNear == null || handle->NextNear == null) { - throw new HLSFPositionChainBreakException(); - } return new hlsfHandle(handle, size); } diff --git a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs index d19a1bdd..9195b314 100644 --- a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs +++ b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs @@ -98,47 +98,6 @@ namespace StgSharp.HighPerformance.Memory return; } - /* - Console.WriteLine("============ Merge start ======================="); - try - { - Console.WriteLine("forward from entry"); - Entry* __cur = entry; - int __cnt = 0; - while (__cur != null && __cur != _spareMemory && __cnt < 512) - { - Console.WriteLine(EntryToString(__cur)); - __cur = __cur->NextNear; - __cnt++; - if (__cur == entry) - { - Console.WriteLine("(looped back to start)"); - break; - } - } - Console.WriteLine(EntryToString(__cur)); - Console.WriteLine("MergeAndRemoveFromBuckets: backward from entry"); - __cur = entry; - __cnt = 0; - while (__cur != null && __cur != _spareMemory && __cnt < 512) - { - Console.WriteLine(EntryToString(__cur)); - __cur = __cur->PreviousNear; - __cnt++; - if (__cur == entry) - { - Console.WriteLine("(looped back to start)"); - break; - } - } - Console.WriteLine(EntryToString(__cur)); - } - catch { } - finally - { - Console.WriteLine("=========================================="); - } - /**/ long size = entry->Size; int originLevel = entry->Level; @@ -228,26 +187,11 @@ namespace StgSharp.HighPerformance.Memory [FieldOffset(0)] internal Entry* NextNear; // Position linked list: next pointer [FieldOffset(8)] internal Entry* PreviousNear; // Position linked list: previous pointer - [FieldOffset(24)] internal int _Level; // Level of the block in the allocator + [FieldOffset(24)] internal int Level; // Level of the block in the allocator [FieldOffset(16)] internal long Size; // Block size information [FieldOffset(36)] public EntryState State; [FieldOffset(28)] public nuint Position; // Memory block position - public int Level - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get => _Level; - [MethodImpl(MethodImplOptions.AggressiveInlining)] - set - { - int size = 64 * (1 << (value * 2)); - if (Size / size > 3 && Size < 64 * 1024 * 1024L) { - throw new HLSFPositionChainBreakException(); - } - _Level = value; - } - } - } internal enum EntryState : uint @@ -260,20 +204,5 @@ namespace StgSharp.HighPerformance.Memory } - #region Entry Operations - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool IsNullOrEmptyEntryIndex(ulong index) - { - return index == 0; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool IsNullOrEmptyEntryIndex(Entry* entry) - { - return entry == null; - } - - #endregion } } diff --git a/StgSharp.Common/HighPerformance/Memory/InroductionToHLSF.md b/StgSharp.Common/HighPerformance/Memory/InroductionToHLSF.md index 92c2da3c..c6ad2fb3 100644 --- a/StgSharp.Common/HighPerformance/Memory/InroductionToHLSF.md +++ b/StgSharp.Common/HighPerformance/Memory/InroductionToHLSF.md @@ -3,12 +3,14 @@ A constant-time (O(1)) allocator for real-time, high-throughput scenarios. It uses layered size classes with segmented buckets and stores all metadata externally for clean user memory and better cache locality. ## Key Points -- O(1) allocate/free with 10 levels: 64B ¡ú 16MB; each level split into 1¡Á/2¡Á/3¡Á segments + +- O(1) allocate/free with 10 levels: 64B x 16MB; each level split into 1x/2x/3x segments - External metadata (`Entry`, `BucketNode`) in dedicated SLABs; user blocks have no headers -- Configurable alignment (¡Ý16B); large block cap per allocation: 32MB +- Configurable alignment (>=16B); large block cap per allocation: 32MB - Single-threaded implementation (external sync required for multi-threaded use) ## Quick Use + ```csharp using var hlsf = new HybridLayerSegregatedFitAllocator(64 * 1024 * 1024); var h = hlsf.Alloc(1024); @@ -16,49 +18,83 @@ Span s = h.BufferHandle; s[0] = 42; hlsf.Free(h); ``` -## Benchmarks (this run) -Environment: .NET 8 (Release), x64, single-thread, arena=3072 MiB. Results from `StgSharpDebug/HlsfStressTest`. +## Core APIs -- Free throughput: 1KB ¡Á 262,144 ¡ú 27,062,260.6 free/s -- Alloc+free pairs: 256B ¡Á 5,000,000 ¡ú 19,256,409.9 ops/s -- Fragmentation cycle (¡Ö512MB): fast recovery in short cycle (see test for details) +### Alloc(uint size) + +- Purpose: Allocate a block of at least `size` bytes. +- Input: `size` in bytes (<= 32MB for internal path; >32MB uses native large allocation). +- Output: `HybridLayerSegregatedFitAllocationHandle` with pointer access and `Span`. +- Behavior: + - Classify into level/segment and pop a matching bucket; escalate segments and levels if empty. + - Fallback to overflow bucket or carve 32MB from spare memory if all buckets are empty. + - Mark selected `Entry` as Alloc; slice any remainder into `Empty` entries and push to buckets. + - For >32MB requests, use `NativeMemory.AlignedAlloc` and mark `EntryState.Large`. + +### Free(HybridLayerSegregatedFitAllocationHandle handle) + +- Purpose: Free a previously allocated block. +- Behavior: + - Mark the corresponding `Entry` as `Empty`. + - Merge with adjacent `Empty` entries in both directions (boundary and level constraints). + - Remove merged neighbors from buckets, update position chain, and optionally promote level. + - Re-enqueue the final `Entry` into the appropriate bucket. + +### Collect() + +- Purpose: Reclaim and rebalance free entries without touching allocated blocks. +- Behavior: + - Consolidate fragmented regions where possible and improve bucket coverage. + - Intended to be cheap; safe to call between benchmark phases. + +## Benchmarks (this run) + +Environment: .NET 8 (Release), x64, single-thread, arena=3072 MiB. Results from StgSharpDebug/HlsfStressTest. + +- Free throughput: 1KB x 262,144 -> 38,928,422.9 free/s +- Alloc+free pairs: 256B x 5,000,000 -> 25,006,976.9 ops/s +- Fragmentation cycle (~512MB): fast recovery in short cycle (see test for details) Representative allocation throughput (higher is better): | Size | Ops | alloc/s (no touch) | alloc/s (touch) | |------|----:|--------------------:|----------------:| -| 64B | 2,000,000 | 7,902,555.2 | 14,176,544.8 | -| 1KB | 1,572,864 | 5,794,161.6 | 7,293,856.5 | -| 4KB | 393,216 | 5,210,132.6 | 6,646,372.3 | -| 32KB | 49,152 | 5,931,432.3 | 7,719,199.1 | -| 256KB | 6,144 | 14,663,484.5 | 10,425,929.1 | -| 512KB | 3,072 | 11,054,336.1 | 16,245,372.8 | -| 1MB | 1,536 | 13,391,456.0 | 9,159,212.9 | -| 2MB | 768 | 17,028,824.8 | 11,962,616.8 | -| 8MB | 192 | 13,061,224.5 | 9,142,857.1 | -| 32MB | 48 | 11,707,317.1 | 9,411,764.7 | +| 64B | 2,000,000 | 8,780,467.7 | 17,524,920.4 | +| 1KB | 1,572,864 | 8,974,865.4 | 9,585,531.6 | +| 4KB | 393,216 | 7,340,392.2 | 6,849,319.4 | +| 32KB | 49,152 | 10,376,625.6 | 7,019,407.9 | +| 256KB | 6,144 | 16,041,775.5 | 12,842,809.4 | +| 512KB | 3,072 | 20,466,355.8 | 14,195,933.5 | +| 1MB | 1,536 | 16,916,299.6 | 13,568,904.6 | +| 2MB | 768 | 17,985,948.5 | 12,569,558.1 | +| 8MB | 192 | 13,426,573.4 | 11,034,482.8 | +| 32MB | 48 | 11,162,790.7 | 9,411,764.7 | Notes: -- Peak no-touch at 2MB (¡Ö17.0M alloc/s). Peak touch at 512KB (¡Ö16.25M alloc/s). + +- Peak no-touch at 2MB (~17.99M alloc/s). Peak touch near 64B (~17.52M alloc/s). - Non-aligned sizes (e.g., 768B, 3KB) typically lower due to slicing/boundary handling. -## Detailed Design +## Detailed Design (summary) +- +- Size classes: levelSizeArray = [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216] (64B x 4^level) +- Segments per level: 1x, 2x, 3x +- Buckets: index = 3 * level + segment (overflow bucket for level > 9) +- Metadata: Entry (external SLAB, position chain), BucketNode (user region, bucket chain) +- Position chain: circular doubly linked list anchored by spare sentinel +- Allocation: search buckets by level/segment with escalation; if none, carve 32MB from spare; set Alloc; slice remainder into buckets +- Slicing: align to level boundaries, pack contiguous blocks forward/backward to minimize overhead +- Free/Merge: bidirectional merge under boundary and level constraints; update buckets; optional level promotion +- Alignment: default 16B; internal blocks aligned by slicing/edge calculation +- Complexity: average O(1) with bounded levels and constant-time bucket ops -### Size Classes and Segments -- Level sizes: `levelSizeArray = [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216]` (64B ¡¤ 4^L) -- Level selection: - - `GetLevelFromSize(uint S)`: `level = max(0, (log2(S) - 6) / 2)` using `BitOperations.LeadingZeroCount` -- Segment selection (within a level): - - `DetermineSegmentIndex(size, level) = (int)((size - 1) / levelSizeArray[level])` ¡ú 0/1/2 -- Bucket index: `index = 3*level + segment` (for L ¡Ü 9); overflow bucket at `^1` for level > 9 - -### Metadata Structures - `Entry` (external SLAB): linkage in position chain, `Position`, `Size`, `Level`, `State` - Invariants: near neighbors must form a consistent double-linked ring; `Level` setter validates size vs. level window - `BucketNode` (in user region): `EntryRef`, `NextLevel`, `PreviousLevel`, `IsInBucket` - Position chain: circular doubly linked list anchored by sentinel `_spareMemory` (`State.Spare`) ### Allocation Flow + 1) Compute `level` and `segment` from requested `size` 2) Try pop bucket (`TryPopLevelForAllocation(level, segment, out entry)`) and escalate: - Advance `segment` 0¡ú1¡ú2, then `level++`, until `MaxLevel` @@ -67,6 +103,7 @@ Notes: 4) If remainder exists in the source region, call `SliceMemory(entry->NextNear, remainder)` to produce free chunks into buckets ### Slicing Logic (`SliceMemory`) + - Inputs: `next` entry position, `sizeToSlice` - Compute `offset` and `end` relative to base buffer; pick starting level by trailing-zero count of `offset`: - `i = (BitOperations.TrailingZeroCount(offset) - 6) / 2`, clamped to `¡Ü MaxLevel` @@ -77,6 +114,7 @@ Notes: - For each packed region: allocate new `Entry`, insert before `next`, mark `Empty`, and push corresponding `BucketNode` into the `(level, segment)` bucket (`PushBucketToLevel(i, count-1, b)`) ### Free and Merge (`MergeAndRemoveFromBuckets`) + - Given an `Entry` to free (turned `Empty` by caller): - Merge left while neighbor `p` is `Empty`, same `Level`, and within boundary (`AssertBoundary`) - Remove `p` from bucket (`RemoveFromBucket`), expand current, fix `Position`, unlink `p` @@ -85,25 +123,30 @@ Notes: - Ensure `BucketNode.EntryRef` points to the merged entry; caller re-enqueues into the proper bucket ### Buckets + - `PushBucketToLevel(level, segment, node)`: push to bucket head, set `IsInBucket`, maintain prev/next - `TryPopLevel(level, segment, out node)`: pop head; if empty, return false - `RemoveFromBucket(level, segment, node)`: unlink from middle or use `TryPopLevel` if it is the head ### Large Allocations and Spare Memory + - Requests > 32MB use `NativeMemory.AlignedAlloc(size, align)` and mark `State.Large` - When buckets are empty, `TryAllocateNew32MBBlock` carves a 32MB region from `_spareMemory`, builds an `Empty` entry, and advances the spare pointer ### Alignment + - Default `_align = 16`; `AlignOfLevel(level)` returns `min(levelSizeArray[level], _align)` - Internal blocks align to level boundaries by design of slicing/edge calculations ### Complexity and Invariants + - Allocate/free are O(1) on average with bounded levels (¡Ü10) and constant-time bucket ops - Invariants guarded by exceptions (e.g., `HLSFPositionChainBreakException`) and pointer consistency checks ## Profiling (hotspots, current workloads) -- `SliceMemory` ¡Ö22% (self ¡Ö17%) ¡ª slicing and bucket enqueue dominate allocation path -- `MergeAndRemoveFromBuckets` ¡Ö13% (self ¡Ö12%) ¡ª merge + removal dominate free path -- `Collect` ¡Ö7.6% (self ¡Ö7.5%) ¡ª low overall cost -Suggested focus: reduce per-slice math and bucket ops; add fast-paths for fully aligned regions; batch/lazy removals after merges. \ No newline at end of file +- SliceMemory ~22% (self ~17%) ¡ª slicing and bucket enqueue dominate allocation path +- MergeAndRemoveFromBuckets ~13% (self ~12%) ¡ª merge + removal dominate free path +- Collect ~7.6% (self ~7.5%) ¡ª low overall cost + +Focus: reduce per-slice math and bucket ops; add fast-paths for aligned regions; batch/lazy removals after merges. \ No newline at end of file diff --git a/StgSharp.Common/Mathematics/Graphic/Matrix44.cs b/StgSharp.Common/Mathematics/Graphic/Matrix44.cs index d0b21fdd..194f4f09 100644 --- a/StgSharp.Common/Mathematics/Graphic/Matrix44.cs +++ b/StgSharp.Common/Mathematics/Graphic/Matrix44.cs @@ -25,6 +25,7 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- +using StgSharp.HighPerformance; using StgSharp.Mathematics.Numeric; using System; using System.Collections; @@ -45,6 +46,7 @@ namespace StgSharp.Mathematics.Graphic [FieldOffset(0)] internal ColumnSet4 mat; [FieldOffset(16 * sizeof(float))] internal ColumnSet4 transpose; + [FieldOffset(0)] internal MatrixKernel kernel; internal Matrix44(ColumnSet4 mat) { @@ -147,13 +149,17 @@ namespace StgSharp.Mathematics.Graphic public static Matrix44 Unit => new Matrix44(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1); - public static Matrix44 WNegativeUnit => new Matrix44(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, -1); + public static Matrix44 WNegativeUnit => new Matrix44(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, + 0, 0, -1); - public static Matrix44 XNegativeUnit => new Matrix44(-1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1); + public static Matrix44 XNegativeUnit => new Matrix44(-1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, + 0, 0, 1); - public static Matrix44 YNegativeUnit => new Matrix44(1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1); + public static Matrix44 YNegativeUnit => new Matrix44(1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 0, + 0, 0, 1); - public static Matrix44 ZNegativeUnit => new Matrix44(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1); + public static Matrix44 ZNegativeUnit => new Matrix44(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, -1, 0, 0, + 0, 0, 1); public ReadOnlySpan AsSpan() { @@ -186,8 +192,8 @@ namespace StgSharp.Mathematics.Graphic public override int GetHashCode() { - return HashCode.Combine(mat.colum0.GetHashCode(), mat.colum1.GetHashCode(), mat.colum2.GetHashCode(), - mat.colum3.GetHashCode()); + return HashCode.Combine(mat.colum0.GetHashCode(), mat.colum1.GetHashCode(), + mat.colum2.GetHashCode(), mat.colum3.GetHashCode()); } public override string ToString() @@ -224,8 +230,8 @@ namespace StgSharp.Mathematics.Graphic [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe Matrix44 operator *(Matrix44 mat, float value) { - return new Matrix44(mat.mat.colum0 * value, mat.mat.colum1 * value, mat.mat.colum2 * value, - mat.mat.colum3 * value); + return new Matrix44(mat.mat.colum0 * value, mat.mat.colum1 * value, + mat.mat.colum2 * value, mat.mat.colum3 * value); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs index 7d26f234..7d76854d 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs @@ -41,7 +41,10 @@ namespace StgSharp.Mathematics.Numeric public static class MatrixCompute { - public static unsafe Matrix Add(Matrix left, Matrix right, Matrix ans) + public static unsafe Matrix Add( + Matrix left, + Matrix right, + Matrix ans) { if (left.ColumnLength != right.ColumnLength || left.RowLength != right.RowLength || @@ -86,11 +89,11 @@ namespace StgSharp.Mathematics.Numeric return ans; } - public static unsafe void Fill(Matrix ans, T value) where T : unmanaged, INumber + public static unsafe void Fill(Matrix ans, float value) { long count = (long)ans.KernelColumnLength * ans.KernelRowLength; ScalarPacket* scalar = MatrixParallelFactory.CreateScalarPacket(); - scalar->Data(0) = value; + scalar->Data(0) = value; if (count <= 256) { NativeIntrinsic.Intrinsic.f32_fill(ans.Buffer, scalar, count); @@ -116,6 +119,19 @@ namespace StgSharp.Mathematics.Numeric taskGroup.Dispose(); } + public static unsafe Matrix Mul( + Matrix left, + Matrix right, + Matrix ans) + { + if (left.RowLength != right.ColumnLength || + left.RowLength != ans.RowLength || + right.ColumnLength != ans.ColumnLength) { + throw new IndexOutOfRangeException("Matrix dimensions must agree."); + } + throw new NotImplementedException(); + } + public static unsafe Matrix Mul(Matrix left, float right, Matrix ans) { if (left.ColumnLength != ans.ColumnLength || left.RowLength != ans.RowLength) { @@ -159,7 +175,10 @@ namespace StgSharp.Mathematics.Numeric return ans; } - public static unsafe Matrix Sub(Matrix left, Matrix right, Matrix ans) + public static unsafe Matrix Sub( + Matrix left, + Matrix right, + Matrix ans) { if (left.ColumnLength != right.ColumnLength || left.RowLength != right.RowLength || diff --git a/StgSharp.Native/StgSharp.Native.vcxproj b/StgSharp.Native/StgSharp.Native.vcxproj index e5db6017..47d66f39 100644 --- a/StgSharp.Native/StgSharp.Native.vcxproj +++ b/StgSharp.Native/StgSharp.Native.vcxproj @@ -148,6 +148,7 @@ + diff --git a/StgSharp.Native/StgSharp.Native.vcxproj.filters b/StgSharp.Native/StgSharp.Native.vcxproj.filters index 13ad698b..3abc766e 100644 --- a/StgSharp.Native/StgSharp.Native.vcxproj.filters +++ b/StgSharp.Native/StgSharp.Native.vcxproj.filters @@ -51,6 +51,9 @@ æºæ–‡ä»¶ + + æºæ–‡ä»¶ + diff --git a/StgSharp.Native/sn-head/sn_intrinsic.h b/StgSharp.Native/sn-head/sn_intrinsic.h index 996bc7a4..0ccff731 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.h @@ -250,6 +250,9 @@ INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int lengt #pragma endregion +typedef void(SN_DECL *BUILD_PANEL_PROC)(mat_panel *panel, mat_kernel *kernel, int col_length, + int row_length, int col_index, int row_index); + // Reordered to match C# IntrinsicContext field order // C# order: city_hash_simplify, f32_add, f32_fill, f32_fma, f32_scalar_mul, f32_sub, f32_transpose, factorial_simd, index_pair, normalize_3 typedef struct sn_intrinsic { diff --git a/StgSharp.Native/sn-head/sn_matkernel.h b/StgSharp.Native/sn-head/sn_matkernel.h index 168c740f..42803f48 100644 --- a/StgSharp.Native/sn-head/sn_matkernel.h +++ b/StgSharp.Native/sn-head/sn_matkernel.h @@ -7,6 +7,7 @@ #define ZMMCOUNT(T) sizeof(T) / 4 #include +#include #define MAT_KERNEL(T) \ union { \ @@ -77,7 +78,29 @@ - Column c occupies vec[c][0..CH-1]. - Row r inside column c is located at vec[c][r / R] with lane (r % R). */ + +/* Global zero kernel declaration (defined in dllmain.c), aligned to 64 bytes */ +#ifdef __cplusplus +extern "C" { +#endif +__declspec(align(64)) extern MAT_KERNEL(uint64_t) SN_ZERO_KERNEL; +#ifdef __cplusplus +} +#endif + #define PANEL_COL_VEC(panel_obj, c, chunk) ((panel_obj).vec[(int)(c)][(int)(chunk)]) /* Scalar flat index helper for column-major addressing */ #define PANEL_ELEM(panel_obj, T, T_VEC, MINQ, r, c) \ ((panel_obj).flat[(int)(c) * PANEL_Q(T, T_VEC, MINQ) + (int)(r)]) + +#define BUILD_PANEL(arch, element) build_panel_##arch##_##element + +#include "sn_internal.h" + +#define DECLARE_BUILD_PANEL(arch, element) \ + INTERNAL void SN_DECL BUILD_PANEL(arch, element)( \ + MAT_PANEL_##arch(element) *restrict panel, \ + MAT_KERNEL(element) const *const matrix, int col_length, int row_length, \ + int col_index, int row_index) + +DECLARE_BUILD_PANEL(AVX, float); diff --git a/StgSharp.Native/sn-src/dllmain.c b/StgSharp.Native/sn-src/dllmain.c index 13a88745..c4af815c 100644 --- a/StgSharp.Native/sn-src/dllmain.c +++ b/StgSharp.Native/sn-src/dllmain.c @@ -1,12 +1,25 @@ - -#include "StgSharpNative.h" +#include "StgSharpNative.h" +#include "sn_matkernel.h" #ifdef _WIN32 +/* Define and initialize a global zero kernel aligned to 64 bytes for universal ISA usage */ +__declspec(align(64)) MAT_KERNEL(uint64_t) SN_ZERO_KERNEL; + +static void init_zero_kernel(void) +{ + for (int c = 0; c < 4; ++c) { + for (int r = 0; r < 4; ++r) { + SN_ZERO_KERNEL.m[r][c] = 0.0; + } + } +} + BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) { switch (ul_reason_for_call) { case DLL_PROCESS_ATTACH: + init_zero_kernel(); return 1; case DLL_THREAD_ATTACH: return 1; @@ -21,6 +34,7 @@ BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserv #else attribute((constructor)) void DllMain() { + init_zero_kernel(); } #endif // _WIN32 diff --git a/StgSharp.Native/sn-src/k_build_panel.c b/StgSharp.Native/sn-src/k_build_panel.c new file mode 100644 index 00000000..7802efa4 --- /dev/null +++ b/StgSharp.Native/sn-src/k_build_panel.c @@ -0,0 +1,42 @@ +#include "sn_matkernel.h" + +DECLARE_BUILD_PANEL(AVX, float) +{ + const int Q = 8; /* fixed for AVX float panel */ + + /* Neighbor kernels; substitute global zero kernel where out-of-bounds */ + const int has_right = (col_index + 1) < col_length; + const int has_bottom = (row_index + 1) < row_length; + + const MAT_KERNEL(float) *const k_lt = matrix + (col_index * row_length + row_index); + const MAT_KERNEL(float) *const k_rt = + has_right ? (matrix + ((col_index + 1) * row_length + row_index)) : + (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; + const MAT_KERNEL(float) *const k_lb = + has_bottom ? (matrix + (col_index * row_length + (row_index + 1))) : + (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; + const MAT_KERNEL(float) *const k_rb = + (has_right && has_bottom) ? + (matrix + ((col_index + 1) * row_length + (row_index + 1))) : + (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; + + register __m128 top128, bot128; + register __m256 col256; + + for (int pc = 0; pc < Q; ++pc) { + const int is_right_half = (pc >= 4); + const int kc_col = pc & 3; /* column index inside 4x4 kernel */ + + const MAT_KERNEL(float) *src_top = is_right_half ? k_rt : k_lt; + const MAT_KERNEL(float) *src_bot = is_right_half ? k_rb : k_lb; + + /* Vector loads: each kernel column is a compact __m128 */ + top128 = src_top->xmm[kc_col]; + bot128 = src_bot->xmm[kc_col]; + + /* Combine into an AVX column: low 128 = top, high 128 = bottom */ + col256 = _mm256_set_m128(bot128, top128); + + PANEL_COL_VEC((*panel), pc, 0) = col256; + } +} diff --git a/StgSharpDebug/Program.cs b/StgSharpDebug/Program.cs index 81cc177e..d3b1c84a 100644 --- a/StgSharpDebug/Program.cs +++ b/StgSharpDebug/Program.cs @@ -25,6 +25,7 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- +using StgSharp.HighPerformance; using StgSharp.HighPerformance.Memory; using StgSharp.Internal; using StgSharp.Mathematics.Numeric; @@ -34,7 +35,7 @@ namespace StgSharp internal static partial class Program { - private static string testExpPath = @"D:\\Programmer\\stepcode\\data\\ap203\\ap203.exp"; + private static string testExpPath = @"D:\Programmer\stepcode\data\ap203\ap203.exp"; internal static unsafe void Main(string[] args) { @@ -49,6 +50,7 @@ namespace StgSharp // TestHlsf(); // HlsfStressTest.Run(); + M128 v = new M128(); /**/ HybridLayerSegregatedFitAllocator hlsf; unchecked {