working on panel building, preparing for panel compute

2025-12-03 18:14:45 +08:00
parent 38dba88331
commit 588fa867af
14 changed files with 338 additions and 212 deletions
@@ -48,6 +48,7 @@ namespace StgSharp.HighPerformance
        internal M128(Vector4 v)
        {
            Unsafe.SkipInit(out this);
+            Unsafe.SkipInit(out Vec);
            Vec = v.AsVector128();
        }

@@ -57,7 +58,7 @@ namespace StgSharp.HighPerformance
            Vec = v;
        }

-        public ref T AsRef<T>() where T: unmanaged, INumber<T>
+        public ref T AsRef<T>() where T : unmanaged, INumber<T>
        {
            return ref Unsafe.As<byte, T>(ref Buffer[0]);
        }
@@ -68,7 +69,7 @@ namespace StgSharp.HighPerformance
            return Vec.AsVector4();
        }

-        public void BroadCastFrom<T>(T value) where T: unmanaged, INumber<T>
+        public void BroadCastFrom<T>(T value) where T : unmanaged, INumber<T>
        {
            Vec = Vector128.Create<T>(value).AsSingle();
        }
@@ -84,7 +85,7 @@ namespace StgSharp.HighPerformance
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public ref T Member<T>(int index) where T: unmanaged, INumber<T>
+        public ref T Member<T>(int index) where T : unmanaged, INumber<T>
        {
            return ref Unsafe.Add(ref AsRef<T>(), index);
        }
@@ -115,65 +116,5 @@ namespace StgSharp.HighPerformance
            return left.Vec == right.Vec;
        }

-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static explicit operator Vector128<byte>(M128 m)
-        {
-            return m.Vec.As<float, byte>();
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static explicit operator Vector128<double>(M128 m)
-        {
-            return m.Vec.As<float, double>();
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static explicit operator Vector128<float>(M128 m)
-        {
-            return m.Vec.As<float, float>();
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static explicit operator Vector128<int>(M128 m)
-        {
-            return m.Vec.As<float, int>();
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static explicit operator Vector128<long>(M128 m)
-        {
-            return m.Vec.As<float, long>();
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static explicit operator Vector128<sbyte>(M128 m)
-        {
-            return m.Vec.As<float, sbyte>();
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static explicit operator Vector128<short>(M128 m)
-        {
-            return m.Vec.As<float, short>();
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static explicit operator Vector128<uint>(M128 m)
-        {
-            return m.Vec.As<float, uint>();
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static explicit operator Vector128<ulong>(M128 m)
-        {
-            return m.Vec.As<float, ulong>();
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static explicit operator Vector128<ushort>(M128 m)
-        {
-            return m.Vec.As<float, ushort>();
-        }
-
    }
 }
@@ -32,11 +32,17 @@ using System.Runtime.InteropServices;

 namespace StgSharp.HighPerformance
 {
-    public interface IMatrixPanel<T> : IDisposable where T: unmanaged,INumber<T>
+    #pragma warning disable CA1000
+
+    public interface IMatrixPanel<T> : IDisposable where T : unmanaged,INumber<T>
    {

+        ref T this[int column, int row] { get; }
+
        static abstract int MaxElementCount { get; }

+        static abstract int ElementSideLength { get; }
+
        static abstract nuint Size { get; }

        void Dispose();
@@ -50,26 +56,39 @@ namespace StgSharp.HighPerformance
    /// <typeparam name="T">
    ///   Type of matrix element.
    /// </typeparam>
-    [InlineArray(16)]
-    public unsafe struct MatrixPanelSSE<T> : IMatrixPanel<T> where T: unmanaged, INumber<T>
+    public unsafe struct MatrixPanelSSE<T> : IMatrixPanel<T> where T : unmanaged, INumber<T>
    {

-        private M128 vector;
+        private T element;

        [Obsolete("Compute kernel should not be created from new(), use Create instead", true)]
        public MatrixPanelSSE() { }

-        public static int MaxElementCount
+        public ref T this[int column, int row]
        {
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
            get
            {
-                int size = sizeof(T);
-                return 16 * 16 / size;
+                int index = row * ElementSideLength + column;
+                return ref Unsafe.Add(ref element, index);
            }
        }

-        public static unsafe nuint Size { get; } = 16 * 16;
+        public static int ElementSideLength
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get;
+        } = int.Max(4, 16 / sizeof(T));
+
+        public static int MaxElementCount
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get;
+        } = ElementSideLength * ElementSideLength;
+
+        public static unsafe nuint Size { get; } = (nuint)(MaxElementCount * sizeof(T));
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int ChunkCount() => (ElementSideLength + LanesPerVector() - 1) / LanesPerVector();

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public static MatrixPanelSSE<T>* Create()
@@ -91,6 +110,25 @@ namespace StgSharp.HighPerformance
            }
        }

+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public ref M128 GetVectorChunk(int column, int chunk)
+        {
+            int lanes = LanesPerVector();
+            int rowStart = chunk * lanes;
+            ref T colBase = ref this[column, 0];
+            return ref Unsafe.As<T, M128>(ref Unsafe.Add(ref colBase, rowStart));
+        }
+
+        // SIMD access helpers (column-major)
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int LanesPerVector() => 16 / sizeof(T);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void SetVectorChunk(int column, int chunk, in M128 value)
+        {
+            GetVectorChunk(column, chunk) = value;
+        }
+
    }

    /// <summary>
@@ -100,31 +138,44 @@ namespace StgSharp.HighPerformance
    /// <typeparam name="T">
    ///   Type of matrix element.
    /// </typeparam>
-    [InlineArray(16)]
-    public unsafe struct MatrixPanelAVX<T> : IMatrixPanel<T> where T: unmanaged, INumber<T>
+    public unsafe struct MatrixPanelAVX<T> : IMatrixPanel<T> where T : unmanaged, INumber<T>
    {

-        private M256 _buffer;
+        private T element;

        [Obsolete("Compute kernel should not be created from new(), use Create instead", true)]
        public MatrixPanelAVX() { }

-        public static unsafe int MaxElementCount
+        public ref T this[int column, int row]
        {
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
            get
            {
-                int size = sizeof(T);
-                return size < 4 ? 128 / size : 16;
+                int index = row * ElementSideLength + column;
+                return ref Unsafe.Add(ref element, index);
            }
        }

-        public static unsafe nuint Size { get; } = 32 * 16;
+        public static int ElementSideLength
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get;
+        } = int.Max(8, 32 / sizeof(T));
+
+        public static unsafe int MaxElementCount
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get;
+        } = ElementSideLength * ElementSideLength;
+
+        public static unsafe nuint Size { get; } = (nuint)(MaxElementCount * sizeof(T));
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int ChunkCount() => (ElementSideLength + LanesPerVector() - 1) / LanesPerVector();

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public static MatrixPanelAVX<T>* Create()
        {
-            MatrixPanelAVX<T>* kernel = (MatrixPanelAVX<T>*)NativeMemory.AlignedAlloc(Size, 16);
+            MatrixPanelAVX<T>* kernel = (MatrixPanelAVX<T>*)NativeMemory.AlignedAlloc(Size, 32);
            return kernel;
        }

@@ -141,6 +192,25 @@ namespace StgSharp.HighPerformance
            }
        }

+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public ref M256 GetVectorChunk(int column, int chunk)
+        {
+            int lanes = LanesPerVector();
+            int rowStart = chunk * lanes;
+            ref T colBase = ref this[column, 0];
+            return ref Unsafe.As<T, M256>(ref Unsafe.Add(ref colBase, rowStart));
+        }
+
+        // SIMD access helpers for AVX (column-major)
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int LanesPerVector() => 32 / sizeof(T);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void SetVectorChunk(int column, int chunk, in M256 value)
+        {
+            GetVectorChunk(column, chunk) = value;
+        }
+
    }

    /// <summary>
@@ -150,31 +220,45 @@ namespace StgSharp.HighPerformance
    /// <typeparam name="T">
    ///   Type of matrix element.
    /// </typeparam>
-    [InlineArray(16)]
-    public unsafe struct MatrixComputeKernelAVX512<T> : IMatrixPanel<T> where T: unmanaged, INumber<T>
+    public unsafe struct MatrixComputeKernelAVX512<T> : IMatrixPanel<T>
+        where T : unmanaged, INumber<T>
    {

-        private M512 _buffer;
+        private T element;

        [Obsolete("Compute kernel should not be created from new(), use Create instead", true)]
        public MatrixComputeKernelAVX512() { }

-        public static unsafe int MaxElementCount
+        public ref T this[int column, int row]
        {
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
            get
            {
-                int size = sizeof(T);
-                return size < 4 ? 128 / size : 16;
+                int index = row * ElementSideLength + column;
+                return ref Unsafe.Add(ref element, index);
            }
        }

-        public static unsafe nuint Size { get; } = 32 * 16;
+        public static int ElementSideLength
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get;
+        } = int.Max(8, 64 / sizeof(T));
+
+        public static unsafe int MaxElementCount
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get;
+        } = ElementSideLength * ElementSideLength;
+
+        public static unsafe nuint Size { get; } = (nuint)(MaxElementCount * sizeof(T));
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int ChunkCount() => (ElementSideLength + LanesPerVector() - 1) / LanesPerVector();

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public static MatrixComputeKernelAVX512<T>* Create()
        {
-            MatrixComputeKernelAVX512<T>* kernel = (MatrixComputeKernelAVX512<T>*)NativeMemory.AlignedAlloc(Size, 16);
+            MatrixComputeKernelAVX512<T>* kernel = (MatrixComputeKernelAVX512<T>*)NativeMemory.AlignedAlloc(Size, 64);
            return kernel;
        }

@@ -191,5 +275,24 @@ namespace StgSharp.HighPerformance
            }
        }

+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public ref M512 GetVectorChunk(int column, int chunk)
+        {
+            int lanes = LanesPerVector();
+            int rowStart = chunk * lanes;
+            ref T colBase = ref this[column, 0];
+            return ref Unsafe.As<T, M512>(ref Unsafe.Add(ref colBase, rowStart));
+        }
+
+        // SIMD access helpers for AVX-512 (column-major)
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int LanesPerVector() => 64 / sizeof(T);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void SetVectorChunk(int column, int chunk, in M512 value)
+        {
+            GetVectorChunk(column, chunk) = value;
+        }
+#pragma warning restore CA1000
    }
 }
@@ -86,9 +86,6 @@ namespace StgSharp.HighPerformance.Memory
                    SliceMemory(handle->NextNear, remainder);
                }
            }
-            if (handle->PreviousNear == null || handle->NextNear == null) {
-                throw new HLSFPositionChainBreakException();
-            }
            return new hlsfHandle(handle, size);
        }

@@ -98,47 +98,6 @@ namespace StgSharp.HighPerformance.Memory
                return;
            }

-            /*
-            Console.WriteLine("============ Merge start =======================");
-            try
-            {
-                Console.WriteLine("forward from entry");
-                Entry* __cur = entry;
-                int __cnt = 0;
-                while (__cur != null && __cur != _spareMemory && __cnt < 512)
-                {
-                    Console.WriteLine(EntryToString(__cur));
-                    __cur = __cur->NextNear;
-                    __cnt++;
-                    if (__cur == entry)
-                    {
-                        Console.WriteLine("(looped back to start)");
-                        break;
-                    }
-                }
-                Console.WriteLine(EntryToString(__cur));
-                Console.WriteLine("MergeAndRemoveFromBuckets: backward from entry");
-                __cur = entry;
-                __cnt = 0;
-                while (__cur != null && __cur != _spareMemory && __cnt < 512)
-                {
-                    Console.WriteLine(EntryToString(__cur));
-                    __cur = __cur->PreviousNear;
-                    __cnt++;
-                    if (__cur == entry)
-                    {
-                        Console.WriteLine("(looped back to start)");
-                        break;
-                    }
-                }
-                Console.WriteLine(EntryToString(__cur));
-            }
-            catch { }
-            finally
-            {
-                Console.WriteLine("==========================================");
-            }
-            /**/

            long size = entry->Size;
            int originLevel = entry->Level;
@@ -228,26 +187,11 @@ namespace StgSharp.HighPerformance.Memory
            [FieldOffset(0)]  internal Entry* NextNear;      // Position linked list: next pointer
            [FieldOffset(8)] internal Entry* PreviousNear;  // Position linked list: previous pointer

-            [FieldOffset(24)] internal int _Level;           // Level of the block in the allocator
+            [FieldOffset(24)] internal int Level;           // Level of the block in the allocator
            [FieldOffset(16)] internal long Size;           // Block size information
            [FieldOffset(36)] public EntryState State;
            [FieldOffset(28)]  public nuint Position;        // Memory block position

-            public int Level
-            {
-                [MethodImpl(MethodImplOptions.AggressiveInlining)]
-                get => _Level;
-                [MethodImpl(MethodImplOptions.AggressiveInlining)]
-                set
-                {
-                    int size = 64 * (1 << (value * 2));
-                    if (Size / size > 3 && Size < 64 * 1024 * 1024L) {
-                        throw new HLSFPositionChainBreakException();
-                    }
-                    _Level = value;
-                }
-            }
-
        }

        internal enum EntryState : uint
@@ -260,20 +204,5 @@ namespace StgSharp.HighPerformance.Memory

        }

-        #region Entry Operations
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static bool IsNullOrEmptyEntryIndex(ulong index)
-        {
-            return index == 0;
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static bool IsNullOrEmptyEntryIndex(Entry* entry)
-        {
-            return entry == null;
-        }
-
-        #endregion
    }
 }
@@ -3,12 +3,14 @@
 A constant-time (O(1)) allocator for real-time, high-throughput scenarios. It uses layered size classes with segmented buckets and stores all metadata externally for clean user memory and better cache locality.

 ## Key Points
- O(1) allocate/free with 10 levels: 64B ★ 16MB; each level split into 1～/2～/3～ segments
+
+- O(1) allocate/free with 10 levels: 64B x 16MB; each level split into 1x/2x/3x segments
 - External metadata (`Entry`, `BucketNode`) in dedicated SLABs; user blocks have no headers
- Configurable alignment (－16B); large block cap per allocation: 32MB
+- Configurable alignment (>=16B); large block cap per allocation: 32MB
 - Single-threaded implementation (external sync required for multi-threaded use)

 ## Quick Use
+
 ```csharp
 using var hlsf = new HybridLayerSegregatedFitAllocator(64 * 1024 * 1024);
 var h = hlsf.Alloc(1024);
@@ -16,49 +18,83 @@ Span<byte> s = h.BufferHandle; s[0] = 42;
 hlsf.Free(h);
 ```

-## Benchmarks (this run)
-Environment: .NET 8 (Release), x64, single-thread, arena=3072 MiB. Results from `StgSharpDebug/HlsfStressTest`.
+## Core APIs

- Free throughput: 1KB ～ 262,144 ★ 27,062,260.6 free/s
- Alloc+free pairs: 256B ～ 5,000,000 ★ 19,256,409.9 ops/s
- Fragmentation cycle (「512MB): fast recovery in short cycle (see test for details)
+### Alloc(uint size)
+
+- Purpose: Allocate a block of at least `size` bytes.
+- Input: `size` in bytes (<= 32MB for internal path; >32MB uses native large allocation).
+- Output: `HybridLayerSegregatedFitAllocationHandle` with pointer access and `Span<byte>`.
+- Behavior:
+  - Classify into level/segment and pop a matching bucket; escalate segments and levels if empty.
+  - Fallback to overflow bucket or carve 32MB from spare memory if all buckets are empty.
+  - Mark selected `Entry` as Alloc; slice any remainder into `Empty` entries and push to buckets.
+  - For >32MB requests, use `NativeMemory.AlignedAlloc` and mark `EntryState.Large`.
+
+### Free(HybridLayerSegregatedFitAllocationHandle handle)
+
+- Purpose: Free a previously allocated block.
+- Behavior:
+  - Mark the corresponding `Entry` as `Empty`.
+  - Merge with adjacent `Empty` entries in both directions (boundary and level constraints).
+  - Remove merged neighbors from buckets, update position chain, and optionally promote level.
+  - Re-enqueue the final `Entry` into the appropriate bucket.
+
+### Collect()
+
+- Purpose: Reclaim and rebalance free entries without touching allocated blocks.
+- Behavior:
+  - Consolidate fragmented regions where possible and improve bucket coverage.
+  - Intended to be cheap; safe to call between benchmark phases.
+
+## Benchmarks (this run)
+
+Environment: .NET 8 (Release), x64, single-thread, arena=3072 MiB. Results from StgSharpDebug/HlsfStressTest.
+
+- Free throughput: 1KB x 262,144 -> 38,928,422.9 free/s
+- Alloc+free pairs: 256B x 5,000,000 -> 25,006,976.9 ops/s
+- Fragmentation cycle (~512MB): fast recovery in short cycle (see test for details)

 Representative allocation throughput (higher is better):

 | Size | Ops | alloc/s (no touch) | alloc/s (touch) |
 |------|----:|--------------------:|----------------:|
-| 64B   | 2,000,000 | 7,902,555.2  | 14,176,544.8 |
-| 1KB   | 1,572,864 | 5,794,161.6  | 7,293,856.5  |
-| 4KB   |   393,216 | 5,210,132.6  | 6,646,372.3  |
-| 32KB  |    49,152 | 5,931,432.3  | 7,719,199.1  |
-| 256KB |     6,144 | 14,663,484.5 | 10,425,929.1 |
-| 512KB |     3,072 | 11,054,336.1 | 16,245,372.8 |
-| 1MB   |     1,536 | 13,391,456.0 | 9,159,212.9  |
-| 2MB   |       768 | 17,028,824.8 | 11,962,616.8 |
-| 8MB   |       192 | 13,061,224.5 | 9,142,857.1  |
-| 32MB  |        48 | 11,707,317.1 | 9,411,764.7  |
+| 64B   | 2,000,000 | 8,780,467.7  | 17,524,920.4 |
+| 1KB   | 1,572,864 | 8,974,865.4  | 9,585,531.6  |
+| 4KB   |   393,216 | 7,340,392.2  | 6,849,319.4  |
+| 32KB  |    49,152 | 10,376,625.6 | 7,019,407.9  |
+| 256KB |     6,144 | 16,041,775.5 | 12,842,809.4 |
+| 512KB |     3,072 | 20,466,355.8 | 14,195,933.5 |
+| 1MB   |     1,536 | 16,916,299.6 | 13,568,904.6 |
+| 2MB   |       768 | 17,985,948.5 | 12,569,558.1 |
+| 8MB   |       192 | 13,426,573.4 | 11,034,482.8 |
+| 32MB  |        48 | 11,162,790.7 | 9,411,764.7  |

 Notes:
- Peak no-touch at 2MB (「17.0M alloc/s). Peak touch at 512KB (「16.25M alloc/s).
+
+- Peak no-touch at 2MB (~17.99M alloc/s). Peak touch near 64B (~17.52M alloc/s).
 - Non-aligned sizes (e.g., 768B, 3KB) typically lower due to slicing/boundary handling.

-## Detailed Design
+## Detailed Design (summary)
+- 
+- Size classes: levelSizeArray = [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216] (64B x 4^level)
+- Segments per level: 1x, 2x, 3x
+- Buckets: index = 3 * level + segment (overflow bucket for level > 9)
+- Metadata: Entry (external SLAB, position chain), BucketNode (user region, bucket chain)
+- Position chain: circular doubly linked list anchored by spare sentinel
+- Allocation: search buckets by level/segment with escalation; if none, carve 32MB from spare; set Alloc; slice remainder into buckets
+- Slicing: align to level boundaries, pack contiguous blocks forward/backward to minimize overhead
+- Free/Merge: bidirectional merge under boundary and level constraints; update buckets; optional level promotion
+- Alignment: default 16B; internal blocks aligned by slicing/edge calculation
+- Complexity: average O(1) with bounded levels and constant-time bucket ops

-### Size Classes and Segments
- Level sizes: `levelSizeArray = [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216]` (64B ， 4^L)
- Level selection:
-  - `GetLevelFromSize(uint S)`: `level = max(0, (log2(S) - 6) / 2)` using `BitOperations.LeadingZeroCount`
- Segment selection (within a level):
-  - `DetermineSegmentIndex(size, level) = (int)((size - 1) / levelSizeArray[level])` ★ 0/1/2
- Bucket index: `index = 3*level + segment` (for L ＋ 9); overflow bucket at `^1` for level > 9
-
-### Metadata Structures
 - `Entry` (external SLAB): linkage in position chain, `Position`, `Size`, `Level`, `State`
  - Invariants: near neighbors must form a consistent double-linked ring; `Level` setter validates size vs. level window
 - `BucketNode` (in user region): `EntryRef`, `NextLevel`, `PreviousLevel`, `IsInBucket`
 - Position chain: circular doubly linked list anchored by sentinel `_spareMemory` (`State.Spare`)

 ### Allocation Flow
+
 1) Compute `level` and `segment` from requested `size`
 2) Try pop bucket (`TryPopLevelForAllocation(level, segment, out entry)`) and escalate:
   - Advance `segment` 0¡ú1¡ú2, then `level++`, until `MaxLevel`
@@ -67,6 +103,7 @@ Notes:
 4) If remainder exists in the source region, call `SliceMemory(entry->NextNear, remainder)` to produce free chunks into buckets

 ### Slicing Logic (`SliceMemory`)
+
 - Inputs: `next` entry position, `sizeToSlice`
 - Compute `offset` and `end` relative to base buffer; pick starting level by trailing-zero count of `offset`:
  - `i = (BitOperations.TrailingZeroCount(offset) - 6) / 2`, clamped to `¡Ü MaxLevel`
@@ -77,6 +114,7 @@ Notes:
 - For each packed region: allocate new `Entry`, insert before `next`, mark `Empty`, and push corresponding `BucketNode` into the `(level, segment)` bucket (`PushBucketToLevel(i, count-1, b)`)

 ### Free and Merge (`MergeAndRemoveFromBuckets`)
+
 - Given an `Entry` to free (turned `Empty` by caller):
  - Merge left while neighbor `p` is `Empty`, same `Level`, and within boundary (`AssertBoundary`)
    - Remove `p` from bucket (`RemoveFromBucket`), expand current, fix `Position`, unlink `p`
@@ -85,25 +123,30 @@ Notes:
  - Ensure `BucketNode.EntryRef` points to the merged entry; caller re-enqueues into the proper bucket

 ### Buckets
+
 - `PushBucketToLevel(level, segment, node)`: push to bucket head, set `IsInBucket`, maintain prev/next
 - `TryPopLevel(level, segment, out node)`: pop head; if empty, return false
 - `RemoveFromBucket(level, segment, node)`: unlink from middle or use `TryPopLevel` if it is the head

 ### Large Allocations and Spare Memory
+
 - Requests > 32MB use `NativeMemory.AlignedAlloc(size, align)` and mark `State.Large`
 - When buckets are empty, `TryAllocateNew32MBBlock` carves a 32MB region from `_spareMemory`, builds an `Empty` entry, and advances the spare pointer

 ### Alignment
+
 - Default `_align = 16`; `AlignOfLevel(level)` returns `min(levelSizeArray[level], _align)`
 - Internal blocks align to level boundaries by design of slicing/edge calculations

 ### Complexity and Invariants
+
 - Allocate/free are O(1) on average with bounded levels (¡Ü10) and constant-time bucket ops
 - Invariants guarded by exceptions (e.g., `HLSFPositionChainBreakException`) and pointer consistency checks

 ## Profiling (hotspots, current workloads)
- `SliceMemory` 「22% (self 「17%) ！ slicing and bucket enqueue dominate allocation path
- `MergeAndRemoveFromBuckets` 「13% (self 「12%) ！ merge + removal dominate free path
- `Collect` 「7.6% (self 「7.5%) ！ low overall cost

-Suggested focus: reduce per-slice math and bucket ops; add fast-paths for fully aligned regions; batch/lazy removals after merges.
+- SliceMemory ~22% (self ~17%) ¡ª slicing and bucket enqueue dominate allocation path
+- MergeAndRemoveFromBuckets ~13% (self ~12%) ¡ª merge + removal dominate free path
+- Collect ~7.6% (self ~7.5%) ¡ª low overall cost
+
+Focus: reduce per-slice math and bucket ops; add fast-paths for aligned regions; batch/lazy removals after merges. 
@@ -25,6 +25,7 @@
 //     
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
+using StgSharp.HighPerformance;
 using StgSharp.Mathematics.Numeric;
 using System;
 using System.Collections;
@@ -45,6 +46,7 @@ namespace StgSharp.Mathematics.Graphic

        [FieldOffset(0)] internal ColumnSet4 mat;
        [FieldOffset(16 * sizeof(float))] internal ColumnSet4 transpose;
+        [FieldOffset(0)] internal MatrixKernel<float> kernel;

        internal Matrix44(ColumnSet4 mat)
        {
@@ -147,13 +149,17 @@ namespace StgSharp.Mathematics.Graphic

        public static Matrix44 Unit => new Matrix44(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1);

-        public static Matrix44 WNegativeUnit => new Matrix44(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, -1);
+        public static Matrix44 WNegativeUnit => new Matrix44(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0,
+                                                             0, 0, -1);

-        public static Matrix44 XNegativeUnit => new Matrix44(-1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1);
+        public static Matrix44 XNegativeUnit => new Matrix44(-1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0,
+                                                             0, 0, 1);

-        public static Matrix44 YNegativeUnit => new Matrix44(1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1);
+        public static Matrix44 YNegativeUnit => new Matrix44(1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 0,
+                                                             0, 0, 1);

-        public static Matrix44 ZNegativeUnit => new Matrix44(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1);
+        public static Matrix44 ZNegativeUnit => new Matrix44(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, -1, 0, 0,
+                                                             0, 0, 1);

        public ReadOnlySpan<float> AsSpan()
        {
@@ -186,8 +192,8 @@ namespace StgSharp.Mathematics.Graphic

        public override int GetHashCode()
        {
-            return HashCode.Combine(mat.colum0.GetHashCode(), mat.colum1.GetHashCode(), mat.colum2.GetHashCode(),
-                                    mat.colum3.GetHashCode());
+            return HashCode.Combine(mat.colum0.GetHashCode(), mat.colum1.GetHashCode(),
+                                    mat.colum2.GetHashCode(), mat.colum3.GetHashCode());
        }

        public override string ToString()
@@ -224,8 +230,8 @@ namespace StgSharp.Mathematics.Graphic
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public static unsafe Matrix44 operator *(Matrix44 mat, float value)
        {
-            return new Matrix44(mat.mat.colum0 * value, mat.mat.colum1 * value, mat.mat.colum2 * value,
-                                mat.mat.colum3 * value);
+            return new Matrix44(mat.mat.colum0 * value, mat.mat.colum1 * value,
+                                mat.mat.colum2 * value, mat.mat.colum3 * value);
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -41,7 +41,10 @@ namespace StgSharp.Mathematics.Numeric
    public static class MatrixCompute
    {

-        public static unsafe Matrix<float> Add(Matrix<float> left, Matrix<float> right, Matrix<float> ans)
+        public static unsafe Matrix<float> Add(
+                                           Matrix<float> left,
+                                           Matrix<float> right,
+                                           Matrix<float> ans)
        {
            if (left.ColumnLength != right.ColumnLength ||
                left.RowLength != right.RowLength ||
@@ -86,11 +89,11 @@ namespace StgSharp.Mathematics.Numeric
            return ans;
        }

-        public static unsafe void Fill<T>(Matrix<float> ans, T value) where T : unmanaged, INumber<T>
+        public static unsafe void Fill(Matrix<float> ans, float value)
        {
            long count = (long)ans.KernelColumnLength * ans.KernelRowLength;
            ScalarPacket* scalar = MatrixParallelFactory.CreateScalarPacket();
-            scalar->Data<T>(0) = value;
+            scalar->Data<float>(0) = value;
            if (count <= 256)
            {
                NativeIntrinsic.Intrinsic.f32_fill(ans.Buffer, scalar, count);
@@ -116,6 +119,19 @@ namespace StgSharp.Mathematics.Numeric
            taskGroup.Dispose();
        }

+        public static unsafe Matrix<float> Mul(
+                                           Matrix<float> left,
+                                           Matrix<float> right,
+                                           Matrix<float> ans)
+        {
+            if (left.RowLength != right.ColumnLength ||
+                left.RowLength != ans.RowLength ||
+                right.ColumnLength != ans.ColumnLength) {
+                throw new IndexOutOfRangeException("Matrix dimensions must agree.");
+            }
+            throw new NotImplementedException();
+        }
+
        public static unsafe Matrix<float> Mul(Matrix<float> left, float right, Matrix<float> ans)
        {
            if (left.ColumnLength != ans.ColumnLength || left.RowLength != ans.RowLength) {
@@ -159,7 +175,10 @@ namespace StgSharp.Mathematics.Numeric
            return ans;
        }

-        public static unsafe Matrix<float> Sub(Matrix<float> left, Matrix<float> right, Matrix<float> ans)
+        public static unsafe Matrix<float> Sub(
+                                           Matrix<float> left,
+                                           Matrix<float> right,
+                                           Matrix<float> ans)
        {
            if (left.ColumnLength != right.ColumnLength ||
                left.RowLength != right.RowLength ||
@@ -148,6 +148,7 @@
    </Link>
  </ItemDefinitionGroup>
  <ItemGroup>
+    <ClCompile Include="sn-src\k_build_panel.c" />
    <ClCompile Include="sn-src\k_basicop.c" />
    <ClCompile Include="sn-src\dllmain.c" />
    <ClCompile Include="sn-src\k_bufop.c" />
@@ -51,6 +51,9 @@
    <ClCompile Include="sn-src\k_bufop.c">
      <Filter>源文件</Filter>
    </ClCompile>
+    <ClCompile Include="sn-src\k_build_panel.c">
+      <Filter>源文件</Filter>
+    </ClCompile>
  </ItemGroup>
  <ItemGroup>
    <ClInclude Include="sn-head\gl.h">
@@ -250,6 +250,9 @@ INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int lengt

 #pragma endregion

+typedef void(SN_DECL *BUILD_PANEL_PROC)(mat_panel *panel, mat_kernel *kernel, int col_length,
+                                        int row_length, int col_index, int row_index);
+
 // Reordered to match C# IntrinsicContext field order
 // C# order: city_hash_simplify, f32_add, f32_fill, f32_fma, f32_scalar_mul, f32_sub, f32_transpose, factorial_simd, index_pair, normalize_3
 typedef struct sn_intrinsic {
@@ -7,6 +7,7 @@
 #define ZMMCOUNT(T) sizeof(T) / 4

 #include <immintrin.h>
+#include <stdint.h>

 #define MAT_KERNEL(T)                        \
        union {                              \
@@ -77,7 +78,29 @@
   - Column c occupies vec[c][0..CH-1].
   - Row r inside column c is located at vec[c][r / R] with lane (r % R).
 */
+
+/* Global zero kernel declaration (defined in dllmain.c), aligned to 64 bytes */
+#ifdef __cplusplus
+extern "C" {
+#endif
+__declspec(align(64)) extern MAT_KERNEL(uint64_t) SN_ZERO_KERNEL;
+#ifdef __cplusplus
+}
+#endif
+
 #define PANEL_COL_VEC(panel_obj, c, chunk) ((panel_obj).vec[(int)(c)][(int)(chunk)])
 /* Scalar flat index helper for column-major addressing */
 #define PANEL_ELEM(panel_obj, T, T_VEC, MINQ, r, c) \
        ((panel_obj).flat[(int)(c) * PANEL_Q(T, T_VEC, MINQ) + (int)(r)])
+
+#define BUILD_PANEL(arch, element) build_panel_##arch##_##element
+
+#include "sn_internal.h"
+
+#define DECLARE_BUILD_PANEL(arch, element)                                               \
+        INTERNAL void SN_DECL BUILD_PANEL(arch, element)(                                \
+                MAT_PANEL_##arch(element) *restrict panel,                               \
+                MAT_KERNEL(element) const *const matrix, int col_length, int row_length, \
+                int col_index, int row_index)
+
+DECLARE_BUILD_PANEL(AVX, float);
@@ -1,12 +1,25 @@
-
-#include "StgSharpNative.h"
+#include "StgSharpNative.h"
+#include "sn_matkernel.h"

 #ifdef _WIN32

+/* Define and initialize a global zero kernel aligned to 64 bytes for universal ISA usage */
+__declspec(align(64)) MAT_KERNEL(uint64_t) SN_ZERO_KERNEL;
+
+static void init_zero_kernel(void)
+{
+        for (int c = 0; c < 4; ++c) {
+                for (int r = 0; r < 4; ++r) {
+                        SN_ZERO_KERNEL.m[r][c] = 0.0;
+                }
+        }
+}
+
 BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved)
 {
        switch (ul_reason_for_call) {
        case DLL_PROCESS_ATTACH:
+                init_zero_kernel();
                return 1;
        case DLL_THREAD_ATTACH:
                return 1;
@@ -21,6 +34,7 @@ BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserv
 #else
 attribute((constructor)) void DllMain()
 {
+        init_zero_kernel();
 }

 #endif // _WIN32
@@ -0,0 +1,42 @@
+#include "sn_matkernel.h"
+
+DECLARE_BUILD_PANEL(AVX, float)
+{
+        const int Q = 8; /* fixed for AVX float panel */
+
+        /* Neighbor kernels; substitute global zero kernel where out-of-bounds */
+        const int has_right = (col_index + 1) < col_length;
+        const int has_bottom = (row_index + 1) < row_length;
+
+        const MAT_KERNEL(float) *const k_lt = matrix + (col_index * row_length + row_index);
+        const MAT_KERNEL(float) *const k_rt =
+                has_right ? (matrix + ((col_index + 1) * row_length + row_index)) :
+                            (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
+        const MAT_KERNEL(float) *const k_lb =
+                has_bottom ? (matrix + (col_index * row_length + (row_index + 1))) :
+                             (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
+        const MAT_KERNEL(float) *const k_rb =
+                (has_right && has_bottom) ?
+                        (matrix + ((col_index + 1) * row_length + (row_index + 1))) :
+                        (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
+
+        register __m128 top128, bot128;
+        register __m256 col256;
+
+        for (int pc = 0; pc < Q; ++pc) {
+                const int is_right_half = (pc >= 4);
+                const int kc_col = pc & 3; /* column index inside 4x4 kernel */
+
+                const MAT_KERNEL(float) *src_top = is_right_half ? k_rt : k_lt;
+                const MAT_KERNEL(float) *src_bot = is_right_half ? k_rb : k_lb;
+
+                /* Vector loads: each kernel column is a compact __m128 */
+                top128 = src_top->xmm[kc_col];
+                bot128 = src_bot->xmm[kc_col];
+
+                /* Combine into an AVX column: low 128 = top, high 128 = bottom */
+                col256 = _mm256_set_m128(bot128, top128);
+
+                PANEL_COL_VEC((*panel), pc, 0) = col256;
+        }
+}
@@ -25,6 +25,7 @@
 //     
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
+using StgSharp.HighPerformance;
 using StgSharp.HighPerformance.Memory;
 using StgSharp.Internal;
 using StgSharp.Mathematics.Numeric;
@@ -34,7 +35,7 @@ namespace StgSharp
    internal static partial class Program
    {

-        private static string testExpPath = @"D:\\Programmer\\stepcode\\data\\ap203\\ap203.exp";
+        private static string testExpPath = @"D:\Programmer\stepcode\data\ap203\ap203.exp";

        internal static unsafe void Main(string[] args)
        {
@@ -49,6 +50,7 @@ namespace StgSharp
                // TestHlsf();

                // HlsfStressTest.Run();
+                M128 v = new M128();
                /**/
                HybridLayerSegregatedFitAllocator hlsf;
                unchecked {