From c64f7ae9f6e9a5611f8015d4e2be48186b1714c9 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:28:53 +0800 Subject: [PATCH] L4 theoretical work --- doc/L4_design.md | 762 ------------------ .../HighPerformance/Memory/L4.Design.md | 136 ++++ .../HighPerformance/Memory/L4.Map.cs | 8 +- .../Memory/L4.PredictorQueue.cs | 12 + .../HighPerformance/Memory/L4.cs | 12 + .../HighPerformance/Memory/L4Predictor.cs | 37 + 6 files changed, 204 insertions(+), 763 deletions(-) delete mode 100644 doc/L4_design.md create mode 100644 src/Nitload.Common/HighPerformance/Memory/L4.Design.md create mode 100644 src/Nitload.Common/HighPerformance/Memory/L4.PredictorQueue.cs create mode 100644 src/Nitload.Common/HighPerformance/Memory/L4Predictor.cs diff --git a/doc/L4_design.md b/doc/L4_design.md deleted file mode 100644 index 4f46c369..00000000 --- a/doc/L4_design.md +++ /dev/null @@ -1,762 +0,0 @@ -# L4 Software Cache Design Specification - -> Project: StgSharp -> Author: Nitload -> Version: Draft v0.3 -> Date: 2026-03-10 - ---- - -## 1. Purpose - -### 1.1 Problem Background - -StgSharp's matrix computation system is migrating from a "single global context table" to a -"per-thread context" architecture to adapt to the scheduling characteristics of hybrid-architecture -CPUs (e.g., Intel P-Core / E-Core). After the migration, each thread independently holds its own -computation context, but matrix data may still be allocated by the HLSF allocator or global -`malloc` on a remote NUMA node, resulting in significantly increased cross-node access latency. - -At the same time, in matrix multiplication (FMA), the core row data of matrix A in the -checkerboard-shaped kernel grid layout must be repacked (Packed) into a micro-kernel-friendly -contiguous arrangement. In libraries such as OpenBLAS, this Pack operation is re-executed on -every tile switch, whereas L4 can cache the Pack results to avoid repeated reordering. - -Additionally, the segmented accumulation (segment) of matrix multiplication requires storing -intermediate partial sums locally to avoid cross-NUMA writes to the target matrix on every -segment. L4 can serve as a local accumulation buffer for partial sums, writing them back in a -single operation after all segments complete locally. - -### 1.2 Solution - -The L4 cache is a purely software-implemented **data caching layer** logically positioned -between the CPU hardware caches (L1/L2/L3) and main memory. Its core responsibilities: - -1. **Pack result caching**: Cache the repack results of FMA core rows in matrix A to avoid repeated Pack operations -2. **NUMA migration cache**: Migrate data from remote NUMA nodes to local memory and cache it, eliminating cross-node latency -3. **Segment accumulation buffer**: Temporarily store partial sums from FMA segmented computation to reduce remote write operations -4. **PLU intermediate data buffer**: Provide a local contiguous workspace for PLU decomposition - -### 1.3 Design Goals - -- Provide NUMA-friendly local data access for per-thread matrix computation contexts -- Fixed capacity (64 MB), no runtime expansion, deterministic latency -- Transparent pointer redirection: upper-layer code queries via original pointers; L4 returns local cache pointers -- Cache eviction policy based on **RRPV mask + windowed OPT prediction** -- Support both **StreamTransparent** and **CacheBuffer** working modes -- Support **writable cache lines** that invoke registered callback functions for pre-processing and write-back upon eviction or explicit flush - ---- - -## 2. Overall Architecture - -### 2.1 Three-Component Model - -``` -┌���─────────────────────────────────────────────────────────────────┐ -│ L4 │ -│ │ -│ ┌──────────────┐ ┌────────────────────┐ ┌──────────────────┐ │ -│ │ CacheLine │ │ L4 Core │ │ Predictor │ │ -│ │ Handle │ │ Cache │ │ Prefetch │ │ -│ │ │ │ │ │ Predictor │ │ -│ │ • local addr │ │ • Swiss Table │ │ • 2048-entry buf │ │ -│ │ • ref count │ │ • Slab Allocator │ │ • threshold 1024 │ │ -│ │ • Dispose │ │ • RRPV mask │ │ • windowed OPT │ │ -│ │ drops ref │ │ • SIMD eviction │ │ • active key set │ │ -│ │ │ │ • writeback mgmt │ │ │ │ -│ └──────────────┘ └────────────────────┘ └──────────────────┘ │ -│ │ -└──────────────────────────────────────────────────────────────────┘ -``` - -### 2.2 Data Access Path - -``` -Compute thread accessing data: - - ┌─────────────────────────────────────────────────────────┐ - │ Read-only path: L4.Map(originKey) │ - │ ├─ StreamTransparent + no repack needed? │ - │ │ └─ Yes → return original pointer directly │ - │ │ (zero overhead) │ - │ │ │ - │ └─ CacheBuffer mode │ - │ ├─ Swiss Table lookup (2.6ns, lock-free) │ - │ ├─ Hit → update RRPV=0, return CacheLine handle │ - │ └─ Miss → SpinWait for Predictor prefetch │ - ├─────────────────────────────────────────────────────────┤ - │ Writable path: L4.MapWritable(originKey, preprocess, writeback) │ - │ ├─ Hit → return existing partial sum cache line │ - │ └─ Miss → allocate zero-initialized cache line, │ - │ register writeback callbacks │ - │ cache line marked dirty (dirty=1) │ - │ eviction invokes preprocess + writeback │ - ├─────────────────────────────────────────────────────────┤ - │ After computation: │ - │ 3. CacheLine.Dispose() (refCount--) │ - │ 4. L4.StepComplete() │ - │ ├─ Predictor.AdvanceExecutionPoint() │ - │ └─ Predictor.NeedsPrefetch? │ - │ └─ Yes → CAS-acquire lock, execute prefetch │ - └─────────────────────────────────────────────────────────┘ -``` - -### 2.3 Key Parameters - -| Parameter | Value | Description | -|------------------------|--------------------------|------------------------------------------------------| -| Total cache capacity | 64 MB | Per-NUMA-node contiguous memory | -| Cache line size | 8192 Bytes | = 64 × `double MAT_KERNEL` | -| Cache line count | 8192 | 64MB / 8192B | -| Mapping table type | Swiss Table | 1024 buckets × 16 slots, SIMD-accelerated | -| Cache line pool | Slab Allocator | Non-concurrent, fixed capacity, no auto-expansion | -| Eviction policy | RRPV mask + windowed OPT | SoA layout + SSE2 batch scan | -| Prefetch policy | Predictor windowed OPT | 2048-entry buffer, 1024-step lookahead | -| Writeback policy | Callback-driven | Invokes registered callbacks on dirty eviction/flush | - ---- - -## 3. Key Components - -### 3.1 Swiss Table (Pointer Mapping Table) - -**Responsibility**: Maintains the `original pointer → cache pointer` mapping. - -- Key: original pointer `nint`, pointing to data allocated by HLSF/malloc on a remote node -- Value: cache pointer `nint`, pointing to locally Slab-allocated cache line -- Hash: hardware `CRC32C` instruction on the 64-bit pointer - - Low 10 bits → bucket index (1024 buckets) - - Bits 10-16 → H2 control byte (used for SIMD pre-filtering) -- Scan: SSE2 `pmovmskb` instruction for parallel comparison of 16 control bytes -- Tombstone mechanism: deleted entries set ctrl to `0xFE`, preserving probe chain integrity -- Capacity: 1024 × 16 = 16,384 mappings, significantly larger than total cache line count (8192 lines), - ensuring extremely low hash collision rate - -**Measured Performance** (Ryzen 7 6800H, .NET 8, BenchmarkDotNet): - -| Operation | Swiss Table | Dictionary | Speedup | -|--------------------------|-------------|-------------|---------| -| Zipfian Lookup (8192×) | 20.0 μs | 76.1 μs | 3.8× | -| Uniform Lookup (8192×) | 18.6 μs | 50.6 μs | 2.7× | -| Insert (4096 entries) | 18.5 μs | 46.5 μs | 2.5× | -| SteadyState (4096 ops) | 25.3 μs | 68.3 μs | 2.7× | - -Single lookup latency ≈ 2.6 ns, GC allocation = 0. - -### 3.2 Slab Allocator (Cache Line Pool) - -**Responsibility**: Manages allocation and reclamation of fixed-size (8192B) cache lines. - -- Mode: **Non-concurrent, fixed capacity** (`FixedSlabAllocator` mode), - reusing the existing `SlabAllocator.FromBuffer()` interface -- No auto-expansion: capacity is fixed at initialization (64MB / 8192B = 8192 lines); - allocation failure triggers eviction rather than expansion -- Allocation complexity: O(1), pops from free stack -- Reclamation complexity: O(1), pushes back to free stack -- Alignment: all cache lines are 64B aligned, ensuring no cross-hardware-cache-line boundaries - -### 3.3 Eviction Metadata (SoA Layout + RRPV Mask + Writeback Callbacks) - -**Responsibility**: Tracks per-cache-line reference state, heat, and writeback information to determine eviction priority. - -Replaces the four-level circular linked list from v0.1 with parallel arrays in Structure-of-Arrays layout: - -``` -SoA Layout — Parallel Arrays - - _refFlags[i] : byte Reference count snapshot (0=free, non-zero=held) - _evictionFlags[i] : byte dirty(1bit) + RRPV(2bit) + costClass(1bit) - _originKeys[i] : nuint Original pointer (reverse-deregisters Swiss Table entry on eviction) - _preprocess[i] : nint Writeback pre-processing function pointer (called before dirty eviction) - _writeback[i] : nint Writeback function pointer (called during dirty eviction) - _callbackCtx[i] : nuint Callback context (passed to preprocess/writeback) -``` - -**Eviction Flag Bit Layout** (`_evictionFlags[i]`): - -The RRPV array uses byte storage for each cache line's usage state; the low 2 bits encode the RRPV value. - -**RRPV Semantics**: - -| RRPV | Meaning | Eviction Priority | -|------|---------------------------------|---------------------------| -| 0 | Will be re-accessed very soon | Lowest (do not evict) | -| 1 | Will be accessed shortly | Low | -| 2 | Will be accessed in a while | Medium | -| 3 | May never be accessed again | Highest (evict first) | - -**RRPV Update Events**: - -| Event | Operation | Description | -|--------------------|---------------------------------|------------------------------------------------------| -| Load (insert) | RRPV = 0 | Freshly loaded data will be used soon | -| Hit | `_evictionFlags[i] = 0` | Bits 1-2 cleared, RRPV reset to 0 | -| Eviction scan | Find lines with RRPV=3 to evict | If none found, increment all RRPV +1 (aging) | - -**Advantages over the v0.1 four-level circular linked list**: - -| Comparison | v0.1 Circular List | v0.3 RRPV Mask + Callbacks | -|----------------------|-------------------------------------|--------------------------------------| -| Data structure | 4 doubly-linked lists + next ptrs | Byte array + function pointer arrays | -| Warm-up operation | Unlink + relink (4 pointer writes) | `+= 1` (1-byte write) | -| Aging operation | Per-node list migration | SIMD batch +1 (16 entries/cycle) | -| Eviction scan | Iterate list head (pointer chasing) | SIMD pattern matching (16/cycle) | -| Cache friendliness | Poor (pointer chasing) | Good (contiguous memory scan) | -| Reference protection | None | Lines with refCount > 0 not evicted | -| Writeback support | None | Callback-function driven | - -### 3.4 Writeback Callback Mechanism - -**Responsibility**: Provides flexible writeback strategies for writable cache lines, using native function pointers to avoid managed overhead. - -**Design Principle**: All writable cache lines are treated as dirty by default. On eviction or explicit flush, -L4 itself does not manage the writeback strategy; instead, the bound predictor is responsible for implementing it. - -The predictor is defined via the following interface: - -``` - public interface IPrefetchPredict - { - - int Predict( - Span node - ); - - bool Prefetch( - nuint origin, - ulong policy, - nuint cache - ); - - void WriteBack( - nuint origin, - ulong policy, - nuint cache - ); - - } -``` - -**Typical Callback Implementations**: - -``` -Scenario 1: Segment accumulation (FMA partial sum) - preprocess = null (no pre-processing needed; data is already a double array) - writeback = AccumulateAdd (element-wise dst[i] += src[i], AVX2-vectorized) - -Scenario 2: Writeback of packed intermediate results - preprocess = UnpackKernelRow (reverse-repack from Pack format to row-major order) - writeback = AccumulateAdd - -Scenario 3: PLU decomposition intermediate data - preprocess = null - writeback = DirectCopy (direct overwrite memcpy) -``` - -### 3.5 CacheLine (Cache Line Handle) - -**Responsibility**: A cache line access handle held by the compute thread; prevents in-use lines from being evicted via reference counting. - -```csharp -public ref struct CacheLine : IDisposable -{ - private ref int _activeCount; // L4 global active count - private ref int _refCount; // Per-line reference count - private nuint _handle; // Local cache data address - - // Construction: Interlocked.Increment(ref _activeCount) - // Interlocked.Increment(ref _refCount) - // Dispose: Interlocked.Decrement on both counters -} -``` - -- `ref struct` ensures it cannot escape to the heap -- Reference counts use `Interlocked` atomic operations for thread safety -- Lines with `_refCount > 0` are skipped during eviction scans -- StreamTransparent mode returns a CacheLine that does not occupy a cache slot -- Both read-only and writable paths return the same CacheLine type, - callers decide read/write behavior via `.Address` - -### 3.6 Predictor (Prefetch Predictor) - -**Responsibility**: Provides deterministic prefetch predictions for L4 based on the windowed OPT strategy. - -**Core Principle**: Matrix computation task sequences are deterministic — the scheduler pre-allocates -task blocks `(kr, kc, zz)` for each thread, allowing precise computation of the cache line keys -required at each future step. This gives L4 a predictive capability that hardware caches cannot -possess: **direct visibility into the future**, rather than inferring the future from history as -strategies like Hawkeye do. - -``` -Predictor Sliding Window: - - Task sequence: ... [T_now] [T+1] [T+2] ... [T+1023] ... [T+2047] - ↑ execution point ↑ buffer tail - - Window capacity = 2048 prediction entries - Prefetch trigger threshold = trigger refill when prefetch frontier - leads execution point by fewer than 1024 steps - Refill batch size = 64 entries (avoids excessive single-refill latency) -``` - -**数据结构**： - -```csharp -internal class Predictor -{ - private const int WindowCapacity = 2048; - private const int RefillThreshold = 1024; - - private PrefetchEntry[] _buffer; // Circular prediction buffer - private int _head; // Prefetch frontier (prefetched up to here) - private int _tail; // Production position (scheduler fills up to here) - private int _executed; // Execution point (compute thread has completed up to here) - - private HashSet _activeKeys; // All active keys within the window (deduplicated) -} - -struct PrefetchEntry -{ - public nuint Key; // Cache line identifier - public nuint SourceAddress; // Source data address - public bool NeedsRepack; // Whether non-contiguous repacking is needed -} -``` - -**Cooperation with Eviction Policy**: - -``` -Eviction priority (from highest to lowest priority): - - 1. refCount==0, not in window, RRPV==3, dirty==0 ← First (zero-cost eviction) - 2. refCount==0, not in window, RRPV==3, dirty==1 ← Second (needs writeback but cold) - 3. refCount==0, not in window, RRPV<3, dirty==0 ← Ordered by RRPV - 4. refCount==0, not in window, RRPV<3, dirty==1 ← Ordered by RRPV - 5. refCount==0, in window ← Avoid evicting if possible - 6. refCount>0 ← Never evict -``` - -Windowed OPT means most eviction decisions do not depend on RRPV — "not in window" alone is -sufficient to determine evictability. RRPV provides fine-grained differentiation among -out-of-window lines; the dirty bit preferentially evicts clean lines at equal RRPV to avoid writeback overhead. - -### 3.7 Concurrency Model - -L4 uses a **no-extra-threads** design; all operations are performed by compute threads themselves: - -| Operation | Executor | Concurrency | -|------------------------------------|---------------------------------|--------------------------------------| -| Lookup (Swiss Table read) | Any compute thread | ✅ Fully parallel, lock-free | -| HIT RRPV update | Any compute thread | ✅ Single-byte write, no sync needed | -| refCount increment/decrement | Any compute thread | ✅ Interlocked atomic operations | -| Prefetch refresh (eviction + load) | CAS-winner thread | Serial (only one thread enters) | -| Dirty writeback | CAS-winner thread | Serial (within eviction flow) | -| FlushAllDirty | Single thread after computation | Serial (no concurrency) | - -**Prefetch Refresh CAS Lock**: - -```csharp -private int _refreshLock; // 0=free, 1=locked - -private void TryRefreshPrefetch() -{ - if (Interlocked.CompareExchange(ref _refreshLock, 1, 0) != 0) - return; // Another thread is doing it; current thread continues computing - try { /* eviction + prefetch */ } - finally { Volatile.Write(ref _refreshLock, 0); } -} -``` - -**Swiss Table Concurrent Read Safety Guarantee**: The prefetch refresh thread follows strict write ordering -on insertion: write Slab data first, then write eviction metadata and callback pointers, `sfence`, then -write the Swiss Table control byte and key/value, ensuring that when a compute thread reads the control -byte, the data is already fully visible. - ---- - -## 4. Operation Flows - -### 4.1 Cache Lookup (Read-Only Hit) - -1. Compute CRC32C hash of the original pointer -2. Swiss Table lookup: SIMD scan ctrl → compare key → return cache pointer -3. Update RRPV: `_evictionFlags[slot] &= 0xF9` (RRPV zeroed, dirty bit unchanged) -4. Construct CacheLine handle (refCount++) -5. Return CacheLine; upper-layer code accesses local data via `.Address` - -### 4.2 Cache Lookup (Writable Hit) - -1. Swiss Table lookup hits -2. Update RRPV: `_evictionFlags[slot] &= 0xF9` -3. Construct CacheLine handle (refCount++) -4. Return CacheLine; upper-layer code reads/writes the cache line via `.Address` (e.g., partial sum +=) - -### 4.3 Cache Miss (Read-Only) - -1. Swiss Table lookup misses -2. Compute thread enters `Thread.Sleep(0)` spin loop -3. Wait for Predictor-driven prefetch refresh to load the line into L4 -4. Exit wait loop when Swiss Table shows a hit -5. Construct CacheLine handle and return - -Under normal operation, the Predictor always leads the execution point by 1024+ steps, -misses occur only during cold start or extreme bursts. - -### 4.4 Cache Miss (Writable) - -1. Swiss Table lookup misses -2. Allocate a cache line from Slab -3. If Slab has no free slots → trigger eviction (see 4.6) -4. Zero-initialize the cache line (`Unsafe.InitBlock(dest, 0, 8192)`) -5. Register writeback callbacks: `_preprocess[slot]`, `_writeback[slot]`, `_callbackCtx[slot]` -6. Set eviction flags: `dirty=1, RRPV=0` -7. Write `_originKeys[slot]` -8. `sfence` -9. Write Swiss Table mapping -10. Construct CacheLine handle and return - -### 4.5 Prefetch Refresh - -Triggered by compute threads in `StepComplete()`; CAS lock ensures single-thread execution: - -1. Synchronize refFlags snapshot (`_refCounts[i] > 0 → _refFlags[i] = 1`) -2. Retrieve pending prefetch entries from Predictor (up to 64 at a time) -3. For each entry: - - Swiss Table hit → mark RRPV=0 (hot-line protection) - - Swiss Table miss → execute load: - a. Allocate cache line from Slab - b. If Slab has no free slots → trigger eviction (see 4.6) - c. Execute Pack reorder or memcpy based on `NeedsRepack` flag - d. Write eviction metadata (originKey, RRPV=0, dirty=0, callbacks=null) - e. `sfence` - f. Write Swiss Table mapping - -### 4.6 Cache Eviction (SIMD Scan + Writeback) - -``` -Eviction scan pseudocode: - - Pass 1: Find refCount==0, RRPV==3, dirty==0 (zero-cost eviction) - for i in range(scanCursor, CacheLineCount), step=16: - refChunk = SSE2.Load(refFlags + i) - refFreeMask = SSE2.MoveMask(SSE2.CompareEqual(refChunk, zero)) - - evChunk = SSE2.Load(evictionFlags + i) - // RRPV==3 and dirty==0 → byte & 0x07 == 0x06 - masked = SSE2.And(evChunk, 0x07) - coldCleanMask = SSE2.MoveMask(SSE2.CompareEqual(masked, 0x06)) - - candidates = refFreeMask & coldCleanMask - while candidates != 0: - bit = TrailingZeroCount(candidates) - idx = i + bit - if originKeys[idx] not in Predictor.ActiveKeys: - evict(idx) // No writeback, discard directly - return - candidates &= candidates - 1 - - Pass 2: Find refCount==0, RRPV==3, dirty==1 (needs writeback) - Same as above, but match byte & 0x07 == 0x07 - evict triggers writeback callbacks - - Pass 3: Increment all RRPV+1 (aging), retry - Up to 3 rounds -``` - -**逐出执行**： - -```csharp -private void Evict(int slot) -{ - byte flags = _evictionFlags[slot]; - - // Dirty writeback - if ((flags & 0x01) != 0) - { - nuint origin = _originKeys[slot]; - nuint cached = SlotAddressFromIndex(slot); - nuint ctx = _callbackCtx[slot]; - - // 1. Pre-processing (optional) - var preprocess = (delegate* unmanaged)_preprocess[slot]; - if (preprocess != null) - preprocess(cached, ctx); - - // 2. Writeback - var writeback = (delegate* unmanaged)_writeback[slot]; - if (writeback != null) - writeback(origin, cached, ctx); - } - - // Standard eviction - _map.TryRemove(_originKeys[slot], out _); - _slab.Free(SlotAddressFromIndex(slot)); - _evictionFlags[slot] = 0; - _originKeys[slot] = 0; - _preprocess[slot] = 0; - _writeback[slot] = 0; - _callbackCtx[slot] = 0; -} -``` - -### 4.7 显式刷新 - -```csharp -/// -/// Explicitly writes back the specified dirty cache line. Called after all segments complete. -/// -public void FlushDirty(nuint originKey) -{ - if (_map.TryGet(originKey, out nuint addr)) - { - int slot = SlotIndex(addr); - if ((_evictionFlags[slot] & 0x01) != 0) - { - // Execute pre-processing + writeback - InvokeWriteBack(slot); - _evictionFlags[slot] &= 0xFE; // Clear dirty bit - } - } -} - -/// -/// Writes back all dirty cache lines in bulk. Called after entire matrix computation completes. -/// -public void FlushAllDirty() -{ - for (int i = 0; i < CacheLineCount; i++) - { - if ((_evictionFlags[i] & 0x01) != 0 && _originKeys[i] != 0) - { - InvokeWriteBack(i); - _evictionFlags[i] &= 0xFE; - } - } -} -``` - -### 4.8 Working Mode Switch - -```csharp -public enum L4Mode -{ - /// - /// Pass-through mode: Map() returns the original pointer directly; L4 does not intervene. - /// Suitable for single-NUMA + no-repack scenarios (e.g., vector addition, scalar multiplication). - /// - StreamTransparent, - - /// - /// Buffer mode: full lookup/Pack/cache/eviction pipeline. - /// Suitable for FMA scenarios requiring repacking or multi-NUMA scenarios. - /// - CacheBuffer -} -``` - -Mode is determined at construction time and does not change at runtime. Decision logic: - -- Requires Pack reordering (FMA core rows of matrix A) → CacheBuffer -- Multiple NUMA nodes → CacheBuffer -- Requires segment accumulation → CacheBuffer -- Single NUMA + no reordering + no segment → StreamTransparent - ---- - -## 5. Segment Accumulation Scenario - -### 5.1 Problem - -For matrix multiplication `C = A × B` when the K dimension is large, segmented accumulation is required: - -``` -segment 0: C_partial = A[kr, 0..KC] × B[0..KC, kc] -segment 1: C_partial += A[kr, KC..2KC] × B[KC..2KC, kc] -... -segment N: C_partial += A[kr, N*KC..K] × B[N*KC..K, kc] - -Final: C[kr, kc] = Σ all segment partial sums -``` - -If each segment's result is `+=` directly to the C matrix, -and C resides on a remote NUMA node, every segment incurs cross-node write latency. - -### 5.2 L4 Solution - -Temporarily store C's partial sums in L4's writable cache lines: - -``` -segment 0: c = l4.MapWritable(C_key, null, &AccumulateAdd) - // First miss → allocate zero-initialized cache line - FMA(c.Address, a.Address, b.Address) // c += a*b - c.Dispose() - -segment 1: c = l4.MapWritable(C_key, null, &AccumulateAdd) - // Hit! Returns existing partial sum cache line - FMA(c.Address, a.Address, b.Address) // c += a*b - c.Dispose() - -... after all segments complete ... - -l4.FlushDirty(C_key) -// Calls AccumulateAdd(C_origin, cacheline, ctx) -// → Adds local partial sum back to remote C matrix in one shot -``` - -### 5.3 Benefits - -``` -Without segment buffering (OpenBLAS approach): - Each segment writes C matrix across NUMA - Remote writes per output tile = Zk times - Per-write latency = ~200-300ns (cross-node) - -With L4 segment buffering: - All segment += operations complete locally in L4 cache lines - Partial sums likely remain in L1/L2 (8KB << L1 capacity) - Remote writes per output tile = 1 (final FlushDirty) - Writeback uses AVX2-vectorized stores: 8192B = 32 × 256-bit stores - - Reduction ratio = Zk : 1 -``` - ---- - -## 6. Uniqueness and Advantages - -### 6.1 Deterministic Prefetch (Windowed OPT) - -Essential difference from hardware cache replacement policies (LRU, RRIP, Hawkeye): - -| Policy | Information Source | Prediction Method | Accuracy | -|---------------------|-------------------------|-------------------------------|---------------------------| -| LRU | Past access order | Least recently used | Moderate | -| RRIP | Insert/hit markers | Predict reuse distance | Moderate-High | -| Hawkeye | OPTgen retrospection | Learn past to infer future | High | -| **L4 Predictor** | **Future task queue** | **Direct future visibility** | **Perfect** (within window) | - -L4's Predictor is a **deterministic Oracle** — it does not predict; it knows. -Because matrix computation task sequences are fully determined at scheduler allocation time. - -### 6.2 NUMA Adaptability - -The same code automatically gains NUMA benefits on different hardware configurations: - -| Hardware Configuration | L4 Behavior | -|-------------------------|-------------------------------------------------------------| -| Single-socket CPU | Pack cache + segment accumulation buffer | -| Dual-socket CPU | Pack cache + NUMA migration cache + segment accum. | -| Hybrid-architecture CPU | Pack cache + cross-domain migration cache + segment accum. | - -### 6.3 Callback-Driven Writeback Strategy - -Using native function pointers for writeback avoids: - -- Virtual method call overhead -- Interface dispatch overhead -- Delegate GC pressure -- Hard-coding specific writeback logic - -Callers register `preprocess` and `writeback` callbacks at `MapWritable` time, -L4 invokes them directly through function pointers on eviction or Flush, -**zero managed overhead, zero virtual dispatch, performance equivalent to C function calls**. - -### 6.4 Reference Count Protection - -Through `CacheLine` handle reference counting, cache lines actively in use by compute threads are -absolutely never evicted (including writeback not being triggered). The SIMD eviction scan skips -lines with `refCount > 0` immediately via the `_refFlags` array, without checking linked lists or acquiring locks. - -### 6.5 Cache Line Size Aligned to Matrix Kernel - -The 8192B cache line size is not arbitrary: - -- 1 `double MAT_KERNEL` (4×4 double matrix) = 128 Bytes -- 1 cache line = 64 `MAT_KERNEL` units -- Matrix multiplication micro-kernels operate on data sizes perfectly aligned to cache lines -- Compute kernel functions operate on cache lines as units; no cross-cache-line accesses occur - -### 6.6 Fixed-Capacity Determinism - -- Slab allocator never expands → memory footprint upper bound is known (64 MB) -- SIMD eviction scan → latency upper bound is known (at most 3 aging rounds × 512 SIMD comparisons) -- Swiss Table fixed 16,384 slots → lookup latency upper bound is known -- Writeback callback latency is controlled by the caller → predictable -- Worst-case L4 latency = lookup + eviction (incl. writeback) + allocation + memcpy, all statically computable - -### 6.7 Reuse of Existing StgSharp Infrastructure - -| L4 Component | Reused StgSharp Infrastructure | -|-----------------------|-------------------------------------------------------| -| Mapping table | `SwissTable` (implemented and benchmark-validated) | -| Mapping table SIMD | `IBucketScanner` + `SIMDID` architecture detection | -| Cache line alloc | `SlabAllocator.FromBuffer()` | -| Hash computation | Hardware `CRC32C` instruction | -| Memory allocation | `NativeMemory.AlignedAlloc` | -| Future multi-platform | `IBucketScanner` + NRGA compilation pipeline | - -### 6.8 Originality Analysis - -After surveying existing mainstream numerical computation libraries, L4's design has -**no direct precedent or equivalent in known public implementations**. - -#### NUMA Strategy Comparison with Existing Libraries - -| Library / Framework | NUMA Strategy | Software Cache Layer | Mapping Table | Eviction Policy | Dirty Writeback | -|---------------------|-------------------------------------|---------------------|-----------------|-------------------------|--------------------| -| Intel MKL | Thread pinning + `numactl` | None | None | None | None | -| OpenBLAS | Thread affinity + block alignment | None | None | None | None | -| BLIS | Multi-level block alignment L1/L2/L3| None | None | None | None | -| Eigen | No explicit NUMA handling | None | None | None | None | -| cuBLAS / rocBLAS | GPU memory (hardware-solved) | None | None | None | None | -| **StgSharp L4** | **Software-managed thread-local cache** | **Yes** | **Swiss Table** | **RRPV + windowed OPT** | **Callback-driven** | - ---- - -## 7. 从 v0.1 到 v0.3 的变更摘要 - -| 项目 | v0.1 | v0.3 | -|-----------------|-------------------------------|---------------------------------------| -| 逐出数据结构 | 4 级环形双向链表 | RRPV 2-bit 掩码 + 8KB SoA 数组 | -| 逐出扫描方式 | 链表头指针遍历 | SSE2 批量模式匹配（16 行/周期） | -| 热���更新 | 链表节点摘插（4 指针写） | 单字节位运算 `&= 0x03` | -| 老化操作 | 逐节点移链表 | SIMD 批量 +1 | -| 引用保护 | 无 | refCount > 0 时 SIMD 扫描跳过 | -| 预取策略 | 无 | Predictor 窗口 OPT（2048 条缓冲） | -| 线程模型 | L4 服务线程 + Command Queue | 无额外线程，CAS 抢锁 | -| 命令队列 | 链式 CommandCache 分段 | 不需要（预取刷新替代命令驱动） | -| 缓存行元数据 | Slab 分配的 CacheLineHead 结构 | SoA 平行数组 + 函数指针数组 | -| 工作模式 | 单一模式 | StreamTransparent / CacheBuffer 切换 | -| 写回策略 | 无（只读缓存） | dirty 位 + preprocess/writeback 回调 | -| 写回机制 | 无 | 本机函数指针，零托管开销 | - ---- - -## 8. 未来工作 - -- [ ] L4 根类型重构（组合 Swiss Table、Slab、SoA 元数据、Predictor） -- [ ] Predictor 与矩阵乘法调度器的集成 -- [ ] Segment 累加写回的 AVX2 向量化实现 -- [ ] SIMD 逐出扫描的 AVX2 升级（32 行/周期） -- [ ] Swiss Table ARM NEON 实现 -- [ ] Swiss Table AVX-512 升级（64 槽扫描从 16 次降至 4 次） -- [ ] 墓碑累积监控与自动 rehash 策略 -- [ ] 性能基准测试（NUMA 本地 vs 远端访问延迟对比） -- [ ] Segment 累加 vs 直接写回的 benchmark 对比 -- [ ] 与逐线程矩阵上下文的集成验证 -- [ ] 双路 6242R 环境下的 NUMA 搬运性能验证 -- [ ] 远期：PPC64LE 平台的 VSX 原生 Swiss Table 扫描支持 - ---- - -## 9. 参考 - -- Google Abseil Swiss Tables Design Notes -- HP Labs RRIP (Re-Reference Interval Prediction) 论文 -- Hawkeye: Efficient Cache Replacement via OPTgen (ISCA 2017) -- StgSharp HLSF Allocator 设计文档 -- StgSharp SIMD Architecture (`SIMDID` 体系) -- StgSharp NRGA Roadmap -- StgSharp `SlabAllocator` 实现 -- StgSharp `SwissTable` 实现及 Benchmark diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Design.md b/src/Nitload.Common/HighPerformance/Memory/L4.Design.md new file mode 100644 index 00000000..03e71bde --- /dev/null +++ b/src/Nitload.Common/HighPerformance/Memory/L4.Design.md @@ -0,0 +1,136 @@ +# L4 运行模型 + +> 工作稿，2026-10-14。本文替代旧版 L4 设计草案，区分当前代码、已讨论方向和仍待确认的问题。 + +## 1. 范围与职责 + +每个 `L4` 实例绑定一个 NUMA 节点，拥有本地 arena，其中放置 TLSF 分配器状态和 L4 Swiss 映射表的双缓冲区。 + +- **L4** 负责本地分配、映射、缓冲区发布和阶段协调。 +- **Swiss** 保存并查询 L4 使用的映射/预测条目，不解释 source layout。 +- **Predictor** 解释自己的 source spec，生成 packed 数据，并负责格式相关的软件写回或撤销。 +- **PredictorMonitor** 计划负责预测器注册、稳定 ID、执行顺序和生命周期；它与 Swiss 中的预测任务/映射状态是两回事。 + +本文不假设自动写回。若 packed 数据需要写回，必须能识别其所属预测器并显式调用对应逻辑。 + +## 2. AddressSource 与缓存行数据 + +`AddressSource` 是 16 字节 key： + +- `Address`：源地址或 source identity。 +- `Description`：64 位 source/layout spec，由所属预测器解释。 + +当前 Swiss 条目使用该 key 和一个 `ulong` value。源码中的 `CacheLineMetadata` 是 32 字节 TLSF metadata；`L4.Handle` 仍是 16 字节 ABI 占位。引用计数、驱逐保护和写回归属尚未端到端实现。 + +### 当前与候选 KVP 布局 + +- 设计讨论中的 `ushort` 是缓存行计数，不是预测器数量或预测器注册表容量。 +- 当前 Swiss bucket 将 key 与 `ulong` value 分开组织；现有的 8-KVP 交叉存储描述的是数据存储布局，不是 H1/hash 探测分组。 +- 一个候选布局是把 value 扩成 16 字节：保留现有 `ulong` cache-line metadata，再增加 8 字节存放预测计数等状态。与 16 字节 `AddressSource` 组成 32 字节 KVP，每条 64 字节 cache line 放两个 KVP。 +- 该方案旨在减少当前 8-KVP 交叉存储造成的 cache-line 分散/冲突。它只调整 KVP 的存储编排，不意味着 H1 探测组从当前 16 槽改成 8 槽。此方案尚未实现，需实测对查找和替换扫描的影响。 + +## 3. Swiss 映射 + +当前表有 1024 组，每组 16 个槽位。控制字节扫描使用 SSE2；x64 路径使用 CRC32C 混合两个 64 位 key 字段。两个表缓冲区都分配在 L4 arena 内。 + +published buffer 地址以 `ulong` 保存： + +- `TryGet` 用一次 `Volatile.Read` 捕获 published pointer，并在整个查找中使用该快照。 +- `SwapBuffers` 用 `Interlocked.Exchange` 发布下一缓冲区，返回的旧 published pointer 成为下一轮 inactive write buffer。 +- 原子发布指针不代表旧 buffer 可以立即复用。Map/swap 阶段协议必须先确保读者不再访问旧 buffer，writer 才能清空或覆盖它。 + +当前只实现 x64/SSE4.2 后端，其他 ISA 后端尚未实现。 + +## 4. 预测器接口与阶段 + +当前接口骨架位于 `L4Predictor.cs`： + +- `IL4Predictor` 是非泛型管理接口，暴露预测器 ID 和同步的 `PredictMission(in Span)`。 +- `IL4Predictor` 为预测器实现提供强类型 mission API。 +- `PredictMission` 填充调用方提供的 span，并返回有效预测数。span 可以由调用方 `stackalloc`，不得逃逸出本次同步调用。 +- `PredictorMonitor` 目前是空占位；预测器注册和生命周期尚未实现。 + +当前讨论中的 L4 生命周期分为不同阶段： + +1. **注册预测器**：monitor 按稳定顺序保存异构预测器。固定容量数组（讨论过 256）是候选，其容量与缓存行计数无关。 +2. **Prefetch**：预测器按依赖顺序执行。每个 L4 同时只有一个活动 prefetch worker，但 prefetch 可与 Map 重叠，也可能跨 NUMA 节点访问目标数据。 +3. **发布**：完成后的 inactive Swiss buffer 被原子发布。Swiss 保存预测/映射任务状态；预测器注册表不是预测任务队列。 +4. **Map**：调用方查询已发布的 `AddressSource` 并取得 L4 handle。 +5. **撤销/写回**：所属预测器解释 packed 数据，并按需执行软件写回或撤销。 + +预测器接口本身是同步的。若预测结果需要在 `PredictMission` 返回后或另一线程继续使用，必须先复制到生命周期足够长的自有存储中。异步边界应位于拥有的 batch/work item 周围，而不能让 stack span 逃逸。 + +## 5. 预测窗口与 Map 同步 + +两个阈值含义不同： + +- `_low_predict = 512`：预测触发/补充阈值。 +- `_predict_slow_path_count = 2 * LogicalCoreCount`：按 L4 绑定 NUMA 节点逻辑处理器数量计算的提前慢路径窗口。 + +当前 Map 的分流意图如下： + +| 剩余预测数 | Map 行为 | +|---|---| +| 小于 512 | 尝试成为预测 writer；预测可能完成并发布新 buffer。竞争失败的调用走带读者计数的查找路径。 | +| 512 到 `512 + _predict_slow_path_count` 之间 | 走带计数的慢查找路径，以便 swap 等待这些读者退出。 | +| 大于或等于 `512 + _predict_slow_path_count` | 走无计数的直接查找快路径。 | + +计数读者协议先等待 swap gate 打开，再增加 reader count 并复查 gate。如果 gate 在两步之间关闭，则撤销计数并重试。swap 方关闭 gate、等待已计数读者归零、交换 buffer、发布新的剩余预测数，最后重新打开 gate。 + +无计数快路径依赖预测提前量和阶段时序。这是性能启发式，本身不能证明一个被抢占的 reader 不会在旧 buffer 复用时仍继续访问。无计数 reader 与 buffer 生命周期之间的不变量，以及快慢路径边界，都仍需验证。 + +当前 `Predict()` 返回 `int`，但实现仍是返回 `0` 的空 stub，因此预测数生命周期尚未端到端运行。 + +## 6. 预测器注册、任务状态与在途缓冲 + +这三者职责不同，不应合并成一个队列： + +- **预测器管理数组**：`PredictorMonitor` 的固定槽位数组，保存注册顺序、稳定 ID 和生命周期状态。容量上限（曾讨论 256）是预测器数量，与缓存行计数无关。预测器按依赖顺序依次执行，一个 L4 同一时刻只有一个 predictor worker 处于 prefetch 阶段，因此不需要多 lane 或 thread-local lane 归属。 +- **Swiss 预测任务状态**：保存/查找待映射的 `AddressSource` 条目；这是预测任务/映射状态，不是预测器注册表。 +- **在途预测器缓冲**：预测器已经产出预测，但对应实际工作尚未完成时，保留该预测器及其 mission/写回上下文。它与注册槽位不同：注册槽位描述“有哪些预测器”，在途缓冲描述“哪些预测工作还未完成”。具体容量、完成标记和回收时机尚待设计。 + +`PredictMission(in Span)` 是同步接口，L4 可传入 stackalloc span。若返回后仍需跨线程或跨 NUMA 处理其内容，必须将内容复制到拥有明确生命周期的缓冲区；不得让 stack span 逃逸。由于预测器顺序有依赖，待处理预测必须按既定顺序交接。若采用异步 worker，队列应有界且不阻塞 Map 快路径。 + +## 7. 驱逐与写回策略 + +L4 不得推断或解码预测器的 packed 格式。若丢弃一个尚未写回、且对应实际工作仍在途的预测条目会丢失必要状态，就不能静默驱逐它。按缓存行的保护/引用字段或 sidecar 正在讨论中，引用/owner metadata 布局尚未定稿。 + +owner ID 可用于路由显式写回；但若多个预测器能独立声明同一条目，单一 owner 字段不够。引用计数可保护共享条目，而写回路径仍需获得预测器 identity/source spec。 + +## 8. 当前状态与待决问题 + +**源码中已实现：** + +- NUMA 绑定的 L4 arena 和 TLSF 包装。 +- 双缓冲 Swiss 表及原子 published pointer 交换。 +- x64 CRC32C hash 和 SSE2 控制字节扫描。 +- 固定预测触发阈值，以及按 NUMA 节点逻辑处理器数量计算的提前慢路径窗口。 +- 慢路径的 Map reader gate/counter。 +- 同步的泛型/非泛型预测器接口骨架。 + +**尚未实现或尚未验证：** + +- PredictorMonitor 注册表容量、注册/注销和生命周期状态转换。 +- 实际预测算法与有效预测条目的生成。 +- 端到端撤销/写回归属和缓存行驱逐保护。 +- Prefetch 与 Map 跨 NUMA 重叠时的自有 queue/batch 交接方式。 +- 无计数 Map 快路径不会与 published buffer 复用重叠的证明/测试。 +- 覆盖预测器发布、Map、撤销和 Dispose 的并发压力测试。 + +## 9. L4 的四个核心操作 + +### 预测（Prefetch） + +Monitor 按注册顺序选择预测器，调用其同步预测接口；预测器解释自己的 source spec，将预测项写入调用方提供的 span。Swiss 管理预测/映射条目。若预测项对应的实际工作尚未完成，则放入在途预测器缓冲，保留后续完成、撤销或写回所需的归属信息。预测期间若没有可用的 cache-line 槽位，替换/保护策略由后续设计决定，不能隐式丢弃仍在途的条目。 + +### 映射（Map） + +计算 worker 按预测器调度获取 `AddressSource`，通过 Swiss 查询 NUMA 本地的 packed 数据并得到 L4 handle。Map 不解释 `Description`，也不负责预测器注册或写回。 + +### 注册预测器 + +将预测器放入 `PredictorMonitor` 的管理槽位，分配稳定 ID 和执行顺序，并纳入生命周期监视。注册表槽位与 Swiss 预测条目、在途预测工作是不同资源。 + +### 移除预测器 + +先停止向该预测器分派新 mission，再等待其在途预测对应的实际工作完成或被撤销；需要的软件写回完成后，才能释放在途状态并回收注册槽位。具体等待与写回协议尚待设计。 \ No newline at end of file diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Map.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Map.cs index 8e210473..6e9c4a4e 100644 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Map.cs +++ b/src/Nitload.Common/HighPerformance/Memory/L4.Map.cs @@ -18,6 +18,7 @@ namespace Nitload.Common.HighPerformance.Memory { private const int _low_predict = 512; + private readonly int _predict_slow_path_count; private int _predict_count; private int _swiss_reader_counter; @@ -29,7 +30,8 @@ namespace Nitload.Common.HighPerformance.Memory ) { ulong handle; - if (Volatile.Read(ref _predict_count) < _low_predict) + int predictCount = Volatile.Read(ref _predict_count); + if (predictCount < _low_predict) { if (Interlocked.CompareExchange(ref _swiss_writer_lock, 1, 0) == 0) { @@ -57,6 +59,10 @@ namespace Nitload.Common.HighPerformance.Memory } else if (!TryGetAddress(source, out handle)) { throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}]."); } + } else if (predictCount < _low_predict + _predict_slow_path_count) + { + if (!TryGetAddress(source, out handle)) + throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}]."); } else if (!_addressMap.TryGet(source, out handle)) { throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}]."); } diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.PredictorQueue.cs b/src/Nitload.Common/HighPerformance/Memory/L4.PredictorQueue.cs new file mode 100644 index 00000000..a9e83119 --- /dev/null +++ b/src/Nitload.Common/HighPerformance/Memory/L4.PredictorQueue.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Nitload.Common.HighPerformance.Memory +{ + public class L4 + { + } +} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.cs b/src/Nitload.Common/HighPerformance/Memory/L4.cs index 8fa59c19..89615e35 100644 --- a/src/Nitload.Common/HighPerformance/Memory/L4.cs +++ b/src/Nitload.Common/HighPerformance/Memory/L4.cs @@ -30,6 +30,7 @@ namespace Nitload.Common.HighPerformance.Memory ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize); ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize); _bindedNuma = bindedNuma; + _predict_slow_path_count = GetPredictSlowPathCount(bindedNuma); nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); ByteCapacity = normalizedBufferSize; nuint swissBufferOffset = normalizedBufferSize + TLSF.RequestedAllocation; @@ -83,5 +84,16 @@ namespace Nitload.Common.HighPerformance.Memory internal partial int Predict(); + private static int GetPredictSlowPathCount(int bindedNuma) + { + foreach (NumaNode node in Numa.AllNumaNode) + { + if (node.NumaNodeId == (uint)bindedNuma) + return checked(node.LogicalCoreCount * 2); + } + + throw new ArgumentOutOfRangeException(nameof(bindedNuma), "No topology information exists for the bound NUMA node."); + } + } } diff --git a/src/Nitload.Common/HighPerformance/Memory/L4Predictor.cs b/src/Nitload.Common/HighPerformance/Memory/L4Predictor.cs new file mode 100644 index 00000000..cde02196 --- /dev/null +++ b/src/Nitload.Common/HighPerformance/Memory/L4Predictor.cs @@ -0,0 +1,37 @@ +// ----------------------------------------------------------------------------- +// file="L4Predictor" +// Project: StgSharp +// Copyright (c) Nitload. +// SPDX-License-Identifier: MIT +// ----------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Nitload.Common.HighPerformance.Memory +{ + public interface IL4Predictor + { + + int Id { get; set; } + + int PredictMission( + in Span RequiredBuffer + ); + + } + + public interface IL4Predictor : IL4Predictor + { + + bool TryGetNextMission( + out TMission mission + ); + + } + + internal class PredictorMonitor { } +}