Compare commits
29
Commits
e04adf594a
...
giga
| Author | SHA1 | Date | |
|---|---|---|---|
 Youmu Liu
|
91cf6a43cd | ||
|
Youmu Liu
|
9867e1f262 | ||
|
Youmu Liu
|
c64f7ae9f6 | ||
|
Youmu Liu
|
6bc7224d4c | ||
 youmuliu
|
b7f1d3d2ba
|
||
|
youmuliu
|
76a9dfa5af
|
||
|
youmuliu
|
2bef14493f
|
||
|
Youmu Liu
|
0cef5aa517 | ||
|
Youmu Liu
|
172f64c6f9 | ||
|
Youmu Liu
|
3a2b123478 | ||
|
Youmu Liu
|
2e7c3040f4 | ||
|
Nitload
|
ddd83b1b05 | ||
|
Youmu Liu
|
1845a00390 | ||
|
youmuliu
|
e24efd9f9c
|
||
|
youmuliu
|
9fc23c29fb
|
||
|
Youmu Liu
|
194e869724 | ||
|
Youmu Liu
|
7ddfc36ac5 | ||
|
Youmu Liu
|
2c17f6c108 | ||
|
youmuliu
|
3d1d475e25
|
||
|
Youmu Liu
|
044b153463 | ||
|
Youmu Liu
|
6e6bd7794b | ||
|
youmuliu
|
7a820261cf
|
||
|
youmuliu
|
fa1186cd25
|
||
|
youmuliu
|
e49b14f037
|
||
|
youmuliu
|
052afa1dd4
|
||
|
youmuliu
|
1d3d3c10dd
|
||
|
youmuliu
|
902be141d0
|
||
|
youmuliu
|
b5b7c75b5e
|
||
|
copilot-swe-agent[bot]
|
aa7394503e |
@@ -0,0 +1,5 @@
|
||||
root = true
|
||||
|
||||
[*.cs]
|
||||
dotnet_diagnostic.IDE0032.severity = none
|
||||
dotnet_diagnostic.IDE0055.severity = none
|
||||
+1
-1
@@ -29,7 +29,7 @@ build.ninja
|
||||
BenchmarkDotNet.Artifacts/
|
||||
|
||||
# Khronos OpenGL Registry input is supplied locally; keep only its instructions.
|
||||
/tools/Nitload.GenerateGL/xml/*
|
||||
/tools/*GenerateGL/xml/*
|
||||
!/tools/Nitload.GenerateGL/xml/README.md
|
||||
|
||||
/src/Nitload.Native/lib/*
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
<NativeCMakeBuildDir Condition="'$(NativeCMakeBuildDir)'==''">$(SolutionRoot)cmake_build/$(NativeCMakePreset)</NativeCMakeBuildDir>
|
||||
<_NativeCMakeOutputDir>$([System.String]::Copy('$(NativeBuildDir)').Replace('\', '/'))</_NativeCMakeOutputDir>
|
||||
</PropertyGroup>
|
||||
<Exec Command="cmake --preset "$(NativeCMakePreset)" -S "$(SolutionRoot)src/Nitload.Native" -B "$(NativeCMakeBuildDir)" "-DNIF_BUILD_OUTPUT_DIR:PATH=$(_NativeCMakeOutputDir)" "-DNIF_EXPECTED_RID:STRING=$(BuildRid)"" />
|
||||
<Exec Command="cmake --preset "$(NativeCMakePreset)" -S "$(SolutionRoot)src/Nitload.Native" -B "$(NativeCMakeBuildDir)" "-DNFN_BUILD_OUTPUT_DIR:PATH=$(_NativeCMakeOutputDir)" "-DNFN_EXPECTED_RID:STRING=$(BuildRid)"" />
|
||||
<Exec Command="cmake --build "$(NativeCMakeBuildDir)" --target "$(NativeLibName)"" />
|
||||
<Error Condition="!Exists('$(NativeBuildDir)$(NativeFileName)')"
|
||||
Text="CMake did not produce the expected native library: $(NativeBuildDir)$(NativeFileName). Check the target OS, architecture and output name." />
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Nitload StgSharp Team
|
||||
Copyright (c) 2024 Nitload
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -20,9 +20,9 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
StgSharp program uses the following open source projects,
|
||||
I'd like to refer to them here and show my sincere appreciation,
|
||||
while the names are sorted in no particular order
|
||||
The nFramework project uses the following open-source projects.
|
||||
Their respective license notices are reproduced below,
|
||||
and the projects are listed in no particular order.
|
||||
|
||||
1. GLFW
|
||||
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
<projectUrl>https://github.com/Nitload-NSI/StgSharp</projectUrl>
|
||||
<description>No des right now</description>
|
||||
<releaseNotes>Summary of changes made in this release of the package.</releaseNotes>
|
||||
<copyright>Nitload reserve</copyright>
|
||||
<copyright>Copyright (c) 2024 Nitload</copyright>
|
||||
<tags>OpenGL C#</tags>
|
||||
</metadata>
|
||||
<files>
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
# Nitload
|
||||
|
||||
|
||||
|
||||
# nFramework
|
||||
|
||||
Nitload (formerly StgSharp) is an experimental .NET framework for graphics, numerical computing, language analysis, and other performance-sensitive applications. It began as the foundation of a next-generation STG (shoot 'em up) engine, but the current repository focuses on reusable low-level infrastructure rather than a complete, ready-to-use game engine.
|
||||
|
||||
The managed libraries now use the `Nitload.*` namespace and project prefix. Repository URLs, the logo filename, and some legacy test artifacts still retain the StgSharp name.
|
||||
nFramework (formerly StgSharp, now also nitload.Framework) is an experimental framework built on .NET for graphics, numerical computing, language analysis, and other performance-sensitive applications. It combines managed C# libraries with a C native runtime, targets .NET 8, and provides Windows and Linux native implementations.
|
||||
|
||||
## Overview
|
||||
|
||||
The framework combines managed C# APIs with a C native library. Its current work includes OpenGL bindings and rendering infrastructure, generic matrix and vector types, SIMD-accelerated native kernels, custom allocators and collections, regular-language analysis, and state-oriented UI infrastructure. Nitload targets .NET 8; platform and architecture coverage varies by module, with Windows and Linux as the principal native targets.
|
||||
nFramework combines managed C# APIs with a C native library. Its current work includes OpenGL bindings and rendering infrastructure, generic matrix and vector types, SIMD-accelerated native kernels, custom allocators and collections, regular-language analysis, and state-oriented UI infrastructure. nFramework targets .NET 8; platform and architecture coverage varies by module, with Windows and Linux as the principal native targets.
|
||||
|
||||
## Core Modules
|
||||
|
||||
@@ -139,7 +137,7 @@ The benchmark reports normalized time per allocation/free operation. Each iterat
|
||||
|
||||
## Installation
|
||||
|
||||
Nitload is currently in development and requires compilation from source. **NuGet packages are not yet available** - you must build the project from source code.
|
||||
nFramework is currently in development and requires compilation from source. **NuGet packages are not yet available** - you must build the project from source code.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
@@ -245,7 +243,7 @@ allocator.Free(handle);
|
||||
|
||||
## Contributing
|
||||
|
||||
Nitload is developed by Nitload Space. Contributions are welcome, particularly in:
|
||||
nFramework is developed by Nitload Space under the Nitload brand. Contributions are welcome, particularly in:
|
||||
|
||||
- Performance optimizations
|
||||
- Cross-platform compatibility
|
||||
@@ -254,7 +252,7 @@ Nitload is developed by Nitload Space. Contributions are welcome, particularly i
|
||||
|
||||
## License
|
||||
|
||||
Nitload is licensed under the MIT License. See [LICENSE](LICENSE) for details.
|
||||
nFramework is licensed under the MIT License. See [LICENSE](LICENSE) for details.
|
||||
|
||||
## Future Plans
|
||||
|
||||
|
||||
@@ -1,762 +0,0 @@
|
||||
# L4 Software Cache Design Specification
|
||||
|
||||
> Project: StgSharp
|
||||
> Author: Nitload
|
||||
> Version: Draft v0.3
|
||||
> Date: 2026-03-10
|
||||
|
||||
---
|
||||
|
||||
## 1. Purpose
|
||||
|
||||
### 1.1 Problem Background
|
||||
|
||||
StgSharp's matrix computation system is migrating from a "single global context table" to a
|
||||
"per-thread context" architecture to adapt to the scheduling characteristics of hybrid-architecture
|
||||
CPUs (e.g., Intel P-Core / E-Core). After the migration, each thread independently holds its own
|
||||
computation context, but matrix data may still be allocated by the HLSF allocator or global
|
||||
`malloc` on a remote NUMA node, resulting in significantly increased cross-node access latency.
|
||||
|
||||
At the same time, in matrix multiplication (FMA), the core row data of matrix A in the
|
||||
checkerboard-shaped kernel grid layout must be repacked (Packed) into a micro-kernel-friendly
|
||||
contiguous arrangement. In libraries such as OpenBLAS, this Pack operation is re-executed on
|
||||
every tile switch, whereas L4 can cache the Pack results to avoid repeated reordering.
|
||||
|
||||
Additionally, the segmented accumulation (segment) of matrix multiplication requires storing
|
||||
intermediate partial sums locally to avoid cross-NUMA writes to the target matrix on every
|
||||
segment. L4 can serve as a local accumulation buffer for partial sums, writing them back in a
|
||||
single operation after all segments complete locally.
|
||||
|
||||
### 1.2 Solution
|
||||
|
||||
The L4 cache is a purely software-implemented **data caching layer** logically positioned
|
||||
between the CPU hardware caches (L1/L2/L3) and main memory. Its core responsibilities:
|
||||
|
||||
1. **Pack result caching**: Cache the repack results of FMA core rows in matrix A to avoid repeated Pack operations
|
||||
2. **NUMA migration cache**: Migrate data from remote NUMA nodes to local memory and cache it, eliminating cross-node latency
|
||||
3. **Segment accumulation buffer**: Temporarily store partial sums from FMA segmented computation to reduce remote write operations
|
||||
4. **PLU intermediate data buffer**: Provide a local contiguous workspace for PLU decomposition
|
||||
|
||||
### 1.3 Design Goals
|
||||
|
||||
- Provide NUMA-friendly local data access for per-thread matrix computation contexts
|
||||
- Fixed capacity (64 MB), no runtime expansion, deterministic latency
|
||||
- Transparent pointer redirection: upper-layer code queries via original pointers; L4 returns local cache pointers
|
||||
- Cache eviction policy based on **RRPV mask + windowed OPT prediction**
|
||||
- Support both **StreamTransparent** and **CacheBuffer** working modes
|
||||
- Support **writable cache lines** that invoke registered callback functions for pre-processing and write-back upon eviction or explicit flush
|
||||
|
||||
---
|
||||
|
||||
## 2. Overall Architecture
|
||||
|
||||
### 2.1 Three-Component Model
|
||||
|
||||
```
|
||||
┌���─────────────────────────────────────────────────────────────────┐
|
||||
│ L4 │
|
||||
│ │
|
||||
│ ┌──────────────┐ ┌────────────────────┐ ┌──────────────────┐ │
|
||||
│ │ CacheLine │ │ L4 Core │ │ Predictor │ │
|
||||
│ │ Handle │ │ Cache │ │ Prefetch │ │
|
||||
│ │ │ │ │ │ Predictor │ │
|
||||
│ │ • local addr │ │ • Swiss Table │ │ • 2048-entry buf │ │
|
||||
│ │ • ref count │ │ • Slab Allocator │ │ • threshold 1024 │ │
|
||||
│ │ • Dispose │ │ • RRPV mask │ │ • windowed OPT │ │
|
||||
│ │ drops ref │ │ • SIMD eviction │ │ • active key set │ │
|
||||
│ │ │ │ • writeback mgmt │ │ │ │
|
||||
│ └──────────────┘ └────────────────────┘ └──────────────────┘ │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 2.2 Data Access Path
|
||||
|
||||
```
|
||||
Compute thread accessing data:
|
||||
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ Read-only path: L4.Map(originKey) │
|
||||
│ ├─ StreamTransparent + no repack needed? │
|
||||
│ │ └─ Yes → return original pointer directly │
|
||||
│ │ (zero overhead) │
|
||||
│ │ │
|
||||
│ └─ CacheBuffer mode │
|
||||
│ ├─ Swiss Table lookup (2.6ns, lock-free) │
|
||||
│ ├─ Hit → update RRPV=0, return CacheLine handle │
|
||||
│ └─ Miss → SpinWait for Predictor prefetch │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ Writable path: L4.MapWritable(originKey, preprocess, writeback) │
|
||||
│ ├─ Hit → return existing partial sum cache line │
|
||||
│ └─ Miss → allocate zero-initialized cache line, │
|
||||
│ register writeback callbacks │
|
||||
│ cache line marked dirty (dirty=1) │
|
||||
│ eviction invokes preprocess + writeback │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ After computation: │
|
||||
│ 3. CacheLine.Dispose() (refCount--) │
|
||||
│ 4. L4.StepComplete() │
|
||||
│ ├─ Predictor.AdvanceExecutionPoint() │
|
||||
│ └─ Predictor.NeedsPrefetch? │
|
||||
│ └─ Yes → CAS-acquire lock, execute prefetch │
|
||||
└─────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 2.3 Key Parameters
|
||||
|
||||
| Parameter | Value | Description |
|
||||
|------------------------|--------------------------|------------------------------------------------------|
|
||||
| Total cache capacity | 64 MB | Per-NUMA-node contiguous memory |
|
||||
| Cache line size | 8192 Bytes | = 64 × `double MAT_KERNEL` |
|
||||
| Cache line count | 8192 | 64MB / 8192B |
|
||||
| Mapping table type | Swiss Table | 1024 buckets × 16 slots, SIMD-accelerated |
|
||||
| Cache line pool | Slab Allocator | Non-concurrent, fixed capacity, no auto-expansion |
|
||||
| Eviction policy | RRPV mask + windowed OPT | SoA layout + SSE2 batch scan |
|
||||
| Prefetch policy | Predictor windowed OPT | 2048-entry buffer, 1024-step lookahead |
|
||||
| Writeback policy | Callback-driven | Invokes registered callbacks on dirty eviction/flush |
|
||||
|
||||
---
|
||||
|
||||
## 3. Key Components
|
||||
|
||||
### 3.1 Swiss Table (Pointer Mapping Table)
|
||||
|
||||
**Responsibility**: Maintains the `original pointer → cache pointer` mapping.
|
||||
|
||||
- Key: original pointer `nint`, pointing to data allocated by HLSF/malloc on a remote node
|
||||
- Value: cache pointer `nint`, pointing to locally Slab-allocated cache line
|
||||
- Hash: hardware `CRC32C` instruction on the 64-bit pointer
|
||||
- Low 10 bits → bucket index (1024 buckets)
|
||||
- Bits 10-16 → H2 control byte (used for SIMD pre-filtering)
|
||||
- Scan: SSE2 `pmovmskb` instruction for parallel comparison of 16 control bytes
|
||||
- Tombstone mechanism: deleted entries set ctrl to `0xFE`, preserving probe chain integrity
|
||||
- Capacity: 1024 × 16 = 16,384 mappings, significantly larger than total cache line count (8192 lines),
|
||||
ensuring extremely low hash collision rate
|
||||
|
||||
**Measured Performance** (Ryzen 7 6800H, .NET 8, BenchmarkDotNet):
|
||||
|
||||
| Operation | Swiss Table | Dictionary | Speedup |
|
||||
|--------------------------|-------------|-------------|---------|
|
||||
| Zipfian Lookup (8192×) | 20.0 μs | 76.1 μs | 3.8× |
|
||||
| Uniform Lookup (8192×) | 18.6 μs | 50.6 μs | 2.7× |
|
||||
| Insert (4096 entries) | 18.5 μs | 46.5 μs | 2.5× |
|
||||
| SteadyState (4096 ops) | 25.3 μs | 68.3 μs | 2.7× |
|
||||
|
||||
Single lookup latency ≈ 2.6 ns, GC allocation = 0.
|
||||
|
||||
### 3.2 Slab Allocator (Cache Line Pool)
|
||||
|
||||
**Responsibility**: Manages allocation and reclamation of fixed-size (8192B) cache lines.
|
||||
|
||||
- Mode: **Non-concurrent, fixed capacity** (`FixedSlabAllocator<T>` mode),
|
||||
reusing the existing `SlabAllocator.FromBuffer<T>()` interface
|
||||
- No auto-expansion: capacity is fixed at initialization (64MB / 8192B = 8192 lines);
|
||||
allocation failure triggers eviction rather than expansion
|
||||
- Allocation complexity: O(1), pops from free stack
|
||||
- Reclamation complexity: O(1), pushes back to free stack
|
||||
- Alignment: all cache lines are 64B aligned, ensuring no cross-hardware-cache-line boundaries
|
||||
|
||||
### 3.3 Eviction Metadata (SoA Layout + RRPV Mask + Writeback Callbacks)
|
||||
|
||||
**Responsibility**: Tracks per-cache-line reference state, heat, and writeback information to determine eviction priority.
|
||||
|
||||
Replaces the four-level circular linked list from v0.1 with parallel arrays in Structure-of-Arrays layout:
|
||||
|
||||
```
|
||||
SoA Layout — Parallel Arrays
|
||||
|
||||
_refFlags[i] : byte Reference count snapshot (0=free, non-zero=held)
|
||||
_evictionFlags[i] : byte dirty(1bit) + RRPV(2bit) + costClass(1bit)
|
||||
_originKeys[i] : nuint Original pointer (reverse-deregisters Swiss Table entry on eviction)
|
||||
_preprocess[i] : nint Writeback pre-processing function pointer (called before dirty eviction)
|
||||
_writeback[i] : nint Writeback function pointer (called during dirty eviction)
|
||||
_callbackCtx[i] : nuint Callback context (passed to preprocess/writeback)
|
||||
```
|
||||
|
||||
**Eviction Flag Bit Layout** (`_evictionFlags[i]`):
|
||||
|
||||
The RRPV array uses byte storage for each cache line's usage state; the low 2 bits encode the RRPV value.
|
||||
|
||||
**RRPV Semantics**:
|
||||
|
||||
| RRPV | Meaning | Eviction Priority |
|
||||
|------|---------------------------------|---------------------------|
|
||||
| 0 | Will be re-accessed very soon | Lowest (do not evict) |
|
||||
| 1 | Will be accessed shortly | Low |
|
||||
| 2 | Will be accessed in a while | Medium |
|
||||
| 3 | May never be accessed again | Highest (evict first) |
|
||||
|
||||
**RRPV Update Events**:
|
||||
|
||||
| Event | Operation | Description |
|
||||
|--------------------|---------------------------------|------------------------------------------------------|
|
||||
| Load (insert) | RRPV = 0 | Freshly loaded data will be used soon |
|
||||
| Hit | `_evictionFlags[i] = 0` | Bits 1-2 cleared, RRPV reset to 0 |
|
||||
| Eviction scan | Find lines with RRPV=3 to evict | If none found, increment all RRPV +1 (aging) |
|
||||
|
||||
**Advantages over the v0.1 four-level circular linked list**:
|
||||
|
||||
| Comparison | v0.1 Circular List | v0.3 RRPV Mask + Callbacks |
|
||||
|----------------------|-------------------------------------|--------------------------------------|
|
||||
| Data structure | 4 doubly-linked lists + next ptrs | Byte array + function pointer arrays |
|
||||
| Warm-up operation | Unlink + relink (4 pointer writes) | `+= 1` (1-byte write) |
|
||||
| Aging operation | Per-node list migration | SIMD batch +1 (16 entries/cycle) |
|
||||
| Eviction scan | Iterate list head (pointer chasing) | SIMD pattern matching (16/cycle) |
|
||||
| Cache friendliness | Poor (pointer chasing) | Good (contiguous memory scan) |
|
||||
| Reference protection | None | Lines with refCount > 0 not evicted |
|
||||
| Writeback support | None | Callback-function driven |
|
||||
|
||||
### 3.4 Writeback Callback Mechanism
|
||||
|
||||
**Responsibility**: Provides flexible writeback strategies for writable cache lines, using native function pointers to avoid managed overhead.
|
||||
|
||||
**Design Principle**: All writable cache lines are treated as dirty by default. On eviction or explicit flush,
|
||||
L4 itself does not manage the writeback strategy; instead, the bound predictor is responsible for implementing it.
|
||||
|
||||
The predictor is defined via the following interface:
|
||||
|
||||
```
|
||||
public interface IPrefetchPredict
|
||||
{
|
||||
|
||||
int Predict(
|
||||
Span<CacheLinePrediction> node
|
||||
);
|
||||
|
||||
bool Prefetch(
|
||||
nuint origin,
|
||||
ulong policy,
|
||||
nuint cache
|
||||
);
|
||||
|
||||
void WriteBack(
|
||||
nuint origin,
|
||||
ulong policy,
|
||||
nuint cache
|
||||
);
|
||||
|
||||
}
|
||||
```
|
||||
|
||||
**Typical Callback Implementations**:
|
||||
|
||||
```
|
||||
Scenario 1: Segment accumulation (FMA partial sum)
|
||||
preprocess = null (no pre-processing needed; data is already a double array)
|
||||
writeback = AccumulateAdd (element-wise dst[i] += src[i], AVX2-vectorized)
|
||||
|
||||
Scenario 2: Writeback of packed intermediate results
|
||||
preprocess = UnpackKernelRow (reverse-repack from Pack format to row-major order)
|
||||
writeback = AccumulateAdd
|
||||
|
||||
Scenario 3: PLU decomposition intermediate data
|
||||
preprocess = null
|
||||
writeback = DirectCopy (direct overwrite memcpy)
|
||||
```
|
||||
|
||||
### 3.5 CacheLine (Cache Line Handle)
|
||||
|
||||
**Responsibility**: A cache line access handle held by the compute thread; prevents in-use lines from being evicted via reference counting.
|
||||
|
||||
```csharp
|
||||
public ref struct CacheLine : IDisposable
|
||||
{
|
||||
private ref int _activeCount; // L4 global active count
|
||||
private ref int _refCount; // Per-line reference count
|
||||
private nuint _handle; // Local cache data address
|
||||
|
||||
// Construction: Interlocked.Increment(ref _activeCount)
|
||||
// Interlocked.Increment(ref _refCount)
|
||||
// Dispose: Interlocked.Decrement on both counters
|
||||
}
|
||||
```
|
||||
|
||||
- `ref struct` ensures it cannot escape to the heap
|
||||
- Reference counts use `Interlocked` atomic operations for thread safety
|
||||
- Lines with `_refCount > 0` are skipped during eviction scans
|
||||
- StreamTransparent mode returns a CacheLine that does not occupy a cache slot
|
||||
- Both read-only and writable paths return the same CacheLine type,
|
||||
callers decide read/write behavior via `.Address`
|
||||
|
||||
### 3.6 Predictor (Prefetch Predictor)
|
||||
|
||||
**Responsibility**: Provides deterministic prefetch predictions for L4 based on the windowed OPT strategy.
|
||||
|
||||
**Core Principle**: Matrix computation task sequences are deterministic — the scheduler pre-allocates
|
||||
task blocks `(kr, kc, zz)` for each thread, allowing precise computation of the cache line keys
|
||||
required at each future step. This gives L4 a predictive capability that hardware caches cannot
|
||||
possess: **direct visibility into the future**, rather than inferring the future from history as
|
||||
strategies like Hawkeye do.
|
||||
|
||||
```
|
||||
Predictor Sliding Window:
|
||||
|
||||
Task sequence: ... [T_now] [T+1] [T+2] ... [T+1023] ... [T+2047]
|
||||
↑ execution point ↑ buffer tail
|
||||
|
||||
Window capacity = 2048 prediction entries
|
||||
Prefetch trigger threshold = trigger refill when prefetch frontier
|
||||
leads execution point by fewer than 1024 steps
|
||||
Refill batch size = 64 entries (avoids excessive single-refill latency)
|
||||
```
|
||||
|
||||
**数据结构**：
|
||||
|
||||
```csharp
|
||||
internal class Predictor
|
||||
{
|
||||
private const int WindowCapacity = 2048;
|
||||
private const int RefillThreshold = 1024;
|
||||
|
||||
private PrefetchEntry[] _buffer; // Circular prediction buffer
|
||||
private int _head; // Prefetch frontier (prefetched up to here)
|
||||
private int _tail; // Production position (scheduler fills up to here)
|
||||
private int _executed; // Execution point (compute thread has completed up to here)
|
||||
|
||||
private HashSet<nuint> _activeKeys; // All active keys within the window (deduplicated)
|
||||
}
|
||||
|
||||
struct PrefetchEntry
|
||||
{
|
||||
public nuint Key; // Cache line identifier
|
||||
public nuint SourceAddress; // Source data address
|
||||
public bool NeedsRepack; // Whether non-contiguous repacking is needed
|
||||
}
|
||||
```
|
||||
|
||||
**Cooperation with Eviction Policy**:
|
||||
|
||||
```
|
||||
Eviction priority (from highest to lowest priority):
|
||||
|
||||
1. refCount==0, not in window, RRPV==3, dirty==0 ← First (zero-cost eviction)
|
||||
2. refCount==0, not in window, RRPV==3, dirty==1 ← Second (needs writeback but cold)
|
||||
3. refCount==0, not in window, RRPV<3, dirty==0 ← Ordered by RRPV
|
||||
4. refCount==0, not in window, RRPV<3, dirty==1 ← Ordered by RRPV
|
||||
5. refCount==0, in window ← Avoid evicting if possible
|
||||
6. refCount>0 ← Never evict
|
||||
```
|
||||
|
||||
Windowed OPT means most eviction decisions do not depend on RRPV — "not in window" alone is
|
||||
sufficient to determine evictability. RRPV provides fine-grained differentiation among
|
||||
out-of-window lines; the dirty bit preferentially evicts clean lines at equal RRPV to avoid writeback overhead.
|
||||
|
||||
### 3.7 Concurrency Model
|
||||
|
||||
L4 uses a **no-extra-threads** design; all operations are performed by compute threads themselves:
|
||||
|
||||
| Operation | Executor | Concurrency |
|
||||
|------------------------------------|---------------------------------|--------------------------------------|
|
||||
| Lookup (Swiss Table read) | Any compute thread | ✅ Fully parallel, lock-free |
|
||||
| HIT RRPV update | Any compute thread | ✅ Single-byte write, no sync needed |
|
||||
| refCount increment/decrement | Any compute thread | ✅ Interlocked atomic operations |
|
||||
| Prefetch refresh (eviction + load) | CAS-winner thread | Serial (only one thread enters) |
|
||||
| Dirty writeback | CAS-winner thread | Serial (within eviction flow) |
|
||||
| FlushAllDirty | Single thread after computation | Serial (no concurrency) |
|
||||
|
||||
**Prefetch Refresh CAS Lock**:
|
||||
|
||||
```csharp
|
||||
private int _refreshLock; // 0=free, 1=locked
|
||||
|
||||
private void TryRefreshPrefetch()
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _refreshLock, 1, 0) != 0)
|
||||
return; // Another thread is doing it; current thread continues computing
|
||||
try { /* eviction + prefetch */ }
|
||||
finally { Volatile.Write(ref _refreshLock, 0); }
|
||||
}
|
||||
```
|
||||
|
||||
**Swiss Table Concurrent Read Safety Guarantee**: The prefetch refresh thread follows strict write ordering
|
||||
on insertion: write Slab data first, then write eviction metadata and callback pointers, `sfence`, then
|
||||
write the Swiss Table control byte and key/value, ensuring that when a compute thread reads the control
|
||||
byte, the data is already fully visible.
|
||||
|
||||
---
|
||||
|
||||
## 4. Operation Flows
|
||||
|
||||
### 4.1 Cache Lookup (Read-Only Hit)
|
||||
|
||||
1. Compute CRC32C hash of the original pointer
|
||||
2. Swiss Table lookup: SIMD scan ctrl → compare key → return cache pointer
|
||||
3. Update RRPV: `_evictionFlags[slot] &= 0xF9` (RRPV zeroed, dirty bit unchanged)
|
||||
4. Construct CacheLine handle (refCount++)
|
||||
5. Return CacheLine; upper-layer code accesses local data via `.Address`
|
||||
|
||||
### 4.2 Cache Lookup (Writable Hit)
|
||||
|
||||
1. Swiss Table lookup hits
|
||||
2. Update RRPV: `_evictionFlags[slot] &= 0xF9`
|
||||
3. Construct CacheLine handle (refCount++)
|
||||
4. Return CacheLine; upper-layer code reads/writes the cache line via `.Address` (e.g., partial sum +=)
|
||||
|
||||
### 4.3 Cache Miss (Read-Only)
|
||||
|
||||
1. Swiss Table lookup misses
|
||||
2. Compute thread enters `Thread.Sleep(0)` spin loop
|
||||
3. Wait for Predictor-driven prefetch refresh to load the line into L4
|
||||
4. Exit wait loop when Swiss Table shows a hit
|
||||
5. Construct CacheLine handle and return
|
||||
|
||||
Under normal operation, the Predictor always leads the execution point by 1024+ steps,
|
||||
misses occur only during cold start or extreme bursts.
|
||||
|
||||
### 4.4 Cache Miss (Writable)
|
||||
|
||||
1. Swiss Table lookup misses
|
||||
2. Allocate a cache line from Slab
|
||||
3. If Slab has no free slots → trigger eviction (see 4.6)
|
||||
4. Zero-initialize the cache line (`Unsafe.InitBlock(dest, 0, 8192)`)
|
||||
5. Register writeback callbacks: `_preprocess[slot]`, `_writeback[slot]`, `_callbackCtx[slot]`
|
||||
6. Set eviction flags: `dirty=1, RRPV=0`
|
||||
7. Write `_originKeys[slot]`
|
||||
8. `sfence`
|
||||
9. Write Swiss Table mapping
|
||||
10. Construct CacheLine handle and return
|
||||
|
||||
### 4.5 Prefetch Refresh
|
||||
|
||||
Triggered by compute threads in `StepComplete()`; CAS lock ensures single-thread execution:
|
||||
|
||||
1. Synchronize refFlags snapshot (`_refCounts[i] > 0 → _refFlags[i] = 1`)
|
||||
2. Retrieve pending prefetch entries from Predictor (up to 64 at a time)
|
||||
3. For each entry:
|
||||
- Swiss Table hit → mark RRPV=0 (hot-line protection)
|
||||
- Swiss Table miss → execute load:
|
||||
a. Allocate cache line from Slab
|
||||
b. If Slab has no free slots → trigger eviction (see 4.6)
|
||||
c. Execute Pack reorder or memcpy based on `NeedsRepack` flag
|
||||
d. Write eviction metadata (originKey, RRPV=0, dirty=0, callbacks=null)
|
||||
e. `sfence`
|
||||
f. Write Swiss Table mapping
|
||||
|
||||
### 4.6 Cache Eviction (SIMD Scan + Writeback)
|
||||
|
||||
```
|
||||
Eviction scan pseudocode:
|
||||
|
||||
Pass 1: Find refCount==0, RRPV==3, dirty==0 (zero-cost eviction)
|
||||
for i in range(scanCursor, CacheLineCount), step=16:
|
||||
refChunk = SSE2.Load(refFlags + i)
|
||||
refFreeMask = SSE2.MoveMask(SSE2.CompareEqual(refChunk, zero))
|
||||
|
||||
evChunk = SSE2.Load(evictionFlags + i)
|
||||
// RRPV==3 and dirty==0 → byte & 0x07 == 0x06
|
||||
masked = SSE2.And(evChunk, 0x07)
|
||||
coldCleanMask = SSE2.MoveMask(SSE2.CompareEqual(masked, 0x06))
|
||||
|
||||
candidates = refFreeMask & coldCleanMask
|
||||
while candidates != 0:
|
||||
bit = TrailingZeroCount(candidates)
|
||||
idx = i + bit
|
||||
if originKeys[idx] not in Predictor.ActiveKeys:
|
||||
evict(idx) // No writeback, discard directly
|
||||
return
|
||||
candidates &= candidates - 1
|
||||
|
||||
Pass 2: Find refCount==0, RRPV==3, dirty==1 (needs writeback)
|
||||
Same as above, but match byte & 0x07 == 0x07
|
||||
evict triggers writeback callbacks
|
||||
|
||||
Pass 3: Increment all RRPV+1 (aging), retry
|
||||
Up to 3 rounds
|
||||
```
|
||||
|
||||
**逐出执行**：
|
||||
|
||||
```csharp
|
||||
private void Evict(int slot)
|
||||
{
|
||||
byte flags = _evictionFlags[slot];
|
||||
|
||||
// Dirty writeback
|
||||
if ((flags & 0x01) != 0)
|
||||
{
|
||||
nuint origin = _originKeys[slot];
|
||||
nuint cached = SlotAddressFromIndex(slot);
|
||||
nuint ctx = _callbackCtx[slot];
|
||||
|
||||
// 1. Pre-processing (optional)
|
||||
var preprocess = (delegate* unmanaged<nuint, nuint, void>)_preprocess[slot];
|
||||
if (preprocess != null)
|
||||
preprocess(cached, ctx);
|
||||
|
||||
// 2. Writeback
|
||||
var writeback = (delegate* unmanaged<nuint, nuint, nuint, void>)_writeback[slot];
|
||||
if (writeback != null)
|
||||
writeback(origin, cached, ctx);
|
||||
}
|
||||
|
||||
// Standard eviction
|
||||
_map.TryRemove(_originKeys[slot], out _);
|
||||
_slab.Free(SlotAddressFromIndex(slot));
|
||||
_evictionFlags[slot] = 0;
|
||||
_originKeys[slot] = 0;
|
||||
_preprocess[slot] = 0;
|
||||
_writeback[slot] = 0;
|
||||
_callbackCtx[slot] = 0;
|
||||
}
|
||||
```
|
||||
|
||||
### 4.7 显式刷新
|
||||
|
||||
```csharp
|
||||
/// <summary>
|
||||
/// Explicitly writes back the specified dirty cache line. Called after all segments complete.
|
||||
/// </summary>
|
||||
public void FlushDirty(nuint originKey)
|
||||
{
|
||||
if (_map.TryGet(originKey, out nuint addr))
|
||||
{
|
||||
int slot = SlotIndex(addr);
|
||||
if ((_evictionFlags[slot] & 0x01) != 0)
|
||||
{
|
||||
// Execute pre-processing + writeback
|
||||
InvokeWriteBack(slot);
|
||||
_evictionFlags[slot] &= 0xFE; // Clear dirty bit
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes back all dirty cache lines in bulk. Called after entire matrix computation completes.
|
||||
/// </summary>
|
||||
public void FlushAllDirty()
|
||||
{
|
||||
for (int i = 0; i < CacheLineCount; i++)
|
||||
{
|
||||
if ((_evictionFlags[i] & 0x01) != 0 && _originKeys[i] != 0)
|
||||
{
|
||||
InvokeWriteBack(i);
|
||||
_evictionFlags[i] &= 0xFE;
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 4.8 Working Mode Switch
|
||||
|
||||
```csharp
|
||||
public enum L4Mode
|
||||
{
|
||||
/// <summary>
|
||||
/// Pass-through mode: Map() returns the original pointer directly; L4 does not intervene.
|
||||
/// Suitable for single-NUMA + no-repack scenarios (e.g., vector addition, scalar multiplication).
|
||||
/// </summary>
|
||||
StreamTransparent,
|
||||
|
||||
/// <summary>
|
||||
/// Buffer mode: full lookup/Pack/cache/eviction pipeline.
|
||||
/// Suitable for FMA scenarios requiring repacking or multi-NUMA scenarios.
|
||||
/// </summary>
|
||||
CacheBuffer
|
||||
}
|
||||
```
|
||||
|
||||
Mode is determined at construction time and does not change at runtime. Decision logic:
|
||||
|
||||
- Requires Pack reordering (FMA core rows of matrix A) → CacheBuffer
|
||||
- Multiple NUMA nodes → CacheBuffer
|
||||
- Requires segment accumulation → CacheBuffer
|
||||
- Single NUMA + no reordering + no segment → StreamTransparent
|
||||
|
||||
---
|
||||
|
||||
## 5. Segment Accumulation Scenario
|
||||
|
||||
### 5.1 Problem
|
||||
|
||||
For matrix multiplication `C = A × B` when the K dimension is large, segmented accumulation is required:
|
||||
|
||||
```
|
||||
segment 0: C_partial = A[kr, 0..KC] × B[0..KC, kc]
|
||||
segment 1: C_partial += A[kr, KC..2KC] × B[KC..2KC, kc]
|
||||
...
|
||||
segment N: C_partial += A[kr, N*KC..K] × B[N*KC..K, kc]
|
||||
|
||||
Final: C[kr, kc] = Σ all segment partial sums
|
||||
```
|
||||
|
||||
If each segment's result is `+=` directly to the C matrix,
|
||||
and C resides on a remote NUMA node, every segment incurs cross-node write latency.
|
||||
|
||||
### 5.2 L4 Solution
|
||||
|
||||
Temporarily store C's partial sums in L4's writable cache lines:
|
||||
|
||||
```
|
||||
segment 0: c = l4.MapWritable(C_key, null, &AccumulateAdd)
|
||||
// First miss → allocate zero-initialized cache line
|
||||
FMA(c.Address, a.Address, b.Address) // c += a*b
|
||||
c.Dispose()
|
||||
|
||||
segment 1: c = l4.MapWritable(C_key, null, &AccumulateAdd)
|
||||
// Hit! Returns existing partial sum cache line
|
||||
FMA(c.Address, a.Address, b.Address) // c += a*b
|
||||
c.Dispose()
|
||||
|
||||
... after all segments complete ...
|
||||
|
||||
l4.FlushDirty(C_key)
|
||||
// Calls AccumulateAdd(C_origin, cacheline, ctx)
|
||||
// → Adds local partial sum back to remote C matrix in one shot
|
||||
```
|
||||
|
||||
### 5.3 Benefits
|
||||
|
||||
```
|
||||
Without segment buffering (OpenBLAS approach):
|
||||
Each segment writes C matrix across NUMA
|
||||
Remote writes per output tile = Zk times
|
||||
Per-write latency = ~200-300ns (cross-node)
|
||||
|
||||
With L4 segment buffering:
|
||||
All segment += operations complete locally in L4 cache lines
|
||||
Partial sums likely remain in L1/L2 (8KB << L1 capacity)
|
||||
Remote writes per output tile = 1 (final FlushDirty)
|
||||
Writeback uses AVX2-vectorized stores: 8192B = 32 × 256-bit stores
|
||||
|
||||
Reduction ratio = Zk : 1
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Uniqueness and Advantages
|
||||
|
||||
### 6.1 Deterministic Prefetch (Windowed OPT)
|
||||
|
||||
Essential difference from hardware cache replacement policies (LRU, RRIP, Hawkeye):
|
||||
|
||||
| Policy | Information Source | Prediction Method | Accuracy |
|
||||
|---------------------|-------------------------|-------------------------------|---------------------------|
|
||||
| LRU | Past access order | Least recently used | Moderate |
|
||||
| RRIP | Insert/hit markers | Predict reuse distance | Moderate-High |
|
||||
| Hawkeye | OPTgen retrospection | Learn past to infer future | High |
|
||||
| **L4 Predictor** | **Future task queue** | **Direct future visibility** | **Perfect** (within window) |
|
||||
|
||||
L4's Predictor is a **deterministic Oracle** — it does not predict; it knows.
|
||||
Because matrix computation task sequences are fully determined at scheduler allocation time.
|
||||
|
||||
### 6.2 NUMA Adaptability
|
||||
|
||||
The same code automatically gains NUMA benefits on different hardware configurations:
|
||||
|
||||
| Hardware Configuration | L4 Behavior |
|
||||
|-------------------------|-------------------------------------------------------------|
|
||||
| Single-socket CPU | Pack cache + segment accumulation buffer |
|
||||
| Dual-socket CPU | Pack cache + NUMA migration cache + segment accum. |
|
||||
| Hybrid-architecture CPU | Pack cache + cross-domain migration cache + segment accum. |
|
||||
|
||||
### 6.3 Callback-Driven Writeback Strategy
|
||||
|
||||
Using native function pointers for writeback avoids:
|
||||
|
||||
- Virtual method call overhead
|
||||
- Interface dispatch overhead
|
||||
- Delegate GC pressure
|
||||
- Hard-coding specific writeback logic
|
||||
|
||||
Callers register `preprocess` and `writeback` callbacks at `MapWritable` time,
|
||||
L4 invokes them directly through function pointers on eviction or Flush,
|
||||
**zero managed overhead, zero virtual dispatch, performance equivalent to C function calls**.
|
||||
|
||||
### 6.4 Reference Count Protection
|
||||
|
||||
Through `CacheLine` handle reference counting, cache lines actively in use by compute threads are
|
||||
absolutely never evicted (including writeback not being triggered). The SIMD eviction scan skips
|
||||
lines with `refCount > 0` immediately via the `_refFlags` array, without checking linked lists or acquiring locks.
|
||||
|
||||
### 6.5 Cache Line Size Aligned to Matrix Kernel
|
||||
|
||||
The 8192B cache line size is not arbitrary:
|
||||
|
||||
- 1 `double MAT_KERNEL` (4×4 double matrix) = 128 Bytes
|
||||
- 1 cache line = 64 `MAT_KERNEL` units
|
||||
- Matrix multiplication micro-kernels operate on data sizes perfectly aligned to cache lines
|
||||
- Compute kernel functions operate on cache lines as units; no cross-cache-line accesses occur
|
||||
|
||||
### 6.6 Fixed-Capacity Determinism
|
||||
|
||||
- Slab allocator never expands → memory footprint upper bound is known (64 MB)
|
||||
- SIMD eviction scan → latency upper bound is known (at most 3 aging rounds × 512 SIMD comparisons)
|
||||
- Swiss Table fixed 16,384 slots → lookup latency upper bound is known
|
||||
- Writeback callback latency is controlled by the caller → predictable
|
||||
- Worst-case L4 latency = lookup + eviction (incl. writeback) + allocation + memcpy, all statically computable
|
||||
|
||||
### 6.7 Reuse of Existing StgSharp Infrastructure
|
||||
|
||||
| L4 Component | Reused StgSharp Infrastructure |
|
||||
|-----------------------|-------------------------------------------------------|
|
||||
| Mapping table | `SwissTable` (implemented and benchmark-validated) |
|
||||
| Mapping table SIMD | `IBucketScanner` + `SIMDID` architecture detection |
|
||||
| Cache line alloc | `SlabAllocator.FromBuffer<T>()` |
|
||||
| Hash computation | Hardware `CRC32C` instruction |
|
||||
| Memory allocation | `NativeMemory.AlignedAlloc` |
|
||||
| Future multi-platform | `IBucketScanner` + NRGA compilation pipeline |
|
||||
|
||||
### 6.8 Originality Analysis
|
||||
|
||||
After surveying existing mainstream numerical computation libraries, L4's design has
|
||||
**no direct precedent or equivalent in known public implementations**.
|
||||
|
||||
#### NUMA Strategy Comparison with Existing Libraries
|
||||
|
||||
| Library / Framework | NUMA Strategy | Software Cache Layer | Mapping Table | Eviction Policy | Dirty Writeback |
|
||||
|---------------------|-------------------------------------|---------------------|-----------------|-------------------------|--------------------|
|
||||
| Intel MKL | Thread pinning + `numactl` | None | None | None | None |
|
||||
| OpenBLAS | Thread affinity + block alignment | None | None | None | None |
|
||||
| BLIS | Multi-level block alignment L1/L2/L3| None | None | None | None |
|
||||
| Eigen | No explicit NUMA handling | None | None | None | None |
|
||||
| cuBLAS / rocBLAS | GPU memory (hardware-solved) | None | None | None | None |
|
||||
| **StgSharp L4** | **Software-managed thread-local cache** | **Yes** | **Swiss Table** | **RRPV + windowed OPT** | **Callback-driven** |
|
||||
|
||||
---
|
||||
|
||||
## 7. 从 v0.1 到 v0.3 的变更摘要
|
||||
|
||||
| 项目 | v0.1 | v0.3 |
|
||||
|-----------------|-------------------------------|---------------------------------------|
|
||||
| 逐出数据结构 | 4 级环形双向链表 | RRPV 2-bit 掩码 + 8KB SoA 数组 |
|
||||
| 逐出扫描方式 | 链表头指针遍历 | SSE2 批量模式匹配（16 行/周期） |
|
||||
| 热���更新 | 链表节点摘插（4 指针写） | 单字节位运算 `&= 0x03` |
|
||||
| 老化操作 | 逐节点移链表 | SIMD 批量 +1 |
|
||||
| 引用保护 | 无 | refCount > 0 时 SIMD 扫描跳过 |
|
||||
| 预取策略 | 无 | Predictor 窗口 OPT（2048 条缓冲） |
|
||||
| 线程模型 | L4 服务线程 + Command Queue | 无额外线程，CAS 抢锁 |
|
||||
| 命令队列 | 链式 CommandCache 分段 | 不需要（预取刷新替代命令驱动） |
|
||||
| 缓存行元数据 | Slab 分配的 CacheLineHead 结构 | SoA 平行数组 + 函数指针数组 |
|
||||
| 工作模式 | 单一模式 | StreamTransparent / CacheBuffer 切换 |
|
||||
| 写回策略 | 无（只读缓存） | dirty 位 + preprocess/writeback 回调 |
|
||||
| 写回机制 | 无 | 本机函数指针，零托管开销 |
|
||||
|
||||
---
|
||||
|
||||
## 8. 未来工作
|
||||
|
||||
- [ ] L4 根类型重构（组合 Swiss Table、Slab、SoA 元数据、Predictor）
|
||||
- [ ] Predictor 与矩阵乘法调度器的集成
|
||||
- [ ] Segment 累加写回的 AVX2 向量化实现
|
||||
- [ ] SIMD 逐出扫描的 AVX2 升级（32 行/周期）
|
||||
- [ ] Swiss Table ARM NEON 实现
|
||||
- [ ] Swiss Table AVX-512 升级（64 槽扫描从 16 次降至 4 次）
|
||||
- [ ] 墓碑累积监控与自动 rehash 策略
|
||||
- [ ] 性能基准测试（NUMA 本地 vs 远端访问延迟对比）
|
||||
- [ ] Segment 累加 vs 直接写回的 benchmark 对比
|
||||
- [ ] 与逐线程矩阵上下文的集成验证
|
||||
- [ ] 双路 6242R 环境下的 NUMA 搬运性能验证
|
||||
- [ ] 远期：PPC64LE 平台的 VSX 原生 Swiss Table 扫描支持
|
||||
|
||||
---
|
||||
|
||||
## 9. 参考
|
||||
|
||||
- Google Abseil Swiss Tables Design Notes
|
||||
- HP Labs RRIP (Re-Reference Interval Prediction) 论文
|
||||
- Hawkeye: Efficient Cache Replacement via OPTgen (ISCA 2017)
|
||||
- StgSharp HLSF Allocator 设计文档
|
||||
- StgSharp SIMD Architecture (`SIMDID` 体系)
|
||||
- StgSharp NRGA Roadmap
|
||||
- StgSharp `SlabAllocator<T>` 实现
|
||||
- StgSharp `SwissTable` 实现及 Benchmark
|
||||
@@ -0,0 +1,11 @@
|
||||
# nFramework Icon
|
||||
|
||||
`nFramework-icon.svg` is the icon-only version of the nFramework mark.
|
||||
|
||||
Copyright (c) 2026 Nitload.
|
||||
|
||||
The artwork in this directory is licensed under the [Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International License (CC BY-NC-ND 4.0)](https://creativecommons.org/licenses/by-nc-nd/4.0/).
|
||||
|
||||
You may copy and redistribute the unmodified artwork for noncommercial purposes when you give appropriate credit and link to the license. This license does not permit distributing adaptations, and it does not grant permission for commercial use. Contact the copyright holder for other permissions.
|
||||
|
||||
The repository's root MIT License applies to software, not the artwork in this directory. This copyright license does not grant trademark rights in the Nitload or nFramework names or marks, including the former StgSharp name, or imply endorsement.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 28 KiB |
@@ -0,0 +1,6 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="616" height="104" viewBox="0 0 616 104" role="img" aria-labelledby="title desc">
|
||||
<title id="title">nFramework</title>
|
||||
<desc id="desc">Horizontal nFramework logo with the upright punctuation icon and an IBM Plex Mono Medium wordmark.</desc>
|
||||
<g fill="#59B2DA" transform="translate(13.6 13.6) scale(0.8)"><polygon points="8,8 31,8 31,20 20,20 20,76 31,76 31,88 8,88" /><polygon points="97,8 120,8 120,88 97,88 97,76 108,76 108,20 97,20" /><polygon points="40,16 53,16 53,45 40,45" /><circle cx="80" cy="76" r="8" /></g>
|
||||
<g fill="#E9EEF5"><path d="M146.384 78.524V39.308H154.668V45.996H155.048Q155.656 44.476 156.606 43.07Q157.556 41.664 158.924 40.638Q160.292 39.612 162.116 39.004Q163.94 38.396 166.372 38.396Q172.3 38.396 175.948 42.272Q179.596 46.148 179.596 53.292V78.524H171.312V54.508Q171.312 45.16 163.18 45.16Q161.584 45.16 160.064 45.578Q158.544 45.996 157.328 46.832Q156.112 47.668 155.39 48.96Q154.668 50.252 154.668 51.924V78.524Z"/><path d="M192.136 78.524V25.476H226.26V32.62H200.648V48.048H223.904V55.192H200.648V78.524Z"/><path d="M236.976 71.836H247.388V45.996H236.976V39.308H255.672V49.72H256.128Q256.66 47.668 257.648 45.768Q258.636 43.868 260.156 42.462Q261.676 41.056 263.88 40.182Q266.084 39.308 269.048 39.308H273.76V47.212H266.008Q261.448 47.212 258.56 49.91Q255.672 52.608 255.672 57.168V71.836H269.884V78.524H236.976Z"/><path d="M314.192 78.524Q310.62 78.524 308.834 76.7Q307.048 74.876 306.668 71.912H306.288Q305.148 75.484 302.146 77.46Q299.144 79.436 294.508 79.436Q288.504 79.436 284.932 76.282Q281.36 73.128 281.36 67.58Q281.36 61.88 285.578 58.878Q289.796 55.876 298.536 55.876H306.06V52.836Q306.06 44.856 297.624 44.856Q293.824 44.856 291.468 46.338Q289.112 47.82 287.516 50.252L282.576 46.224Q284.248 43.032 288.2 40.714Q292.152 38.396 298.384 38.396Q305.832 38.396 310.088 41.968Q314.344 45.54 314.344 52.304V71.988H319.284V78.524ZM296.864 73.356Q300.892 73.356 303.476 71.532Q306.06 69.708 306.06 66.82V61.12H298.688Q289.72 61.12 289.72 66.516V68.036Q289.72 70.696 291.62 72.026Q293.52 73.356 296.864 73.356Z"/><path d="M325.744 78.524V39.308H332.812V44.4H333.192Q334.104 41.892 335.814 40.144Q337.524 38.396 340.716 38.396Q343.832 38.396 345.542 40.106Q347.252 41.816 347.708 44.78H348.012Q349.076 42.12 350.976 40.258Q352.876 38.396 356.448 38.396Q361.008 38.396 362.794 41.588Q364.58 44.78 364.58 50.936V78.524H357.512V52Q357.512 47.896 356.524 46.338Q355.536 44.78 353.332 44.78Q351.356 44.78 350.026 46.11Q348.696 47.44 348.696 50.1V78.524H341.628V52Q341.628 47.896 340.678 46.338Q339.728 44.78 337.524 44.78Q335.472 44.78 334.142 46.11Q332.812 47.44 332.812 50.1V78.524Z"/><path d="M391.636 79.436Q387.152 79.436 383.58 77.992Q380.008 76.548 377.576 73.888Q375.144 71.228 373.852 67.466Q372.56 63.704 372.56 58.992Q372.56 54.28 373.89 50.48Q375.22 46.68 377.614 43.982Q380.008 41.284 383.428 39.84Q386.848 38.396 391.028 38.396Q395.132 38.396 398.476 39.802Q401.82 41.208 404.176 43.792Q406.532 46.376 407.786 49.948Q409.04 53.52 409.04 57.852V60.968H380.92V62.26Q380.92 66.896 383.808 69.822Q386.696 72.748 391.788 72.748Q395.588 72.748 398.286 71.114Q400.984 69.48 402.656 66.744L407.824 71.304Q405.848 74.724 401.782 77.08Q397.716 79.436 391.636 79.436ZM391.028 44.704Q388.824 44.704 386.962 45.464Q385.1 46.224 383.77 47.63Q382.44 49.036 381.68 50.936Q380.92 52.836 380.92 55.116V55.648H400.604V54.888Q400.604 50.252 397.982 47.478Q395.36 44.704 391.028 44.704Z"/><path d="M415.12 39.308H422.188L425.988 72.292H426.672L432.524 39.308H440.58L446.356 72.292H447.04L450.916 39.308H457.68L452.208 78.524H442.024L436.78 48.124H436.096L430.776 78.524H420.592Z"/><path d="M482 79.436Q477.668 79.436 474.21 77.992Q470.752 76.548 468.358 73.888Q465.964 71.228 464.672 67.428Q463.38 63.628 463.38 58.916Q463.38 54.204 464.672 50.404Q465.964 46.604 468.358 43.944Q470.752 41.284 474.21 39.84Q477.668 38.396 482 38.396Q486.332 38.396 489.79 39.84Q493.248 41.284 495.642 43.944Q498.036 46.604 499.328 50.404Q500.62 54.204 500.62 58.916Q500.62 63.628 499.328 67.428Q498.036 71.228 495.642 73.888Q493.248 76.548 489.79 77.992Q486.332 79.436 482 79.436ZM482 72.9Q486.56 72.9 489.258 70.126Q491.956 67.352 491.956 61.88V55.952Q491.956 50.48 489.258 47.706Q486.56 44.932 482 44.932Q477.44 44.932 474.742 47.706Q472.044 50.48 472.044 55.952V61.88Q472.044 67.352 474.742 70.126Q477.44 72.9 482 72.9Z"/><path d="M510.576 71.836H520.988V45.996H510.576V39.308H529.272V49.72H529.728Q530.26 47.668 531.248 45.768Q532.236 43.868 533.756 42.462Q535.276 41.056 537.48 40.182Q539.684 39.308 542.648 39.308H547.36V47.212H539.608Q535.048 47.212 532.16 49.91Q529.272 52.608 529.272 57.168V71.836H543.484V78.524H510.576Z"/><path d="M557.088 22.284H565.372V56.256H565.752L571.908 49.948L582.928 39.308H592.732L576.848 54.736L594.48 78.524H584.524L570.844 59.448L565.372 64.54V78.524H557.088Z"/></g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 4.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 28 KiB |
@@ -0,0 +1,6 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="616" height="104" viewBox="0 0 616 104" role="img" aria-labelledby="title desc">
|
||||
<title id="title">nFramework</title>
|
||||
<desc id="desc">Horizontal nFramework logo with the upright punctuation icon and an IBM Plex Mono Medium wordmark.</desc>
|
||||
<g fill="#59B2DA" transform="translate(13.6 13.6) scale(0.8)"><polygon points="8,8 31,8 31,20 20,20 20,76 31,76 31,88 8,88" /><polygon points="97,8 120,8 120,88 97,88 97,76 108,76 108,20 97,20" /><polygon points="40,16 53,16 53,45 40,45" /><circle cx="80" cy="76" r="8" /></g>
|
||||
<g fill="#14202D"><path d="M146.384 78.524V39.308H154.668V45.996H155.048Q155.656 44.476 156.606 43.07Q157.556 41.664 158.924 40.638Q160.292 39.612 162.116 39.004Q163.94 38.396 166.372 38.396Q172.3 38.396 175.948 42.272Q179.596 46.148 179.596 53.292V78.524H171.312V54.508Q171.312 45.16 163.18 45.16Q161.584 45.16 160.064 45.578Q158.544 45.996 157.328 46.832Q156.112 47.668 155.39 48.96Q154.668 50.252 154.668 51.924V78.524Z"/><path d="M192.136 78.524V25.476H226.26V32.62H200.648V48.048H223.904V55.192H200.648V78.524Z"/><path d="M236.976 71.836H247.388V45.996H236.976V39.308H255.672V49.72H256.128Q256.66 47.668 257.648 45.768Q258.636 43.868 260.156 42.462Q261.676 41.056 263.88 40.182Q266.084 39.308 269.048 39.308H273.76V47.212H266.008Q261.448 47.212 258.56 49.91Q255.672 52.608 255.672 57.168V71.836H269.884V78.524H236.976Z"/><path d="M314.192 78.524Q310.62 78.524 308.834 76.7Q307.048 74.876 306.668 71.912H306.288Q305.148 75.484 302.146 77.46Q299.144 79.436 294.508 79.436Q288.504 79.436 284.932 76.282Q281.36 73.128 281.36 67.58Q281.36 61.88 285.578 58.878Q289.796 55.876 298.536 55.876H306.06V52.836Q306.06 44.856 297.624 44.856Q293.824 44.856 291.468 46.338Q289.112 47.82 287.516 50.252L282.576 46.224Q284.248 43.032 288.2 40.714Q292.152 38.396 298.384 38.396Q305.832 38.396 310.088 41.968Q314.344 45.54 314.344 52.304V71.988H319.284V78.524ZM296.864 73.356Q300.892 73.356 303.476 71.532Q306.06 69.708 306.06 66.82V61.12H298.688Q289.72 61.12 289.72 66.516V68.036Q289.72 70.696 291.62 72.026Q293.52 73.356 296.864 73.356Z"/><path d="M325.744 78.524V39.308H332.812V44.4H333.192Q334.104 41.892 335.814 40.144Q337.524 38.396 340.716 38.396Q343.832 38.396 345.542 40.106Q347.252 41.816 347.708 44.78H348.012Q349.076 42.12 350.976 40.258Q352.876 38.396 356.448 38.396Q361.008 38.396 362.794 41.588Q364.58 44.78 364.58 50.936V78.524H357.512V52Q357.512 47.896 356.524 46.338Q355.536 44.78 353.332 44.78Q351.356 44.78 350.026 46.11Q348.696 47.44 348.696 50.1V78.524H341.628V52Q341.628 47.896 340.678 46.338Q339.728 44.78 337.524 44.78Q335.472 44.78 334.142 46.11Q332.812 47.44 332.812 50.1V78.524Z"/><path d="M391.636 79.436Q387.152 79.436 383.58 77.992Q380.008 76.548 377.576 73.888Q375.144 71.228 373.852 67.466Q372.56 63.704 372.56 58.992Q372.56 54.28 373.89 50.48Q375.22 46.68 377.614 43.982Q380.008 41.284 383.428 39.84Q386.848 38.396 391.028 38.396Q395.132 38.396 398.476 39.802Q401.82 41.208 404.176 43.792Q406.532 46.376 407.786 49.948Q409.04 53.52 409.04 57.852V60.968H380.92V62.26Q380.92 66.896 383.808 69.822Q386.696 72.748 391.788 72.748Q395.588 72.748 398.286 71.114Q400.984 69.48 402.656 66.744L407.824 71.304Q405.848 74.724 401.782 77.08Q397.716 79.436 391.636 79.436ZM391.028 44.704Q388.824 44.704 386.962 45.464Q385.1 46.224 383.77 47.63Q382.44 49.036 381.68 50.936Q380.92 52.836 380.92 55.116V55.648H400.604V54.888Q400.604 50.252 397.982 47.478Q395.36 44.704 391.028 44.704Z"/><path d="M415.12 39.308H422.188L425.988 72.292H426.672L432.524 39.308H440.58L446.356 72.292H447.04L450.916 39.308H457.68L452.208 78.524H442.024L436.78 48.124H436.096L430.776 78.524H420.592Z"/><path d="M482 79.436Q477.668 79.436 474.21 77.992Q470.752 76.548 468.358 73.888Q465.964 71.228 464.672 67.428Q463.38 63.628 463.38 58.916Q463.38 54.204 464.672 50.404Q465.964 46.604 468.358 43.944Q470.752 41.284 474.21 39.84Q477.668 38.396 482 38.396Q486.332 38.396 489.79 39.84Q493.248 41.284 495.642 43.944Q498.036 46.604 499.328 50.404Q500.62 54.204 500.62 58.916Q500.62 63.628 499.328 67.428Q498.036 71.228 495.642 73.888Q493.248 76.548 489.79 77.992Q486.332 79.436 482 79.436ZM482 72.9Q486.56 72.9 489.258 70.126Q491.956 67.352 491.956 61.88V55.952Q491.956 50.48 489.258 47.706Q486.56 44.932 482 44.932Q477.44 44.932 474.742 47.706Q472.044 50.48 472.044 55.952V61.88Q472.044 67.352 474.742 70.126Q477.44 72.9 482 72.9Z"/><path d="M510.576 71.836H520.988V45.996H510.576V39.308H529.272V49.72H529.728Q530.26 47.668 531.248 45.768Q532.236 43.868 533.756 42.462Q535.276 41.056 537.48 40.182Q539.684 39.308 542.648 39.308H547.36V47.212H539.608Q535.048 47.212 532.16 49.91Q529.272 52.608 529.272 57.168V71.836H543.484V78.524H510.576Z"/><path d="M557.088 22.284H565.372V56.256H565.752L571.908 49.948L582.928 39.308H592.732L576.848 54.736L594.48 78.524H584.524L570.844 59.448L565.372 64.54V78.524H557.088Z"/></g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 4.8 KiB |
@@ -0,0 +1,11 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="128" height="104" viewBox="0 0 128 104" role="img" aria-labelledby="title desc">
|
||||
<title id="title">nFramework icon</title>
|
||||
<desc id="desc">The geometric nFramework mark, with a diagonal slash suggesting an N.</desc>
|
||||
<metadata>Copyright (c) 2026 Nitload. Licensed under CC BY-NC-ND 4.0: https://creativecommons.org/licenses/by-nc-nd/4.0/</metadata>
|
||||
<g fill="#59B2DA" transform="translate(13.6 13.6) scale(0.8)">
|
||||
<polygon points="8,8 31,8 31,20 20,20 20,76 31,76 31,88 8,88" />
|
||||
<polygon points="97,8 120,8 120,88 97,88 97,76 108,76 108,20 97,20" />
|
||||
<polygon points="40,16 53,16 53,45 40,45" />
|
||||
<circle cx="80" cy="76" r="8" />
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 709 B |
@@ -9,16 +9,23 @@ using BenchmarkDotNet.Configs;
|
||||
using BenchmarkDotNet.Jobs;
|
||||
using BenchmarkDotNet.Loggers;
|
||||
using BenchmarkDotNet.Running;
|
||||
using BenchmarkDotNet.Toolchains.InProcess.Emit;
|
||||
using Nitload.Benchmark;
|
||||
|
||||
if (args.Length > 0 && args[0] == "--disasm-swiss")
|
||||
{
|
||||
IConfig disassemblyConfig = ManualConfig.Create(DefaultConfig.Instance)
|
||||
.AddLogger(ConsoleLogger.Default)
|
||||
.AddJob(Job.ShortRun.WithWarmupCount(6));
|
||||
BenchmarkRunner.Run<SwissX86DisassemblyBench>(disassemblyConfig);
|
||||
return;
|
||||
}
|
||||
|
||||
IConfig config = ManualConfig.Create(DefaultConfig.Instance)
|
||||
.AddLogger(ConsoleLogger.Default)
|
||||
.AddJob(Job.Default
|
||||
.WithToolchain(InProcessEmitToolchain.Instance)
|
||||
.WithLaunchCount(1)
|
||||
.WithWarmupCount(6)
|
||||
.WithIterationCount(15)
|
||||
.WithInvocationCount(3)
|
||||
.WithUnrollFactor(1));
|
||||
_ = BenchmarkRunner.Run<TlsfAllocatorBench>(config);
|
||||
BenchmarkSwitcher.FromAssembly(typeof(TlsfAllocatorBench).Assembly).Run(args, config);
|
||||
|
||||
@@ -1,485 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SwissBenchmark"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using BenchmarkDotNet.Attributes;
|
||||
using Nitload.Common.Collections;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace Nitload.Benchmark
|
||||
{
|
||||
/// <summary>
|
||||
/// Benchmark comparing SwissTable against <see cref="Dictionary{TKey, TValue}" /> under
|
||||
/// realistic L4 software cache address-mapping workloads. <para>/// L4 scenario
|
||||
/// characteristics: /// - Keys are HLSF/malloc pointer addresses (large, 1024-byte aligned)
|
||||
/// /// - Access pattern is Zipfian-skewed (20% hot cache lines receive 80% of lookups) ///
|
||||
/// - Steady state: high hit-rate lookups dominate, with rare evict+refill on miss /// -
|
||||
/// Working set: 512–4096 active cache line mappings /// </para>
|
||||
/// </summary>
|
||||
[MemoryDiagnoser]
|
||||
public class SwissTableBench
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// L4 cache line size in bytes (per L4_design.md).
|
||||
/// </summary>
|
||||
private const int CacheLineSize = 1024;
|
||||
|
||||
/// <summary>
|
||||
/// Simulated heap base address for pointer-like keys.
|
||||
/// </summary>
|
||||
private const long HeapBase = 0x7FFF_0000_0000L;
|
||||
|
||||
/// <summary>
|
||||
/// Number of lookup operations per Lookup benchmark call.
|
||||
/// </summary>
|
||||
private const int LookupIterations = 8192;
|
||||
|
||||
/// <summary>
|
||||
/// Fraction of steady-state ops that are cache misses.
|
||||
/// </summary>
|
||||
private const double MissRate = 0.05;
|
||||
|
||||
/// <summary>
|
||||
/// Number of steady-state operations per call. Each op is either a lookup (hit) or an
|
||||
/// evict+insert+lookup (miss).
|
||||
/// </summary>
|
||||
private const int SteadyStateOps = 4096;
|
||||
|
||||
/// <summary>
|
||||
/// Indices into _residentKeys for eviction targets (cold entries, tail region).
|
||||
/// </summary>
|
||||
private int[] _evictOrder = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Additional keys not in the table, for triggering misses.
|
||||
/// </summary>
|
||||
private nuint[] _externalKeys = null!;
|
||||
|
||||
// ── Pre-generated key arrays (pointer-like, 1024-byte aligned) ──
|
||||
|
||||
/// <summary>
|
||||
/// Keys for initial table fill (simulates cached addresses).
|
||||
/// </summary>
|
||||
private nuint[] _residentKeys = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Pre-built steady-state operation script. Each entry: positive index → lookup
|
||||
/// _residentKeys[idx], negative value → cache miss trigger (evict + insert from
|
||||
/// _externalKeys).
|
||||
/// </summary>
|
||||
private int[] _steadyStateScript = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Uniform random lookup sequence (baseline comparison).
|
||||
/// </summary>
|
||||
private nuint[] _uniformLookups = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Zipfian-skewed lookup sequence (hot 20% accessed 80% of time).
|
||||
/// </summary>
|
||||
private nuint[] _zipfianLookups = null!;
|
||||
private Dictionary<nuint, nuint> _dictLookup = null!;
|
||||
private Dictionary<nuint, nuint> _freshDict = null!;
|
||||
|
||||
// ── Fresh instances for insert/mixed (rebuilt each iteration) ──
|
||||
|
||||
private SwissTable _freshSwiss = null!;
|
||||
|
||||
// ── Pre-filled lookup targets ──
|
||||
|
||||
private SwissTable _swissLookup = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Active cache line count. Matches L4 capacity range: 5 MB / 1024B ≈ 5120, 10 MB /
|
||||
/// 1024B ≈ 10240. 512 and 1024 test low-occupancy; 4096 tests near-capacity.
|
||||
/// </summary>
|
||||
[Params(512, 1024, 4096)]
|
||||
public int KeyCount;
|
||||
|
||||
[GlobalCleanup]
|
||||
public void Cleanup()
|
||||
{
|
||||
_dictLookup.Clear();
|
||||
_dictLookup.TrimExcess();
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════
|
||||
// Insert — bulk fill with scattered pointer addresses
|
||||
// Simulates L4 initial cache warm-up phase.
|
||||
// ════════════════════════════════════════════════════════════
|
||||
|
||||
[Benchmark]
|
||||
public void Dictionary_Insert()
|
||||
{
|
||||
Dictionary<nuint, nuint> dict = _freshDict;
|
||||
nuint[] keys = _residentKeys;
|
||||
for (int i = 0; i < keys.Length; i++) {
|
||||
dict[keys[i]] = keys[i] ^ unchecked((nuint)0xDEAD_BEEF);
|
||||
}
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════
|
||||
// Lookup — Uniform (baseline comparison)
|
||||
// ════════════════════════════════════════════════════════════
|
||||
|
||||
[Benchmark]
|
||||
public nuint Dictionary_Lookup_Uniform()
|
||||
{
|
||||
nuint sum = 0;
|
||||
nuint[] keys = _uniformLookups;
|
||||
Dictionary<nuint, nuint> dict = _dictLookup;
|
||||
for (int i = 0; i < keys.Length; i++)
|
||||
{
|
||||
if (dict.TryGetValue(keys[i], out nuint value)) {
|
||||
sum += value;
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════
|
||||
// Lookup — Zipfian (L4 hot path, skewed access)
|
||||
// 20% of cache lines receive 80% of accesses.
|
||||
// This is the most representative L4 benchmark.
|
||||
// ════════════════════════════════════════════════════════════
|
||||
|
||||
[Benchmark(Baseline = true)]
|
||||
public nuint Dictionary_Lookup_Zipfian()
|
||||
{
|
||||
nuint sum = 0;
|
||||
nuint[] keys = _zipfianLookups;
|
||||
Dictionary<nuint, nuint> dict = _dictLookup;
|
||||
for (int i = 0; i < keys.Length; i++)
|
||||
{
|
||||
if (dict.TryGetValue(keys[i], out nuint value)) {
|
||||
sum += value;
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════
|
||||
// SteadyState — the KEY L4 benchmark
|
||||
// Pre-filled table at capacity, then 4096 operations:
|
||||
// ~95% are lookups (cache hits)
|
||||
// ~5% are miss → evict cold entry → insert new entry → lookup new
|
||||
// This directly simulates L4's main runtime loop.
|
||||
// ════════════════════════════════════════════════════════════
|
||||
|
||||
[Benchmark]
|
||||
public nuint Dictionary_SteadyState()
|
||||
{
|
||||
nuint sum = 0;
|
||||
Dictionary<nuint, nuint> dict = _freshDict;
|
||||
nuint[] resident = _residentKeys;
|
||||
nuint[] external = _externalKeys;
|
||||
int[] script = _steadyStateScript;
|
||||
int[] evict = _evictOrder;
|
||||
int evictCursor = 0;
|
||||
int externalCursor = 0;
|
||||
|
||||
for (int i = 0; i < script.Length; i++)
|
||||
{
|
||||
int op = script[i];
|
||||
if (op >= 0)
|
||||
{
|
||||
// Cache hit: lookup existing entry
|
||||
if (dict.TryGetValue(resident[op], out nuint value))
|
||||
{
|
||||
sum += value;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// Cache miss: evict a cold entry, insert a new one, then lookup
|
||||
int evictIdx = evict[evictCursor % evict.Length];
|
||||
evictCursor++;
|
||||
dict.Remove(resident[evictIdx]);
|
||||
|
||||
int extIdx = externalCursor % external.Length;
|
||||
externalCursor++;
|
||||
nuint newKey = external[extIdx];
|
||||
nuint newVal = newKey ^ unchecked((nuint)0xDEAD_BEEF);
|
||||
dict[newKey] = newVal;
|
||||
|
||||
if (dict.TryGetValue(newKey, out nuint v)) {
|
||||
sum += v;
|
||||
}
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
[IterationSetup(Targets = [
|
||||
nameof(SwissTable_Insert),
|
||||
nameof(SwissTable_SteadyState),
|
||||
nameof(Dictionary_Insert),
|
||||
nameof(Dictionary_SteadyState)
|
||||
])]
|
||||
public void IterationSetup()
|
||||
{
|
||||
_freshSwiss = SwissTable.Create();
|
||||
_freshDict = new Dictionary<nuint, nuint>(KeyCount);
|
||||
|
||||
// Pre-fill for steady-state benchmarks
|
||||
for (int i = 0; i < KeyCount; i++)
|
||||
{
|
||||
nuint key = _residentKeys[i];
|
||||
nuint val = key ^ unchecked((nuint)0xDEAD_BEEF);
|
||||
_freshSwiss.TryAddOrSet(key, val, out _);
|
||||
_freshDict[key] = val;
|
||||
}
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════
|
||||
// Setup / Teardown
|
||||
// ════════════════════════════════════════════════════════════
|
||||
|
||||
[GlobalSetup]
|
||||
public void Setup()
|
||||
{
|
||||
Random rng = new Random(42);
|
||||
|
||||
// ── Generate pointer-like keys (HeapBase + random_offset * 1024) ──
|
||||
_residentKeys = GeneratePointerKeys(KeyCount, rng);
|
||||
_externalKeys = GeneratePointerKeys(KeyCount, rng); // disjoint set for miss simulation
|
||||
|
||||
// ── Build lookup sequences ──
|
||||
_zipfianLookups = GenerateZipfianLookups(_residentKeys, LookupIterations, rng);
|
||||
_uniformLookups = GenerateUniformLookups(_residentKeys, LookupIterations, rng);
|
||||
|
||||
// ── Build steady-state operation script ──
|
||||
_steadyStateScript = GenerateSteadyStateScript(KeyCount, SteadyStateOps, MissRate, rng);
|
||||
|
||||
// ── Build eviction order (cold tail entries first) ──
|
||||
_evictOrder = new int[KeyCount];
|
||||
for (int i = 0; i < KeyCount; i++) {
|
||||
_evictOrder[i] = i;
|
||||
}
|
||||
|
||||
// Shuffle so eviction isn't just sequential
|
||||
Shuffle(_evictOrder, rng);
|
||||
|
||||
// ── Pre-fill lookup tables ──
|
||||
_swissLookup = SwissTable.Create();
|
||||
_dictLookup = new Dictionary<nuint, nuint>(KeyCount);
|
||||
|
||||
for (int i = 0; i < KeyCount; i++)
|
||||
{
|
||||
nuint key = _residentKeys[i];
|
||||
nuint val = key ^ unchecked((nuint)0xDEAD_BEEF); // simulated cache pointer
|
||||
_swissLookup.TryAddOrSet(key, val, out _);
|
||||
_dictLookup[key] = val;
|
||||
}
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public void SwissTable_Insert()
|
||||
{
|
||||
SwissTable swiss = _freshSwiss;
|
||||
nuint[] keys = _residentKeys;
|
||||
for (int i = 0; i < keys.Length; i++) {
|
||||
swiss.TryAddOrSet(keys[i], keys[i] ^ unchecked((nuint)0xDEAD_BEEF), out _);
|
||||
}
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public nuint SwissTable_Lookup_Uniform()
|
||||
{
|
||||
nuint sum = 0;
|
||||
nuint[] keys = _uniformLookups;
|
||||
SwissTable swiss = _swissLookup;
|
||||
for (int i = 0; i < keys.Length; i++)
|
||||
{
|
||||
if (swiss.TryGet(keys[i], out nuint value)) {
|
||||
sum += value;
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public nuint SwissTable_Lookup_Zipfian()
|
||||
{
|
||||
nuint sum = 0;
|
||||
nuint[] keys = _zipfianLookups;
|
||||
SwissTable swiss = _swissLookup;
|
||||
for (int i = 0; i < keys.Length; i++)
|
||||
{
|
||||
if (swiss.TryGet(keys[i], out nuint value)) {
|
||||
sum += value;
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public nuint SwissTable_SteadyState()
|
||||
{
|
||||
nuint sum = 0;
|
||||
SwissTable swiss = _freshSwiss;
|
||||
nuint[] resident = _residentKeys;
|
||||
nuint[] external = _externalKeys;
|
||||
int[] script = _steadyStateScript;
|
||||
int[] evict = _evictOrder;
|
||||
int evictCursor = 0;
|
||||
int externalCursor = 0;
|
||||
|
||||
for (int i = 0; i < script.Length; i++)
|
||||
{
|
||||
int op = script[i];
|
||||
if (op >= 0)
|
||||
{
|
||||
// Cache hit: lookup existing entry
|
||||
if (swiss.TryGet(resident[op], out nuint value))
|
||||
{
|
||||
sum += value;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// Cache miss: evict a cold entry, insert a new one, then lookup
|
||||
int evictIdx = evict[evictCursor % evict.Length];
|
||||
evictCursor++;
|
||||
swiss.TryRemove(resident[evictIdx], out _);
|
||||
|
||||
int extIdx = externalCursor % external.Length;
|
||||
externalCursor++;
|
||||
nuint newKey = external[extIdx];
|
||||
nuint newVal = newKey ^ unchecked((nuint)0xDEAD_BEEF);
|
||||
swiss.TryAddOrSet(newKey, newVal, out _);
|
||||
|
||||
if (swiss.TryGet(newKey, out nuint v)) {
|
||||
sum += v;
|
||||
}
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════
|
||||
// Data generation helpers
|
||||
// ════════════════════════════════════════════════════════════
|
||||
|
||||
/// <summary>
|
||||
/// Generate <paramref name="count" /> unique pointer-like keys. Each key = HeapBase +
|
||||
/// randomOffset * CacheLineSize, simulating HLSF-allocated 1024-byte aligned addresses
|
||||
/// scattered across a ~1 GB virtual address range.
|
||||
/// </summary>
|
||||
private static nuint[] GeneratePointerKeys(
|
||||
int count,
|
||||
Random rng
|
||||
)
|
||||
{
|
||||
nuint[] keys = new nuint[count];
|
||||
HashSet<long> used = new HashSet<long>(count);
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
long offset;
|
||||
do
|
||||
{
|
||||
// Random offset within ~1M cache lines (1 GB range)
|
||||
offset = (long)rng.Next(0, 1 << 20) * CacheLineSize;
|
||||
} while (!used.Add(offset));
|
||||
keys[i] = (nuint)(HeapBase + offset);
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generate a steady-state operation script. Each entry: non-negative value = index into
|
||||
/// _residentKeys (lookup/hit), negative value (-1) = cache miss trigger (evict → insert →
|
||||
/// lookup). Lookup indices are Zipfian-skewed.
|
||||
/// </summary>
|
||||
private static int[] GenerateSteadyStateScript(
|
||||
int keyCount,
|
||||
int opCount,
|
||||
double missRate,
|
||||
Random rng
|
||||
)
|
||||
{
|
||||
int hotSetSize = Math.Max(1, keyCount / 5);
|
||||
int[] script = new int[opCount];
|
||||
for (int i = 0; i < opCount; i++)
|
||||
{
|
||||
if (rng.NextDouble() < missRate)
|
||||
{
|
||||
script[i] = -1; // miss
|
||||
} else
|
||||
{
|
||||
// Zipfian hit
|
||||
if (rng.NextDouble() < 0.8)
|
||||
{
|
||||
script[i] = rng.Next(0, hotSetSize);
|
||||
} else
|
||||
{
|
||||
script[i] = rng.Next(hotSetSize, keyCount);
|
||||
}
|
||||
}
|
||||
}
|
||||
return script;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generate uniform random lookup sequence (baseline for comparison with Zipfian).
|
||||
/// </summary>
|
||||
private static nuint[] GenerateUniformLookups(
|
||||
nuint[] sourceKeys,
|
||||
int count,
|
||||
Random rng
|
||||
)
|
||||
{
|
||||
nuint[] lookups = new nuint[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
lookups[i] = sourceKeys[rng.Next(0, sourceKeys.Length)];
|
||||
}
|
||||
return lookups;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generate Zipfian-skewed lookup sequence: 80% of accesses hit the hottest 20% of keys
|
||||
/// (sorted by index as "hotness proxy"), 20% of accesses hit the remaining 80%. This
|
||||
/// models matrix computation where a small panel working set is reused heavily.
|
||||
/// </summary>
|
||||
private static nuint[] GenerateZipfianLookups(
|
||||
nuint[] sourceKeys,
|
||||
int count,
|
||||
Random rng
|
||||
)
|
||||
{
|
||||
int hotSetSize = Math.Max(1, sourceKeys.Length / 5);
|
||||
nuint[] lookups = new nuint[count];
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
if (rng.NextDouble() < 0.8)
|
||||
{
|
||||
lookups[i] = sourceKeys[rng.Next(0, hotSetSize)];
|
||||
} else
|
||||
{
|
||||
lookups[i] = sourceKeys[rng.Next(hotSetSize, sourceKeys.Length)];
|
||||
}
|
||||
}
|
||||
return lookups;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fisher-Yates shuffle.
|
||||
/// </summary>
|
||||
private static void Shuffle(
|
||||
int[] array,
|
||||
Random rng
|
||||
)
|
||||
{
|
||||
for (int i = array.Length - 1; i > 0; i--)
|
||||
{
|
||||
int j = rng.Next(0, i + 1);
|
||||
(array[i], array[j]) = (array[j], array[i]);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SwissX86Bench"
|
||||
// Project: nFramework
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using BenchmarkDotNet.Attributes;
|
||||
using BenchmarkDotNet.Configs;
|
||||
using Nitload.Common.HighPerformance.Memory;
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.Benchmark
|
||||
{
|
||||
[MemoryDiagnoser]
|
||||
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
|
||||
[CategoriesColumn]
|
||||
public unsafe class SwissX86Bench
|
||||
{
|
||||
|
||||
private const int QueryCount = 1 << 21;
|
||||
private const int ConcurrentQueryCount = QueryCount * 6;
|
||||
private const int MaximumResidentCount = 8192;
|
||||
|
||||
private byte* _buffer;
|
||||
private L4.SwissTable _swiss;
|
||||
private Dictionary<L4.AddressSource, ulong> _dictionary = null!;
|
||||
private ConcurrentDictionary<L4.AddressSource, ulong> _concurrentDictionary = null!;
|
||||
private L4.AddressSource[] _hitQueries = [];
|
||||
private L4.AddressSource[] _missQueries = [];
|
||||
private L4.AddressSource[] _writerQueries = [];
|
||||
private ulong[] _writerValues = [];
|
||||
private byte* _readBuffer;
|
||||
private byte* _writeBuffer;
|
||||
private ParallelOptions _parallelOptions = null!;
|
||||
private Func<ulong> _parallelLocalInit = null!;
|
||||
private Func<int, ParallelLoopState, ulong, ulong> _swissParallelBody = null!;
|
||||
private Func<int, ParallelLoopState, ulong, ulong> _concurrentDictionaryParallelBody = null!;
|
||||
private Action<ulong> _parallelLocalFinally = null!;
|
||||
private long _parallelChecksum;
|
||||
|
||||
[Params(25, 50, 100)]
|
||||
public int LoadPercent { get; set; }
|
||||
|
||||
private int ResidentCount => MaximumResidentCount * LoadPercent / 100;
|
||||
|
||||
[GlobalSetup]
|
||||
public void Setup()
|
||||
{
|
||||
if (!Sse42.X64.IsSupported)
|
||||
throw new PlatformNotSupportedException("The Swiss X86 benchmark requires x64 SSE4.2.");
|
||||
|
||||
_buffer = (byte*)NativeMemory.AlignedAlloc((nuint)L4.BufferSize, 64);
|
||||
if (_buffer == null)
|
||||
throw new OutOfMemoryException("Could not allocate Swiss benchmark tables.");
|
||||
|
||||
_swiss = L4.SwissTable.Create(_buffer);
|
||||
_dictionary = new Dictionary<L4.AddressSource, ulong>(ResidentCount);
|
||||
_concurrentDictionary = new ConcurrentDictionary<L4.AddressSource, ulong>(
|
||||
Math.Max(1, Math.Min(Environment.ProcessorCount, 8)),
|
||||
ResidentCount);
|
||||
|
||||
L4.AddressSource[] residentKeys = new L4.AddressSource[ResidentCount];
|
||||
for (int i = 0; i < ResidentCount; i++)
|
||||
{
|
||||
L4.AddressSource key = MakeKey(i);
|
||||
ulong value = (ulong)i + 1;
|
||||
residentKeys[i] = key;
|
||||
if (!_swiss.TryAddOrSet(key, value))
|
||||
throw new InvalidOperationException($"Swiss insertion failed while seeding key {i}.");
|
||||
_dictionary.Add(key, value);
|
||||
_concurrentDictionary.TryAdd(key, value);
|
||||
}
|
||||
|
||||
_swiss.SwapBuffers();
|
||||
for (int i = 0; i < ResidentCount; i++)
|
||||
{
|
||||
if (!_swiss.TryAddOrSet(residentKeys[i], (ulong)i + 1))
|
||||
throw new InvalidOperationException($"Swiss writer-buffer seeding failed for key {i}.");
|
||||
}
|
||||
|
||||
_readBuffer = _swiss.ReadBufferForLockFreeBenchmark;
|
||||
_writeBuffer = _swiss.WriteBufferForLockFreeBenchmark;
|
||||
|
||||
_hitQueries = new L4.AddressSource[QueryCount];
|
||||
_missQueries = new L4.AddressSource[QueryCount];
|
||||
_writerQueries = new L4.AddressSource[QueryCount];
|
||||
_writerValues = new ulong[QueryCount];
|
||||
Random random = new(0x51_15_2026);
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
{
|
||||
_hitQueries[i] = residentKeys[random.Next(ResidentCount)];
|
||||
_missQueries[i] = MakeKey(MaximumResidentCount + i);
|
||||
_writerQueries[i] = residentKeys[random.Next(ResidentCount)];
|
||||
_writerValues[i] = (ulong)i + 0x10000;
|
||||
}
|
||||
|
||||
_parallelOptions = new ParallelOptions
|
||||
{
|
||||
MaxDegreeOfParallelism = Math.Max(2, Math.Min(Environment.ProcessorCount, 8))
|
||||
};
|
||||
_parallelLocalInit = static() => 0UL;
|
||||
_swissParallelBody = ReadSwissParallel;
|
||||
_concurrentDictionaryParallelBody = ReadConcurrentDictionaryParallel;
|
||||
_parallelLocalFinally = MergeParallelChecksum;
|
||||
|
||||
ValidateLookupResults();
|
||||
}
|
||||
|
||||
[GlobalCleanup]
|
||||
public void Cleanup()
|
||||
{
|
||||
_swiss.Dispose();
|
||||
NativeMemory.AlignedFree(_buffer);
|
||||
_buffer = null;
|
||||
}
|
||||
|
||||
[Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Hit lookup")]
|
||||
public ulong DictionaryHitLookup()
|
||||
{
|
||||
ulong checksum = 0;
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
checksum += _dictionary[_hitQueries[i]];
|
||||
return checksum;
|
||||
}
|
||||
|
||||
[Benchmark(OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Hit lookup")]
|
||||
public ulong SwissLockFreeHitLookup()
|
||||
{
|
||||
ulong checksum = 0;
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
{
|
||||
if (!L4.SwissReaderX86.TryGet(_readBuffer, _hitQueries[i], out ulong value))
|
||||
throw new InvalidOperationException("Swiss table missed a seeded hit key.");
|
||||
checksum += value;
|
||||
}
|
||||
return checksum;
|
||||
}
|
||||
|
||||
[Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Miss lookup")]
|
||||
public int DictionaryMissLookup()
|
||||
{
|
||||
int misses = 0;
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
misses += _dictionary.TryGetValue(_missQueries[i], out _) ? 0 : 1;
|
||||
return misses;
|
||||
}
|
||||
|
||||
[Benchmark(OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Miss lookup")]
|
||||
public int SwissLockFreeMissLookup()
|
||||
{
|
||||
int misses = 0;
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
misses += L4.SwissReaderX86.TryGet(_readBuffer, _missQueries[i], out _) ? 0 : 1;
|
||||
return misses;
|
||||
}
|
||||
|
||||
[Benchmark(Baseline = true, OperationsPerInvoke = ConcurrentQueryCount)]
|
||||
[BenchmarkCategory("Concurrent hit lookup")]
|
||||
public ulong ConcurrentDictionaryConcurrentHitLookup()
|
||||
{
|
||||
Interlocked.Exchange(ref _parallelChecksum, 0);
|
||||
Parallel.For(
|
||||
0,
|
||||
ConcurrentQueryCount,
|
||||
_parallelOptions,
|
||||
_parallelLocalInit,
|
||||
_concurrentDictionaryParallelBody,
|
||||
_parallelLocalFinally);
|
||||
return unchecked((ulong)Interlocked.Read(ref _parallelChecksum));
|
||||
}
|
||||
|
||||
[Benchmark(OperationsPerInvoke = ConcurrentQueryCount)]
|
||||
[BenchmarkCategory("Concurrent hit lookup")]
|
||||
public ulong SwissLockFreeConcurrentHitLookup()
|
||||
{
|
||||
Interlocked.Exchange(ref _parallelChecksum, 0);
|
||||
Parallel.For(
|
||||
0,
|
||||
ConcurrentQueryCount,
|
||||
_parallelOptions,
|
||||
_parallelLocalInit,
|
||||
_swissParallelBody,
|
||||
_parallelLocalFinally);
|
||||
return unchecked((ulong)Interlocked.Read(ref _parallelChecksum));
|
||||
}
|
||||
|
||||
[Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Upsert existing")]
|
||||
public ulong DictionaryUpsertExisting()
|
||||
{
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
_dictionary[_writerQueries[i]] = _writerValues[i];
|
||||
return _dictionary[_writerQueries[^1]];
|
||||
}
|
||||
|
||||
[Benchmark(OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Upsert existing")]
|
||||
public ulong SwissUpsertExisting()
|
||||
{
|
||||
ulong failures = 0;
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
failures += L4.SwissWriterX86.TryAddOrSet(_writeBuffer, _writerQueries[i], _writerValues[i]) ? 0UL : 1UL;
|
||||
if (failures != 0 || !L4.SwissReaderX86.TryGet(_writeBuffer, _writerQueries[^1], out ulong value))
|
||||
throw new InvalidOperationException("Swiss upsert failed to preserve the final value.");
|
||||
return value;
|
||||
}
|
||||
|
||||
[Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Remove and reinsert")]
|
||||
public ulong DictionaryRemoveAndReinsert()
|
||||
{
|
||||
ulong checksum = 0;
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
{
|
||||
L4.AddressSource key = _writerQueries[i];
|
||||
ulong value = _dictionary[key];
|
||||
_dictionary.Remove(key);
|
||||
_dictionary.Add(key, value + 1);
|
||||
checksum += value;
|
||||
}
|
||||
return checksum;
|
||||
}
|
||||
|
||||
[Benchmark(OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Remove and reinsert")]
|
||||
public ulong SwissRemoveAndReinsert()
|
||||
{
|
||||
ulong checksum = 0;
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
{
|
||||
L4.AddressSource key = _writerQueries[i];
|
||||
if (!L4.SwissWriterX86.TryRemove(_writeBuffer, key, out ulong value) ||
|
||||
!L4.SwissWriterX86.TryAddOrSet(_writeBuffer, key, value + 1)) {
|
||||
throw new InvalidOperationException("Swiss remove/reinsert failed.");
|
||||
}
|
||||
checksum += value;
|
||||
}
|
||||
return checksum;
|
||||
}
|
||||
|
||||
private static L4.AddressSource MakeKey(int index)
|
||||
{
|
||||
return new L4.AddressSource
|
||||
{
|
||||
Address = 0x1000_0000UL + ((ulong)index * 64UL),
|
||||
Description = ((ulong)index * 0x9e37_79b9UL) ^ 0xd1b5_4a32_d192_ed03UL
|
||||
};
|
||||
}
|
||||
|
||||
private ulong ReadSwissParallel(int index, ParallelLoopState state, ulong checksum)
|
||||
{
|
||||
int queryIndex = index & (QueryCount - 1);
|
||||
if (!L4.SwissReaderX86.TryGet(_readBuffer, _hitQueries[queryIndex], out ulong value))
|
||||
throw new InvalidOperationException("Swiss table missed a seeded concurrent hit key.");
|
||||
return checksum + value;
|
||||
}
|
||||
|
||||
private ulong ReadConcurrentDictionaryParallel(int index, ParallelLoopState state, ulong checksum)
|
||||
{
|
||||
int queryIndex = index & (QueryCount - 1);
|
||||
if (!_concurrentDictionary.TryGetValue(_hitQueries[queryIndex], out ulong value))
|
||||
throw new InvalidOperationException("ConcurrentDictionary missed a seeded hit key.");
|
||||
return checksum + value;
|
||||
}
|
||||
|
||||
private void MergeParallelChecksum(ulong checksum)
|
||||
{
|
||||
_ = Interlocked.Add(ref _parallelChecksum, unchecked((long)checksum));
|
||||
}
|
||||
|
||||
private void ValidateLookupResults()
|
||||
{
|
||||
for (int i = 0; i < _hitQueries.Length; i++)
|
||||
{
|
||||
bool dictionaryFound = _dictionary.TryGetValue(_hitQueries[i], out ulong dictionaryValue);
|
||||
bool swissFound = L4.SwissReaderX86.TryGet(_readBuffer, _hitQueries[i], out ulong swissValue);
|
||||
if (!dictionaryFound || !swissFound || dictionaryValue != swissValue)
|
||||
throw new InvalidOperationException("Swiss and Dictionary hit results differ.");
|
||||
|
||||
if (!_concurrentDictionary.TryGetValue(_hitQueries[i], out ulong concurrentValue) ||
|
||||
concurrentValue != dictionaryValue)
|
||||
throw new InvalidOperationException("ConcurrentDictionary and Dictionary hit results differ.");
|
||||
}
|
||||
|
||||
for (int i = 0; i < _missQueries.Length; i++)
|
||||
{
|
||||
bool dictionaryFound = _dictionary.TryGetValue(_missQueries[i], out _);
|
||||
bool swissFound = L4.SwissReaderX86.TryGet(_readBuffer, _missQueries[i], out _);
|
||||
if (dictionaryFound || swissFound)
|
||||
throw new InvalidOperationException("Swiss and Dictionary miss results differ.");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SwissX86DisassemblyBench"
|
||||
// Project: nFramework
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using BenchmarkDotNet.Attributes;
|
||||
using BenchmarkDotNet.Diagnosers;
|
||||
using Nitload.Common.HighPerformance.Memory;
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
|
||||
namespace Nitload.Benchmark
|
||||
{
|
||||
[DisassemblyDiagnoser(maxDepth: 3, exportHtml: true)]
|
||||
public unsafe class SwissX86DisassemblyBench
|
||||
{
|
||||
|
||||
private const int ResidentCount = 8192;
|
||||
|
||||
private byte* _buffer;
|
||||
private L4.SwissTable _table;
|
||||
private byte* _readBuffer;
|
||||
private byte* _writeBuffer;
|
||||
private L4.AddressSource _hitKey;
|
||||
private L4.AddressSource _missKey;
|
||||
private ulong _value;
|
||||
|
||||
[GlobalSetup]
|
||||
public void Setup()
|
||||
{
|
||||
if (!Sse42.X64.IsSupported)
|
||||
throw new PlatformNotSupportedException("The Swiss X86 disassembly probe requires x64 SSE4.2.");
|
||||
|
||||
_buffer = (byte*)NativeMemory.AlignedAlloc((nuint)L4.BufferSize, 64);
|
||||
if (_buffer == null)
|
||||
throw new OutOfMemoryException("Could not allocate Swiss disassembly buffers.");
|
||||
|
||||
_table = L4.SwissTable.Create(_buffer);
|
||||
for (int index = 0; index < ResidentCount; index++)
|
||||
{
|
||||
if (!L4.SwissWriterX86.TryAddOrSet(_table.WriteBufferForLockFreeBenchmark,
|
||||
MakeKey(index), (ulong)index + 1))
|
||||
throw new InvalidOperationException($"Failed to seed Swiss key {index}.");
|
||||
}
|
||||
|
||||
_table.SwapBuffers();
|
||||
_readBuffer = _table.ReadBufferForLockFreeBenchmark;
|
||||
_writeBuffer = _table.WriteBufferForLockFreeBenchmark;
|
||||
for (int index = 0; index < ResidentCount; index++)
|
||||
{
|
||||
if (!L4.SwissWriterX86.TryAddOrSet(_writeBuffer, MakeKey(index), (ulong)index + 1))
|
||||
throw new InvalidOperationException($"Failed to seed inactive Swiss key {index}.");
|
||||
}
|
||||
|
||||
_hitKey = MakeKey(ResidentCount / 2);
|
||||
_missKey = MakeKey(ResidentCount + 1);
|
||||
_value = 0x1234_5678;
|
||||
|
||||
if (!L4.SwissReaderX86.TryGet(_readBuffer, _hitKey, out _) ||
|
||||
L4.SwissReaderX86.TryGet(_readBuffer, _missKey, out _))
|
||||
throw new InvalidOperationException("Swiss reader setup validation failed.");
|
||||
}
|
||||
|
||||
[GlobalCleanup]
|
||||
public void Cleanup()
|
||||
{
|
||||
_table.Dispose();
|
||||
NativeMemory.AlignedFree(_buffer);
|
||||
_buffer = null;
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool RawReaderHit()
|
||||
{
|
||||
return L4.SwissReaderX86.TryGet(_readBuffer, _hitKey, out _value);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool RawReaderMiss()
|
||||
{
|
||||
return L4.SwissReaderX86.TryGet(_readBuffer, _missKey, out _value);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool RawWriterUpsertExisting()
|
||||
{
|
||||
return L4.SwissWriterX86.TryAddOrSet(_writeBuffer, _hitKey, _value);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool RawWriterRemoveAndReinsert()
|
||||
{
|
||||
return L4.SwissWriterX86.TryRemove(_writeBuffer, _hitKey, out ulong previousValue) &&
|
||||
L4.SwissWriterX86.TryAddOrSet(_writeBuffer, _hitKey, previousValue);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool TableReaderHit()
|
||||
{
|
||||
return _table.TryGet(_hitKey, out _value);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool TableReaderMiss()
|
||||
{
|
||||
return _table.TryGet(_missKey, out _value);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool TableWriterUpsertExisting()
|
||||
{
|
||||
return _table.TryAddOrSet(_hitKey, _value);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool TableWriterRemoveAndReinsert()
|
||||
{
|
||||
return _table.TryRemove(_hitKey, out ulong previousValue) &&
|
||||
_table.TryAddOrSet(_hitKey, previousValue);
|
||||
}
|
||||
|
||||
private static L4.AddressSource MakeKey(int index)
|
||||
{
|
||||
return new L4.AddressSource
|
||||
{
|
||||
Address = 0x1000_0000UL + ((ulong)index * 64UL),
|
||||
Description = ((ulong)index * 0x9e37_79b9UL) ^ 0xd1b5_4a32_d192_ed03UL
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SwissTable.X86"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Numerics;
|
||||
using System.Runtime.Intrinsics;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
|
||||
namespace Nitload.Common.Collections
|
||||
{
|
||||
public partial class SwissTable
|
||||
{
|
||||
|
||||
private unsafe class BucketScannerX86 : IBucketScanner
|
||||
{
|
||||
|
||||
public bool TryFind(
|
||||
nuint key,
|
||||
uint hash,
|
||||
Bucket* bucket,
|
||||
out nuint value
|
||||
)
|
||||
{
|
||||
byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F);
|
||||
Vector128<byte> ctrlVector = Vector128.Create(ctrl);
|
||||
Vector128<byte> bucketCtrlVector = Sse2.LoadVector128((byte*)&bucket->Control);
|
||||
Vector128<byte> cmpResult = Sse2.CompareEqual(ctrlVector, bucketCtrlVector);
|
||||
int mask = Sse2.MoveMask(cmpResult);
|
||||
while (mask != 0)
|
||||
{
|
||||
int bitPos = BitOperations.TrailingZeroCount(mask);
|
||||
int index = bitPos;
|
||||
if (bucket->Keys[index] == key)
|
||||
{
|
||||
value = bucket->Values[index];
|
||||
return true;
|
||||
}
|
||||
mask &= mask - 1;
|
||||
}
|
||||
value = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
public bool TryRemove(
|
||||
nuint key,
|
||||
uint hash,
|
||||
Bucket* bucket,
|
||||
out nuint value
|
||||
)
|
||||
{
|
||||
byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F);
|
||||
Vector128<byte> ctrlVector = Vector128.Create(ctrl);
|
||||
|
||||
Vector128<byte> bucketCtrl = Sse2.LoadVector128((byte*)&bucket->Control);
|
||||
Vector128<byte> matchMask = Sse2.CompareEqual(ctrlVector, bucketCtrl);
|
||||
int matches = Sse2.MoveMask(matchMask);
|
||||
|
||||
while (matches != 0)
|
||||
{
|
||||
int bitPos = BitOperations.TrailingZeroCount(matches);
|
||||
int index = bitPos;
|
||||
if (bucket->Keys[index] == key)
|
||||
{
|
||||
value = bucket->Values[index];
|
||||
bucket->Control[index] = 0xFE; // mark as DELETED
|
||||
// no need to remove Key/Value
|
||||
return true;
|
||||
}
|
||||
matches &= matches - 1;
|
||||
}
|
||||
value = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
public bool TrySet(
|
||||
nuint key,
|
||||
uint hash,
|
||||
Bucket* bucket,
|
||||
nuint value,
|
||||
out bool isNewKey
|
||||
)
|
||||
{
|
||||
byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F); // H2: upper 8 bits (highest bit is naturally 0)
|
||||
Vector128<byte> ctrlVector = Vector128.Create(ctrl);
|
||||
|
||||
Vector128<byte> bucketCtrl = Sse2.LoadVector128(
|
||||
(byte*)&bucket->Control);
|
||||
|
||||
// === Phase 1: Look up an existing key ===
|
||||
Vector128<byte> matchMask = Sse2.CompareEqual(ctrlVector, bucketCtrl);
|
||||
int matches = Sse2.MoveMask(matchMask);
|
||||
|
||||
while (matches != 0)
|
||||
{
|
||||
int bitPos = BitOperations.TrailingZeroCount(matches);
|
||||
int index = bitPos;
|
||||
if (bucket->Keys[index] == key)
|
||||
{
|
||||
// Key already exists -> update value
|
||||
bucket->Values[index] = value;
|
||||
isNewKey = false;
|
||||
return true;
|
||||
}
|
||||
matches &= matches - 1;
|
||||
}
|
||||
|
||||
// === Phase 2: Track an available insertion slot ===
|
||||
int emptyOrDeleted = Sse2.MoveMask(bucketCtrl);
|
||||
if (emptyOrDeleted == 0)
|
||||
{
|
||||
isNewKey = false;
|
||||
return false; // bucket full
|
||||
}
|
||||
|
||||
int slot = BitOperations.TrailingZeroCount(emptyOrDeleted);
|
||||
bucket->Control[slot] = ctrl;
|
||||
bucket->Keys[slot] = key;
|
||||
bucket->Values[slot] = value;
|
||||
isNewKey = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,224 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SwissTable"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Numerics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Intrinsics.Arm;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
|
||||
namespace Nitload.Common.Collections
|
||||
{
|
||||
/// <summary>
|
||||
///
|
||||
/// </summary>
|
||||
public sealed unsafe partial class SwissTable : IDisposable
|
||||
{
|
||||
|
||||
private const int BucketCount = 1 << EntryMaskWidth;
|
||||
private const int BucketSize = 16;
|
||||
private const uint CrcMagic = 0xffff9961; // Magic number for CRC32C, used in hash computation
|
||||
private const int EntryMaskWidth = 10; // 1024 entries total, 16 per bucket
|
||||
public const int MaxCapacity = BucketCount * BucketSize; // 256 entries per bucket, 1024 buckets
|
||||
|
||||
public static readonly nuint RequestedNativeMemorySize = (nuint)Unsafe.SizeOf<Entry>();
|
||||
|
||||
private readonly Entry* _entries;
|
||||
private readonly Action<nuint> _freeHandle;
|
||||
private bool _disposed;
|
||||
|
||||
private SwissTable(
|
||||
Entry* buffer,
|
||||
Action<nuint> freeHandle
|
||||
)
|
||||
{
|
||||
_entries = buffer;
|
||||
_freeHandle = freeHandle;
|
||||
}
|
||||
|
||||
public static SwissTable Create()
|
||||
{
|
||||
Entry* buffer = (Entry*)NativeMemory.AlignedAlloc((nuint)Unsafe.SizeOf<Entry>(), 16);
|
||||
for (int i = 0; i < BucketCount; i++)
|
||||
{
|
||||
// Set all control bytes to 0xFF (empty)
|
||||
Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize);
|
||||
}
|
||||
return new SwissTable(buffer, handle => NativeMemory.AlignedFree((void*)handle));
|
||||
}
|
||||
|
||||
public static SwissTable Create(
|
||||
Func<nuint, nuint> allocator,
|
||||
Action<nuint> freeHandle
|
||||
)
|
||||
{
|
||||
Entry* buffer = (Entry*)allocator(RequestedNativeMemorySize);
|
||||
for (int i = 0; i < BucketCount; i++)
|
||||
{
|
||||
// Set all control bytes to 0xFF (empty)
|
||||
Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize);
|
||||
}
|
||||
return new SwissTable(buffer, freeHandle);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed) {
|
||||
return;
|
||||
}
|
||||
_disposed = true;
|
||||
_freeHandle((nuint)_entries);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
public bool TryAddOrSet(
|
||||
nuint key,
|
||||
nuint value,
|
||||
out bool isNewKey
|
||||
)
|
||||
{
|
||||
uint hash = BitOperations.Crc32C(CrcMagic, (ulong)key);
|
||||
ushort entry = (ushort)(hash & (BucketCount - 1));
|
||||
Bucket* bucket = &(*_entries)[entry];
|
||||
return IBucketScanner.Current.TrySet(key, hash, bucket, value, out isNewKey);
|
||||
}
|
||||
|
||||
public bool TryGet(
|
||||
nuint key,
|
||||
out nuint value
|
||||
)
|
||||
{
|
||||
uint hash = BitOperations.Crc32C(CrcMagic, key);
|
||||
ushort entry = (ushort)(hash & (BucketCount - 1));
|
||||
Bucket* bucket = &(*_entries)[entry];
|
||||
return IBucketScanner.Current.TryFind(key, hash, bucket, out value);
|
||||
/*
|
||||
* hash = CRC32 of key
|
||||
* entry = low 8 of hash
|
||||
* ctrl = 8-15 of hash
|
||||
* index = rest of hash
|
||||
* find value of hash
|
||||
*/
|
||||
}
|
||||
|
||||
public bool TryRemove(
|
||||
nuint key,
|
||||
out nuint value
|
||||
)
|
||||
{
|
||||
uint hash = BitOperations.Crc32C(CrcMagic, key);
|
||||
ushort entry = (ushort)(hash & (BucketCount - 1));
|
||||
Bucket* bucket = &(*_entries)[entry];
|
||||
return IBucketScanner.Current.TryRemove(key, hash, bucket, out value);
|
||||
}
|
||||
|
||||
internal static SwissTable Create(
|
||||
nuint bufferPtr,
|
||||
Action<nuint> FreeHandle
|
||||
)
|
||||
{
|
||||
Entry* buffer = (Entry*)bufferPtr;
|
||||
for (int i = 0; i < BucketCount; i++)
|
||||
{
|
||||
// Set all control bytes to 0xFF (empty)
|
||||
Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize);
|
||||
}
|
||||
return new SwissTable(buffer, FreeHandle);
|
||||
}
|
||||
|
||||
~SwissTable()
|
||||
{
|
||||
Dispose();
|
||||
}
|
||||
|
||||
[InlineArray(BucketCount)]
|
||||
private struct Entry
|
||||
{
|
||||
|
||||
public Bucket Bucket;
|
||||
|
||||
}
|
||||
|
||||
private unsafe struct Bucket
|
||||
{
|
||||
|
||||
public BucketControl Control;
|
||||
public BucketKeyStorage Keys;
|
||||
public BucketValueStorage Values;
|
||||
|
||||
}
|
||||
|
||||
private interface IBucketScanner
|
||||
{
|
||||
|
||||
static IBucketScanner Current { get; } = Create();
|
||||
|
||||
bool TryFind(
|
||||
nuint key,
|
||||
uint Hash,
|
||||
Bucket* bucket,
|
||||
out nuint value
|
||||
);
|
||||
|
||||
unsafe bool TryRemove(
|
||||
nuint key,
|
||||
uint hash,
|
||||
Bucket* bucket,
|
||||
out nuint value
|
||||
);
|
||||
|
||||
bool TrySet(
|
||||
nuint key,
|
||||
uint hash,
|
||||
Bucket* bucket,
|
||||
nuint value,
|
||||
out bool isNewKey
|
||||
);
|
||||
|
||||
private static IBucketScanner Create()
|
||||
{
|
||||
if (Sse42.IsSupported)
|
||||
{
|
||||
return new BucketScannerX86();
|
||||
} else if (ArmBase.IsSupported)
|
||||
{
|
||||
throw new NotSupportedException("ARM architecture is currently not supported for bucket scanning. Please use an x86 platform.");
|
||||
} else
|
||||
{
|
||||
throw new NotSupportedException("No suitable bucket scanner implementation found for the current platform.");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
[InlineArray(BucketSize)]
|
||||
private struct BucketControl
|
||||
{
|
||||
|
||||
public byte Control;
|
||||
|
||||
}
|
||||
|
||||
[InlineArray(BucketSize)]
|
||||
private struct BucketKeyStorage
|
||||
{
|
||||
|
||||
public nuint value;
|
||||
|
||||
}
|
||||
|
||||
[InlineArray(BucketSize)]
|
||||
private struct BucketValueStorage
|
||||
{
|
||||
|
||||
public nuint value;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -15,4 +15,5 @@ global using System.Threading;
|
||||
[assembly: InternalsVisibleTo("Nitload.RegularAnalysis")]
|
||||
[assembly: InternalsVisibleTo("Nitload.Graphics")]
|
||||
[assembly: InternalsVisibleTo("Nitload.Mathematics")]
|
||||
[assembly: InternalsVisibleTo("Nitload.Benchmark")]
|
||||
[assembly: InternalsVisibleTo("StgSharpDebug")]
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Nitload.Common.HighPerformance.ProcessorAbstraction;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
@@ -23,14 +22,12 @@ namespace Nitload.Common.HighPerformance.Memory
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Explicit, Size = 32, Pack = 32)]
|
||||
private unsafe struct CacheLineMetadata
|
||||
private struct CacheLineMetadata
|
||||
{
|
||||
|
||||
internal const uint MaxBufferSize = 0xFFFFFFFF - 64U;
|
||||
|
||||
[FieldOffset(0)] private readonly M256 FillingMask;
|
||||
|
||||
[FieldOffset(24)] internal nuint Origin;
|
||||
[FieldOffset(16)] internal AllocationState State;
|
||||
|
||||
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
|
||||
|
||||
@@ -48,33 +45,25 @@ namespace Nitload.Common.HighPerformance.Memory
|
||||
[FieldOffset(8)] public uint PositionMask;
|
||||
[FieldOffset(12)] public uint SizeMask;
|
||||
|
||||
#endregion
|
||||
|
||||
#region mask
|
||||
|
||||
[FieldOffset(16)] internal AllocationState State;
|
||||
[FieldOffset(17)] internal byte ReversedProfile;
|
||||
[FieldOffset(18)] internal ushort CustomizeProfile;
|
||||
[FieldOffset(20)] internal int RefCount;
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
#pragma warning disable IDE0032
|
||||
[StructLayout(LayoutKind.Explicit)]
|
||||
[StructLayout(LayoutKind.Explicit, Size = 16)]
|
||||
public unsafe struct Handle : IDisposable
|
||||
{
|
||||
|
||||
[FieldOffset(8)] private readonly CacheLineMetadata* _head;
|
||||
[FieldOffset(0)] private nuint _handle;
|
||||
|
||||
private Handle(
|
||||
nuint handle,
|
||||
CacheLineMetadata* cacheRefCount
|
||||
[FieldOffset(8)] private nuint _reserved;
|
||||
|
||||
internal Handle(
|
||||
nuint handle,
|
||||
nuint reserved
|
||||
)
|
||||
{
|
||||
_handle = handle;
|
||||
_head = cacheRefCount;
|
||||
_reserved = reserved;
|
||||
}
|
||||
|
||||
[Obsolete("You should not create a CacheLineHandle instance outside a L4", true)]
|
||||
@@ -88,7 +77,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
||||
return;
|
||||
}
|
||||
_handle = 0;
|
||||
_ = Interlocked.Decrement(ref _head->RefCount);
|
||||
_reserved = 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Debug"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
/**/
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
public void DebugAgeAll()
|
||||
{
|
||||
// AgeAll();
|
||||
}
|
||||
|
||||
public bool DebugContains(
|
||||
nuint origin
|
||||
)
|
||||
{
|
||||
return _map.TryGet(origin, out _);
|
||||
}
|
||||
|
||||
public int DebugEvict(
|
||||
int count = 1
|
||||
)
|
||||
{
|
||||
return Evict(count);
|
||||
}
|
||||
|
||||
public unsafe DebugSnapshot GetDebugSnapshot()
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
/*
|
||||
int maxId = _maxId;
|
||||
byte[] rrpv = new byte[maxId];
|
||||
byte* p = (byte*)_eviction;
|
||||
for (int i = 0; i < maxId; i++) {
|
||||
rrpv[i] = p[i];
|
||||
}
|
||||
return new DebugSnapshot(_activeCount, _aheadCount, maxId, rrpv);
|
||||
/**/
|
||||
}
|
||||
|
||||
public readonly struct DebugSnapshot
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// RRPV value for each line（0~3）, length == MaxId
|
||||
/// </summary>
|
||||
public readonly byte[] RrpvSnapshot;
|
||||
/// <summary>
|
||||
/// Count of activated cache line handle
|
||||
/// </summary>
|
||||
public readonly int ActiveCount;
|
||||
|
||||
/// <summary>
|
||||
/// Count of predictor ahead of current map position（aheadCount）
|
||||
/// </summary>
|
||||
public readonly int AheadCount;
|
||||
|
||||
/// <summary>
|
||||
/// Max ID of the line being using or once used（maxId）
|
||||
/// </summary>
|
||||
public readonly int MaxId;
|
||||
|
||||
internal DebugSnapshot(
|
||||
int activeCount,
|
||||
int aheadCount,
|
||||
int maxId,
|
||||
byte[] rrpv
|
||||
)
|
||||
{
|
||||
ActiveCount = activeCount;
|
||||
AheadCount = aheadCount;
|
||||
MaxId = maxId;
|
||||
RrpvSnapshot = rrpv;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
/**/
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
# L4 运行模型
|
||||
|
||||
> 工作稿，2026-10-14。本文替代旧版 L4 设计草案，区分当前代码、已讨论方向和仍待确认的问题。
|
||||
|
||||
## 1. 范围与职责
|
||||
|
||||
每个 `L4` 实例绑定一个 NUMA 节点，拥有本地 arena，其中放置 TLSF 分配器状态和 L4 Swiss 映射表的双缓冲区。
|
||||
|
||||
- **L4** 负责本地分配、映射、缓冲区发布和阶段协调。
|
||||
- **Swiss** 保存并查询 L4 使用的映射/预测条目，不解释 source layout。
|
||||
- **Predictor** 解释自己的 source spec，生成 packed 数据，并负责格式相关的软件写回或撤销。
|
||||
- **PredictorMonitor** 计划负责预测器注册、稳定 ID、执行顺序和生命周期；它与 Swiss 中的预测任务/映射状态是两回事。
|
||||
|
||||
本文不假设自动写回。若 packed 数据需要写回，必须能识别其所属预测器并显式调用对应逻辑。
|
||||
|
||||
## 2. AddressSource 与缓存行数据
|
||||
|
||||
`AddressSource` 是 16 字节 key：
|
||||
|
||||
- `Address`：源地址或 source identity。
|
||||
- `Description`：64 位 source/layout spec，由所属预测器解释。
|
||||
|
||||
当前 Swiss 条目使用该 key 和一个 `ulong` value。源码中的 `CacheLineMetadata` 是 32 字节 TLSF metadata；`L4.Handle` 仍是 16 字节 ABI 占位。引用计数、驱逐保护和写回归属尚未端到端实现。
|
||||
|
||||
### 当前与候选 KVP 布局
|
||||
|
||||
- 设计讨论中的 `ushort` 是缓存行计数，不是预测器数量或预测器注册表容量。
|
||||
- 当前 Swiss bucket 将 key 与 `ulong` value 分开组织；现有的 8-KVP 交叉存储描述的是数据存储布局，不是 H1/hash 探测分组。
|
||||
- 一个候选布局是把 value 扩成 16 字节：保留现有 `ulong` cache-line metadata，再增加 8 字节存放预测计数等状态。与 16 字节 `AddressSource` 组成 32 字节 KVP，每条 64 字节 cache line 放两个 KVP。
|
||||
- 该方案旨在减少当前 8-KVP 交叉存储造成的 cache-line 分散/冲突。它只调整 KVP 的存储编排，不意味着 H1 探测组从当前 16 槽改成 8 槽。此方案尚未实现，需实测对查找和替换扫描的影响。
|
||||
|
||||
## 3. Swiss 映射
|
||||
|
||||
当前表有 1024 组，每组 16 个槽位。控制字节扫描使用 SSE2；x64 路径使用 CRC32C 混合两个 64 位 key 字段。两个表缓冲区都分配在 L4 arena 内。
|
||||
|
||||
published buffer 地址以 `ulong` 保存：
|
||||
|
||||
- `TryGet` 用一次 `Volatile.Read` 捕获 published pointer，并在整个查找中使用该快照。
|
||||
- `SwapBuffers` 用 `Interlocked.Exchange` 发布下一缓冲区，返回的旧 published pointer 成为下一轮 inactive write buffer。
|
||||
- 原子发布指针不代表旧 buffer 可以立即复用。Map/swap 阶段协议必须先确保读者不再访问旧 buffer，writer 才能清空或覆盖它。
|
||||
|
||||
当前只实现 x64/SSE4.2 后端，其他 ISA 后端尚未实现。
|
||||
|
||||
## 4. 预测器接口与阶段
|
||||
|
||||
当前接口骨架位于 `L4Predictor.cs`：
|
||||
|
||||
- `IL4Predictor` 是非泛型管理接口，暴露预测器 ID 和同步的 `PredictMission(in Span<AddressSource>)`。
|
||||
- `IL4Predictor<TMission>` 为预测器实现提供强类型 mission API。
|
||||
- `PredictMission` 填充调用方提供的 span，并返回有效预测数。span 可以由调用方 `stackalloc`，不得逃逸出本次同步调用。
|
||||
- `PredictorMonitor` 目前是空占位；预测器注册和生命周期尚未实现。
|
||||
|
||||
当前讨论中的 L4 生命周期分为不同阶段：
|
||||
|
||||
1. **注册预测器**：monitor 按稳定顺序保存异构预测器。固定容量数组（讨论过 256）是候选，其容量与缓存行计数无关。
|
||||
2. **Prefetch**：预测器按依赖顺序执行。每个 L4 同时只有一个活动 prefetch worker，但 prefetch 可与 Map 重叠，也可能跨 NUMA 节点访问目标数据。
|
||||
3. **发布**：完成后的 inactive Swiss buffer 被原子发布。Swiss 保存预测/映射任务状态；预测器注册表不是预测任务队列。
|
||||
4. **Map**：调用方查询已发布的 `AddressSource` 并取得 L4 handle。
|
||||
5. **撤销/写回**：所属预测器解释 packed 数据，并按需执行软件写回或撤销。
|
||||
|
||||
预测器接口本身是同步的。若预测结果需要在 `PredictMission` 返回后或另一线程继续使用，必须先复制到生命周期足够长的自有存储中。异步边界应位于拥有的 batch/work item 周围，而不能让 stack span 逃逸。
|
||||
|
||||
## 5. 预测窗口与 Map 同步
|
||||
|
||||
两个阈值含义不同：
|
||||
|
||||
- `_low_predict = 512`：预测触发/补充阈值。
|
||||
- `_predict_slow_path_count = 2 * LogicalCoreCount`：按 L4 绑定 NUMA 节点逻辑处理器数量计算的提前慢路径窗口。
|
||||
|
||||
当前 Map 的分流意图如下：
|
||||
|
||||
| 剩余预测数 | Map 行为 |
|
||||
|---|---|
|
||||
| 小于 512 | 尝试成为预测 writer；预测可能完成并发布新 buffer。竞争失败的调用走带读者计数的查找路径。 |
|
||||
| 512 到 `512 + _predict_slow_path_count` 之间 | 走带计数的慢查找路径，以便 swap 等待这些读者退出。 |
|
||||
| 大于或等于 `512 + _predict_slow_path_count` | 走无计数的直接查找快路径。 |
|
||||
|
||||
计数读者协议先等待 swap gate 打开，再增加 reader count 并复查 gate。如果 gate 在两步之间关闭，则撤销计数并重试。swap 方关闭 gate、等待已计数读者归零、交换 buffer、发布新的剩余预测数，最后重新打开 gate。
|
||||
|
||||
无计数快路径依赖预测提前量和阶段时序。这是性能启发式，本身不能证明一个被抢占的 reader 不会在旧 buffer 复用时仍继续访问。无计数 reader 与 buffer 生命周期之间的不变量，以及快慢路径边界，都仍需验证。
|
||||
|
||||
当前 `Predict()` 返回 `int`，但实现仍是返回 `0` 的空 stub，因此预测数生命周期尚未端到端运行。
|
||||
|
||||
## 6. 预测器注册、任务状态与在途缓冲
|
||||
|
||||
预测器调度和预测任务执行是两条相关但不同的生命周期，不能把“预测器已预测完”当成“它产生的工作已完成”。
|
||||
|
||||
- **待调度预测器（多个）**：保存在 PredictorMonitor 的有序管理数组/队列中，等待轮到自己。固定槽位容量（曾讨论 256）表示预测器注册上限，与缓存行计数无关。
|
||||
- **活动预测器（最多一个）**：当前 L4 正在执行其同步 `PredictMission`。预测器之间按依赖顺序切换，不需要多 lane 或 thread-local lane 归属。
|
||||
- **在途预测批次（可以多个）**：某预测器已经产出候选，但候选对应的实际任务尚未全部执行。此时要保留稳定的 predictor ID/代次、mission 或写回上下文；该记录属于任务 bag，不等同于预测器注册槽位。`PredictMission` 按预测器依赖顺序串行调用；前一批次的实际任务可以仍在途，是否与后续批次重叠由任务依赖决定。
|
||||
- **已完成批次**：对应实际工作全部完成，且所需软件写回或显式丢弃已完成。此时释放在途记录及其引用资源，不再由 L4 保存该批次。预测器本身是否注销是另一项生命周期操作。
|
||||
|
||||
Swiss 负责预测任务/映射条目的查找状态；Monitor 负责预测器注册和调度；在途预测器缓冲负责尚未完成的预测工作及其归属。三者不可混为同一个队列或数组。
|
||||
|
||||
`PredictMission(in Span<AddressSource>)` 是同步接口，L4 可传入 stackalloc span。若返回后仍需跨线程或跨 NUMA 处理其内容，必须将内容复制到拥有明确生命周期的缓冲区；不得让 stack span 逃逸。由于预测器顺序有依赖，待处理预测必须按既定顺序交接。若采用异步 worker，队列应有界且不阻塞 Map 快路径。
|
||||
|
||||
## 7. 驱逐与写回策略
|
||||
|
||||
L4 不得推断或解码预测器的 packed 格式。若丢弃一个尚未写回、且对应实际工作仍在途的预测条目会丢失必要状态，就不能静默驱逐它；必须由缓存行保护/引用状态阻止替换，或由 Monitor/预测器先完成写回、撤销或显式释放。相关状态可放在 KVP metadata 或 sidecar，布局尚未定稿。
|
||||
|
||||
owner ID 可用于路由显式写回；但若多个预测器能独立声明同一条目，单一 owner 字段不够。引用计数可保护共享条目，而写回路径仍需获得预测器 identity/source spec。
|
||||
|
||||
## 8. 当前状态与待决问题
|
||||
|
||||
**源码中已实现：**
|
||||
|
||||
- NUMA 绑定的 L4 arena 和 TLSF 包装。
|
||||
- 双缓冲 Swiss 表及原子 published pointer 交换。
|
||||
- x64 CRC32C hash 和 SSE2 控制字节扫描。
|
||||
- 固定预测触发阈值，以及按 NUMA 节点逻辑处理器数量计算的提前慢路径窗口。
|
||||
- 慢路径的 Map reader gate/counter。
|
||||
- 同步的泛型/非泛型预测器接口骨架。
|
||||
|
||||
**尚未实现或尚未验证：**
|
||||
|
||||
- PredictorMonitor 注册表容量、注册/注销和生命周期状态转换。
|
||||
- 实际预测算法与有效预测条目的生成。
|
||||
- 端到端撤销/写回归属和缓存行驱逐保护。
|
||||
- Prefetch 与 Map 跨 NUMA 重叠时的自有 queue/batch 交接方式。
|
||||
- 无计数 Map 快路径不会与 published buffer 复用重叠的证明/测试。
|
||||
- 覆盖预测器发布、Map、撤销和 Dispose 的并发压力测试。
|
||||
|
||||
## 9. L4 的四个核心操作
|
||||
|
||||
### 预测（Prefetch）
|
||||
|
||||
Monitor 按注册顺序选择预测器，调用其同步预测接口；预测器解释自己的 source spec，将预测项写入调用方提供的 span。Swiss 管理预测/映射条目。若预测项对应的实际工作尚未完成，则放入在途预测器缓冲，保留后续完成、撤销或写回所需的归属信息。预测期间若没有可用的 cache-line 槽位，替换/保护策略由后续设计决定，不能隐式丢弃仍在途的条目。
|
||||
|
||||
### 映射（Map）
|
||||
|
||||
计算 worker 按预测器调度获取 `AddressSource`，通过 Swiss 查询 NUMA 本地的 packed 数据并得到 L4 handle。Map 不解释 `Description`，也不负责预测器注册或写回。
|
||||
|
||||
### 注册预测器
|
||||
|
||||
将预测器放入 `PredictorMonitor` 的管理槽位，分配稳定 ID 和执行顺序，并纳入生命周期监视。注册表槽位与 Swiss 预测条目、在途预测工作是不同资源。
|
||||
|
||||
### 移除预测器
|
||||
|
||||
先停止向该预测器分派新 mission，再等待其在途预测对应的实际工作完成或被撤销；需要的软件写回完成后，才能释放在途状态并回收注册槽位。具体等待与写回协议尚待设计。
|
||||
@@ -1,64 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Evict"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
///
|
||||
/// </summary>
|
||||
/// <param name="count">
|
||||
///
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// Count of lines not evicted
|
||||
/// </returns>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private unsafe int Evict(
|
||||
int count = 1
|
||||
)
|
||||
{
|
||||
count = (count < 0) ? 0 : count;
|
||||
int* pPtr = _predictCount, mPtr = _mapCount;
|
||||
|
||||
|
||||
int remain = count;
|
||||
for (int i = 0; i < CacheLineCount; i++)
|
||||
{
|
||||
if (pPtr[i] > mPtr[i])
|
||||
{
|
||||
continue;
|
||||
}
|
||||
remain -= TryEvictAt(i) ? 1 : 0;
|
||||
}
|
||||
return remain;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private unsafe bool TryEvictAt(
|
||||
int index
|
||||
)
|
||||
{
|
||||
CacheLineMetadata* head = &_head[index];
|
||||
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
|
||||
{
|
||||
// EvictLine(i + index);
|
||||
_ = _map.TryRemove(head->Origin, out _);
|
||||
IL4Predict predict = CurrentPredict;
|
||||
predict.WriteBack(head->Origin, head->CustomizeProfile, new ReadOnlySpan<byte>((byte*)(ulong)(head->PositionMask + _baseAddress), (int)(ulong)head->SizeMask));
|
||||
_bufferAllocator.Free(index);
|
||||
_mapCount[index] = 0;
|
||||
_predictCount[index] = int.MaxValue;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Exceptions"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
#pragma warning disable CA1032 // Implement standard exception constructors
|
||||
|
||||
public sealed class L4PredictUnregisteredException : Exception
|
||||
{
|
||||
|
||||
public L4PredictUnregisteredException(
|
||||
int id
|
||||
)
|
||||
: base($"L4 predictor with id {id} is null. Maybe it has been unregistered.") { }
|
||||
|
||||
}
|
||||
|
||||
public sealed class L4PredictConflictException : Exception
|
||||
{
|
||||
|
||||
public L4PredictConflictException(
|
||||
nuint basePtr,
|
||||
ushort policy,
|
||||
int current
|
||||
)
|
||||
: base($"L4 predictor with id {current} has already registered a cache line with base pointer {basePtr} and policy {policy}.") { }
|
||||
|
||||
}
|
||||
#pragma warning restore CA1032 // Implement standard exception constructors
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Map"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class L4
|
||||
{
|
||||
|
||||
private const int _low_predict = 512;
|
||||
|
||||
private int _predict_count;
|
||||
private readonly int _predict_slow_path_count;
|
||||
private int _swiss_reader_counter;
|
||||
private int _swiss_swap_lock;
|
||||
private int _swiss_writer_lock;
|
||||
|
||||
public partial Handle Map(
|
||||
AddressSource source
|
||||
)
|
||||
{
|
||||
ulong handle;
|
||||
int predictCount = Volatile.Read(ref _predict_count);
|
||||
if (predictCount < _low_predict)
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _swiss_writer_lock, 1, 0) == 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
int effectivePredictionCount = Predict();
|
||||
if (effectivePredictionCount > 0)
|
||||
{
|
||||
_ = Interlocked.Exchange(ref _swiss_swap_lock, 1);
|
||||
while (Volatile.Read(ref _swiss_reader_counter) != 0) { }
|
||||
|
||||
_addressMap.SwapBuffers();
|
||||
Volatile.Write(ref _predict_count, effectivePredictionCount);
|
||||
_addressMap.TryRehash();
|
||||
}
|
||||
|
||||
if (_addressMap.TryGet(source, out handle)) {
|
||||
throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}].");
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_ = Interlocked.Exchange(ref _swiss_swap_lock, 0);
|
||||
_ = Interlocked.Exchange(ref _swiss_writer_lock, 0);
|
||||
}
|
||||
} else if (!TryGetAddress(source, out handle)) {
|
||||
throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}].");
|
||||
}
|
||||
} else if (predictCount < _low_predict + _predict_slow_path_count)
|
||||
{
|
||||
if (!TryGetAddress(source, out handle)) {
|
||||
throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}].");
|
||||
}
|
||||
} else if (!_addressMap.TryGet(source, out handle)) {
|
||||
throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}].");
|
||||
}
|
||||
|
||||
CacheLineMetadata* metadata = (CacheLineMetadata*)handle;
|
||||
|
||||
// process metadata, unfinished
|
||||
uint offset = metadata->PositionMask;
|
||||
nuint address = /*base ptr + */ offset;
|
||||
_ = Interlocked.Decrement(ref _predict_count);
|
||||
|
||||
return new Handle(address, (nuint)handle);
|
||||
}
|
||||
|
||||
private bool TryGetAddress(
|
||||
AddressSource source,
|
||||
out ulong handle
|
||||
)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
while (Volatile.Read(ref _swiss_swap_lock) != 0) { }
|
||||
|
||||
_ = Interlocked.Increment(ref _swiss_reader_counter);
|
||||
if (Volatile.Read(ref _swiss_swap_lock) == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
_ = Interlocked.Decrement(ref _swiss_reader_counter);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return _addressMap.TryGet(source, out handle);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_ = Interlocked.Decrement(ref _swiss_reader_counter);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -5,10 +5,7 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Nitload.Common.Collections;
|
||||
using Nitload.Common.HighPerformance.Memory;
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
@@ -16,22 +13,12 @@ using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
public unsafe partial class L4
|
||||
{
|
||||
|
||||
private IL4Predict CurrentPredict { get; set; }
|
||||
|
||||
public void ResetPredict(
|
||||
IL4Predict predict
|
||||
)
|
||||
internal partial int Predict()
|
||||
{
|
||||
if (CurrentPredict is null)
|
||||
{
|
||||
CurrentPredict = predict;
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: write back all and register new predictor
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Predictor"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
[Flags]
|
||||
public enum PredictResult
|
||||
{
|
||||
|
||||
Success = 1,
|
||||
Exhausted = 2,
|
||||
|
||||
}
|
||||
|
||||
public interface IL4Predict
|
||||
{
|
||||
|
||||
PredictResult Predict(
|
||||
Span<CacheLineDescription> node,
|
||||
out int count
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Materializes the cache line identified by <paramref name="origin" /> into <paramref ///
|
||||
/// name="cache" />.
|
||||
/// </summary>
|
||||
/// <param name="origin">
|
||||
/// Source address associated with the cache line.
|
||||
/// </param>
|
||||
/// <param name="policy">
|
||||
/// Mapping policy associated with the cache line.
|
||||
/// </param>
|
||||
/// <param name="cache">
|
||||
/// Destination cache line buffer.
|
||||
/// </param>
|
||||
void Prefetch(
|
||||
nuint origin,
|
||||
uint policy,
|
||||
Span<byte> cache
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Writes the cache line in <paramref name="cache" /> back to <paramref name="origin" />.
|
||||
/// </summary>
|
||||
/// <param name="origin">
|
||||
/// Source address associated with the cache line.
|
||||
/// </param>
|
||||
/// <param name="policy">
|
||||
/// Mapping policy associated with the cache line.
|
||||
/// </param>
|
||||
/// <param name="cache">
|
||||
/// Source cache line buffer.
|
||||
/// </param>
|
||||
void WriteBack(
|
||||
nuint origin,
|
||||
uint policy,
|
||||
ReadOnlySpan<byte> cache
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Explicit)]
|
||||
public struct CacheLineDescription : IEquatable<CacheLineDescription>
|
||||
{
|
||||
|
||||
public override readonly bool Equals(
|
||||
object? obj
|
||||
)
|
||||
{
|
||||
return (obj is CacheLineDescription prediction) &&
|
||||
Address.Equals(prediction.Address) &&
|
||||
(MapPolicy == prediction.MapPolicy);
|
||||
}
|
||||
|
||||
public readonly bool Equals(
|
||||
CacheLineDescription other
|
||||
)
|
||||
{
|
||||
return
|
||||
(Address == other.Address) && (MapPolicy == other.MapPolicy);
|
||||
}
|
||||
|
||||
public override readonly int GetHashCode()
|
||||
{
|
||||
return (int)ComposePredictionKey64(Address, MapPolicy);
|
||||
}
|
||||
|
||||
public readonly nuint GetHashCodeLong()
|
||||
{
|
||||
return ComposePredictionKey64(Address, MapPolicy);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static nuint ComposePredictionKey64(
|
||||
nuint address,
|
||||
ushort mapPolicy
|
||||
)
|
||||
{
|
||||
nuint x = address;
|
||||
nuint y = mapPolicy;
|
||||
unchecked
|
||||
{
|
||||
nuint mixed = x ^ (y * ((nuint)0x9e3779b97f4a7c15UL));
|
||||
mixed ^= mixed >> 30;
|
||||
mixed *= (nuint)0xbf58476d1ce4e5b9UL;
|
||||
mixed ^= mixed >> 27;
|
||||
mixed *= (nuint)0x94d049bb133111ebUL;
|
||||
mixed ^= mixed >> 31;
|
||||
return mixed;
|
||||
}
|
||||
}
|
||||
|
||||
public static bool operator !=(
|
||||
CacheLineDescription left,
|
||||
CacheLineDescription right
|
||||
)
|
||||
{
|
||||
return !(left == right);
|
||||
}
|
||||
public static bool operator ==(
|
||||
CacheLineDescription left,
|
||||
CacheLineDescription right
|
||||
)
|
||||
{
|
||||
return left.Equals(right);
|
||||
}
|
||||
|
||||
#pragma warning disable CA1051
|
||||
[FieldOffset(0)]public nuint Address;
|
||||
[FieldOffset(8)]public uint Size;
|
||||
[FieldOffset(12)]public ushort MapPolicy;
|
||||
[FieldOffset(14)]internal short Predictor;
|
||||
#pragma warning restore CA1051
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.PredictorQueue"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
private readonly IL4Predictor[] _ring = new IL4Predictor[256];
|
||||
private volatile int _consume;
|
||||
private volatile int _enter;
|
||||
private volatile int _predict;
|
||||
|
||||
private int _queue_sync;
|
||||
|
||||
internal void DropPredictor(
|
||||
IL4Predictor predictor
|
||||
)
|
||||
{
|
||||
if (predictor is null) {
|
||||
return;
|
||||
}
|
||||
SpinWait sp = new();
|
||||
while (Interlocked.CompareExchange(ref _queue_sync, 1, 0) != 0) {
|
||||
sp.SpinOnce();
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
unchecked
|
||||
{
|
||||
for (int i = _consume; i < _predict; i++)
|
||||
{
|
||||
byte idx = (byte)i;
|
||||
if (ReferenceEquals(predictor, _ring[idx]))
|
||||
{
|
||||
int end = _predict;
|
||||
for (int move = i; move < end - 1; move++)
|
||||
{
|
||||
_ring[(byte)move] = _ring[(byte)(move + 1)];
|
||||
}
|
||||
_ring[(byte)(end - 1)] = default!;
|
||||
_predict = end - 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
Interlocked.Exchange(ref _queue_sync, 0);
|
||||
}
|
||||
}
|
||||
|
||||
internal void RegistPredictor(
|
||||
IL4Predictor predictor
|
||||
)
|
||||
{
|
||||
if (predictor is null) {
|
||||
return;
|
||||
}
|
||||
SpinWait sp = new();
|
||||
while (Interlocked.CompareExchange(ref _queue_sync, 1, 0) != 0) {
|
||||
sp.SpinOnce();
|
||||
}
|
||||
|
||||
WaitForSpare();
|
||||
|
||||
_consume++;
|
||||
|
||||
byte idx = (byte)_consume;
|
||||
_ring[idx] = predictor;
|
||||
}
|
||||
|
||||
private void WaitForSpare()
|
||||
{
|
||||
int spin_count = 0;
|
||||
|
||||
// wait until the queue has space
|
||||
while (_enter - _consume >= 256)
|
||||
{
|
||||
switch (spin_count)
|
||||
{
|
||||
case < 1024:
|
||||
spin_count++;
|
||||
break;
|
||||
case < 65536:
|
||||
spin_count++;
|
||||
Thread.SpinWait(1);
|
||||
break;
|
||||
default:
|
||||
Thread.Sleep(1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Prefetch"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.ComponentModel;
|
||||
using System.Numerics;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class L4
|
||||
{
|
||||
|
||||
private partial void Prefetch()
|
||||
{
|
||||
if (CurrentPredict is null) {
|
||||
return;
|
||||
}
|
||||
IL4Predict predict = CurrentPredict;
|
||||
Span<CacheLineDescription> span = new(_prediction, PredictionCount);
|
||||
PredictResult result = predict.Predict(span, out int actualCount);
|
||||
_exhausted = (result & PredictResult.Exhausted) != 0;
|
||||
if (actualCount <= 0)
|
||||
{
|
||||
_exhausted = true;
|
||||
return;
|
||||
}
|
||||
int missIndex = 0;
|
||||
for (int i = 0; i < actualCount; i++)
|
||||
{
|
||||
ref CacheLineDescription handle = ref span[i];
|
||||
nuint address = handle.Address;
|
||||
if (!_map.TryGet(address, out _))
|
||||
{
|
||||
// miss and record
|
||||
span[missIndex] = handle;
|
||||
missIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
// now prefetch the misses
|
||||
for (int i = 0; i < missIndex; i++)
|
||||
{
|
||||
CacheLineMetadata* head;
|
||||
ref CacheLineDescription pHandle = ref span[i];
|
||||
nuint address = pHandle.Address;
|
||||
int index;
|
||||
if (_map.TryGet(address, out nuint handle))
|
||||
{
|
||||
head = (CacheLineMetadata*)handle;
|
||||
index = (int)(head - _head);
|
||||
_predictCount[index] = 0; // reset eviction counter
|
||||
|
||||
#region update predictor belonging
|
||||
|
||||
if (_predictCount[index] > short.MaxValue / 2)
|
||||
{
|
||||
int predictCount = Volatile.Read(ref _predictCount[index]);
|
||||
int mapCount = Volatile.Read(ref _mapCount[index]);
|
||||
int decrement = (int.Min(predictCount, mapCount) + 1) / 2;
|
||||
_ = Interlocked.Add(ref _predictCount[index], -decrement);
|
||||
_ = Interlocked.Add(ref _mapCount[index], -decrement);
|
||||
}
|
||||
_predictCount[index]++;
|
||||
|
||||
#endregion
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
int retryCount = 0, remain = missIndex - i;
|
||||
const int maxRetry = 1024;
|
||||
while (!_bufferAllocator.TryAlloc(pHandle.Size, out index))
|
||||
{
|
||||
remain = Evict(remain);
|
||||
retryCount++;
|
||||
if (retryCount > maxRetry) {
|
||||
throw new OverflowException($"Failed to evict cache line after {maxRetry} retries.");
|
||||
}
|
||||
if (retryCount > maxRetry / 2) {
|
||||
Thread.Sleep(1);
|
||||
}
|
||||
}
|
||||
head = _head + index;
|
||||
head->Origin = address;
|
||||
head->CustomizeProfile = pHandle.MapPolicy;
|
||||
head->RefCount = 0;
|
||||
|
||||
_predictCount[index] = 1;
|
||||
_mapCount[index] = 1;
|
||||
|
||||
// pre fetch them here
|
||||
predict.Prefetch(head->Origin, head->CustomizeProfile, new Span<byte>((byte*)(head->PositionMask + _baseAddress), (int)head->SizeMask));
|
||||
_ = _map.TryAddOrSet(address, (nuint)head, out _);
|
||||
}
|
||||
_ = Interlocked.Add(ref _aheadCount, actualCount);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.SwissTable"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Buffers;
|
||||
using System.Numerics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Intrinsics;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public sealed unsafe partial class L4
|
||||
{
|
||||
|
||||
private const int BucketCapacity = 16;
|
||||
private const int GroupCount = 1 << GroupCountLog;
|
||||
private const int GroupCountLog = 10;
|
||||
private const int IdealCount = GroupCount * BucketCapacity;//16384
|
||||
private const int OffsetOfBucket = IdealCount;
|
||||
|
||||
private static readonly int TableBufferSize =
|
||||
IdealCount + (GroupCount * Unsafe.SizeOf<SwissBucket>());
|
||||
internal static readonly int BufferSize = TableBufferSize * 2;
|
||||
|
||||
private enum SwissControl : byte
|
||||
{
|
||||
|
||||
Empty = 0b10000000,
|
||||
Deleted = 0b10000001
|
||||
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct AddressSource : IEquatable<AddressSource>
|
||||
{
|
||||
|
||||
public ulong Address;
|
||||
public ulong Description;
|
||||
|
||||
public readonly bool Equals(
|
||||
AddressSource other
|
||||
)
|
||||
{
|
||||
return this == other;
|
||||
}
|
||||
|
||||
public override readonly bool Equals(
|
||||
object? obj
|
||||
)
|
||||
{
|
||||
return obj is AddressSource source && Equals(source);
|
||||
}
|
||||
|
||||
public override readonly int GetHashCode()
|
||||
{
|
||||
return HashCode.Combine(Address, Description);
|
||||
}
|
||||
|
||||
public static bool operator !=(
|
||||
AddressSource left,
|
||||
AddressSource right
|
||||
)
|
||||
{
|
||||
return !(left == right);
|
||||
}
|
||||
|
||||
public static bool operator ==(
|
||||
AddressSource left,
|
||||
AddressSource right
|
||||
)
|
||||
{
|
||||
return left.Address == right.Address && left.Description == right.Description;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
[InlineArray(BucketCapacity)]
|
||||
internal struct KeyArray
|
||||
{
|
||||
|
||||
private AddressSource _element0;
|
||||
|
||||
}
|
||||
|
||||
internal struct SwissBucket
|
||||
{
|
||||
|
||||
public KeyArray Keys;
|
||||
public fixed ulong Values[BucketCapacity];
|
||||
|
||||
}
|
||||
|
||||
internal unsafe struct SwissEnumerator(
|
||||
byte* buffer
|
||||
)
|
||||
{
|
||||
|
||||
private readonly SwissBucket* _buckets = (SwissBucket*)(buffer + IdealCount);
|
||||
private readonly Vector128<byte>* _control_bytes = (Vector128<byte>*)buffer;
|
||||
private int _current_group;
|
||||
private int _current_mask;
|
||||
private int _current_offset;
|
||||
private bool _group_loaded;
|
||||
|
||||
public (AddressSource key, ulong value) Current
|
||||
{
|
||||
get
|
||||
{
|
||||
SwissBucket* bucket = &_buckets[_current_group];
|
||||
AddressSource key = bucket->Keys[_current_offset];
|
||||
ulong value = bucket->Values[_current_offset];
|
||||
return (key, value);
|
||||
}
|
||||
}
|
||||
|
||||
public bool MoveNext()
|
||||
{
|
||||
while (_current_group < GroupCount)
|
||||
{
|
||||
if (!_group_loaded)
|
||||
{
|
||||
if (Sse42.IsSupported)
|
||||
{
|
||||
_current_mask = SwissReaderX86.CompareOccupiedMask(*(_control_bytes + _current_group));
|
||||
} else
|
||||
{
|
||||
throw new PlatformNotSupportedException();
|
||||
}
|
||||
_group_loaded = true;
|
||||
}
|
||||
|
||||
if (_current_mask != 0)
|
||||
{
|
||||
_current_offset = BitOperations.TrailingZeroCount(_current_mask);
|
||||
_current_mask &= _current_mask - 1;
|
||||
return true;
|
||||
}
|
||||
|
||||
_current_group++;
|
||||
_group_loaded = false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal unsafe struct SwissTable : IDisposable
|
||||
{
|
||||
|
||||
private byte* _buffer_write;
|
||||
private int _delete_count;
|
||||
private ulong _published_buffer;
|
||||
|
||||
private SwissTable(
|
||||
byte* buffers
|
||||
)
|
||||
{
|
||||
_published_buffer = (ulong)buffers;
|
||||
_buffer_write = buffers + TableBufferSize;
|
||||
InitializeBuffer(buffers);
|
||||
InitializeBuffer(_buffer_write);
|
||||
}
|
||||
|
||||
internal byte* ReadBufferForLockFreeBenchmark => (byte*)Volatile.Read(ref _published_buffer);
|
||||
|
||||
internal readonly byte* WriteBufferForLockFreeBenchmark => _buffer_write;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_ = Interlocked.Exchange(ref _published_buffer, 0);
|
||||
_buffer_write = null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Publishes the write buffer. The caller must provide the phase barrier around this
|
||||
/// operation.
|
||||
/// </summary>
|
||||
public void SwapBuffers()
|
||||
{
|
||||
ulong previousBuffer = Interlocked.Exchange(ref _published_buffer, (ulong)_buffer_write);
|
||||
_buffer_write = (byte*)previousBuffer;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes to the inactive buffer. The caller must serialize all writes.
|
||||
/// </summary>
|
||||
public bool TryAddOrSet(
|
||||
AddressSource key,
|
||||
ulong value
|
||||
)
|
||||
{
|
||||
if (Sse42.IsSupported) {
|
||||
return SwissWriterX86.TryAddOrSet(_buffer_write, key, value, ref _delete_count);
|
||||
}
|
||||
|
||||
throw new NotImplementedException("The Swiss backend is not implemented for this ISA.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the published buffer. The caller must prevent swaps until all reads finish.
|
||||
/// </summary>
|
||||
public bool TryGet(
|
||||
AddressSource key,
|
||||
out ulong value
|
||||
)
|
||||
{
|
||||
byte* publishedBuffer = (byte*)Volatile.Read(ref _published_buffer);
|
||||
if (Sse42.IsSupported) {
|
||||
return SwissReaderX86.TryGet(publishedBuffer, key, out value);
|
||||
}
|
||||
|
||||
throw new NotImplementedException("The Swiss backend is not implemented for this ISA.");
|
||||
}
|
||||
|
||||
public void TryRehash()
|
||||
{
|
||||
if (_delete_count < 2048)
|
||||
{
|
||||
NativeMemory.Copy(_buffer_write, (byte*)_published_buffer, (nuint)TableBufferSize);
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
SwissEnumerator e = new((byte*)_published_buffer);
|
||||
InitializeBuffer(_buffer_write);
|
||||
if (Sse42.IsSupported)
|
||||
{
|
||||
while (e.MoveNext())
|
||||
{
|
||||
(AddressSource key, ulong value) = e.Current;
|
||||
if (!SwissWriterX86.TryAddOrSet(_buffer_write, key, value, ref _delete_count))
|
||||
{
|
||||
throw new InvalidOperationException("The Swiss table could not fit all active entries during rehash.");
|
||||
}
|
||||
}
|
||||
_delete_count = 0;
|
||||
} else
|
||||
{
|
||||
throw new PlatformNotSupportedException();
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
NativeMemory.Copy(_buffer_write, (byte*)_published_buffer, (nuint)TableBufferSize);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes from the inactive buffer. The caller must serialize all writes.
|
||||
/// </summary>
|
||||
public bool TryRemove(
|
||||
AddressSource key,
|
||||
out ulong value
|
||||
)
|
||||
{
|
||||
if (Sse42.IsSupported)
|
||||
{
|
||||
return SwissWriterX86.TryRemove(_buffer_write, key, out value, ref _delete_count);
|
||||
} else
|
||||
{
|
||||
throw new NotImplementedException("The Swiss backend is not implemented for this ISA.");
|
||||
}
|
||||
}
|
||||
|
||||
internal static SwissTable Create(
|
||||
byte* buffers
|
||||
)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(buffers);
|
||||
return new SwissTable(buffers);
|
||||
}
|
||||
|
||||
private static void InitializeBuffer(
|
||||
byte* buffer
|
||||
)
|
||||
{
|
||||
Unsafe.InitBlock(buffer, (byte)SwissControl.Empty, IdealCount);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Swiss_X86"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Numerics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.Intrinsics;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
private const uint CrcMagic = 0x7f1c9961;
|
||||
|
||||
internal static class SwissHashX64
|
||||
{
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static uint Compute(
|
||||
AddressSource key
|
||||
)
|
||||
{
|
||||
ulong hash = CrcMagic;
|
||||
hash = Sse42.X64.Crc32(hash, key.Address);
|
||||
hash = Sse42.X64.Crc32(hash, key.Description);
|
||||
return (uint)hash;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal readonly unsafe struct SwissWriterX86
|
||||
{
|
||||
|
||||
public static bool TryAddOrSet(
|
||||
byte* buffer,
|
||||
AddressSource key,
|
||||
ulong value,
|
||||
ref int delete_count
|
||||
)
|
||||
{
|
||||
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
|
||||
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
|
||||
|
||||
uint hash = SwissHashX64.Compute(key);
|
||||
uint begin = hash % GroupCount;
|
||||
|
||||
uint first_tomb_group = 0;
|
||||
int first_tomb_idx = -1;
|
||||
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
|
||||
|
||||
for (int i = 0; i < GroupCount; i++)
|
||||
{
|
||||
Vector128<byte>* control_bytes_ptr = control_bytes + begin;
|
||||
Vector128<byte> control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)control_bytes_ptr);
|
||||
|
||||
Vector128<byte> target_hash_v128 = Vector128.Create(target_hash);
|
||||
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
|
||||
Vector128<byte> target_tomb_v128 = Vector128.Create((byte)SwissControl.Deleted);
|
||||
|
||||
Vector128<byte> cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128);
|
||||
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
|
||||
Vector128<byte> cmp_tomb = Sse2.CompareEqual(control_bytes_v128, target_tomb_v128);
|
||||
|
||||
int hash_mask = Sse2.MoveMask(cmp_hash);
|
||||
int empty_mask = Sse2.MoveMask(cmp_empty);
|
||||
int tomb_mask = Sse2.MoveMask(cmp_tomb);
|
||||
|
||||
if (tomb_mask != 0 && first_tomb_idx < 0)
|
||||
{
|
||||
int index = BitOperations.TrailingZeroCount(tomb_mask);
|
||||
first_tomb_group = begin;
|
||||
first_tomb_idx = index;
|
||||
}
|
||||
while (hash_mask != 0)
|
||||
{
|
||||
int index = BitOperations.TrailingZeroCount(hash_mask);
|
||||
SwissBucket* bucket = &buckets[begin];
|
||||
if (bucket->Keys[index] == key)
|
||||
{
|
||||
bucket->Values[index] = value;
|
||||
return true;
|
||||
}
|
||||
hash_mask &= ~(1 << index);
|
||||
}
|
||||
while (empty_mask != 0)
|
||||
{
|
||||
int index = BitOperations.TrailingZeroCount(empty_mask);
|
||||
if (first_tomb_idx < 0)
|
||||
{
|
||||
SwissBucket* bucket = &buckets[begin];
|
||||
bucket->Keys[index] = key;
|
||||
bucket->Values[index] = value;
|
||||
((byte*)control_bytes_ptr)[index] = target_hash;
|
||||
} else
|
||||
{
|
||||
SwissBucket* bucket = &buckets[first_tomb_group];
|
||||
bucket->Keys[first_tomb_idx] = key;
|
||||
bucket->Values[first_tomb_idx] = value;
|
||||
((byte*)(control_bytes + first_tomb_group))[first_tomb_idx] = target_hash;
|
||||
delete_count--;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
begin = (begin + 1) & (GroupCount - 1);
|
||||
}
|
||||
if (first_tomb_idx >= 0)
|
||||
{
|
||||
SwissBucket* bucket = &buckets[first_tomb_group];
|
||||
bucket->Keys[first_tomb_idx] = key;
|
||||
bucket->Values[first_tomb_idx] = value;
|
||||
((byte*)(control_bytes + first_tomb_group))[first_tomb_idx] = target_hash;
|
||||
delete_count--;
|
||||
return true;
|
||||
}
|
||||
value = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
public static bool TryRemove(
|
||||
byte* buffer,
|
||||
AddressSource key,
|
||||
out ulong value,
|
||||
ref int delete_count
|
||||
)
|
||||
{
|
||||
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
|
||||
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
|
||||
uint hash = SwissHashX64.Compute(key);
|
||||
uint begin = hash % GroupCount;
|
||||
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
|
||||
|
||||
for (int i = 0; i < GroupCount; i++)
|
||||
{
|
||||
Vector128<byte> control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)(control_bytes + begin));
|
||||
|
||||
Vector128<byte> target_hash_v128 = Vector128.Create(target_hash);
|
||||
Vector128<byte> cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128);
|
||||
|
||||
int mask = Sse2.MoveMask(cmp_hash);
|
||||
|
||||
while (mask != 0)
|
||||
{
|
||||
int index = BitOperations.TrailingZeroCount(mask);
|
||||
SwissBucket* bucket = &buckets[begin];
|
||||
if (bucket->Keys[index] == key)
|
||||
{
|
||||
((byte*)(control_bytes + begin))[index] = (byte)SwissControl.Deleted;
|
||||
value = bucket->Values[index];
|
||||
delete_count++;
|
||||
return true;
|
||||
}
|
||||
mask &= ~(1 << index);
|
||||
}
|
||||
|
||||
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
|
||||
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
|
||||
int empty_mask = Sse2.MoveMask(cmp_empty);
|
||||
if (empty_mask != 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
begin = (begin + 1) & (GroupCount - 1);
|
||||
}
|
||||
value = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal readonly unsafe struct SwissReaderX86
|
||||
{
|
||||
|
||||
private static readonly Vector128<byte> _empty_mask = Vector128.Create((byte)SwissControl.Empty);
|
||||
private static readonly Vector128<byte> _tomb_mask = Vector128.Create((byte)SwissControl.Deleted);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static int CompareOccupiedMask(
|
||||
Vector128<byte> control_bytes_v128
|
||||
)
|
||||
{
|
||||
Vector128<byte> cmp_eql = Sse2.CompareEqual(control_bytes_v128, _empty_mask);
|
||||
Vector128<byte> cmp_del = Sse2.CompareEqual(control_bytes_v128, _tomb_mask);
|
||||
int specialMask = Sse2.MoveMask(Sse2.Or(cmp_eql, cmp_del));
|
||||
return (ushort)~(ushort)specialMask;
|
||||
}
|
||||
|
||||
public static bool TryGet(
|
||||
byte* buffer,
|
||||
AddressSource key,
|
||||
out ulong value
|
||||
)
|
||||
{
|
||||
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
|
||||
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
|
||||
|
||||
uint hash = SwissHashX64.Compute(key);
|
||||
uint begin = hash % GroupCount;
|
||||
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
|
||||
|
||||
for (int i = 0; i < GroupCount; i++)
|
||||
{
|
||||
Vector128<byte> control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)(control_bytes + begin));
|
||||
Vector128<byte> target_hash_v128 = Vector128.Create(target_hash);
|
||||
Vector128<byte> cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128);
|
||||
int mask = Sse2.MoveMask(cmp_hash);
|
||||
while (mask != 0)
|
||||
{
|
||||
int index = BitOperations.TrailingZeroCount(mask);
|
||||
SwissBucket* bucket = &buckets[begin];
|
||||
if (bucket->Keys[index] == key)
|
||||
{
|
||||
value = bucket->Values[index];
|
||||
return true;
|
||||
}
|
||||
mask &= ~(1 << index);
|
||||
}
|
||||
|
||||
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
|
||||
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
|
||||
if (Sse2.MoveMask(cmp_empty) != 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
begin = (begin + 1) & (GroupCount - 1);
|
||||
}
|
||||
value = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -19,20 +19,21 @@ namespace Nitload.Common.HighPerformance.Memory
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
private unsafe class TLSF
|
||||
private unsafe struct TLSF
|
||||
{
|
||||
|
||||
public const int MaxCacheLineCount = 8192;
|
||||
|
||||
public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * CacheLineMetadata.SizeOf;
|
||||
|
||||
private readonly short[] _bucket;
|
||||
public static readonly nuint RequestedAllocation =
|
||||
((nuint)MaxCacheLineCount + 1u) * CacheLineMetadata.SizeOf;
|
||||
private readonly CacheLineMetadata* _spareMemory;
|
||||
private readonly int _align;
|
||||
private readonly int _maxLevel;
|
||||
|
||||
private int _metadataEnd;
|
||||
private readonly Ptr64 _basePtr;
|
||||
|
||||
private fixed short _bucket[64];
|
||||
private readonly uint _maxAllocSize;
|
||||
private readonly uint _totalSize;
|
||||
|
||||
@@ -48,7 +49,9 @@ namespace Nitload.Common.HighPerformance.Memory
|
||||
_maxAllocSize = uint.Max(64, (~64U) & (uint.Min(maxBlockSize, CacheLineMetadata.MaxBufferSize) + 63U));
|
||||
_maxLevel = int.MaxValue;
|
||||
_maxLevel = GetLevel(_maxAllocSize);
|
||||
_bucket = new short[GetLevel(_maxAllocSize) + 2];
|
||||
|
||||
// _bucket = new short[GetLevel(_maxAllocSize) + 2];
|
||||
_bucket[63] = (short)(GetLevel(_maxAllocSize) + 2);
|
||||
_spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize);
|
||||
_metadataEnd = 0;
|
||||
|
||||
@@ -268,7 +271,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
||||
idx = candidate->NextLevel;
|
||||
}
|
||||
}
|
||||
for (int i = level + 1; i < _bucket.Length; i++)
|
||||
for (int i = level + 1; i < _bucket[63]; i++)
|
||||
{
|
||||
int idx = _bucket[i];
|
||||
if (idx != 0)
|
||||
|
||||
@@ -5,49 +5,20 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Nitload.Common.Collections;
|
||||
using Nitload.Common.HighPerformance.Memory;
|
||||
using Nitload.Common.HighPerformance.ProcessorAbstraction;
|
||||
using System.Data;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public sealed unsafe partial class L4 : IDisposable
|
||||
{
|
||||
|
||||
private const int CacheLineCount = 8192;
|
||||
private const nuint PageSize = 4096;
|
||||
private const int PredictionCount = 2048;
|
||||
private const int PredictionLeastAhead = 1024;
|
||||
|
||||
/// <summary>
|
||||
/// Array of metadata for each cache line, also used as metadata for tlsf allocator.
|
||||
/// </summary>
|
||||
private readonly CacheLineMetadata* _head;
|
||||
/// <summary>
|
||||
/// Count of cache lines mapped for each prediction, used for eviction policy.
|
||||
/// </summary>
|
||||
private readonly int* _mapCount;
|
||||
/// <summary>
|
||||
/// Count of times each cache line is predicted, used for eviction policy.
|
||||
/// </summary>
|
||||
private readonly int* _predictCount;
|
||||
/// <summary>
|
||||
/// Original memory view for prediction span.
|
||||
/// </summary>
|
||||
private readonly CacheLineDescription* _prediction;
|
||||
private bool _disposed;
|
||||
private bool _exhausted;
|
||||
|
||||
private int _aheadCount;
|
||||
|
||||
private readonly int _bindedNuma;
|
||||
private volatile int _prefetching;
|
||||
private readonly nuint _baseAddress;
|
||||
private readonly SwissTable _map;
|
||||
|
||||
private readonly TLSF _bufferAllocator;
|
||||
private SwissTable _addressMap;
|
||||
private TLSF _bufferAllocator;
|
||||
|
||||
public L4(
|
||||
int bindedNuma,
|
||||
@@ -59,130 +30,70 @@ namespace Nitload.Common.HighPerformance.Memory
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize);
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize);
|
||||
_bindedNuma = bindedNuma;
|
||||
|
||||
// init order is fallow by align requirement of each component.
|
||||
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineMetadata>()) * CacheLineCount; // 16 byte align
|
||||
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
|
||||
nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
|
||||
nuint predictionCountSize = sizeof(int) * CacheLineCount; // 16 byte align
|
||||
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
|
||||
_predict_slow_path_count = GetPredictSlowPathCount(bindedNuma);
|
||||
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1));
|
||||
ByteCapacity = normalizedBufferSize;
|
||||
|
||||
AllocSize = headSize + mapCountSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize;
|
||||
|
||||
// malloc all memory at once, and free at once in Dispose.
|
||||
// This is to avoid fragmentation and improve performance.
|
||||
nuint swissBufferOffset = normalizedBufferSize + TLSF.RequestedAllocation;
|
||||
AllocSize = swissBufferOffset + (nuint)BufferSize;
|
||||
_baseAddress = (nuint)Numa.Alloc(bindedNuma, AllocSize);
|
||||
nuint ptr = _baseAddress;
|
||||
|
||||
// init all components with the buffer, and move the buffer pointer forward.
|
||||
nuint _buffer = _baseAddress;
|
||||
_bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress);
|
||||
_buffer += normalizedBufferSize;
|
||||
_head = (CacheLineMetadata*)ptr;
|
||||
ptr += headSize;
|
||||
_prediction = (CacheLineDescription*)ptr;
|
||||
ptr += predictionSize;
|
||||
_map = SwissTable.Create(ptr, _ => { });
|
||||
ptr += mapCountSize;
|
||||
_predictCount = (int*)ptr;
|
||||
ptr += predictionCountSize;
|
||||
_mapCount = (int*)ptr;
|
||||
ptr += predictionCountSize;
|
||||
|
||||
M512* mPtr = (M512*)_head;
|
||||
for (uint i = 1; i < CacheLineCount; i++) {
|
||||
mPtr->BroadCastFrom<ulong>(0);
|
||||
}
|
||||
_addressMap = SwissTable.Create((byte*)(_baseAddress + swissBufferOffset));
|
||||
}
|
||||
|
||||
public nuint ByteCapacity { get; init; }
|
||||
|
||||
public nuint AllocSize { get; init; }
|
||||
|
||||
// public IL4Predict PrefetchPredict { get; set; }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ContinueOnce(
|
||||
Handle line
|
||||
)
|
||||
{
|
||||
if (line.Address > 0) {
|
||||
_ = Interlocked.Decrement(ref _aheadCount);
|
||||
}
|
||||
}
|
||||
|
||||
[System.Diagnostics.CodeAnalysis.SuppressMessage(
|
||||
"Design",
|
||||
"CA1063:Implement IDisposable correctly",
|
||||
Justification = """
|
||||
Only owns native memory via Numa.Free();
|
||||
no managed IDisposable fields.
|
||||
Dispose(bool) pattern is unnecessary.
|
||||
""")]
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed) {
|
||||
return;
|
||||
}
|
||||
_disposed = true;
|
||||
_map.Dispose();
|
||||
#pragma warning disable CA1816
|
||||
GC.SuppressFinalize(_bufferAllocator);
|
||||
#pragma warning restore CA1816
|
||||
|
||||
|
||||
// init order is fallow by align requirement of each component.
|
||||
_addressMap.Dispose();
|
||||
Numa.Free((void*)_baseAddress, _bindedNuma, AllocSize);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public Handle Map(
|
||||
ref CacheLineDescription prediction
|
||||
)
|
||||
public partial Handle Map(
|
||||
AddressSource source
|
||||
);
|
||||
|
||||
internal void Collect()
|
||||
{
|
||||
if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (_aheadCount < PredictionLeastAhead)
|
||||
{
|
||||
// do prefetch in work load now, sync
|
||||
Prefetch();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_prefetching = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
while (true)
|
||||
{
|
||||
if (_map.TryGet(prediction.GetHashCodeLong(), out nuint entryHandle))
|
||||
{
|
||||
// cache hit
|
||||
CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle;
|
||||
if (Interlocked.Increment(ref entry->RefCount) > 0)
|
||||
{
|
||||
_predictCount[entry - _head] = 0;
|
||||
_ = Interlocked.Decrement(ref _aheadCount);
|
||||
_ = Interlocked.Decrement(ref _mapCount[entry - _head]);
|
||||
nuint* hPtr = stackalloc nuint[2] { entry->PositionMask + _baseAddress, (nuint)entry };
|
||||
return *(Handle*)hPtr;
|
||||
} else
|
||||
{
|
||||
_ = Interlocked.Decrement(ref entry->RefCount);
|
||||
}
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
_bufferAllocator.Collect();
|
||||
}
|
||||
|
||||
private partial void Prefetch();
|
||||
internal void Free(
|
||||
int handle
|
||||
)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
_bufferAllocator.Free(handle);
|
||||
}
|
||||
|
||||
internal bool TryAllocate(
|
||||
uint size,
|
||||
out int handle
|
||||
)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
return _bufferAllocator.TryAlloc(size, out handle);
|
||||
}
|
||||
|
||||
internal partial int Predict();
|
||||
|
||||
private static int GetPredictSlowPathCount(int bindedNuma)
|
||||
{
|
||||
foreach (NumaNode node in Numa.AllNumaNode)
|
||||
{
|
||||
if (node.NumaNodeId == (uint)bindedNuma)
|
||||
return checked(node.LogicalCoreCount * 2);
|
||||
}
|
||||
|
||||
throw new ArgumentOutOfRangeException(nameof(bindedNuma), "No topology information exists for the bound NUMA node.");
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4Predictor"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public interface IL4Predictor
|
||||
{
|
||||
|
||||
int Id { get; set; }
|
||||
|
||||
int PredictMission(
|
||||
in Span<L4.AddressSource> RequiredBuffer
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
public interface IL4Predictor<TMission> : IL4Predictor
|
||||
{
|
||||
|
||||
bool TryGetNextMission(
|
||||
out TMission mission
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
internal class PredictorMonitor { }
|
||||
}
|
||||
@@ -16,22 +16,21 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
|
||||
public readonly struct LogicalCore
|
||||
{
|
||||
|
||||
public readonly uint GlobalCpuId;
|
||||
public readonly uint NumaGroupId;
|
||||
|
||||
/// <summary>
|
||||
/// Stays 0 on linux.
|
||||
/// </summary>
|
||||
public readonly uint ProcessorGroupId;
|
||||
public readonly uint SocketId;
|
||||
public readonly uint CpuIndex;
|
||||
public readonly uint NumaNodeId;
|
||||
|
||||
internal LogicalCore(
|
||||
uint globalCpuId,
|
||||
uint numaGroupId,
|
||||
uint processorGroupId,
|
||||
uint socketId,
|
||||
uint cpuIndex,
|
||||
uint numaNodeId
|
||||
)
|
||||
{
|
||||
GlobalCpuId = globalCpuId;
|
||||
NumaGroupId = numaGroupId;
|
||||
ProcessorGroupId = processorGroupId;
|
||||
SocketId = socketId;
|
||||
CpuIndex = cpuIndex;
|
||||
NumaNodeId = numaNodeId;
|
||||
}
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
|
||||
if (_numaNodes.Length == 0) {
|
||||
throw new InvalidOperationException("Numa info has not been loaded.");
|
||||
}
|
||||
if ((numaId < 0) || (numaId >= _numaNodes.Length)) {
|
||||
if ((numaId < 0) || !ContainsNode((uint)numaId)) {
|
||||
throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID.");
|
||||
}
|
||||
return NumaAllocNative(size, numaId);
|
||||
@@ -69,22 +69,24 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
|
||||
if (_numaNodes.Length == 0) {
|
||||
throw new InvalidOperationException("Numa info has not been loaded.");
|
||||
}
|
||||
if ((numaId < 0) || (numaId >= _numaNodes.Length)) {
|
||||
if ((numaId < 0) || !ContainsNode((uint)numaId)) {
|
||||
throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID.");
|
||||
}
|
||||
NumaFreeNative(address, size, numaId);
|
||||
}
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_get_numa_each")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_get_numa_each")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
public static partial int GetNumaEach(
|
||||
uint numaId,
|
||||
Span<ulong> CpuIndexInNuma,
|
||||
Span<uint> globalCpuIdInNuma
|
||||
Span<uint> processorGroupIds,
|
||||
Span<uint> socketIds,
|
||||
Span<uint> cpuIndices,
|
||||
Span<uint> numaNodeIds
|
||||
);
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_get_numa_overall")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_get_numa_overall")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
public static partial int GetNumaOverall(
|
||||
@@ -97,10 +99,10 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
|
||||
if (_numaNodes.Length == 0) {
|
||||
throw new InvalidOperationException("Numa info has not been loaded.");
|
||||
}
|
||||
L4[] buffers = new L4[NumaNodeCount];
|
||||
for (int i = 0; i < NumaNodeCount; i++)
|
||||
L4[] buffers = new L4[_numaNodes.Length];
|
||||
for (int i = 0; i < _numaNodes.Length; i++)
|
||||
{
|
||||
L4 buffer = new L4(i);
|
||||
L4 buffer = new L4((int)_numaNodes[i].NumaNodeId);
|
||||
buffers[i] = buffer;
|
||||
}
|
||||
return buffers;
|
||||
@@ -108,23 +110,38 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
|
||||
|
||||
internal static void Initialize()
|
||||
{
|
||||
Span<int> coreCountInNuma = stackalloc int[(Environment.ProcessorCount / 32) + 1];
|
||||
int numaNodeCount = 0;
|
||||
if (0 == GetNumaOverall(ref numaNodeCount, coreCountInNuma)) {
|
||||
if (0 == GetNumaOverall(ref numaNodeCount, default)) {
|
||||
throw new InvalidOperationException("Failed to get NUMA overall information.");
|
||||
}
|
||||
_numaNodes = new NumaNode[numaNodeCount];
|
||||
for (uint i = 0; i < numaNodeCount; i++) {
|
||||
_numaNodes[i] = InitNumaNode(i, coreCountInNuma[(int)i]);
|
||||
int[] coreCountInNuma = new int[numaNodeCount];
|
||||
if (0 == GetNumaOverall(ref numaNodeCount, coreCountInNuma)) {
|
||||
throw new InvalidOperationException("Failed to get NUMA node processor counts.");
|
||||
}
|
||||
int activeNodeCount = 0;
|
||||
for (int i = 0; i < coreCountInNuma.Length; i++) {
|
||||
if (coreCountInNuma[i] > 0) {
|
||||
activeNodeCount++;
|
||||
}
|
||||
}
|
||||
NumaNodeCount = activeNodeCount;
|
||||
_numaNodes = new NumaNode[activeNodeCount];
|
||||
int nodeIndex = 0;
|
||||
for (uint numaId = 0; numaId < numaNodeCount; numaId++) {
|
||||
int coreCount = coreCountInNuma[(int)numaId];
|
||||
if (coreCount == 0) {
|
||||
continue;
|
||||
}
|
||||
_numaNodes[nodeIndex++] = InitNumaNode(numaId, coreCount);
|
||||
}
|
||||
}
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_set_thread_affinity")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_set_thread_affinity")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
internal static partial void SetThreadAffinity(
|
||||
int coreId,
|
||||
int numaId
|
||||
int cpuIndex,
|
||||
int processorGroupId
|
||||
);
|
||||
|
||||
[MethodImpl(MethodImplOptions.NoInlining)]
|
||||
@@ -133,12 +150,27 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
|
||||
int count
|
||||
)
|
||||
{
|
||||
Span<uint> globalIndex = stackalloc uint[count];
|
||||
_ = GetNumaEach(numaId, null, globalIndex);
|
||||
return new(numaId, count, globalIndex);
|
||||
Span<uint> processorGroupIds = stackalloc uint[count];
|
||||
Span<uint> socketIds = stackalloc uint[count];
|
||||
Span<uint> cpuIndices = stackalloc uint[count];
|
||||
Span<uint> numaNodeIds = stackalloc uint[count];
|
||||
if (0 == GetNumaEach(numaId, processorGroupIds, socketIds, cpuIndices, numaNodeIds)) {
|
||||
throw new InvalidOperationException($"Failed to enumerate processors for NUMA node {numaId}.");
|
||||
}
|
||||
return new(numaId, count, processorGroupIds, socketIds, cpuIndices, numaNodeIds);
|
||||
}
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_numa_alloc")]
|
||||
private static bool ContainsNode(uint numaId)
|
||||
{
|
||||
foreach (NumaNode node in _numaNodes) {
|
||||
if (node.NumaNodeId == numaId) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_numa_alloc")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static unsafe partial void* NumaAllocNative(
|
||||
@@ -146,7 +178,7 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
|
||||
int numaId
|
||||
);
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_numa_free")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_numa_free")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static unsafe partial void NumaFreeNative(
|
||||
|
||||
@@ -22,16 +22,26 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
|
||||
internal NumaNode(
|
||||
uint numaId,
|
||||
int count,
|
||||
Span<uint> logicalCoreIds
|
||||
Span<uint> processorGroupIds,
|
||||
Span<uint> socketIds,
|
||||
Span<uint> cpuIndices,
|
||||
Span<uint> numaNodeIds
|
||||
|
||||
)
|
||||
{
|
||||
NumaNodeId = numaId;
|
||||
_logicalCores = new LogicalCore[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
_logicalCores[i] = new LogicalCore(1, numaId, logicalCoreIds[i]);
|
||||
_logicalCores[i] = new LogicalCore(
|
||||
processorGroupIds[i],
|
||||
socketIds[i],
|
||||
cpuIndices[i],
|
||||
numaNodeIds[i]);
|
||||
}
|
||||
}
|
||||
|
||||
public uint NumaNodeId { get; }
|
||||
|
||||
public int LogicalCoreCount => _logicalCores.Length;
|
||||
|
||||
public ReadOnlySpan<LogicalCore> LogicalCores => _logicalCores;
|
||||
|
||||
@@ -37,7 +37,7 @@ namespace Nitload.Threading
|
||||
LogicalCore core
|
||||
)
|
||||
{
|
||||
SetThreadAffinity(core.GlobalCpuId, core.NumaNodeId);
|
||||
SetThreadAffinity(core.CpuIndex, core.ProcessorGroupId);
|
||||
}
|
||||
|
||||
[Conditional("DEBUG")]
|
||||
@@ -51,12 +51,12 @@ namespace Nitload.Threading
|
||||
|
||||
#pragma warning disable CA5393
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_set_thread_affinity")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_set_thread_affinity")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static partial void SetThreadAffinity(
|
||||
uint globalCoreId,
|
||||
uint numaNodeId
|
||||
uint cpuIndex,
|
||||
uint processorGroupId
|
||||
);
|
||||
|
||||
#pragma warning restore CA5393
|
||||
|
||||
@@ -97,7 +97,7 @@ namespace Nitload
|
||||
}
|
||||
|
||||
Thread.Sleep(10);
|
||||
Numa.SetThreadAffinity((int)threadConfig.BindedCoreIndex, (int)threadConfig.BindedNumaNode);
|
||||
Numa.SetThreadAffinity((int)threadConfig.CpuIndex, (int)threadConfig.ProcessorGroupId);
|
||||
while (mainProvider.IsRunning)
|
||||
{
|
||||
long elapsedTicks;
|
||||
|
||||
@@ -14,15 +14,15 @@ using System.Threading;
|
||||
namespace Nitload.Common.Timing
|
||||
{
|
||||
public class TimeSourceStartupConfiguration(
|
||||
nuint bindedNumaNode = 0,
|
||||
nuint bindedCoreIndex = 0,
|
||||
nuint processorGroupId = 0,
|
||||
nuint cpuIndex = 0,
|
||||
ThreadPriority threadPriority = ThreadPriority.Highest
|
||||
)
|
||||
{
|
||||
|
||||
public nuint BindedNumaNode { get; } = bindedNumaNode;
|
||||
public nuint ProcessorGroupId { get; } = processorGroupId;
|
||||
|
||||
public nuint BindedCoreIndex { get; } = bindedCoreIndex;
|
||||
public nuint CpuIndex { get; } = cpuIndex;
|
||||
|
||||
public ThreadPriority Priority { get; } = threadPriority;
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ namespace Nitload.Graphics
|
||||
|
||||
public static GLFWFunction glfwContext => _glfw;
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_load_glfw_functions")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_load_glfw_functions")]
|
||||
internal static unsafe partial void glfwLoadLibrary(
|
||||
GLFWFunction* context
|
||||
);
|
||||
|
||||
@@ -170,7 +170,7 @@ namespace Nitload.Mathematics.Numeric
|
||||
|
||||
#region C invoke
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_compare_simd")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_compare_simd")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static partial int CompareSIMDID(
|
||||
@@ -178,7 +178,7 @@ namespace Nitload.Mathematics.Numeric
|
||||
SIMDID right
|
||||
);
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_load_intrinsic_function")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_load_intrinsic_function")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static partial void LoadIntrinsicFunction(
|
||||
@@ -186,7 +186,7 @@ namespace Nitload.Mathematics.Numeric
|
||||
ulong id
|
||||
);
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_get_unite_simd")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_get_unite_simd")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static partial SIMDID UniteSIMDID(
|
||||
|
||||
@@ -155,12 +155,12 @@ namespace Nitload.Mathematics.Numeric
|
||||
}
|
||||
|
||||
#pragma warning disable CA5393
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_get_simd_level_local")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_get_simd_level_local")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
internal static partial ulong GetSimdId();
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_load_intrinsic_function")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_load_intrinsic_function")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static unsafe partial void LoadIntrinsicFunction(
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
CompileFlags:
|
||||
CompilationDatabase: ../..
|
||||
CompilationDatabase: .
|
||||
|
||||
@@ -3,91 +3,68 @@ project(Nitload.Native LANGUAGES C)
|
||||
|
||||
include(${CMAKE_CURRENT_LIST_DIR}/ExternalLibs.cmake OPTIONAL)
|
||||
|
||||
set(NIF_TARGET_OS "auto" CACHE STRING "Target OS identifier: auto, windows, linux, macos, ios, android, unknown")
|
||||
set_property(CACHE NIF_TARGET_OS PROPERTY STRINGS auto windows linux macos ios android unknown)
|
||||
set(NFN_TARGET_OS "auto" CACHE STRING "Target OS identifier: auto, windows, linux, macos, ios, android, unknown")
|
||||
set_property(CACHE NFN_TARGET_OS PROPERTY STRINGS auto windows linux macos ios android unknown)
|
||||
|
||||
set(NIF_TARGET_ARCH "auto" CACHE STRING "Target ISA identifier: auto, x86_64, arm64, riscv64, ppc64le")
|
||||
set_property(CACHE NIF_TARGET_ARCH PROPERTY STRINGS auto x86_64 arm64 riscv64 ppc64le)
|
||||
set(NFN_TARGET_ARCH "auto" CACHE STRING "Target ISA identifier: auto, x86_64, arm64, riscv64, ppc64le")
|
||||
set_property(CACHE NFN_TARGET_ARCH PROPERTY STRINGS auto x86_64 arm64 riscv64 ppc64le)
|
||||
|
||||
nif_resolve_target_identifiers(
|
||||
"${NIF_TARGET_OS}"
|
||||
"${NIF_TARGET_ARCH}"
|
||||
NIF_RESOLVED_OS_NAME
|
||||
NIF_RESOLVED_OS_DEFINE
|
||||
NIF_RESOLVED_ARCH_NAME
|
||||
NIF_RESOLVED_ARCH_DEFINE)
|
||||
nfn_resolve_target_identifiers(
|
||||
"${NFN_TARGET_OS}"
|
||||
"${NFN_TARGET_ARCH}"
|
||||
NFN_RESOLVED_OS_NAME
|
||||
NFN_RESOLVED_OS_DEFINE
|
||||
NFN_RESOLVED_ARCH_NAME
|
||||
NFN_RESOLVED_ARCH_DEFINE)
|
||||
|
||||
set(NIF_NATIVE_SOURCES
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/dllmain.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/load_glfw.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/load_intrinsic.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/m.bufop.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/m.solve.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/m.transpose.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/mk.fma.pack.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/mk.fma.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/mk.p_scan.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/s.factorial.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/s.hash.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/simdid_compare.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/simdid.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/sk.fma.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/thread.linux.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/thread.windows.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/vector.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/vector_dot.c)
|
||||
# CONFIGURE_DEPENDS makes CMake reconfigure when files are added or removed,
|
||||
# so native sources, headers, and templates need no per-file CMake edits.
|
||||
file(GLOB_RECURSE NFN_NATIVE_SOURCES CONFIGURE_DEPENDS
|
||||
${CMAKE_CURRENT_LIST_DIR}/nfn-src/*.c)
|
||||
|
||||
set(NIF_NATIVE_HEADERS
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/glfw_function.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/nif_internal.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/nif_intrinsic.context.base.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/nif_intrinsic.context.x86.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/nif_intrinsic.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/nif_intrinsic.matkernel.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/nif_intrinsic.std.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/nif_target.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/NitloadFrameWork.h)
|
||||
file(GLOB_RECURSE NFN_NATIVE_HEADERS CONFIGURE_DEPENDS
|
||||
${CMAKE_CURRENT_LIST_DIR}/nfn-head/*.h)
|
||||
|
||||
set(NIF_NATIVE_TEMPLATES
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-template/nif_template.fma.x86.h)
|
||||
file(GLOB_RECURSE NFN_NATIVE_TEMPLATES CONFIGURE_DEPENDS
|
||||
${CMAKE_CURRENT_LIST_DIR}/nfn-template/*.h)
|
||||
|
||||
# Nitload.Native now lives under <root>/src/Nitload.Native, so walk up two
|
||||
# levels (project dir -> src -> solution root) to reach the solution root.
|
||||
get_filename_component(NIF_NATIVE_SRC_ROOT ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
|
||||
get_filename_component(NIF_SOLUTION_ROOT ${NIF_NATIVE_SRC_ROOT} DIRECTORY)
|
||||
get_filename_component(NFN_NATIVE_SRC_ROOT ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
|
||||
get_filename_component(NFN_SOLUTION_ROOT ${NFN_NATIVE_SRC_ROOT} DIRECTORY)
|
||||
|
||||
if(NIF_RESOLVED_OS_NAME STREQUAL "windows")
|
||||
set(NIF_OS_TAG win)
|
||||
elseif(NIF_RESOLVED_OS_NAME STREQUAL "linux")
|
||||
set(NIF_OS_TAG linux)
|
||||
if(NFN_RESOLVED_OS_NAME STREQUAL "windows")
|
||||
set(NFN_OS_TAG win)
|
||||
elseif(NFN_RESOLVED_OS_NAME STREQUAL "linux")
|
||||
set(NFN_OS_TAG linux)
|
||||
else()
|
||||
set(NIF_OS_TAG unknown)
|
||||
set(NFN_OS_TAG unknown)
|
||||
endif()
|
||||
|
||||
if(NIF_RESOLVED_ARCH_NAME STREQUAL "x86_64")
|
||||
set(NIF_ARCH_TAG x64)
|
||||
if(NFN_RESOLVED_ARCH_NAME STREQUAL "x86_64")
|
||||
set(NFN_ARCH_TAG x64)
|
||||
else()
|
||||
set(NIF_ARCH_TAG ${NIF_RESOLVED_ARCH_NAME})
|
||||
set(NFN_ARCH_TAG ${NFN_RESOLVED_ARCH_NAME})
|
||||
endif()
|
||||
|
||||
set(NIF_EXPECTED_RID "" CACHE STRING "Optional managed target RID to validate before deployment")
|
||||
if(NIF_EXPECTED_RID AND NOT NIF_EXPECTED_RID STREQUAL "${NIF_OS_TAG}-${NIF_ARCH_TAG}")
|
||||
message(FATAL_ERROR "Native target ${NIF_OS_TAG}-${NIF_ARCH_TAG} does not match managed target ${NIF_EXPECTED_RID}. Use a matching compiler/toolchain and a separate build directory.")
|
||||
set(NFN_EXPECTED_RID "" CACHE STRING "Optional managed target RID to validate before deployment")
|
||||
if(NFN_EXPECTED_RID AND NOT NFN_EXPECTED_RID STREQUAL "${NFN_OS_TAG}-${NFN_ARCH_TAG}")
|
||||
message(FATAL_ERROR "Native target ${NFN_OS_TAG}-${NFN_ARCH_TAG} does not match managed target ${NFN_EXPECTED_RID}. Use a matching compiler/toolchain and a separate build directory.")
|
||||
endif()
|
||||
|
||||
set(NIF_BUILD_OUTPUT_DIR "${NIF_SOLUTION_ROOT}/bin/${NIF_OS_TAG}-${NIF_ARCH_TAG}"
|
||||
set(NFN_BUILD_OUTPUT_DIR "${NFN_SOLUTION_ROOT}/bin/${NFN_OS_TAG}-${NFN_ARCH_TAG}"
|
||||
CACHE PATH "Native artifact directory shared with managed build deployment")
|
||||
# Compile-time intermediates (native PDB / .obj side files) live inside the CMake
|
||||
# build tree (cmake_build/<preset>) so they get cleaned up with it.
|
||||
set(NIF_OBJECT_ROOT_DIR ${CMAKE_CURRENT_BINARY_DIR}/obj/${PROJECT_NAME})
|
||||
set(NIF_NATIVE_PROJECT_FILES
|
||||
${NIF_NATIVE_SOURCES}
|
||||
${NIF_NATIVE_HEADERS}
|
||||
${NIF_NATIVE_TEMPLATES})
|
||||
set(NFN_OBJECT_ROOT_DIR ${CMAKE_CURRENT_BINARY_DIR}/obj/${PROJECT_NAME})
|
||||
set(NFN_NATIVE_PROJECT_FILES
|
||||
${NFN_NATIVE_SOURCES}
|
||||
${NFN_NATIVE_HEADERS}
|
||||
${NFN_NATIVE_TEMPLATES})
|
||||
|
||||
add_library(${PROJECT_NAME} SHARED ${NIF_NATIVE_PROJECT_FILES})
|
||||
add_library(${PROJECT_NAME} SHARED ${NFN_NATIVE_PROJECT_FILES})
|
||||
|
||||
source_group(TREE ${CMAKE_CURRENT_LIST_DIR} FILES ${NIF_NATIVE_PROJECT_FILES})
|
||||
source_group(TREE ${CMAKE_CURRENT_LIST_DIR} FILES ${NFN_NATIVE_PROJECT_FILES})
|
||||
|
||||
set_target_properties(${PROJECT_NAME} PROPERTIES
|
||||
C_STANDARD 11
|
||||
@@ -98,10 +75,10 @@ set_target_properties(${PROJECT_NAME} PROPERTIES
|
||||
RELEASE_POSTFIX ""
|
||||
RELWITHDEBINFO_POSTFIX ""
|
||||
MINSIZEREL_POSTFIX ""
|
||||
RUNTIME_OUTPUT_DIRECTORY "${NIF_BUILD_OUTPUT_DIR}"
|
||||
LIBRARY_OUTPUT_DIRECTORY "${NIF_BUILD_OUTPUT_DIR}"
|
||||
ARCHIVE_OUTPUT_DIRECTORY "${NIF_BUILD_OUTPUT_DIR}"
|
||||
PDB_OUTPUT_DIRECTORY "${NIF_BUILD_OUTPUT_DIR}")
|
||||
RUNTIME_OUTPUT_DIRECTORY "${NFN_BUILD_OUTPUT_DIR}"
|
||||
LIBRARY_OUTPUT_DIRECTORY "${NFN_BUILD_OUTPUT_DIR}"
|
||||
ARCHIVE_OUTPUT_DIRECTORY "${NFN_BUILD_OUTPUT_DIR}"
|
||||
PDB_OUTPUT_DIRECTORY "${NFN_BUILD_OUTPUT_DIR}")
|
||||
|
||||
# P/Invoke loads Nitload.Native.dll on Windows, including GNU-style Clang toolchains.
|
||||
if(WIN32)
|
||||
@@ -111,22 +88,22 @@ endif()
|
||||
foreach(_config Debug Release RelWithDebInfo MinSizeRel)
|
||||
string(TOUPPER ${_config} _config_upper)
|
||||
set_target_properties(${PROJECT_NAME} PROPERTIES
|
||||
RUNTIME_OUTPUT_DIRECTORY_${_config_upper} ${NIF_BUILD_OUTPUT_DIR}
|
||||
LIBRARY_OUTPUT_DIRECTORY_${_config_upper} ${NIF_BUILD_OUTPUT_DIR}
|
||||
ARCHIVE_OUTPUT_DIRECTORY_${_config_upper} ${NIF_BUILD_OUTPUT_DIR}
|
||||
PDB_OUTPUT_DIRECTORY_${_config_upper} ${NIF_BUILD_OUTPUT_DIR}
|
||||
COMPILE_PDB_OUTPUT_DIRECTORY_${_config_upper} ${NIF_OBJECT_ROOT_DIR}/${_config}/${NIF_ARCH_TAG})
|
||||
RUNTIME_OUTPUT_DIRECTORY_${_config_upper} ${NFN_BUILD_OUTPUT_DIR}
|
||||
LIBRARY_OUTPUT_DIRECTORY_${_config_upper} ${NFN_BUILD_OUTPUT_DIR}
|
||||
ARCHIVE_OUTPUT_DIRECTORY_${_config_upper} ${NFN_BUILD_OUTPUT_DIR}
|
||||
PDB_OUTPUT_DIRECTORY_${_config_upper} ${NFN_BUILD_OUTPUT_DIR}
|
||||
COMPILE_PDB_OUTPUT_DIRECTORY_${_config_upper} ${NFN_OBJECT_ROOT_DIR}/${_config}/${NFN_ARCH_TAG})
|
||||
endforeach()
|
||||
|
||||
set_target_properties(${PROJECT_NAME} PROPERTIES
|
||||
COMPILE_PDB_OUTPUT_DIRECTORY ${NIF_OBJECT_ROOT_DIR}/$<CONFIG>/${NIF_ARCH_TAG})
|
||||
COMPILE_PDB_OUTPUT_DIRECTORY ${NFN_OBJECT_ROOT_DIR}/$<CONFIG>/${NFN_ARCH_TAG})
|
||||
|
||||
if(MSVC OR (CMAKE_C_COMPILER_ID STREQUAL "Clang" AND CMAKE_C_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC"))
|
||||
target_compile_options(${PROJECT_NAME} PRIVATE
|
||||
/Fd${NIF_OBJECT_ROOT_DIR}/$<CONFIG>/${NIF_ARCH_TAG}/)
|
||||
/Fd${NFN_OBJECT_ROOT_DIR}/$<CONFIG>/${NFN_ARCH_TAG}/)
|
||||
endif()
|
||||
|
||||
if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND NIF_RESOLVED_ARCH_NAME STREQUAL "x86_64")
|
||||
if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND NFN_RESOLVED_ARCH_NAME STREQUAL "x86_64")
|
||||
# Keep the intrinsic type system and builtin declarations visible to all TUs.
|
||||
# The runtime dispatch contract still follows T_arch/T_feature at the function level.
|
||||
if(CMAKE_C_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC")
|
||||
@@ -156,19 +133,19 @@ endif()
|
||||
|
||||
target_include_directories(${PROJECT_NAME}
|
||||
PRIVATE
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-template
|
||||
${CMAKE_CURRENT_LIST_DIR}/nfn-head
|
||||
${CMAKE_CURRENT_LIST_DIR}/nfn-template
|
||||
${CMAKE_CURRENT_LIST_DIR}/lib)
|
||||
|
||||
target_compile_definitions(${PROJECT_NAME}
|
||||
PRIVATE
|
||||
NIF_EXTERN_TARGET
|
||||
NIF_OS_TARGET=${NIF_RESOLVED_OS_DEFINE}
|
||||
NIF_ARCH_TARGET=${NIF_RESOLVED_ARCH_DEFINE})
|
||||
NFN_EXTERN_TARGET
|
||||
NFN_OS_TARGET=${NFN_RESOLVED_OS_DEFINE}
|
||||
NFN_ARCH_TARGET=${NFN_RESOLVED_ARCH_DEFINE})
|
||||
|
||||
set(NIF_NATIVE_EXTERNAL_TARGET ${PROJECT_NAME})
|
||||
set(NFN_NATIVE_EXTERNAL_TARGET ${PROJECT_NAME})
|
||||
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/lib)
|
||||
|
||||
message(STATUS "Nitload.Native target OS: ${NIF_RESOLVED_OS_NAME} (${NIF_RESOLVED_OS_DEFINE})")
|
||||
message(STATUS "Nitload.Native target ISA: ${NIF_RESOLVED_ARCH_NAME} (${NIF_RESOLVED_ARCH_DEFINE})")
|
||||
message(STATUS "Nitload.Native output dir: ${NIF_BUILD_OUTPUT_DIR}")
|
||||
message(STATUS "Nitload.Native target OS: ${NFN_RESOLVED_OS_NAME} (${NFN_RESOLVED_OS_DEFINE})")
|
||||
message(STATUS "Nitload.Native target ISA: ${NFN_RESOLVED_ARCH_NAME} (${NFN_RESOLVED_ARCH_DEFINE})")
|
||||
message(STATUS "Nitload.Native output dir: ${NFN_BUILD_OUTPUT_DIR}")
|
||||
|
||||
@@ -9,8 +9,7 @@
|
||||
"cacheVariables": {
|
||||
"CMAKE_BUILD_TYPE": "Debug",
|
||||
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
|
||||
"CMAKE_C_COMPILER": "clang",
|
||||
"CMAKE_CXX_COMPILER": "clang++"
|
||||
"CMAKE_C_COMPILER": "clang"
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -21,8 +20,7 @@
|
||||
"cacheVariables": {
|
||||
"CMAKE_BUILD_TYPE": "Release",
|
||||
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
|
||||
"CMAKE_C_COMPILER": "clang",
|
||||
"CMAKE_CXX_COMPILER": "clang++"
|
||||
"CMAKE_C_COMPILER": "clang"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
||||
@@ -1,74 +1,74 @@
|
||||
function(nif_normalize_os_identifier INPUT_VALUE OUTPUT_NAME OUTPUT_DEFINE)
|
||||
string(TOLOWER "${INPUT_VALUE}" _nif_os)
|
||||
function(nfn_normalize_os_identifier INPUT_VALUE OUTPUT_NAME OUTPUT_DEFINE)
|
||||
string(TOLOWER "${INPUT_VALUE}" _nfn_os)
|
||||
|
||||
if(_nif_os STREQUAL "auto")
|
||||
string(TOLOWER "${CMAKE_SYSTEM_NAME}" _nif_system_name)
|
||||
if(_nif_system_name STREQUAL "windows")
|
||||
set(_nif_os "windows")
|
||||
elseif(_nif_system_name STREQUAL "linux")
|
||||
set(_nif_os "linux")
|
||||
if(_nfn_os STREQUAL "auto")
|
||||
string(TOLOWER "${CMAKE_SYSTEM_NAME}" _nfn_system_name)
|
||||
if(_nfn_system_name STREQUAL "windows")
|
||||
set(_nfn_os "windows")
|
||||
elseif(_nfn_system_name STREQUAL "linux")
|
||||
set(_nfn_os "linux")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(_nif_os STREQUAL "windows")
|
||||
set(_nif_define NIF_OS_WINDOWS)
|
||||
elseif(_nif_os STREQUAL "linux")
|
||||
set(_nif_define NIF_OS_LINUX)
|
||||
if(_nfn_os STREQUAL "windows")
|
||||
set(_nfn_define NFN_OS_WINDOWS)
|
||||
elseif(_nfn_os STREQUAL "linux")
|
||||
set(_nfn_define NFN_OS_LINUX)
|
||||
else()
|
||||
message(FATAL_ERROR "Unsupported NIF_TARGET_OS='${INPUT_VALUE}'.")
|
||||
message(FATAL_ERROR "Unsupported NFN_TARGET_OS='${INPUT_VALUE}'.")
|
||||
endif()
|
||||
|
||||
set(${OUTPUT_NAME} "${_nif_os}" PARENT_SCOPE)
|
||||
set(${OUTPUT_DEFINE} "${_nif_define}" PARENT_SCOPE)
|
||||
set(${OUTPUT_NAME} "${_nfn_os}" PARENT_SCOPE)
|
||||
set(${OUTPUT_DEFINE} "${_nfn_define}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
function(nif_normalize_arch_identifier INPUT_VALUE OUTPUT_NAME OUTPUT_DEFINE)
|
||||
string(TOLOWER "${INPUT_VALUE}" _nif_arch)
|
||||
function(nfn_normalize_arch_identifier INPUT_VALUE OUTPUT_NAME OUTPUT_DEFINE)
|
||||
string(TOLOWER "${INPUT_VALUE}" _nfn_arch)
|
||||
|
||||
if(_nif_arch STREQUAL "auto")
|
||||
string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _nif_processor)
|
||||
if(_nif_processor MATCHES "^(amd64|x86_64|x64)$")
|
||||
set(_nif_arch "x86_64")
|
||||
elseif(_nif_processor MATCHES "^(arm64|aarch64)$")
|
||||
set(_nif_arch "arm64")
|
||||
elseif(_nif_processor STREQUAL "riscv64")
|
||||
set(_nif_arch "riscv64")
|
||||
elseif(_nif_processor MATCHES "^(ppc64le|powerpc64le)$")
|
||||
set(_nif_arch "ppc64le")
|
||||
if(_nfn_arch STREQUAL "auto")
|
||||
string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _nfn_processor)
|
||||
if(_nfn_processor MATCHES "^(amd64|x86_64|x64)$")
|
||||
set(_nfn_arch "x86_64")
|
||||
elseif(_nfn_processor MATCHES "^(arm64|aarch64)$")
|
||||
set(_nfn_arch "arm64")
|
||||
elseif(_nfn_processor STREQUAL "riscv64")
|
||||
set(_nfn_arch "riscv64")
|
||||
elseif(_nfn_processor MATCHES "^(ppc64le|powerpc64le)$")
|
||||
set(_nfn_arch "ppc64le")
|
||||
else()
|
||||
message(FATAL_ERROR "Unable to infer ISA from CMAKE_SYSTEM_PROCESSOR='${CMAKE_SYSTEM_PROCESSOR}'. Set NIF_TARGET_ARCH explicitly.")
|
||||
message(FATAL_ERROR "Unable to infer ISA from CMAKE_SYSTEM_PROCESSOR='${CMAKE_SYSTEM_PROCESSOR}'. Set NFN_TARGET_ARCH explicitly.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(_nif_arch STREQUAL "x86_64")
|
||||
set(_nif_define NIF_ARCH_X86_64)
|
||||
elseif(_nif_arch STREQUAL "arm64")
|
||||
set(_nif_define NIF_ARCH_ARM64)
|
||||
elseif(_nif_arch STREQUAL "riscv64")
|
||||
set(_nif_define NIF_ARCH_RISCV64)
|
||||
elseif(_nif_arch STREQUAL "ppc64le")
|
||||
set(_nif_define NIF_ARCH_PPC64le)
|
||||
if(_nfn_arch STREQUAL "x86_64")
|
||||
set(_nfn_define NFN_ARCH_X86_64)
|
||||
elseif(_nfn_arch STREQUAL "arm64")
|
||||
set(_nfn_define NFN_ARCH_ARM64)
|
||||
elseif(_nfn_arch STREQUAL "riscv64")
|
||||
set(_nfn_define NFN_ARCH_RISCV64)
|
||||
elseif(_nfn_arch STREQUAL "ppc64le")
|
||||
set(_nfn_define NFN_ARCH_PPC64le)
|
||||
else()
|
||||
message(FATAL_ERROR "Unsupported NIF_TARGET_ARCH='${INPUT_VALUE}'.")
|
||||
message(FATAL_ERROR "Unsupported NFN_TARGET_ARCH='${INPUT_VALUE}'.")
|
||||
endif()
|
||||
|
||||
set(${OUTPUT_NAME} "${_nif_arch}" PARENT_SCOPE)
|
||||
set(${OUTPUT_DEFINE} "${_nif_define}" PARENT_SCOPE)
|
||||
set(${OUTPUT_NAME} "${_nfn_arch}" PARENT_SCOPE)
|
||||
set(${OUTPUT_DEFINE} "${_nfn_define}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
function(nif_resolve_target_identifiers INPUT_OS INPUT_ARCH OUTPUT_OS_NAME OUTPUT_OS_DEFINE OUTPUT_ARCH_NAME OUTPUT_ARCH_DEFINE)
|
||||
nif_normalize_os_identifier("${INPUT_OS}" _nif_os_name _nif_os_define)
|
||||
nif_normalize_arch_identifier("${INPUT_ARCH}" _nif_arch_name _nif_arch_define)
|
||||
function(nfn_resolve_target_identifiers INPUT_OS INPUT_ARCH OUTPUT_OS_NAME OUTPUT_OS_DEFINE OUTPUT_ARCH_NAME OUTPUT_ARCH_DEFINE)
|
||||
nfn_normalize_os_identifier("${INPUT_OS}" _nfn_os_name _nfn_os_define)
|
||||
nfn_normalize_arch_identifier("${INPUT_ARCH}" _nfn_arch_name _nfn_arch_define)
|
||||
|
||||
set(${OUTPUT_OS_NAME} "${_nif_os_name}" PARENT_SCOPE)
|
||||
set(${OUTPUT_OS_DEFINE} "${_nif_os_define}" PARENT_SCOPE)
|
||||
set(${OUTPUT_ARCH_NAME} "${_nif_arch_name}" PARENT_SCOPE)
|
||||
set(${OUTPUT_ARCH_DEFINE} "${_nif_arch_define}" PARENT_SCOPE)
|
||||
set(${OUTPUT_OS_NAME} "${_nfn_os_name}" PARENT_SCOPE)
|
||||
set(${OUTPUT_OS_DEFINE} "${_nfn_os_define}" PARENT_SCOPE)
|
||||
set(${OUTPUT_ARCH_NAME} "${_nfn_arch_name}" PARENT_SCOPE)
|
||||
set(${OUTPUT_ARCH_DEFINE} "${_nfn_arch_define}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
function(nif_register_external_package TARGET PACKAGE_NAME PACKAGE_DIR)
|
||||
function(nfn_register_external_package TARGET PACKAGE_NAME PACKAGE_DIR)
|
||||
if(NOT TARGET ${TARGET})
|
||||
message(FATAL_ERROR "nif_register_external_package: target '${TARGET}' does not exist.")
|
||||
message(FATAL_ERROR "nfn_register_external_package: target '${TARGET}' does not exist.")
|
||||
endif()
|
||||
|
||||
if(EXISTS "${PACKAGE_DIR}")
|
||||
@@ -79,7 +79,7 @@ function(nif_register_external_package TARGET PACKAGE_NAME PACKAGE_DIR)
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
function(nif_link_external_binary_if_exists TARGET LIBRARY_PATH)
|
||||
function(nfn_link_external_binary_if_exists TARGET LIBRARY_PATH)
|
||||
if(EXISTS "${LIBRARY_PATH}")
|
||||
target_link_libraries(${TARGET} PRIVATE "${LIBRARY_PATH}")
|
||||
message(STATUS "Linked external binary: ${LIBRARY_PATH}")
|
||||
|
||||
@@ -111,7 +111,7 @@
|
||||
<LanguageStandard_C>stdc11</LanguageStandard_C>
|
||||
<MultiProcessorCompilation>true</MultiProcessorCompilation>
|
||||
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
|
||||
<AdditionalIncludeDirectories>$(ProjectDir)lib;$(ProjectDir)nif-head;$(ProjectDir)nif-template;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<AdditionalIncludeDirectories>$(ProjectDir)lib;$(ProjectDir)nfn-head;$(ProjectDir)nfn-template;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<!-- DLL 应该用 Windows，不是 Console -->
|
||||
@@ -173,23 +173,23 @@
|
||||
<!-- source files -->
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<ItemGroup>
|
||||
<ClCompile Include="nif-src\dllmain.c" />
|
||||
<ClCompile Include="nif-src\m.bufop.x86.c" />
|
||||
<ClCompile Include="nif-src\m.solve.x86.c" />
|
||||
<ClCompile Include="nif-src\m.transpose.x86.c" />
|
||||
<ClCompile Include="nif-src\mk.fma.x86.c" />
|
||||
<ClCompile Include="nif-src\mk.p_scan.x86.c" />
|
||||
<ClCompile Include="nif-src\s.factorial.x86.c" />
|
||||
<ClCompile Include="nif-src\sk.fma.x86.c" />
|
||||
<ClCompile Include="nif-src\s.hash.x86.c" />
|
||||
<ClCompile Include="nif-src\load_intrinsic.x86.c" />
|
||||
<ClCompile Include="nif-src\load_glfw.c" />
|
||||
<ClCompile Include="nif-src\simdid.x86.c" />
|
||||
<ClCompile Include="nif-src\simdid_compare.x86.c" />
|
||||
<ClCompile Include="nif-src\thread.linux.c" />
|
||||
<ClCompile Include="nif-src\thread.windows.c" />
|
||||
<ClCompile Include="nif-src\vector.c" />
|
||||
<ClCompile Include="nif-src\vector_dot.c" />
|
||||
<ClCompile Include="nfn-src\dllmain.c" />
|
||||
<ClCompile Include="nfn-src\m.bufop.x86.c" />
|
||||
<ClCompile Include="nfn-src\m.solve.x86.c" />
|
||||
<ClCompile Include="nfn-src\m.transpose.x86.c" />
|
||||
<ClCompile Include="nfn-src\mk.fma.x86.c" />
|
||||
<ClCompile Include="nfn-src\mk.p_scan.x86.c" />
|
||||
<ClCompile Include="nfn-src\s.factorial.x86.c" />
|
||||
<ClCompile Include="nfn-src\sk.fma.x86.c" />
|
||||
<ClCompile Include="nfn-src\s.hash.x86.c" />
|
||||
<ClCompile Include="nfn-src\load_intrinsic.x86.c" />
|
||||
<ClCompile Include="nfn-src\load_glfw.c" />
|
||||
<ClCompile Include="nfn-src\simdid.x86.c" />
|
||||
<ClCompile Include="nfn-src\simdid_compare.x86.c" />
|
||||
<ClCompile Include="nfn-src\thread.linux.c" />
|
||||
<ClCompile Include="nfn-src\thread.windows.c" />
|
||||
<ClCompile Include="nfn-src\vector.c" />
|
||||
<ClCompile Include="nfn-src\vector_dot.c" />
|
||||
</ItemGroup>
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<!-- header files -->
|
||||
@@ -198,15 +198,15 @@
|
||||
<ClInclude Include="lib\glfw\glfw3.h" />
|
||||
<ClInclude Include="lib\glfw\glfw3native.h" />
|
||||
<ClInclude Include="lib\stbi\stb_image.h" />
|
||||
<ClInclude Include="nif-head\glfw_function.h" />
|
||||
<ClInclude Include="nif-head\nif_internal.h" />
|
||||
<ClInclude Include="nif-head\nif_intrinsic.context.x86.h" />
|
||||
<ClInclude Include="nif-head\nif_intrinsic.std.h" />
|
||||
<ClInclude Include="nif-head\nif_intrinsic.h" />
|
||||
<ClInclude Include="nif-head\nif_intrinsic.matkernel.h" />
|
||||
<ClInclude Include="nif-head\nif_target.h" />
|
||||
<ClInclude Include="nif-head\NitloadFrameWork.h" />
|
||||
<ClInclude Include="nif-template\nif_template.fma.x86.h" />
|
||||
<ClInclude Include="nfn-head\glfw_function.h" />
|
||||
<ClInclude Include="nfn-head\nfn_internal.h" />
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.context.x86.h" />
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.std.h" />
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.h" />
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.matkernel.h" />
|
||||
<ClInclude Include="nfn-head\nfn_target.h" />
|
||||
<ClInclude Include="nfn-head\NitloadFrameWork.h" />
|
||||
<ClInclude Include="nfn-template\nfn_template.fma.x86.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="naming.md" />
|
||||
|
||||
@@ -21,69 +21,69 @@
|
||||
</Filter>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="nif-src\dllmain.c">
|
||||
<ClCompile Include="nfn-src\dllmain.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\s.factorial.x86.c">
|
||||
<ClCompile Include="nfn-src\s.factorial.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\s.hash.x86.c">
|
||||
<ClCompile Include="nfn-src\s.hash.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\vector.c">
|
||||
<ClCompile Include="nfn-src\vector.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\vector_dot.c">
|
||||
<ClCompile Include="nfn-src\vector_dot.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\load_glfw.c">
|
||||
<ClCompile Include="nfn-src\load_glfw.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\sk.fma.x86.c">
|
||||
<ClCompile Include="nfn-src\sk.fma.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\load_intrinsic.x86.c">
|
||||
<ClCompile Include="nfn-src\load_intrinsic.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\m.bufop.x86.c">
|
||||
<ClCompile Include="nfn-src\m.bufop.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\m.solve.x86.c">
|
||||
<ClCompile Include="nfn-src\m.solve.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\simdid.x86.c">
|
||||
<ClCompile Include="nfn-src\simdid.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\mk.fma.x86.c">
|
||||
<ClCompile Include="nfn-src\mk.fma.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\simdid_compare.x86.c">
|
||||
<ClCompile Include="nfn-src\simdid_compare.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\thread.linux.c">
|
||||
<ClCompile Include="nfn-src\thread.linux.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\thread.windows.c">
|
||||
<ClCompile Include="nfn-src\thread.windows.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\m.transpose.x86.c">
|
||||
<ClCompile Include="nfn-src\m.transpose.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\mk.p_scan.x86.c">
|
||||
<ClCompile Include="nfn-src\mk.p_scan.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="nif-head\nif_internal.h">
|
||||
<ClInclude Include="nfn-head\nfn_internal.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-head\NitloadFrameWork.h">
|
||||
<ClInclude Include="nfn-head\NitloadFrameWork.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-head\glfw_function.h">
|
||||
<ClInclude Include="nfn-head\glfw_function.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-head\nif_intrinsic.matkernel.h">
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.matkernel.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="lib\glfw\glfw3.h">
|
||||
@@ -95,19 +95,19 @@
|
||||
<ClInclude Include="lib\stbi\stb_image.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-head\nif_target.h">
|
||||
<ClInclude Include="nfn-head\nfn_target.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-head\nif_intrinsic.context.x86.h">
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.context.x86.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-head\nif_intrinsic.h">
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-head\nif_intrinsic.std.h">
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.std.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-template\nif_template.fma.x86.h">
|
||||
<ClInclude Include="nfn-template\nfn_template.fma.x86.h">
|
||||
<Filter>模板文件</Filter>
|
||||
</ClInclude>
|
||||
</ItemGroup>
|
||||
|
||||
@@ -1,5 +1,18 @@
|
||||
# Nitload.Native
|
||||
|
||||
## Naming convention
|
||||
|
||||
The C implementation uses `nfn` for Nitload Framework Native: exported functions use
|
||||
`nfn_`, macros and CMake cache options use `NFN_`, and native source folders and
|
||||
headers use `nfn-` and `nfn_`. The shared library remains named `Nitload.Native`.
|
||||
The longer prefix also distinguishes these names from Linux Netfilter's `nf_` names.
|
||||
|
||||
This migration renames the exported C symbols from `nif_*` to `nfn_*`. Native
|
||||
binaries and managed assemblies must therefore be rebuilt and deployed together;
|
||||
pre-migration P/Invoke entry points are not compatible with a newly built library.
|
||||
The old `NIF_*` CMake cache options are no longer used. Reconfigure an existing
|
||||
build tree with the corresponding `NFN_*` options, or use a fresh build tree.
|
||||
|
||||
## COMPILE
|
||||
|
||||
Though this project do has a .vcxproj file,
|
||||
@@ -11,15 +24,21 @@ Currently you can only compile this lib via Clang and cmake.
|
||||
From the repository root:
|
||||
|
||||
```powershell
|
||||
# On Windows, add your LLVM bin directory if clang is not already on PATH.
|
||||
$env:PATH = 'C:\Program Files\LLVM\bin;' + $env:PATH
|
||||
cmake --preset clang-debug -S src/Nitload.Native
|
||||
cmake --build cmake_build/clang-debug --target Nitload.Native
|
||||
```
|
||||
|
||||
The presets require `clang` on `PATH`. If an existing CMake cache was configured
|
||||
with an absolute compiler path, make `clang` available on `PATH` before running
|
||||
the preset again. The `NFN_*` options replace the former `NIF_*` options.
|
||||
|
||||
CMake writes `Nitload.Native.dll` and its linker PDB to `bin/win-x64` on Windows x64
|
||||
(Linux uses `bin/linux-x64/libNitload.Native.so`). Debug and release presets currently
|
||||
share this platform directory; the most recent native build is the deployed version.
|
||||
`NIF_BUILD_OUTPUT_DIR` can override the destination. Managed builds pass their expected
|
||||
RID as `NIF_EXPECTED_RID` so an incompatible native target is rejected before deployment.
|
||||
`NFN_BUILD_OUTPUT_DIR` can override the destination. Managed builds pass their expected
|
||||
RID as `NFN_EXPECTED_RID` so an incompatible native target is rejected before deployment.
|
||||
|
||||
`StgSharpDebug.csproj` enables `BuildNativeBeforeManaged` and `RequireNativeLibrary`.
|
||||
Its build runs the incremental CMake build (`clang-debug` for Debug, `clang-release`
|
||||
@@ -32,6 +51,15 @@ Clang/Ninja environment and the local GLFW static library are required for nativ
|
||||
Cross-target builds require a matching toolchain/preset; changing a managed platform alone
|
||||
does not cross-compile the C library.
|
||||
|
||||
## clangd in VS Code
|
||||
|
||||
The `.clangd` file searches for `compile_commands.json` in `src/Nitload.Native`.
|
||||
When using CMake Tools, set `cmake.copyCompileCommands` to
|
||||
`${workspaceFolder}/src/Nitload.Native/compile_commands.json` so the active Debug or Release preset
|
||||
updates that file after configuration. For command-line configuration, copy
|
||||
`cmake_build/<preset>/compile_commands.json` to `src/Nitload.Native`. Restart
|
||||
clangd after changing the database or renaming headers.
|
||||
|
||||
## Code Layout
|
||||
|
||||
Conventions:
|
||||
|
||||
@@ -217,7 +217,7 @@ Detection:
|
||||
- **Unknown hybrid** (flag set but core type unrecognised): leave topology at 00.
|
||||
- **SIMD_DUAL**: set when any of the following: `AVX Feature DUAL=1` (glued YMM), or `AVX512_IMPL=DUAL`.
|
||||
|
||||
`nif_get_unite_simd` computes topology bits [1..0] with bitwise **AND** (conservative intersection — weaker topology wins) and SIMD_DUAL with bitwise **OR** (dual if either core is dual). `nif_compare_simd` compares topology value numerically as step 6 and SIMD_DUAL as the final (step 7) tie-break.
|
||||
`nfn_get_unite_simd` computes topology bits [1..0] with bitwise **AND** (conservative intersection — weaker topology wins) and SIMD_DUAL with bitwise **OR** (dual if either core is dual). `nfn_compare_simd` compares topology value numerically as step 6 and SIMD_DUAL as the final (step 7) tie-break.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
|
||||
if(NOT DEFINED NIF_NATIVE_EXTERNAL_TARGET)
|
||||
message(FATAL_ERROR "NIF_NATIVE_EXTERNAL_TARGET must be set before adding lib/.")
|
||||
if(NOT DEFINED NFN_NATIVE_EXTERNAL_TARGET)
|
||||
message(FATAL_ERROR "NFN_NATIVE_EXTERNAL_TARGET must be set before adding lib/.")
|
||||
endif()
|
||||
|
||||
include(${CMAKE_CURRENT_LIST_DIR}/../ExternalLibs.cmake OPTIONAL)
|
||||
@@ -17,6 +17,6 @@ foreach(_pkg ${LIB_PACKAGE_DIRS})
|
||||
if(EXISTS ${_pkg_dir}/CMakeLists.txt)
|
||||
add_subdirectory(${_pkg_dir})
|
||||
else()
|
||||
nif_register_external_package(${NIF_NATIVE_EXTERNAL_TARGET} ${_pkg} ${_pkg_dir})
|
||||
nfn_register_external_package(${NFN_NATIVE_EXTERNAL_TARGET} ${_pkg} ${_pkg_dir})
|
||||
endif()
|
||||
endforeach()
|
||||
@@ -1,11 +1,11 @@
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
|
||||
nif_register_external_package(${NIF_NATIVE_EXTERNAL_TARGET} glfw ${CMAKE_CURRENT_LIST_DIR})
|
||||
nfn_register_external_package(${NFN_NATIVE_EXTERNAL_TARGET} glfw ${CMAKE_CURRENT_LIST_DIR})
|
||||
|
||||
if(WIN32)
|
||||
if(NOT EXISTS ${CMAKE_CURRENT_LIST_DIR}/glfw3_mt.lib)
|
||||
message(FATAL_ERROR "Required GLFW static library not found: ${CMAKE_CURRENT_LIST_DIR}/glfw3_mt.lib")
|
||||
endif()
|
||||
|
||||
nif_link_external_binary_if_exists(${NIF_NATIVE_EXTERNAL_TARGET} ${CMAKE_CURRENT_LIST_DIR}/glfw3_mt.lib)
|
||||
nfn_link_external_binary_if_exists(${NFN_NATIVE_EXTERNAL_TARGET} ${CMAKE_CURRENT_LIST_DIR}/glfw3_mt.lib)
|
||||
endif()
|
||||
@@ -207,7 +207,7 @@ GLFWAPI const char *glfwGetWin32Monitor(GLFWmonitor *monitor);
|
||||
* GLFW_PLATFORM_UNAVAILABLE.
|
||||
*
|
||||
* @remark The `HDC` associated with the window can be queried with the
|
||||
* [GetDC](https://docs.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-getdc)
|
||||
* [GetDC](https://docs.microsoft.com/en-us/windows/win32/api/winuser/nfn-winuser-getdc)
|
||||
* function.
|
||||
* @code
|
||||
* HDC dc = GetDC(glfwGetWin32Window(window));
|
||||
@@ -234,7 +234,7 @@ GLFWAPI HWND glfwGetWin32Window(GLFWwindow *window);
|
||||
* GLFW_PLATFORM_UNAVAILABLE and @ref GLFW_NO_WINDOW_CONTEXT.
|
||||
*
|
||||
* @remark The `HDC` associated with the window can be queried with the
|
||||
* [GetDC](https://docs.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-getdc)
|
||||
* [GetDC](https://docs.microsoft.com/en-us/windows/win32/api/winuser/nfn-winuser-getdc)
|
||||
* function.
|
||||
* @code
|
||||
* HDC dc = GetDC(glfwGetWin32Window(window));
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
|
||||
nif_register_external_package(${NIF_NATIVE_EXTERNAL_TARGET} stbi ${CMAKE_CURRENT_LIST_DIR})
|
||||
nfn_register_external_package(${NFN_NATIVE_EXTERNAL_TARGET} stbi ${CMAKE_CURRENT_LIST_DIR})
|
||||
@@ -0,0 +1,37 @@
|
||||
#ifndef NFN_NATIVE_H
|
||||
#define NFN_NATIVE_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
#error "This header is not meant to be included in C++ code. Nitload.Native is a C library and should be used with C linkage."
|
||||
#endif
|
||||
|
||||
#include "nfn_internal.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <assert.h>
|
||||
#include <glfw_function.h>
|
||||
|
||||
#define byte char
|
||||
#define cs_char uint16_t
|
||||
#define nfn_null_assert(ptr) assert(ptr != NULL)
|
||||
|
||||
#ifndef NFN_
|
||||
#endif
|
||||
|
||||
NFN_API void NFN_DECL nfn_load_glfw_functions(GLFWFunctionTable *table);
|
||||
NFN_API void NFN_DECL nfn_load_intrinsic_function(void *intrinsic_context, uint64_t id);
|
||||
NFN_API SIMDID NFN_DECL nfn_get_simd_level_global();
|
||||
/* Per-core variant: always re-executes CPUID, never caches.
|
||||
* Must be called on the target core (after affinity is set). */
|
||||
NFN_API SIMDID NFN_DECL nfn_get_simd_level_local();
|
||||
NFN_API SIMDID NFN_DECL nfn_get_unite_simd(SIMDID left, SIMDID right);
|
||||
NFN_API int NFN_DECL nfn_compare_simd(SIMDID left, SIMDID right);
|
||||
NFN_API void NFN_DECL nfn_set_thread_affinity(int cpu_index, int processor_group_id);
|
||||
NFN_API void *NFN_DECL nfn_numa_alloc(size_t size, int numa_id);
|
||||
NFN_API void NFN_DECL nfn_numa_free(void *ptr, size_t size, int numa_id);
|
||||
NFN_API int NFN_DECL nfn_get_numa_each(int numa_id, uint32_t *processor_group_ids,
|
||||
uint32_t *socket_ids, uint32_t *cpu_indices,
|
||||
uint32_t *numa_node_ids);
|
||||
NFN_API int NFN_DECL nfn_get_numa_overall(int *out_numa_count, int *count_of_each_numa);
|
||||
#endif
|
||||
+14
-14
@@ -1,6 +1,6 @@
|
||||
|
||||
#ifndef NIF_INTERNAL
|
||||
#define NIF_INTERNAL
|
||||
#ifndef NFN_INTERNAL
|
||||
#define NFN_INTERNAL
|
||||
|
||||
#include <xmmintrin.h>
|
||||
|
||||
@@ -9,33 +9,33 @@
|
||||
#ifdef _MSC_VER
|
||||
#define _CRT_SECURE_NO_WARNINGS 1
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#define NIF_DECL __cdecl
|
||||
#define NIF_MAT_DECL __cdecl __forceinline
|
||||
#define NIF_API __declspec(dllexport)
|
||||
#define NFN_DECL __cdecl
|
||||
#define NFN_MAT_DECL __cdecl __forceinline
|
||||
#define NFN_API __declspec(dllexport)
|
||||
#define FORCEINLINE __forceinline
|
||||
#define NOINLINE __declspec(noinline)
|
||||
#include <windows.h>
|
||||
#elif defined(__clang__) || defined(__GNUC__)
|
||||
#define NIF_DECL __attribute__((cdecl))
|
||||
#define NIF_MAT_DECL __attribute__((cdecl)) __attribute__((always_inline))
|
||||
#define NIF_API __attribute__((dllexport))
|
||||
#define NFN_DECL __attribute__((cdecl))
|
||||
#define NFN_MAT_DECL __attribute__((cdecl)) __attribute__((always_inline))
|
||||
#define NFN_API __attribute__((dllexport))
|
||||
#define FORCEINLINE __attribute__((always_inline))
|
||||
#define NOINLINE __attribute__((noinline))
|
||||
#else
|
||||
#define NIF_API
|
||||
#define NIF_DECL
|
||||
#define NIF_MAT_DECL
|
||||
#define NFN_API
|
||||
#define NFN_DECL
|
||||
#define NFN_MAT_DECL
|
||||
#define FORCEINLINE
|
||||
#define NOINLINE
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#define NIF_ALIGNAS(n) __declspec(align(n))
|
||||
#define NFN_ALIGNAS(n) __declspec(align(n))
|
||||
#else
|
||||
#define NIF_ALIGNAS(n) __attribute__((aligned(n)))
|
||||
#define NFN_ALIGNAS(n) __attribute__((aligned(n)))
|
||||
#endif
|
||||
|
||||
#define NIF_APIDEF
|
||||
#define NFN_APIDEF
|
||||
|
||||
#define PRIVATE static
|
||||
#define INTERNAL extern
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
#ifndef NIF_INTRINSIC_CONTEXT_BASE
|
||||
#define NIF_INTRINSIC_CONTEXT_BASE
|
||||
#ifndef NFN_INTRINSIC_CONTEXT_BASE
|
||||
#define NFN_INTRINSIC_CONTEXT_BASE
|
||||
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
+107
-107
@@ -1,20 +1,20 @@
|
||||
#ifndef NIF_INTRINSIC_CONTEXT_X86_H
|
||||
#define NIF_INTRINSIC_CONTEXT_X86_H
|
||||
#ifndef NFN_INTRINSIC_CONTEXT_X86_H
|
||||
#define NFN_INTRINSIC_CONTEXT_X86_H
|
||||
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nif_intrinsic.context.base.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.context.base.h"
|
||||
|
||||
#ifndef NIF_X86_TARGET_ATTR_sse
|
||||
#define NIF_STD_TARGET_ATTR_IMPL(T_arch, T_feature) NIF_X86_TARGET_ATTR_##T_arch##T_feature
|
||||
#define NIF_X86_TARGET_ATTR_sse NIF_CLANG_TARGET_ATTR("sse4.2")
|
||||
#define NIF_X86_TARGET_ATTR_avx NIF_CLANG_TARGET_ATTR("avx")
|
||||
#define NIF_X86_TARGET_ATTR_avx_fma NIF_CLANG_TARGET_ATTR("avx,avx2,fma")
|
||||
#define NIF_X86_TARGET_ATTR_512 NIF_CLANG_TARGET_ATTR("avx512f,avx512vl")
|
||||
#ifndef NFN_X86_TARGET_ATTR_sse
|
||||
#define NFN_STD_TARGET_ATTR_IMPL(T_arch, T_feature) NFN_X86_TARGET_ATTR_##T_arch##T_feature
|
||||
#define NFN_X86_TARGET_ATTR_sse NFN_CLANG_TARGET_ATTR("sse4.2")
|
||||
#define NFN_X86_TARGET_ATTR_avx NFN_CLANG_TARGET_ATTR("avx")
|
||||
#define NFN_X86_TARGET_ATTR_avx_fma NFN_CLANG_TARGET_ATTR("avx,avx2,fma")
|
||||
#define NFN_X86_TARGET_ATTR_512 NFN_CLANG_TARGET_ATTR("avx512f,avx512vl")
|
||||
#endif
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
@@ -111,45 +111,45 @@
|
||||
|
||||
INTERNAL void f32_normalize(VEC_SEGMENT(float) * source, VEC_SEGMENT(float) * target);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , k_quality);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , k_quality);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , add);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , add);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , add);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , add);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , add);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , add);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , add);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , add);
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , add);
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , add);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , add);
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , add);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , sub);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , sub);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , sub);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , sub);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , sub);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , sub);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , sub);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , sub);
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , sub);
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , sub);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , sub);
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , sub);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , transpose);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , transpose);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , transpose);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , transpose);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , transpose);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , transpose);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , scalar_mul);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , scalar_mul);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , scalar_mul);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , scalar_mul);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , scalar_mul);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , scalar_mul);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , scalar_mul);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , scalar_mul);
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , scalar_mul);
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , scalar_mul);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , scalar_mul);
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , scalar_mul);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fill);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fill);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fill);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fill);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fill);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fill);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , fill);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , fill);
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , fill);
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , fill);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , fill);
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , fill);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , quality);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , quality);
|
||||
|
||||
/*
|
||||
* pack left matrix data to L4 buffer
|
||||
@@ -166,9 +166,9 @@ NIF_MK_PROC_DECL_STD(float, sse, , quality);
|
||||
* scalar pack carries column length and offset data:
|
||||
* [column length, empty, empty, empty]
|
||||
*/
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fma_pack_left);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fma_pack_left);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fma_pack_left);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fma_pack_left);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fma_pack_left);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fma_pack_left);
|
||||
|
||||
/*
|
||||
* pack right matrix data to L4 buffer
|
||||
@@ -185,104 +185,104 @@ NIF_MK_PROC_DECL_STD(float, 512, , fma_pack_left);
|
||||
* scalar pack carries column length and offset data:
|
||||
* [column length, empty, empty, empty]
|
||||
*/
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fma_pack_right);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fma_pack_right);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fma_pack_right);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fma_pack_right);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fma_pack_right);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fma_pack_right);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fma);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fma);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, _fma, fma);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fma);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fma);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fma);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, _fma, fma);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fma);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fma_unpack);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fma_unpack);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fma_unpack);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fma_unpack);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fma_unpack);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fma_unpack);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , fma_pack_left);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , fma_pack_left);
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , fma_pack_left);
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , fma_pack_left);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , fma_pack_left);
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , fma_pack_left);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , fma);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , fma);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, _fma, fma);
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , fma);
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , fma);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , fma);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, _fma, fma);
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , fma);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , fma_unpack);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , fma_unpack);
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , fma_unpack);
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , fma_unpack);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , fma_unpack);
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , fma_unpack);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, avx, , add);
|
||||
NIF_SK_PROC_DECL_STD(float, 512, , add);
|
||||
NFN_SK_PROC_DECL_STD(float, avx, , add);
|
||||
NFN_SK_PROC_DECL_STD(float, 512, , add);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, sse, , add);
|
||||
NIF_SK_PROC_DECL_STD(double, avx, , add);
|
||||
NIF_SK_PROC_DECL_STD(double, 512, , add);
|
||||
NFN_SK_PROC_DECL_STD(double, sse, , add);
|
||||
NFN_SK_PROC_DECL_STD(double, avx, , add);
|
||||
NFN_SK_PROC_DECL_STD(double, 512, , add);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, sse, , sub);
|
||||
NIF_SK_PROC_DECL_STD(float, avx, , sub);
|
||||
NIF_SK_PROC_DECL_STD(float, 512, , sub);
|
||||
NFN_SK_PROC_DECL_STD(float, sse, , sub);
|
||||
NFN_SK_PROC_DECL_STD(float, avx, , sub);
|
||||
NFN_SK_PROC_DECL_STD(float, 512, , sub);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, sse, , sub);
|
||||
NIF_SK_PROC_DECL_STD(double, avx, , sub);
|
||||
NIF_SK_PROC_DECL_STD(double, 512, , sub);
|
||||
NFN_SK_PROC_DECL_STD(double, sse, , sub);
|
||||
NFN_SK_PROC_DECL_STD(double, avx, , sub);
|
||||
NFN_SK_PROC_DECL_STD(double, 512, , sub);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, sse, , transpose);
|
||||
NIF_SK_PROC_DECL_STD(float, avx, , transpose);
|
||||
NIF_SK_PROC_DECL_STD(float, 512, , transpose);
|
||||
NFN_SK_PROC_DECL_STD(float, sse, , transpose);
|
||||
NFN_SK_PROC_DECL_STD(float, avx, , transpose);
|
||||
NFN_SK_PROC_DECL_STD(float, 512, , transpose);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, sse, , scalar_mul);
|
||||
NIF_SK_PROC_DECL_STD(float, avx, , scalar_mul);
|
||||
NIF_SK_PROC_DECL_STD(float, 512, , scalar_mul);
|
||||
NFN_SK_PROC_DECL_STD(float, sse, , scalar_mul);
|
||||
NFN_SK_PROC_DECL_STD(float, avx, , scalar_mul);
|
||||
NFN_SK_PROC_DECL_STD(float, 512, , scalar_mul);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, sse, , scalar_mul);
|
||||
NIF_SK_PROC_DECL_STD(double, avx, , scalar_mul);
|
||||
NIF_SK_PROC_DECL_STD(double, 512, , scalar_mul);
|
||||
NFN_SK_PROC_DECL_STD(double, sse, , scalar_mul);
|
||||
NFN_SK_PROC_DECL_STD(double, avx, , scalar_mul);
|
||||
NFN_SK_PROC_DECL_STD(double, 512, , scalar_mul);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, sse, , fill);
|
||||
NIF_SK_PROC_DECL_STD(float, avx, , fill);
|
||||
NIF_SK_PROC_DECL_STD(float, 512, , fill);
|
||||
NFN_SK_PROC_DECL_STD(float, sse, , fill);
|
||||
NFN_SK_PROC_DECL_STD(float, avx, , fill);
|
||||
NFN_SK_PROC_DECL_STD(float, 512, , fill);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, sse, , fill);
|
||||
NIF_SK_PROC_DECL_STD(double, avx, , fill);
|
||||
NIF_SK_PROC_DECL_STD(double, 512, , fill);
|
||||
NFN_SK_PROC_DECL_STD(double, sse, , fill);
|
||||
NFN_SK_PROC_DECL_STD(double, avx, , fill);
|
||||
NFN_SK_PROC_DECL_STD(double, 512, , fill);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, sse, , quality);
|
||||
NFN_SK_PROC_DECL_STD(float, sse, , quality);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, sse, , fma);
|
||||
NIF_SK_PROC_DECL_STD(float, avx, , fma);
|
||||
NIF_SK_PROC_DECL_STD(float, avx, _fma, fma);
|
||||
NIF_SK_PROC_DECL_STD(float, 512, , fma);
|
||||
NFN_SK_PROC_DECL_STD(float, sse, , fma);
|
||||
NFN_SK_PROC_DECL_STD(float, avx, , fma);
|
||||
NFN_SK_PROC_DECL_STD(float, avx, _fma, fma);
|
||||
NFN_SK_PROC_DECL_STD(float, 512, , fma);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, sse, , fma);
|
||||
NIF_SK_PROC_DECL_STD(double, avx, , fma);
|
||||
NIF_SK_PROC_DECL_STD(double, avx, _fma, fma);
|
||||
NIF_SK_PROC_DECL_STD(double, 512, , fma);
|
||||
NFN_SK_PROC_DECL_STD(double, sse, , fma);
|
||||
NFN_SK_PROC_DECL_STD(double, avx, , fma);
|
||||
NFN_SK_PROC_DECL_STD(double, avx, _fma, fma);
|
||||
NFN_SK_PROC_DECL_STD(double, 512, , fma);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region scaler_simd
|
||||
|
||||
INTERNAL uint64_t NIF_DECL factorial_simd_sse(int n);
|
||||
INTERNAL uint64_t NFN_DECL factorial_simd_sse(int n);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region dot_product
|
||||
|
||||
INTERNAL void FORCEINLINE NIF_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
|
||||
INTERNAL void FORCEINLINE NFN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
|
||||
__m128 const *vector, __m128 *ans);
|
||||
INTERNAL void NIF_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
INTERNAL void NFN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans);
|
||||
INTERNAL void NIF_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
INTERNAL void NFN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans);
|
||||
INTERNAL void NIF_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
INTERNAL void NFN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region string
|
||||
|
||||
INTERNAL int NIF_DECL city_hash_simplify_sse(byte const *str, int length);
|
||||
INTERNAL int NIF_DECL index_pair_sse(short const *stram, uint32_t target, int length);
|
||||
INTERNAL int NFN_DECL city_hash_simplify_sse(byte const *str, int length);
|
||||
INTERNAL int NFN_DECL index_pair_sse(short const *stram, uint32_t target, int length);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
+24
-24
@@ -4,11 +4,11 @@
|
||||
#define SSC_INTRIN_HEAD
|
||||
|
||||
#include "NitloadFrameWork.h"
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
#include <xmmintrin.h>
|
||||
#include <immintrin.h>
|
||||
#include <smmintrin.h>
|
||||
#include "nif_intrinsic.matkernel.h"
|
||||
#include "nfn_intrinsic.matkernel.h"
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef _MSC_VER
|
||||
@@ -28,7 +28,7 @@
|
||||
((LASTTWO(s3) << 6) | (LASTTWO(s2) << 4) | (LASTTWO(s1) << 2) | (LASTTWO(s0)))
|
||||
#define BIT_CVRT(pack, index, T) *(T *)(pack->data + index)
|
||||
|
||||
#if NIF_IS_ARCH(x86_64)
|
||||
#if NFN_IS_ARCH(x86_64)
|
||||
|
||||
#define vec_128 __m128
|
||||
#define vec_256 __m256
|
||||
@@ -66,7 +66,7 @@ typedef union uint64_u {
|
||||
typedef struct mat_kernel mat_kernel;
|
||||
|
||||
typedef union M128{
|
||||
#if NIF_IS_ARCH(x86_64)
|
||||
#if NFN_IS_ARCH(x86_64)
|
||||
__m128 m128;
|
||||
#endif
|
||||
float f32[4];
|
||||
@@ -76,7 +76,7 @@ typedef union M128{
|
||||
}M128;
|
||||
|
||||
typedef union M256{
|
||||
#if NIF_IS_ARCH(x86_64)
|
||||
#if NFN_IS_ARCH(x86_64)
|
||||
__m256 m256;
|
||||
#endif
|
||||
float f32[8];
|
||||
@@ -86,7 +86,7 @@ typedef union M256{
|
||||
}M256;
|
||||
|
||||
typedef union M512{
|
||||
#if NIF_IS_ARCH(x86_64)
|
||||
#if NFN_IS_ARCH(x86_64)
|
||||
__m512 m512;
|
||||
#endif
|
||||
uint64_t u64[8];
|
||||
@@ -97,24 +97,24 @@ typedef union M512{
|
||||
int64_t i64[8];
|
||||
}M512;
|
||||
|
||||
typedef struct nif_L4_cache_head {
|
||||
typedef struct nfn_L4_cache_head {
|
||||
uint64_t reverse_mask_0;
|
||||
uint32_t position_mask;
|
||||
uint32_t size;
|
||||
uint64_t reverse_mask_1;
|
||||
uint64_t reverse_mask_2;
|
||||
} nif_L4_cache_head;
|
||||
} nfn_L4_cache_head;
|
||||
|
||||
typedef struct nif_L4_handle {
|
||||
typedef struct nfn_L4_handle {
|
||||
void *ptr;
|
||||
nif_L4_cache_head* head;
|
||||
} nif_L4_handle;
|
||||
nfn_L4_cache_head* head;
|
||||
} nfn_L4_handle;
|
||||
|
||||
typedef struct matrix_task {
|
||||
nif_L4_handle buffer_0;
|
||||
nif_L4_handle buffer_1;
|
||||
nif_L4_handle buffer_2;
|
||||
nif_L4_handle buffer_3;
|
||||
nfn_L4_handle buffer_0;
|
||||
nfn_L4_handle buffer_1;
|
||||
nfn_L4_handle buffer_2;
|
||||
nfn_L4_handle buffer_3;
|
||||
union {
|
||||
struct {
|
||||
int32_t reverse_0;
|
||||
@@ -144,18 +144,18 @@ static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mis
|
||||
static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mismatch");
|
||||
#endif
|
||||
|
||||
typedef void(NIF_DECL *UNI_MK_PROC)(matrix_task *task);
|
||||
typedef void(NIF_DECL *UNI_SK_PROC)(mat_kernel const *left, mat_kernel const *right,
|
||||
typedef void(NFN_DECL *UNI_MK_PROC)(matrix_task *task);
|
||||
typedef void(NFN_DECL *UNI_SK_PROC)(mat_kernel const *left, mat_kernel const *right,
|
||||
mat_kernel *restrict ans, uint64_t scalar);
|
||||
|
||||
typedef void(NIF_DECL *VECTORNORMALIZEPROC)(VEC_SEGMENT(float) *source, VEC_SEGMENT(float) *target);
|
||||
typedef void(NIF_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans);
|
||||
typedef uint64_t(NIF_DECL *FACTORIALROC)(int n);
|
||||
typedef void(NFN_DECL *VECTORNORMALIZEPROC)(VEC_SEGMENT(float) *source, VEC_SEGMENT(float) *target);
|
||||
typedef void(NFN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans);
|
||||
typedef uint64_t(NFN_DECL *FACTORIALROC)(int n);
|
||||
|
||||
#pragma region string
|
||||
|
||||
typedef int(NIF_DECL *HASH)(byte const *str, int length);
|
||||
typedef int(NIF_DECL *INDEX_PAIR)(short const *stram, uint32_t target, int length);
|
||||
typedef int(NFN_DECL *HASH)(byte const *str, int length);
|
||||
typedef int(NFN_DECL *INDEX_PAIR)(short const *stram, uint32_t target, int length);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
@@ -168,13 +168,13 @@ typedef struct mat_sk_intrinsic {
|
||||
} mat_sk_intrinsic;
|
||||
|
||||
// Mirrors C# IntrinsicContext field order so managed/unmanaged layouts stay in sync.
|
||||
typedef struct nif_intrinsic {
|
||||
typedef struct nfn_intrinsic {
|
||||
HASH city_hash_simplify;
|
||||
FACTORIALROC factorial_simd;
|
||||
VECTORNORMALIZEPROC vec_f32_normalize_3;
|
||||
mat_mk_intrinsic mat_mk[0x0a];
|
||||
mat_sk_intrinsic mat_sk[0x0a];
|
||||
} nif_intrinsic;
|
||||
} nfn_intrinsic;
|
||||
|
||||
|
||||
typedef enum mat_proc_index {
|
||||
@@ -0,0 +1,54 @@
|
||||
#ifndef NFN_MATKERNEL
|
||||
#define NFN_MATKERNEL
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "nfn_internal.h"
|
||||
|
||||
/* Kernel storage unions for a fixed 4x4 block. */
|
||||
#define ELEMENTSIZE(T) sizeof(T)
|
||||
#define XMMCOUNT(T) (sizeof(T))
|
||||
#define YMMCOUNT(T) (sizeof(T) / 2)
|
||||
#define ZMMCOUNT(T) (sizeof(T) / 4)
|
||||
|
||||
#define NFN_MAT_KERNEL_MEMBER(T) \
|
||||
{ \
|
||||
__m128 x[XMMCOUNT(T)]; \
|
||||
__m256 y[YMMCOUNT(T)]; \
|
||||
__m512 z[ZMMCOUNT(T)]; \
|
||||
T m[4][4]; \
|
||||
}
|
||||
|
||||
#define NFN_MAT_KERNEL_DEFINE(T) \
|
||||
typedef union NFN_MAT_KERNEL_TYPE_##T NFN_MAT_KERNEL_MEMBER(T) NFN_MAT_KERNEL_TYPE_##T
|
||||
|
||||
#define MAT_KERNEL(T) NFN_MAT_KERNEL_TYPE(T)
|
||||
#define NFN_MAT_KERNEL_TYPE(T) NFN_MAT_KERNEL_TYPE_##T
|
||||
|
||||
NFN_MAT_KERNEL_DEFINE(float);
|
||||
NFN_MAT_KERNEL_DEFINE(double);
|
||||
NFN_MAT_KERNEL_DEFINE(uint64_t);
|
||||
|
||||
|
||||
#define NFN_VEC_SEGMENT_MEMBER(T) \
|
||||
{ \
|
||||
__m128 f32_x[XMMCOUNT(T) / 4]; \
|
||||
__m256 f32_y[YMMCOUNT(T) / 4]; \
|
||||
__m512 f32_z[ZMMCOUNT(T) / 4]; \
|
||||
T v[4]; \
|
||||
}
|
||||
|
||||
#define NFN_VEC_SEGMENT_DEFINE(T) \
|
||||
typedef union NFN_VEC_SEGMENT_TYPE_##T NFN_VEC_SEGMENT_MEMBER(T) NFN_VEC_SEGMENT_TYPE_##T
|
||||
|
||||
#define VEC_SEGMENT(T) NFN_VEC_SEGMENT_TYPE(T)
|
||||
#define NFN_VEC_SEGMENT_TYPE(T) NFN_VEC_SEGMENT_TYPE_##T
|
||||
|
||||
NFN_VEC_SEGMENT_DEFINE(float);
|
||||
NFN_VEC_SEGMENT_DEFINE(double);
|
||||
|
||||
/* Global zero kernel declaration (defined in dllmain.c), aligned to 64 bytes */
|
||||
extern __declspec(align(64)) MAT_KERNEL(u64) NFN_ZERO_KERNEL;
|
||||
|
||||
#endif /* NFN_MATKERNEL */
|
||||
@@ -0,0 +1,101 @@
|
||||
#ifndef NFN_INTRINSIC_CALLCONV_STD
|
||||
#define NFN_INTRINSIC_CALLCONV_STD
|
||||
|
||||
#include "nfn_internal.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
|
||||
#ifndef static_assert
|
||||
#define static_assert _Static_assert
|
||||
#endif
|
||||
|
||||
#define NFN_STD_TARGET_ATTR(T_arch, T_feature) NFN_STD_TARGET_ATTR_IMPL(T_arch, T_feature)
|
||||
|
||||
#pragma region mk_proc attributes
|
||||
#define MK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
|
||||
T_type##_multi_kernel_proc_##T_op##_##T_arch##_##T_feature
|
||||
#define GET_MK_PROC_STD(T_type, T_arch, T_feature, T_op) \
|
||||
((UNI_MK_PROC)MK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op))
|
||||
#define NFN_MK_PROC_DECL(T_type, P_name) INTERNAL void NFN_DECL P_name(MAT_TASK(T_type) const *task)
|
||||
#define NFN_MK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
|
||||
INTERNAL NOINLINE NFN_STD_TARGET_ATTR(T_arch, T_feature) void NFN_DECL MK_PROC_NAME_STD( \
|
||||
T_type, T_arch, T_feature, T_op)(MAT_TASK(T_type) const *task)
|
||||
#pragma endregion
|
||||
|
||||
#pragma region sk_proc attributes
|
||||
#define SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
|
||||
T_type##_single_kernel_proc_##T_op##_##T_arch##_##T_feature
|
||||
#define GET_SK_PROC_STD(T_type, T_arch, T_feature, T_op) \
|
||||
((UNI_SK_PROC)SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op))
|
||||
#define NFN_SK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
|
||||
INTERNAL NOINLINE NFN_STD_TARGET_ATTR(T_arch, T_feature) void NFN_DECL SK_PROC_NAME_STD( \
|
||||
T_type, T_arch, T_feature, \
|
||||
T_op)(MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
|
||||
MAT_KERNEL(T_type) *restrict ans, uint64_t scalar)
|
||||
#pragma endregion
|
||||
|
||||
#define _NFN_STD__PROBE_0 0
|
||||
#define _NFN_STD__PROBE_1 1
|
||||
#define NFN_STD__PROBE_PRINT(x) _NFN_STD__PROBE_##x
|
||||
#define NFN_STD__PROBE(x) NFN_STD__PROBE_PRINT(x)
|
||||
|
||||
#pragma region param styles
|
||||
|
||||
#define _NFN_MATPROC_LEFT_RIGHT_ANS 1
|
||||
#define _NFN_MATPROC_RIGHT_ANS 1
|
||||
#define _NFN_MATPROC_LEFT_RIGHT_ANS_SCALAR 1
|
||||
#define _NFN_MATPROC_RIGHT_ANS_SCALAR 1
|
||||
#define _NFN_MATPROC_ANS_SCALAR 1
|
||||
|
||||
#define _NFN_MK_EXPAND_BUFFER_SIZE(i)\
|
||||
int offset_##i = task->offset_##i;\
|
||||
int count_##i = task->count_##i;\
|
||||
|
||||
#define _NFN_MK_EXPAND_RIGHT_ANS \
|
||||
NFN_MAT_KERNEL_LOCAL const *right = (NFN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans = (NFN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr;\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(1); \
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _NFN_MK_EXPAND_LEFT_RIGHT_ANS \
|
||||
NFN_MAT_KERNEL_LOCAL const *left = (NFN_MAT_KERNEL_LOCAL const *)task->buffer_0.ptr; \
|
||||
NFN_MAT_KERNEL_LOCAL const *right = (NFN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans = (NFN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr;\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(0); \
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(1);\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _NFN_MK_EXPAND_RIGHT_ANS_SCALAR \
|
||||
NFN_MAT_KERNEL_LOCAL const *right = (NFN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans = (NFN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
|
||||
M128 scalar = task->scalar_pack;\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(1);\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _NFN_MK_EXPAND_LEFT_RIGHT_ANS_SCALAR \
|
||||
NFN_MAT_KERNEL_LOCAL const *left = (NFN_MAT_KERNEL_LOCAL const *)task->buffer_0.ptr; \
|
||||
NFN_MAT_KERNEL_LOCAL const *right = (NFN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans = (NFN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
|
||||
M128 scalar = task->scalar_pack;\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(0);\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(1);\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _NFN_MK_EXPAND_ANS_SCALAR \
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans = (NFN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
|
||||
M128 scalar = task->scalar_pack;\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _NFN_PARAM_STYLE(T) _NFN_MATPROC_##T
|
||||
#define _NFN_MK_EXPAND_PARAM(style) _NFN_MK_EXPAND_##style
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#define __mk_param_std(param, T) \
|
||||
typedef MAT_KERNEL(T) NFN_MAT_KERNEL_LOCAL; \
|
||||
matrix_task const *_ = (_NFN_PARAM_STYLE(param), task); \
|
||||
_NFN_MK_EXPAND_PARAM(param)
|
||||
|
||||
#define __sk_param_std(T) \
|
||||
typedef MAT_KERNEL(T) NFN_MAT_KERNEL_LOCAL; \
|
||||
(void)scalar
|
||||
#endif // !NFN_INTRINSIC_PARAM_EXPAND
|
||||
@@ -0,0 +1,94 @@
|
||||
#ifndef NFN_TARGET
|
||||
#define NFN_TARGET
|
||||
|
||||
#include "nfn_internal.h"
|
||||
#include <stdint.h>
|
||||
|
||||
#define NFN_OS_WINDOWS 0
|
||||
#define NFN_OS_LINUX 1
|
||||
#define NFN_OS_MACOS 2
|
||||
#define NFN_OS_IOS 3
|
||||
#define NFN_OS_ANDROID 4
|
||||
#define NFN_OS_UNKNOWN 5
|
||||
|
||||
#define NFN_ARCH_X86_64 0
|
||||
#define NFN_ARCH_ARM64 1
|
||||
#define NFN_ARCH_RISCV64 2
|
||||
#define NFN_ARCH_PPC64le 3
|
||||
|
||||
#if defined(_M_X64) || defined(_M_AMD64) || defined(__x86_64__) || defined(__amd64__)
|
||||
#define NFN_CURRENT_ARCH NFN_ARCH_X86_64
|
||||
|
||||
#elif defined(_M_ARM64) || defined(__aarch64__)
|
||||
#define NFN_CURRENT_ARCH NFN_ARCH_ARM64
|
||||
|
||||
#elif defined(_M_ARM) || defined(__arm__)
|
||||
#define NFN_CURRENT_ARCH NFN_ARCH_ARM32
|
||||
|
||||
#elif defined(_M_IX86) || defined(__i386__)
|
||||
#define NFN_CURRENT_ARCH NFN_ARCH_X86_32
|
||||
|
||||
#elif defined(__powerpc64__) && defined(__LITTLE_ENDIAN__)
|
||||
#define NFN_CURRENT_ARCH NFN_ARCH_PPC64LE
|
||||
|
||||
#elif defined(__riscv) && (__riscv_xlen == 64)
|
||||
#define NFN_CURRENT_ARCH NFN_ARCH_RISCV64
|
||||
|
||||
#else
|
||||
#error "SN: Unsupported target architecture"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Target selection policy:
|
||||
* - Default (no NFN_EXTERN_TARGET): fixed Windows + x86_64.
|
||||
* - External (NFN_EXTERN_TARGET): build system must provide NFN_OS and NFN_ARCH.
|
||||
*/
|
||||
#ifndef NFN_EXTERN_TARGET
|
||||
|
||||
#define NFN_OS_TARGET NFN_OS_WINDOWS
|
||||
#define NFN_ARCH_TARGET NFN_ARCH_X86_64
|
||||
|
||||
#else
|
||||
|
||||
#ifndef NFN_OS_TARGET
|
||||
#error "NFN_EXTERN_TARGET is defined but NFN_OS is not provided."
|
||||
#endif
|
||||
|
||||
#ifndef NFN_ARCH_TARGET
|
||||
#error "NFN_EXTERN_TARGET is defined but NFN_ARCH is not provided."
|
||||
#endif
|
||||
|
||||
/* Validate externally supplied values are within known enum constants. */
|
||||
#if (NFN_OS_TARGET != NFN_OS_WINDOWS) && (NFN_OS_TARGET != NFN_OS_LINUX) && \
|
||||
(NFN_OS_TARGET != NFN_OS_MACOS) && (NFN_OS_TARGET != NFN_OS_IOS) && \
|
||||
(NFN_OS_TARGET != NFN_OS_ANDROID) && (NFN_OS_TARGET != NFN_OS_UNKNOWN)
|
||||
#error "NFN_OS has an invalid value. Use NFN_OS_<...> enum constants."
|
||||
#endif
|
||||
|
||||
#if (NFN_ARCH_TARGET != NFN_ARCH_X86_64) && (NFN_ARCH_TARGET != NFN_ARCH_ARM64) && \
|
||||
(NFN_ARCH_TARGET != NFN_ARCH_RISCV64) && (NFN_ARCH_TARGET != NFN_ARCH_PPC64le)
|
||||
#error "NFN_ARCH has an invalid value. Use NFN_ARCH_<...> enum constants."
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#define NFN_IS_OS(os) ((NFN_OS_TARGET) == (os))
|
||||
|
||||
#define NFN_IS_ARCH(arch) ((NFN_ARCH_TARGET) == (arch))
|
||||
|
||||
/*
|
||||
* Per-function ISA selection is required for runtime-dispatched kernels that
|
||||
* mix multiple x86 feature levels in one build target.
|
||||
*/
|
||||
#if defined(__clang__) && ((NFN_ARCH_TARGET) == NFN_ARCH_X86_64)
|
||||
#define NFN_CLANG_TARGET_ATTR(options) __attribute__((target(options)))
|
||||
#else
|
||||
#define NFN_CLANG_TARGET_ATTR(options)
|
||||
#endif
|
||||
|
||||
typedef union SIMDID {
|
||||
uint64_t mask;
|
||||
char make_byte[8];
|
||||
} SIMDID;
|
||||
|
||||
#endif // NFN_TARGET
|
||||
@@ -1,18 +1,18 @@
|
||||
#include "NitloadFrameWork.h"
|
||||
#include "nif_intrinsic.matkernel.h"
|
||||
#include "nfn_intrinsic.matkernel.h"
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
|
||||
/* Define and initialize a global zero kernel aligned to 64 bytes for universal ISA usage */
|
||||
|
||||
__declspec(align(64)) MAT_KERNEL(u64) NIF_ZERO_KERNEL;
|
||||
__declspec(align(64)) MAT_KERNEL(u64) NFN_ZERO_KERNEL;
|
||||
|
||||
static void init_zero_kernel(void)
|
||||
{
|
||||
for (int c = 0; c < 4; ++c) {
|
||||
for (int r = 0; r < 4; ++r) {
|
||||
NIF_ZERO_KERNEL.m[r][c] = 0;
|
||||
NFN_ZERO_KERNEL.m[r][c] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
#include "NitloadFrameWork.h"
|
||||
|
||||
NIF_API void NIF_DECL nif_load_glfw_functions(GLFWFunctionTable *table)
|
||||
NFN_API void NFN_DECL nfn_load_glfw_functions(GLFWFunctionTable *table)
|
||||
{
|
||||
table->glfwCreateCursor = glfwCreateCursor;
|
||||
table->glfwCreateStandardCursor = glfwCreateStandardCursor;
|
||||
+15
-15
@@ -1,28 +1,28 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "NitloadFrameWork.h"
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
|
||||
PRIVATE void load_intrinsic_sse(nif_intrinsic *context, SIMDID id);
|
||||
PRIVATE void load_intrinsic_avx(nif_intrinsic *context, SIMDID id);
|
||||
PRIVATE void load_intrinsic_512(nif_intrinsic *context, SIMDID id);
|
||||
PRIVATE void load_intrinsic_sse(nfn_intrinsic *context, SIMDID id);
|
||||
PRIVATE void load_intrinsic_avx(nfn_intrinsic *context, SIMDID id);
|
||||
PRIVATE void load_intrinsic_512(nfn_intrinsic *context, SIMDID id);
|
||||
|
||||
NIF_API void NIF_DECL nif_load_intrinsic_function(void *intrinsic_context, uint64_t id)
|
||||
NFN_API void NFN_DECL nfn_load_intrinsic_function(void *intrinsic_context, uint64_t id)
|
||||
{
|
||||
nif_null_assert(intrinsic_context);
|
||||
nfn_null_assert(intrinsic_context);
|
||||
|
||||
SIMDID simdid;
|
||||
if (id) {
|
||||
simdid.mask = id;
|
||||
} else {
|
||||
simdid = nif_get_simd_level_global();
|
||||
simdid = nfn_get_simd_level_global();
|
||||
}
|
||||
|
||||
nif_intrinsic *context = (nif_intrinsic *)intrinsic_context;
|
||||
nfn_intrinsic *context = (nfn_intrinsic *)intrinsic_context;
|
||||
|
||||
switch (simdid.make_byte[1]) {
|
||||
case SIMDID_MAIN_LVL_SSE:
|
||||
@@ -38,7 +38,7 @@ NIF_API void NIF_DECL nif_load_intrinsic_function(void *intrinsic_context, uint6
|
||||
}
|
||||
}
|
||||
|
||||
void load_intrinsic_sse(nif_intrinsic *context, SIMDID id)
|
||||
void load_intrinsic_sse(nfn_intrinsic *context, SIMDID id)
|
||||
{
|
||||
context->city_hash_simplify = city_hash_simplify_sse;
|
||||
context->factorial_simd = factorial_simd_sse;
|
||||
@@ -62,7 +62,7 @@ void load_intrinsic_sse(nif_intrinsic *context, SIMDID id)
|
||||
context->vec_f32_normalize_3 = f32_normalize;
|
||||
}
|
||||
|
||||
void load_intrinsic_avx(nif_intrinsic *context, SIMDID id)
|
||||
void load_intrinsic_avx(nfn_intrinsic *context, SIMDID id)
|
||||
{
|
||||
char avx_seg = id.make_byte[4];
|
||||
char is_fma = (avx_seg & SIMDID_AVX_FMA) != 0;
|
||||
@@ -91,7 +91,7 @@ void load_intrinsic_avx(nif_intrinsic *context, SIMDID id)
|
||||
context->vec_f32_normalize_3 = f32_normalize;
|
||||
}
|
||||
|
||||
void load_intrinsic_512(nif_intrinsic *context, SIMDID id)
|
||||
void load_intrinsic_512(nfn_intrinsic *context, SIMDID id)
|
||||
{
|
||||
context->city_hash_simplify = city_hash_simplify_sse;
|
||||
context->factorial_simd = factorial_simd_sse;
|
||||
+29
-29
@@ -1,13 +1,13 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include <immintrin.h>
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fill)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fill)
|
||||
{
|
||||
__mk_param_std(ANS_SCALAR, float);
|
||||
register __m128 vec = _mm_set_ps1(scalar.f32[0]);
|
||||
@@ -19,7 +19,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , fill)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fill)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fill)
|
||||
{
|
||||
__mk_param_std(ANS_SCALAR, float);
|
||||
register __m256 vec = _mm256_broadcast_ss(&scalar.f32[0]);
|
||||
@@ -29,7 +29,7 @@ NIF_MK_PROC_DECL_STD(float, avx, , fill)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fill)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fill)
|
||||
{
|
||||
__mk_param_std(ANS_SCALAR, float);
|
||||
register __m512 vec = _mm512_broadcastss_ps(scalar.m128);
|
||||
@@ -38,7 +38,7 @@ NIF_MK_PROC_DECL_STD(float, 512, , fill)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , fill)
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , fill)
|
||||
{
|
||||
__mk_param_std(ANS_SCALAR, double);
|
||||
register __m128d vec_d = _mm_set1_pd(scalar.f64[0]);
|
||||
@@ -55,7 +55,7 @@ NIF_MK_PROC_DECL_STD(double, sse, , fill)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , fill)
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , fill)
|
||||
{
|
||||
__mk_param_std(ANS_SCALAR, double);
|
||||
register __m256d vec = _mm256_broadcast_sd(&scalar.f64[0]);
|
||||
@@ -67,7 +67,7 @@ NIF_MK_PROC_DECL_STD(double, avx, , fill)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , fill)
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , fill)
|
||||
{
|
||||
__mk_param_std(ANS_SCALAR, double);
|
||||
register __m512d vec = _mm512_set1_pd(scalar.f64[0]);
|
||||
@@ -77,7 +77,7 @@ NIF_MK_PROC_DECL_STD(double, 512, , fill)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , add)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , add)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, float);
|
||||
|
||||
@@ -94,7 +94,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , add)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , add)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , add)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, float);
|
||||
|
||||
@@ -107,7 +107,7 @@ NIF_MK_PROC_DECL_STD(float, avx, , add)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , add)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , add)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, float);
|
||||
|
||||
@@ -116,7 +116,7 @@ NIF_MK_PROC_DECL_STD(float, 512, , add)
|
||||
ans[i].z[0] = c0;
|
||||
}
|
||||
}
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , add)
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , add)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, double);
|
||||
|
||||
@@ -149,7 +149,7 @@ NIF_MK_PROC_DECL_STD(double, sse, , add)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , add)
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , add)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, double);
|
||||
|
||||
@@ -166,7 +166,7 @@ NIF_MK_PROC_DECL_STD(double, avx, , add)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , add)
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , add)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, double);
|
||||
|
||||
@@ -178,7 +178,7 @@ NIF_MK_PROC_DECL_STD(double, 512, , add)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , sub)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , sub)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, float);
|
||||
|
||||
@@ -195,7 +195,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , sub)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , sub)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , sub)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, float);
|
||||
|
||||
@@ -207,7 +207,7 @@ NIF_MK_PROC_DECL_STD(float, avx, , sub)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , sub)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , sub)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, float);
|
||||
|
||||
@@ -217,7 +217,7 @@ NIF_MK_PROC_DECL_STD(float, 512, , sub)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , sub)
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , sub)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, double);
|
||||
|
||||
@@ -242,7 +242,7 @@ NIF_MK_PROC_DECL_STD(double, sse, , sub)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , sub)
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , sub)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, double);
|
||||
|
||||
@@ -254,7 +254,7 @@ NIF_MK_PROC_DECL_STD(double, avx, , sub)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , sub)
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , sub)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, double);
|
||||
|
||||
@@ -264,7 +264,7 @@ NIF_MK_PROC_DECL_STD(double, 512, , sub)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , scalar_mul)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , scalar_mul)
|
||||
{
|
||||
__mk_param_std( RIGHT_ANS_SCALAR, float);
|
||||
|
||||
@@ -282,7 +282,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , scalar_mul)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , scalar_mul)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , scalar_mul)
|
||||
{
|
||||
__mk_param_std( RIGHT_ANS_SCALAR, float);
|
||||
|
||||
@@ -296,7 +296,7 @@ NIF_MK_PROC_DECL_STD(float, avx, , scalar_mul)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , scalar_mul)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , scalar_mul)
|
||||
{
|
||||
__mk_param_std( RIGHT_ANS_SCALAR, float);
|
||||
|
||||
@@ -308,7 +308,7 @@ NIF_MK_PROC_DECL_STD(float, 512, , scalar_mul)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , scalar_mul)
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , scalar_mul)
|
||||
{
|
||||
__mk_param_std( RIGHT_ANS_SCALAR, double);
|
||||
|
||||
@@ -334,7 +334,7 @@ NIF_MK_PROC_DECL_STD(double, sse, , scalar_mul)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , scalar_mul)
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , scalar_mul)
|
||||
{
|
||||
__mk_param_std( RIGHT_ANS_SCALAR, double);
|
||||
|
||||
@@ -352,7 +352,7 @@ NIF_MK_PROC_DECL_STD(double, avx, , scalar_mul)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , scalar_mul)
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , scalar_mul)
|
||||
{
|
||||
__mk_param_std( RIGHT_ANS_SCALAR, double);
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
#include <math.h>
|
||||
|
||||
static const float quality_threshold = 1e-6f;
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , try_plu)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , try_plu)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS_SCALAR, float);
|
||||
|
||||
@@ -115,7 +115,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , try_plu)
|
||||
* layout of scalarpack:
|
||||
*
|
||||
*/
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , quality)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , quality)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS, float);
|
||||
|
||||
@@ -126,7 +126,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , quality)
|
||||
register __m128 col2 = _mm_load_ps((float *)&right->x[2]);
|
||||
register __m128 col3 = _mm_load_ps((float *)&right->x[3]);
|
||||
|
||||
NIF_ALIGNAS(16) float self_dot_src[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
|
||||
NFN_ALIGNAS(16) float self_dot_src[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
|
||||
// try in reigister mix later
|
||||
DOT_F32_SSE(col0, col0, self_dot_src[0]);
|
||||
DOT_F32_SSE(col1, col1, self_dot_src[1]);
|
||||
@@ -153,7 +153,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , quality)
|
||||
return;
|
||||
}
|
||||
|
||||
NIF_ALIGNAS(16) float cross_dot_src[2][4] = { 0 };
|
||||
NFN_ALIGNAS(16) float cross_dot_src[2][4] = { 0 };
|
||||
DOT_F32_SSE(col0, col1, cross_dot_src[0][0]);
|
||||
DOT_F32_SSE(col0, col2, cross_dot_src[0][1]);
|
||||
DOT_F32_SSE(col0, col3, cross_dot_src[0][2]);
|
||||
+8
-8
@@ -1,13 +1,13 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
|
||||
// Matrix transpose implementations
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , transpose)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , transpose)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS, float);
|
||||
|
||||
@@ -26,7 +26,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , transpose)
|
||||
ans->x[3] = _mm_movehl_ps(tmp3, tmp1);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , transpose)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , transpose)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS, float);
|
||||
register __m256 col_low = right->y[0];
|
||||
@@ -42,7 +42,7 @@ NIF_MK_PROC_DECL_STD(float, avx, , transpose)
|
||||
ans->y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , transpose)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , transpose)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS, float);
|
||||
register __m512 matrix = right->z[0];
|
||||
@@ -1,8 +1,8 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
|
||||
/*
|
||||
* pack left matrix data to L4 buffer
|
||||
@@ -19,14 +19,14 @@
|
||||
* scalar pack carries column length and offset data:
|
||||
* [column length, empty, empty, empty]
|
||||
*/
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fma_pack_left)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fma_pack_left)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS_SCALAR, float);
|
||||
// for SSE branch, no need for rounding, a direct copy is sufficient as long as the buffer is large enough to hold the packed data
|
||||
memcpy(ans, right + offset_1, sizeof(MAT_KERNEL(float)) * count_2);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fma_pack_left)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fma_pack_left)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS_SCALAR, float);
|
||||
size_t i = 0;
|
||||
@@ -61,7 +61,7 @@ NIF_MK_PROC_DECL_STD(float, avx, , fma_pack_left)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fma_pack_left)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fma_pack_left)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS_SCALAR, float);
|
||||
size_t i = 0;
|
||||
@@ -154,7 +154,7 @@ NIF_MK_PROC_DECL_STD(float, 512, , fma_pack_left)
|
||||
* scalar pack carries column length and offset data:
|
||||
* [column length, empty, empty, empty]
|
||||
*/
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fma_pack_right)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fma_pack_right)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS_SCALAR, float);
|
||||
int col_length = scalar.i32[0];
|
||||
@@ -163,7 +163,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , fma_pack_right)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fma_pack_right)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fma_pack_right)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS_SCALAR, float);
|
||||
int col_length = scalar.i32[0];
|
||||
@@ -205,7 +205,7 @@ NIF_MK_PROC_DECL_STD(float, avx, , fma_pack_right)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fma_pack_right)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fma_pack_right)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS_SCALAR, float);
|
||||
int col_length = scalar.i32[0];
|
||||
@@ -1,22 +1,22 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
#include <immintrin.h>
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
// #include "nif_intrinsic.h"
|
||||
#include "nif_template.fma.x86.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
// #include "nfn_intrinsic.h"
|
||||
#include "nfn_template.fma.x86.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fma)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, float);
|
||||
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
|
||||
register __m128 c0 = _mm_load_ps((float *)&ans_cur->x[0]);
|
||||
register __m128 c1 = _mm_load_ps((float *)&ans_cur->x[1]);
|
||||
@@ -25,8 +25,8 @@ NIF_MK_PROC_DECL_STD(float, sse, , fma)
|
||||
register __m128 bcol;
|
||||
|
||||
for (size_t z = 0; z < count_0; z++) {
|
||||
NIF_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NIF_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
NFN_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NFN_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
|
||||
const __m128 a0 = _mm_load_ps((float *)&left_base->x[0]);
|
||||
const __m128 a1 = _mm_load_ps((float *)&left_base->x[1]);
|
||||
@@ -68,18 +68,18 @@ NIF_MK_PROC_DECL_STD(float, sse, , fma)
|
||||
_mm_store_ps((float *)&ans_cur->x[3], c3);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fma)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, float);
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]);
|
||||
register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]);
|
||||
register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]);
|
||||
register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]);
|
||||
|
||||
for (size_t z = 0; z < count_0; z++) {
|
||||
NIF_MAT_KERNEL_LOCAL const *b_base = right;
|
||||
NIF_MAT_KERNEL_LOCAL const *a_base = left;
|
||||
NFN_MAT_KERNEL_LOCAL const *b_base = right;
|
||||
NFN_MAT_KERNEL_LOCAL const *a_base = left;
|
||||
|
||||
register __m256 b0 = _mm256_load_ps((float *)&b_base->y[0]);
|
||||
register __m256 b1 = _mm256_load_ps((float *)&b_base->y[1]);
|
||||
@@ -91,10 +91,10 @@ NIF_MK_PROC_DECL_STD(float, avx, , fma)
|
||||
register __m256 a2 = _mm256_load_ps((float *)&a_base->y[2]);
|
||||
register __m256 a3 = _mm256_load_ps((float *)&a_base->y[3]);
|
||||
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(0);
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(1);
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(2);
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(3);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(0);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(1);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(2);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(3);
|
||||
}
|
||||
_mm256_store_ps((float *)&(ans_cur->y[0]), c0);
|
||||
_mm256_store_ps((float *)&(ans_cur->y[1]), c1);
|
||||
@@ -102,18 +102,18 @@ NIF_MK_PROC_DECL_STD(float, avx, , fma)
|
||||
_mm256_store_ps((float *)&(ans_cur->y[3]), c3);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, _fma, fma)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, _fma, fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, float);
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]);
|
||||
register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]);
|
||||
register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]);
|
||||
register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]);
|
||||
|
||||
for (size_t z = 0; z < count_0; z += 2) {
|
||||
const NIF_MAT_KERNEL_LOCAL *a_base = left + z;
|
||||
const NIF_MAT_KERNEL_LOCAL *b_base = right + z;
|
||||
const NFN_MAT_KERNEL_LOCAL *a_base = left + z;
|
||||
const NFN_MAT_KERNEL_LOCAL *b_base = right + z;
|
||||
|
||||
register __m256 b0 = _mm256_load_ps((float *)&b_base->y[0]);
|
||||
register __m256 b1 = _mm256_load_ps((float *)&b_base->y[1]);
|
||||
@@ -125,10 +125,10 @@ NIF_MK_PROC_DECL_STD(float, avx, _fma, fma)
|
||||
register __m256 a2 = _mm256_load_ps((float *)&a_base->y[2]);
|
||||
register __m256 a3 = _mm256_load_ps((float *)&a_base->y[3]);
|
||||
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(0);
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(1);
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(2);
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(3);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(0);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(1);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(2);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(3);
|
||||
}
|
||||
_mm256_store_ps((float *)&(ans_cur->y[0]), c0);
|
||||
_mm256_store_ps((float *)&(ans_cur->y[1]), c1);
|
||||
@@ -136,18 +136,18 @@ NIF_MK_PROC_DECL_STD(float, avx, _fma, fma)
|
||||
_mm256_store_ps((float *)&(ans_cur->y[3]), c3);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fma)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, float);
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
register __m512 c0 = _mm512_load_ps((float *)&ans_cur->z[0]);
|
||||
register __m512 c1 = _mm512_load_ps((float *)&ans_cur->z[0]);
|
||||
register __m512 c2 = _mm512_load_ps((float *)&ans_cur->z[0]);
|
||||
register __m512 c3 = _mm512_load_ps((float *)&ans_cur->z[0]);
|
||||
|
||||
for (size_t z = 0; z < count_0; z += 4) {
|
||||
const NIF_MAT_KERNEL_LOCAL *a_base = left + z;
|
||||
const NIF_MAT_KERNEL_LOCAL *b_base = right + z;
|
||||
const NFN_MAT_KERNEL_LOCAL *a_base = left + z;
|
||||
const NFN_MAT_KERNEL_LOCAL *b_base = right + z;
|
||||
|
||||
register __m512 b0 = _mm512_load_ps((float *)&b_base->z[0]);
|
||||
register __m512 b1 = _mm512_load_ps((float *)&b_base->z[1]);
|
||||
@@ -159,10 +159,10 @@ NIF_MK_PROC_DECL_STD(float, 512, , fma)
|
||||
register __m512 a2 = _mm512_load_ps((float *)&a_base->z[2]);
|
||||
register __m512 a3 = _mm512_load_ps((float *)&a_base->z[3]);
|
||||
|
||||
NIF_MK_FMA_F32_512_CYCLE(0);
|
||||
NIF_MK_FMA_F32_512_CYCLE(1);
|
||||
NIF_MK_FMA_F32_512_CYCLE(2);
|
||||
NIF_MK_FMA_F32_512_CYCLE(3);
|
||||
NFN_MK_FMA_F32_512_CYCLE(0);
|
||||
NFN_MK_FMA_F32_512_CYCLE(1);
|
||||
NFN_MK_FMA_F32_512_CYCLE(2);
|
||||
NFN_MK_FMA_F32_512_CYCLE(3);
|
||||
}
|
||||
_mm512_store_ps((float *)&ans_cur->z[0], c0);
|
||||
_mm512_store_ps((float *)&ans_cur->z[1], c1);
|
||||
@@ -170,11 +170,11 @@ NIF_MK_PROC_DECL_STD(float, 512, , fma)
|
||||
_mm512_store_ps((float *)&ans_cur->z[3], c3);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , fma)
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, double);
|
||||
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
|
||||
register __m128d c0_lo = ans_cur->x[0];
|
||||
register __m128d c0_hi = ans_cur->x[1];
|
||||
@@ -186,18 +186,18 @@ NIF_MK_PROC_DECL_STD(double, sse, , fma)
|
||||
register __m128d c3_hi = ans_cur->x[7];
|
||||
|
||||
for (size_t z = 0; z < count_0; z++) {
|
||||
NIF_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NIF_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
NFN_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NFN_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
const double *right_values = &right_base->m[0][0];
|
||||
const double *left_values = &left_base->m[0][0];
|
||||
|
||||
NIF_MK_FMA_F64_SSE_PHASE(0);
|
||||
NFN_MK_FMA_F64_SSE_PHASE(0);
|
||||
|
||||
NIF_MK_FMA_F64_SSE_PHASE(1);
|
||||
NFN_MK_FMA_F64_SSE_PHASE(1);
|
||||
|
||||
NIF_MK_FMA_F64_SSE_PHASE(2);
|
||||
NFN_MK_FMA_F64_SSE_PHASE(2);
|
||||
|
||||
NIF_MK_FMA_F64_SSE_PHASE(3);
|
||||
NFN_MK_FMA_F64_SSE_PHASE(3);
|
||||
}
|
||||
|
||||
ans_cur->x[0] = c0_lo;
|
||||
@@ -210,11 +210,11 @@ NIF_MK_PROC_DECL_STD(double, sse, , fma)
|
||||
ans_cur->x[7] = c3_hi;
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , fma)
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, double);
|
||||
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
|
||||
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->y[0]);
|
||||
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->y[1]);
|
||||
@@ -223,8 +223,8 @@ NIF_MK_PROC_DECL_STD(double, avx, , fma)
|
||||
register __m256d bcol;
|
||||
|
||||
for (size_t z = 0; z < count_0; z++) {
|
||||
NIF_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NIF_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
NFN_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NFN_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
|
||||
const __m256d a0 = _mm256_load_pd((double *)&left_base->y[0]);
|
||||
const __m256d a1 = _mm256_load_pd((double *)&left_base->y[1]);
|
||||
@@ -282,11 +282,11 @@ NIF_MK_PROC_DECL_STD(double, avx, , fma)
|
||||
_mm256_store_pd((double *)&ans_cur->y[3], c3);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, avx, _fma, fma)
|
||||
NFN_MK_PROC_DECL_STD(double, avx, _fma, fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, double);
|
||||
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
|
||||
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->y[0]);
|
||||
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->y[1]);
|
||||
@@ -295,8 +295,8 @@ NIF_MK_PROC_DECL_STD(double, avx, _fma, fma)
|
||||
register __m256d bcol;
|
||||
|
||||
for (size_t z = 0; z < count_0; z++) {
|
||||
NIF_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NIF_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
NFN_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NFN_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
|
||||
const __m256d a0 = _mm256_load_pd((double *)&left_base->y[0]);
|
||||
const __m256d a1 = _mm256_load_pd((double *)&left_base->y[1]);
|
||||
@@ -338,10 +338,10 @@ NIF_MK_PROC_DECL_STD(double, avx, _fma, fma)
|
||||
_mm256_store_pd((double *)&ans_cur->y[3], c3);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , fma)
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, double);
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
|
||||
register __m512d c0 = _mm512_load_pd((double *)&ans_cur->z[0]);
|
||||
register __m512d c1 = _mm512_load_pd((double *)&ans_cur->z[1]);
|
||||
@@ -349,8 +349,8 @@ NIF_MK_PROC_DECL_STD(double, 512, , fma)
|
||||
register __m512d c3 = _mm512_load_pd((double *)&ans_cur->z[3]);
|
||||
|
||||
for (size_t z = 0; z < count_0; z += 2) {
|
||||
NIF_MAT_KERNEL_LOCAL const *right_base = right + z;
|
||||
NIF_MAT_KERNEL_LOCAL const *left_base = left + z;
|
||||
NFN_MAT_KERNEL_LOCAL const *right_base = right + z;
|
||||
NFN_MAT_KERNEL_LOCAL const *left_base = left + z;
|
||||
|
||||
register __m512d a0 = _mm512_load_pd((double *)&left_base->z[0]);
|
||||
register __m512d a1 = _mm512_load_pd((double *)&left_base->z[1]);
|
||||
@@ -362,10 +362,10 @@ NIF_MK_PROC_DECL_STD(double, 512, , fma)
|
||||
register __m512d b2 = _mm512_load_pd((double *)&right_base->z[2]);
|
||||
register __m512d b3 = _mm512_load_pd((double *)&right_base->z[3]);
|
||||
|
||||
NIF_MK_FMA_F64_AVX2_CYCLE(0);
|
||||
NIF_MK_FMA_F64_AVX2_CYCLE(1);
|
||||
NIF_MK_FMA_F64_AVX2_CYCLE(2);
|
||||
NIF_MK_FMA_F64_AVX2_CYCLE(3);
|
||||
NFN_MK_FMA_F64_AVX2_CYCLE(0);
|
||||
NFN_MK_FMA_F64_AVX2_CYCLE(1);
|
||||
NFN_MK_FMA_F64_AVX2_CYCLE(2);
|
||||
NFN_MK_FMA_F64_AVX2_CYCLE(3);
|
||||
}
|
||||
_mm512_store_pd((double *)&ans_cur->z[0], c0);
|
||||
_mm512_store_pd((double *)&ans_cur->z[1], c1);
|
||||
+10
-10
@@ -1,25 +1,25 @@
|
||||
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "nif_internal.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nif_intrinsic.matkernel.h"
|
||||
#include "nfn_internal.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.matkernel.h"
|
||||
#include <emmintrin.h>
|
||||
#include <immintrin.h>
|
||||
#include <pmmintrin.h>
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , k_quality)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , k_quality)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS_SCALAR, float);
|
||||
}
|
||||
|
||||
INTERNAL NIF_STD_TARGET_ATTR(sse, ) void kp_check_f32(mat_kernel *kernel)
|
||||
INTERNAL NFN_STD_TARGET_ATTR(sse, ) void kp_check_f32(mat_kernel *kernel)
|
||||
{
|
||||
typedef MAT_KERNEL(float) NIF_MAT;
|
||||
NIF_MAT const *k = (NIF_MAT const *)kernel;
|
||||
typedef MAT_KERNEL(float) NFN_MAT;
|
||||
NFN_MAT const *k = (NFN_MAT const *)kernel;
|
||||
float cross[4];
|
||||
cross[0] = k->m[0][0];
|
||||
cross[1] = k->m[1][1];
|
||||
+6
-6
@@ -1,14 +1,14 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "nif_internal.h"
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nfn_internal.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
|
||||
static __m128i facIncrement;
|
||||
static __m128i facBase;
|
||||
|
||||
INTERNAL uint64_t NIF_DECL factorial_simd_sse(int n)
|
||||
INTERNAL uint64_t NFN_DECL factorial_simd_sse(int n)
|
||||
{
|
||||
facIncrement = _mm_set1_epi32(4);
|
||||
facBase = _mm_setr_epi32(1, 2, 3, 4);
|
||||
@@ -20,7 +20,7 @@ INTERNAL uint64_t NIF_DECL factorial_simd_sse(int n)
|
||||
facBaseLocal = _mm_add_epi32(facBaseLocal, facIncrement);
|
||||
baseResult = _mm_mullo_epi32(baseResult, facBaseLocal);
|
||||
}
|
||||
NIF_ALIGNAS(16) int32_t lanes[4];
|
||||
NFN_ALIGNAS(16) int32_t lanes[4];
|
||||
_mm_store_si128((__m128i *)lanes, baseResult);
|
||||
uint64_t result = (uint64_t)lanes[0] * (uint64_t)lanes[1] * (uint64_t)lanes[2] *
|
||||
(uint64_t)lanes[3];
|
||||
@@ -1,10 +1,10 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "NitloadFrameWork.h"
|
||||
|
||||
INTERNAL int NIF_DECL city_hash_simplify_sse(byte const *str, int const length)
|
||||
INTERNAL int NFN_DECL city_hash_simplify_sse(byte const *str, int const length)
|
||||
{
|
||||
uint32_t hash = length, size = length << 1;
|
||||
byte const *end = str + size;
|
||||
@@ -1,13 +1,13 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
/* clang-format off */
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#include "nif_intrinsic.h"
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "NitloadFrameWork.h"
|
||||
static SIMDID global_simdid = { 0 };
|
||||
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <string.h>
|
||||
@@ -126,7 +126,7 @@ static int is_amd_glue_ymm(const char *vendor, int cpuid1_eax)
|
||||
return (family == 0x17) && (model < 0x30);
|
||||
}
|
||||
|
||||
NIF_API SIMDID NIF_DECL nif_get_simd_level_local()
|
||||
NFN_API SIMDID NFN_DECL nfn_get_simd_level_local()
|
||||
{
|
||||
SIMDID simdid = { 0 };
|
||||
|
||||
@@ -314,13 +314,13 @@ NIF_API SIMDID NIF_DECL nif_get_simd_level_local()
|
||||
return simdid;
|
||||
}
|
||||
|
||||
NIF_API SIMDID NIF_DECL nif_get_simd_level_global()
|
||||
NFN_API SIMDID NFN_DECL nfn_get_simd_level_global()
|
||||
{
|
||||
if (global_simdid.mask) {
|
||||
return global_simdid;
|
||||
}
|
||||
|
||||
global_simdid = nif_get_simd_level_local();
|
||||
global_simdid = nfn_get_simd_level_local();
|
||||
return global_simdid;
|
||||
}
|
||||
|
||||
+7
-7
@@ -1,9 +1,9 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "NitloadFrameWork.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
|
||||
/* -----------------------------------------------------------------------
|
||||
* Field extraction helpers.
|
||||
@@ -49,7 +49,7 @@ static int impl_strength(uint16_t avx512)
|
||||
}
|
||||
|
||||
/* -----------------------------------------------------------------------
|
||||
* nif_get_unite_simd
|
||||
* nfn_get_unite_simd
|
||||
*
|
||||
* Produce the conservative intersection of two SIMDIDs — the strongest
|
||||
* capability set that is safe to use on BOTH cores:
|
||||
@@ -60,7 +60,7 @@ static int impl_strength(uint16_t avx512)
|
||||
* - Root / Manu : keep left (must be same arch in practice)
|
||||
* - uArchHi : bitwise OR (HYBRID_P/E union; SIMD_DUAL set if either core is dual)
|
||||
* ----------------------------------------------------------------------- */
|
||||
NIF_API SIMDID NIF_DECL nif_get_unite_simd(SIMDID left, SIMDID right)
|
||||
NFN_API SIMDID NFN_DECL nfn_get_unite_simd(SIMDID left, SIMDID right)
|
||||
{
|
||||
SIMDID out;
|
||||
out.mask = 0;
|
||||
@@ -117,7 +117,7 @@ NIF_API SIMDID NIF_DECL nif_get_unite_simd(SIMDID left, SIMDID right)
|
||||
}
|
||||
|
||||
/* -----------------------------------------------------------------------
|
||||
* nif_compare_simd
|
||||
* nfn_compare_simd
|
||||
*
|
||||
* Compare two SIMDIDs by ISA strength. Returns:
|
||||
* +1 if left is strictly stronger than right
|
||||
@@ -154,7 +154,7 @@ NIF_API SIMDID NIF_DECL nif_get_unite_simd(SIMDID left, SIMDID right)
|
||||
* 7. uArchHi SIMD_DUAL (final tie-break)
|
||||
* Non-dual (SIMD_DUAL=0) > dual (SIMD_DUAL=1).
|
||||
* ----------------------------------------------------------------------- */
|
||||
NIF_API int NIF_DECL nif_compare_simd(SIMDID left, SIMDID right)
|
||||
NFN_API int NFN_DECL nfn_compare_simd(SIMDID left, SIMDID right)
|
||||
{
|
||||
/* 1. MainLevel */
|
||||
unsigned lm = FIELD_MAIN(left);
|
||||
@@ -1,10 +1,10 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
|
||||
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
// float
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, sse, , fma)
|
||||
NFN_SK_PROC_DECL_STD(float, sse, , fma)
|
||||
{
|
||||
__sk_param_std(float);
|
||||
register __m128 c0 = _mm_load_ps((float *)&ans->x[0]);
|
||||
@@ -60,7 +60,7 @@ NIF_SK_PROC_DECL_STD(float, sse, , fma)
|
||||
_mm_store_ps((float *)&ans->x[3], c3);
|
||||
}
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, avx, , fma)
|
||||
NFN_SK_PROC_DECL_STD(float, avx, , fma)
|
||||
{
|
||||
__sk_param_std(float);
|
||||
register __m256 c01 = _mm256_load_ps((float *)&ans->y[0]);
|
||||
@@ -86,7 +86,7 @@ NIF_SK_PROC_DECL_STD(float, avx, , fma)
|
||||
_mm256_store_ps((float *)&ans->y[1], c23);
|
||||
}
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, avx, _fma, fma)
|
||||
NFN_SK_PROC_DECL_STD(float, avx, _fma, fma)
|
||||
{
|
||||
__sk_param_std(float);
|
||||
register __m256 c01 = _mm256_load_ps((float *)&ans->y[0]);
|
||||
@@ -112,7 +112,7 @@ NIF_SK_PROC_DECL_STD(float, avx, _fma, fma)
|
||||
_mm256_store_ps((float *)&ans->y[1], c23);
|
||||
}
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, 512, , fma)
|
||||
NFN_SK_PROC_DECL_STD(float, 512, , fma)
|
||||
{
|
||||
__sk_param_std(float);
|
||||
register __m512 c = _mm512_load_ps((float *)ans);
|
||||
@@ -173,7 +173,7 @@ NIF_SK_PROC_DECL_STD(float, 512, , fma)
|
||||
|
||||
// clang-format on
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, sse, , fma)
|
||||
NFN_SK_PROC_DECL_STD(double, sse, , fma)
|
||||
{
|
||||
__sk_param_std(double);
|
||||
const double *left_values = &left->m[0][0];
|
||||
@@ -204,7 +204,7 @@ NIF_SK_PROC_DECL_STD(double, sse, , fma)
|
||||
}
|
||||
|
||||
// clang-format off
|
||||
NIF_SK_PROC_DECL_STD(double, avx, , fma)
|
||||
NFN_SK_PROC_DECL_STD(double, avx, , fma)
|
||||
{
|
||||
__sk_param_std(double);
|
||||
register __m256d c0 = _mm256_load_pd((double *)&ans->y[0]);
|
||||
@@ -253,7 +253,7 @@ NIF_SK_PROC_DECL_STD(double, avx, , fma)
|
||||
}
|
||||
// clang-format on
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, avx, _fma, fma)
|
||||
NFN_SK_PROC_DECL_STD(double, avx, _fma, fma)
|
||||
{
|
||||
__sk_param_std(double);
|
||||
register __m256d c0 = _mm256_load_pd((double *)&ans->y[0]);
|
||||
@@ -301,7 +301,7 @@ NIF_SK_PROC_DECL_STD(double, avx, _fma, fma)
|
||||
_mm256_store_pd((double *)&ans->y[3], c3);
|
||||
}
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, 512, , fma)
|
||||
NFN_SK_PROC_DECL_STD(double, 512, , fma)
|
||||
{
|
||||
__sk_param_std(double);
|
||||
register __m512d c01 = _mm512_load_pd((double *)&ans->z[0]);
|
||||
+104
-64
@@ -1,8 +1,8 @@
|
||||
#define _GNU_SOURCE
|
||||
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_OS(NIF_OS_LINUX)
|
||||
#if NFN_IS_OS(NFN_OS_LINUX)
|
||||
#include <errno.h>
|
||||
#include <inttypes.h>
|
||||
#include <limits.h>
|
||||
@@ -16,27 +16,30 @@
|
||||
#include <sys/syscall.h>
|
||||
#include <unistd.h>
|
||||
|
||||
typedef int (*nif_linux_id_visitor)(int value, void *context);
|
||||
typedef int (*nfn_linux_id_visitor)(int value, void *context);
|
||||
|
||||
typedef struct nif_linux_counter_context {
|
||||
typedef struct nfn_linux_counter_context {
|
||||
int count;
|
||||
} nif_linux_counter_context;
|
||||
} nfn_linux_counter_context;
|
||||
|
||||
typedef struct nif_linux_max_context {
|
||||
typedef struct nfn_linux_max_context {
|
||||
int max_value;
|
||||
} nif_linux_max_context;
|
||||
} nfn_linux_max_context;
|
||||
|
||||
typedef struct nif_linux_fill_cpu_context {
|
||||
uint64_t *internal;
|
||||
uint32_t *global;
|
||||
typedef struct nfn_linux_fill_cpu_context {
|
||||
uint32_t *processor_group_ids;
|
||||
uint32_t *socket_ids;
|
||||
uint32_t *cpu_indices;
|
||||
uint32_t *numa_node_ids;
|
||||
int numa_id;
|
||||
int index;
|
||||
} nif_linux_fill_cpu_context;
|
||||
} nfn_linux_fill_cpu_context;
|
||||
|
||||
typedef struct nif_linux_fill_node_count_context {
|
||||
typedef struct nfn_linux_fill_node_count_context {
|
||||
int *count_of_each_numa;
|
||||
} nif_linux_fill_node_count_context;
|
||||
} nfn_linux_fill_node_count_context;
|
||||
|
||||
static int nif_linux_read_first_line(char const *path, char *buffer, size_t buffer_size)
|
||||
static int nfn_linux_read_first_line(char const *path, char *buffer, size_t buffer_size)
|
||||
{
|
||||
FILE *file = fopen(path, "r");
|
||||
if (file == NULL)
|
||||
@@ -47,7 +50,7 @@ static int nif_linux_read_first_line(char const *path, char *buffer, size_t buff
|
||||
return line != NULL;
|
||||
}
|
||||
|
||||
static int nif_linux_parse_nonnegative_int(char const **cursor, int *value)
|
||||
static int nfn_linux_parse_nonnegative_int(char const **cursor, int *value)
|
||||
{
|
||||
char *end = NULL;
|
||||
uintmax_t parsed_value = 0;
|
||||
@@ -65,7 +68,7 @@ static int nif_linux_parse_nonnegative_int(char const **cursor, int *value)
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nif_linux_visit_id_list(char const *text, nif_linux_id_visitor visitor, void *context)
|
||||
static int nfn_linux_visit_id_list(char const *text, nfn_linux_id_visitor visitor, void *context)
|
||||
{
|
||||
char const *cursor = text;
|
||||
|
||||
@@ -77,14 +80,14 @@ static int nif_linux_visit_id_list(char const *text, nif_linux_id_visitor visito
|
||||
break;
|
||||
|
||||
int begin = 0;
|
||||
if (!nif_linux_parse_nonnegative_int(&cursor, &begin))
|
||||
if (!nfn_linux_parse_nonnegative_int(&cursor, &begin))
|
||||
return 0;
|
||||
|
||||
int finish = begin;
|
||||
|
||||
if (*cursor == '-') {
|
||||
cursor++;
|
||||
if (!nif_linux_parse_nonnegative_int(&cursor, &finish))
|
||||
if (!nfn_linux_parse_nonnegative_int(&cursor, &finish))
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -103,65 +106,88 @@ static int nif_linux_visit_id_list(char const *text, nif_linux_id_visitor visito
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nif_linux_count_id(int value, void *context)
|
||||
static int nfn_linux_count_id(int value, void *context)
|
||||
{
|
||||
(void)value;
|
||||
((nif_linux_counter_context *)context)->count++;
|
||||
((nfn_linux_counter_context *)context)->count++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nif_linux_track_max_id(int value, void *context)
|
||||
static int nfn_linux_track_max_id(int value, void *context)
|
||||
{
|
||||
nif_linux_max_context *max_context = (nif_linux_max_context *)context;
|
||||
nfn_linux_max_context *max_context = (nfn_linux_max_context *)context;
|
||||
if (value > max_context->max_value)
|
||||
max_context->max_value = value;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nif_linux_fill_cpu_id(int value, void *context)
|
||||
static int nfn_linux_read_socket_id(int cpu_id, uint32_t *socket_id)
|
||||
{
|
||||
nif_linux_fill_cpu_context *fill_context = (nif_linux_fill_cpu_context *)context;
|
||||
char path[160];
|
||||
char buffer[64];
|
||||
char *end = NULL;
|
||||
long parsed_value;
|
||||
snprintf(path, sizeof(path),
|
||||
"/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu_id);
|
||||
if (!nfn_linux_read_first_line(path, buffer, sizeof(buffer)))
|
||||
return 0;
|
||||
|
||||
if (fill_context->internal != NULL)
|
||||
fill_context->internal[fill_context->index] = (uint64_t)fill_context->index;
|
||||
if (fill_context->global != NULL)
|
||||
fill_context->global[fill_context->index] = (uint32_t)value;
|
||||
errno = 0;
|
||||
parsed_value = strtol(buffer, &end, 10);
|
||||
if (end == buffer || errno == ERANGE || parsed_value < 0 || (uintmax_t)parsed_value > UINT32_MAX)
|
||||
return 0;
|
||||
|
||||
*socket_id = (uint32_t)parsed_value;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nfn_linux_fill_cpu_id(int value, void *context)
|
||||
{
|
||||
nfn_linux_fill_cpu_context *fill_context = (nfn_linux_fill_cpu_context *)context;
|
||||
uint32_t socket_id = 0;
|
||||
if (!nfn_linux_read_socket_id(value, &socket_id))
|
||||
return 0;
|
||||
|
||||
fill_context->processor_group_ids[fill_context->index] = 0u;
|
||||
fill_context->socket_ids[fill_context->index] = socket_id;
|
||||
fill_context->cpu_indices[fill_context->index] = (uint32_t)value;
|
||||
fill_context->numa_node_ids[fill_context->index] = (uint32_t)fill_context->numa_id;
|
||||
|
||||
fill_context->index++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nif_linux_count_cpu_list(char const *path)
|
||||
static int nfn_linux_count_cpu_list(char const *path)
|
||||
{
|
||||
char buffer[256];
|
||||
if (!nif_linux_read_first_line(path, buffer, sizeof(buffer)))
|
||||
if (!nfn_linux_read_first_line(path, buffer, sizeof(buffer)))
|
||||
return 0;
|
||||
|
||||
nif_linux_counter_context counter = { 0 };
|
||||
if (!nif_linux_visit_id_list(buffer, nif_linux_count_id, &counter))
|
||||
nfn_linux_counter_context counter = { 0 };
|
||||
if (!nfn_linux_visit_id_list(buffer, nfn_linux_count_id, &counter))
|
||||
return 0;
|
||||
|
||||
return counter.count;
|
||||
}
|
||||
|
||||
static int nif_linux_fill_node_cpu_count(int value, void *context)
|
||||
static int nfn_linux_fill_node_cpu_count(int value, void *context)
|
||||
{
|
||||
char path[128];
|
||||
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", value);
|
||||
|
||||
((nif_linux_fill_node_count_context *)context)->count_of_each_numa[value] =
|
||||
nif_linux_count_cpu_list(path);
|
||||
((nfn_linux_fill_node_count_context *)context)->count_of_each_numa[value] =
|
||||
nfn_linux_count_cpu_list(path);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nif_linux_node_exists(int numa_id)
|
||||
static int nfn_linux_node_exists(int numa_id)
|
||||
{
|
||||
char path[128];
|
||||
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d", numa_id);
|
||||
return access(path, F_OK) == 0;
|
||||
}
|
||||
|
||||
static size_t nif_linux_round_up_to_page_size(size_t size)
|
||||
static size_t nfn_linux_round_up_to_page_size(size_t size)
|
||||
{
|
||||
int64_t page_size_value = (int64_t)sysconf(_SC_PAGESIZE);
|
||||
size_t effective_page_size = (page_size_value > 0) ? (size_t)page_size_value : 4096u;
|
||||
@@ -173,7 +199,7 @@ static size_t nif_linux_round_up_to_page_size(size_t size)
|
||||
return rounded - (rounded % effective_page_size);
|
||||
}
|
||||
|
||||
static int nif_linux_bind_memory_to_node(void *address, size_t size, int numa_id, uint32_t flags)
|
||||
static int nfn_linux_bind_memory_to_node(void *address, size_t size, int numa_id, uint32_t flags)
|
||||
{
|
||||
#if defined(SYS_mbind)
|
||||
size_t bits_per_word = sizeof(unsigned long) * 8u;
|
||||
@@ -197,7 +223,7 @@ static int nif_linux_bind_memory_to_node(void *address, size_t size, int numa_id
|
||||
#endif
|
||||
}
|
||||
|
||||
static int nif_linux_populate_writable_mapping(void *address, size_t size)
|
||||
static int nfn_linux_populate_writable_mapping(void *address, size_t size)
|
||||
{
|
||||
#if defined(MADV_POPULATE_WRITE)
|
||||
if (madvise(address, size, MADV_POPULATE_WRITE) == 0)
|
||||
@@ -211,12 +237,14 @@ static int nif_linux_populate_writable_mapping(void *address, size_t size)
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Pin the calling thread to a single logical processor.
|
||||
* core_id >= 0 : bind to that logical processor (global index, across groups).
|
||||
* core_id < 0 : clear affinity constraint within the current group. */
|
||||
NIF_API void NIF_DECL nif_set_thread_affinity(int core_id, int numa_id)
|
||||
/* Linux has no Windows-style processor groups; processor_group_id is ignored. */
|
||||
NFN_API void NFN_DECL nfn_set_thread_affinity(int cpu_index, int processor_group_id)
|
||||
{
|
||||
int cpu_count = (int)sysconf(_SC_NPROCESSORS_ONLN);
|
||||
(void)processor_group_id;
|
||||
|
||||
int cpu_count = (int)sysconf(_SC_NPROCESSORS_CONF);
|
||||
if (cpu_index >= cpu_count)
|
||||
cpu_count = cpu_index + 1;
|
||||
if (cpu_count <= 0)
|
||||
return;
|
||||
|
||||
@@ -227,25 +255,25 @@ NIF_API void NIF_DECL nif_set_thread_affinity(int core_id, int numa_id)
|
||||
|
||||
CPU_ZERO_S(size, set);
|
||||
|
||||
if (core_id < 0) {
|
||||
if (cpu_index < 0) {
|
||||
for (int i = 0; i < cpu_count; i++)
|
||||
CPU_SET_S((int)i, size, set);
|
||||
} else {
|
||||
CPU_SET_S(core_id, size, set);
|
||||
CPU_SET_S(cpu_index, size, set);
|
||||
}
|
||||
|
||||
pthread_setaffinity_np(pthread_self(), size, set);
|
||||
CPU_FREE(set);
|
||||
}
|
||||
|
||||
NIF_API int NIF_DECL nif_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
|
||||
NFN_API int NFN_DECL nfn_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
|
||||
{
|
||||
char buffer[256];
|
||||
if (!nif_linux_read_first_line("/sys/devices/system/node/online", buffer, sizeof(buffer)))
|
||||
if (!nfn_linux_read_first_line("/sys/devices/system/node/online", buffer, sizeof(buffer)))
|
||||
return 0;
|
||||
|
||||
nif_linux_max_context max_context = { -1 };
|
||||
if (!nif_linux_visit_id_list(buffer, nif_linux_track_max_id, &max_context) ||
|
||||
nfn_linux_max_context max_context = { -1 };
|
||||
if (!nfn_linux_visit_id_list(buffer, nfn_linux_track_max_id, &max_context) ||
|
||||
max_context.max_value < 0)
|
||||
return 0;
|
||||
|
||||
@@ -256,32 +284,44 @@ NIF_API int NIF_DECL nif_get_numa_overall(int *out_numa_count, int *count_of_eac
|
||||
for (int index = 0; index <= max_context.max_value; index++)
|
||||
count_of_each_numa[index] = 0;
|
||||
|
||||
nif_linux_fill_node_count_context fill_context = { count_of_each_numa };
|
||||
if (!nif_linux_visit_id_list(buffer, nif_linux_fill_node_cpu_count, &fill_context))
|
||||
nfn_linux_fill_node_count_context fill_context = { count_of_each_numa };
|
||||
if (!nfn_linux_visit_id_list(buffer, nfn_linux_fill_node_cpu_count, &fill_context))
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
NIF_API int NIF_DECL nif_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global)
|
||||
NFN_API int NFN_DECL nfn_get_numa_each(int numa_id,
|
||||
uint32_t *processor_group_ids,
|
||||
uint32_t *socket_ids,
|
||||
uint32_t *cpu_indices,
|
||||
uint32_t *numa_node_ids)
|
||||
{
|
||||
if (numa_id < 0)
|
||||
if (numa_id < 0 || processor_group_ids == NULL || socket_ids == NULL ||
|
||||
cpu_indices == NULL || numa_node_ids == NULL)
|
||||
return 0;
|
||||
|
||||
char path[128];
|
||||
char buffer[256];
|
||||
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", numa_id);
|
||||
if (!nif_linux_read_first_line(path, buffer, sizeof(buffer)))
|
||||
if (!nfn_linux_read_first_line(path, buffer, sizeof(buffer)))
|
||||
return 0;
|
||||
|
||||
nif_linux_fill_cpu_context fill_context = { internal, global, 0 };
|
||||
return nif_linux_visit_id_list(buffer, nif_linux_fill_cpu_id, &fill_context);
|
||||
nfn_linux_fill_cpu_context fill_context = {
|
||||
processor_group_ids,
|
||||
socket_ids,
|
||||
cpu_indices,
|
||||
numa_node_ids,
|
||||
numa_id,
|
||||
0
|
||||
};
|
||||
return nfn_linux_visit_id_list(buffer, nfn_linux_fill_cpu_id, &fill_context);
|
||||
}
|
||||
|
||||
NIF_API void *NIF_DECL nif_numa_alloc(size_t size, int numa_id)
|
||||
NFN_API void *NFN_DECL nfn_numa_alloc(size_t size, int numa_id)
|
||||
{
|
||||
size_t allocation_size = nif_linux_round_up_to_page_size(size);
|
||||
size_t allocation_size = nfn_linux_round_up_to_page_size(size);
|
||||
if (allocation_size == 0)
|
||||
return NULL;
|
||||
|
||||
@@ -303,9 +343,9 @@ NIF_API void *NIF_DECL nif_numa_alloc(size_t size, int numa_id)
|
||||
bind_flags = MPOL_MF_MOVE | MPOL_MF_STRICT;
|
||||
#endif
|
||||
|
||||
if (!nif_linux_node_exists(numa_id) ||
|
||||
!nif_linux_bind_memory_to_node(memory, allocation_size, numa_id, bind_flags) ||
|
||||
!nif_linux_populate_writable_mapping(memory, allocation_size)) {
|
||||
if (!nfn_linux_node_exists(numa_id) ||
|
||||
!nfn_linux_bind_memory_to_node(memory, allocation_size, numa_id, bind_flags) ||
|
||||
!nfn_linux_populate_writable_mapping(memory, allocation_size)) {
|
||||
munmap(memory, allocation_size);
|
||||
return NULL;
|
||||
}
|
||||
@@ -313,14 +353,14 @@ NIF_API void *NIF_DECL nif_numa_alloc(size_t size, int numa_id)
|
||||
return memory;
|
||||
}
|
||||
|
||||
NIF_API void NIF_DECL nif_numa_free(void *ptr, size_t size, int numa_id)
|
||||
NFN_API void NFN_DECL nfn_numa_free(void *ptr, size_t size, int numa_id)
|
||||
{
|
||||
(void)numa_id;
|
||||
|
||||
if (ptr == NULL || size == 0)
|
||||
return;
|
||||
|
||||
size_t allocation_size = nif_linux_round_up_to_page_size(size);
|
||||
size_t allocation_size = nfn_linux_round_up_to_page_size(size);
|
||||
if (allocation_size != 0)
|
||||
munmap(ptr, allocation_size);
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
#include "nfn_target.h"
|
||||
#include "NitloadFrameWork.h"
|
||||
|
||||
#if NFN_IS_OS(NFN_OS_WINDOWS)
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <windows.h>
|
||||
|
||||
static int nfn_windows_get_socket_map(uint32_t **out_socket_map, DWORD *out_group_count)
|
||||
{
|
||||
DWORD group_count = GetActiveProcessorGroupCount();
|
||||
DWORD buffer_size = 0;
|
||||
uint32_t *socket_map;
|
||||
SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *information;
|
||||
BYTE *cursor;
|
||||
uint32_t socket_id = 0;
|
||||
|
||||
if (group_count == 0)
|
||||
return 0;
|
||||
|
||||
socket_map = (uint32_t *)malloc((size_t)group_count * 64u * sizeof(uint32_t));
|
||||
if (socket_map == NULL)
|
||||
return 0;
|
||||
for (size_t i = 0; i < (size_t)group_count * 64u; i++)
|
||||
socket_map[i] = UINT32_MAX;
|
||||
|
||||
GetLogicalProcessorInformationEx(RelationProcessorPackage, NULL, &buffer_size);
|
||||
if (GetLastError() != ERROR_INSUFFICIENT_BUFFER || buffer_size == 0) {
|
||||
free(socket_map);
|
||||
return 0;
|
||||
}
|
||||
|
||||
information = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)malloc(buffer_size);
|
||||
if (information == NULL ||
|
||||
!GetLogicalProcessorInformationEx(RelationProcessorPackage, information, &buffer_size)) {
|
||||
free(information);
|
||||
free(socket_map);
|
||||
return 0;
|
||||
}
|
||||
|
||||
cursor = (BYTE *)information;
|
||||
while (cursor < (BYTE *)information + buffer_size) {
|
||||
SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *entry =
|
||||
(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)cursor;
|
||||
if (entry->Relationship == RelationProcessorPackage) {
|
||||
PROCESSOR_RELATIONSHIP *package = &entry->Processor;
|
||||
for (WORD i = 0; i < package->GroupCount; i++) {
|
||||
WORD group = package->GroupMask[i].Group;
|
||||
KAFFINITY mask = package->GroupMask[i].Mask;
|
||||
if (group >= group_count)
|
||||
continue;
|
||||
for (DWORD cpu = 0; cpu < 64; cpu++) {
|
||||
if ((mask & ((KAFFINITY)1 << cpu)) != 0)
|
||||
socket_map[(size_t)group * 64u + cpu] = socket_id;
|
||||
}
|
||||
}
|
||||
socket_id++;
|
||||
}
|
||||
if (entry->Size == 0)
|
||||
break;
|
||||
cursor += entry->Size;
|
||||
}
|
||||
|
||||
free(information);
|
||||
*out_socket_map = socket_map;
|
||||
*out_group_count = group_count;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nfn_windows_get_processor_node(WORD group, BYTE cpu, USHORT *node_id)
|
||||
{
|
||||
PROCESSOR_NUMBER processor;
|
||||
memset(&processor, 0, sizeof(processor));
|
||||
processor.Group = group;
|
||||
processor.Number = cpu;
|
||||
return GetNumaProcessorNodeEx(&processor, node_id) != 0;
|
||||
}
|
||||
|
||||
/* Pin the calling thread using its processor group and group-local CPU index. */
|
||||
NFN_API void NFN_DECL nfn_set_thread_affinity(int cpu_index, int processor_group_id)
|
||||
{
|
||||
if (cpu_index < 0 || processor_group_id < 0) {
|
||||
GROUP_AFFINITY current;
|
||||
GetThreadGroupAffinity(GetCurrentThread(), ¤t);
|
||||
DWORD count = GetActiveProcessorCount(current.Group);
|
||||
GROUP_AFFINITY affinity;
|
||||
memset(&affinity, 0, sizeof(affinity));
|
||||
affinity.Group = current.Group;
|
||||
affinity.Mask = (count >= 64) ? ~(KAFFINITY)0 : ((KAFFINITY)1 << count) - 1;
|
||||
SetThreadGroupAffinity(GetCurrentThread(), &affinity, NULL);
|
||||
return;
|
||||
}
|
||||
|
||||
if ((DWORD)processor_group_id >= GetActiveProcessorGroupCount() || cpu_index >= 64)
|
||||
return;
|
||||
USHORT node_id = 0;
|
||||
if (!nfn_windows_get_processor_node((WORD)processor_group_id, (BYTE)cpu_index, &node_id))
|
||||
return;
|
||||
|
||||
GROUP_AFFINITY affinity;
|
||||
memset(&affinity, 0, sizeof(affinity));
|
||||
affinity.Group = (WORD)processor_group_id;
|
||||
affinity.Mask = (KAFFINITY)1 << cpu_index;
|
||||
SetThreadGroupAffinity(GetCurrentThread(), &affinity, NULL);
|
||||
}
|
||||
|
||||
NFN_API int NFN_DECL nfn_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
|
||||
{
|
||||
ULONG highest;
|
||||
if (!GetNumaHighestNodeNumber(&highest))
|
||||
return 0;
|
||||
*out_numa_count = (int)highest + 1;
|
||||
|
||||
if (count_of_each_numa != NULL) {
|
||||
for (ULONG n = 0; n <= highest; n++)
|
||||
count_of_each_numa[n] = 0;
|
||||
}
|
||||
|
||||
DWORD group_count = GetActiveProcessorGroupCount();
|
||||
for (DWORD group = 0; group < group_count; group++) {
|
||||
for (DWORD cpu = 0; cpu < 64; cpu++) {
|
||||
USHORT node_id = 0;
|
||||
if (nfn_windows_get_processor_node((WORD)group, (BYTE)cpu, &node_id) &&
|
||||
count_of_each_numa != NULL && node_id <= highest)
|
||||
count_of_each_numa[node_id]++;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
NFN_API int NFN_DECL nfn_get_numa_each(int numa_id,
|
||||
uint32_t *processor_group_ids,
|
||||
uint32_t *socket_ids,
|
||||
uint32_t *cpu_indices,
|
||||
uint32_t *numa_node_ids)
|
||||
{
|
||||
if (numa_id < 0 || processor_group_ids == NULL || socket_ids == NULL ||
|
||||
cpu_indices == NULL || numa_node_ids == NULL)
|
||||
return 0;
|
||||
|
||||
uint32_t *socket_map = NULL;
|
||||
DWORD group_count = 0;
|
||||
if (!nfn_windows_get_socket_map(&socket_map, &group_count))
|
||||
return 0;
|
||||
|
||||
int index = 0;
|
||||
for (DWORD group = 0; group < group_count; group++) {
|
||||
for (DWORD cpu = 0; cpu < 64; cpu++) {
|
||||
USHORT node_id = 0;
|
||||
if (!nfn_windows_get_processor_node((WORD)group, (BYTE)cpu, &node_id) ||
|
||||
node_id != (USHORT)numa_id)
|
||||
continue;
|
||||
|
||||
uint32_t socket_id = socket_map[(size_t)group * 64u + cpu];
|
||||
if (socket_id == UINT32_MAX) {
|
||||
free(socket_map);
|
||||
return 0;
|
||||
}
|
||||
processor_group_ids[index] = (uint32_t)group;
|
||||
socket_ids[index] = socket_id;
|
||||
cpu_indices[index] = cpu;
|
||||
numa_node_ids[index] = node_id;
|
||||
index++;
|
||||
}
|
||||
}
|
||||
|
||||
free(socket_map);
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
NFN_API void *NFN_DECL nfn_numa_alloc(size_t size, int numa_id)
|
||||
{
|
||||
if (size == 0)
|
||||
return NULL;
|
||||
|
||||
if (numa_id < 0)
|
||||
return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
|
||||
|
||||
ULONG highest = 0;
|
||||
if (!GetNumaHighestNodeNumber(&highest) || (ULONG)numa_id > highest)
|
||||
return NULL;
|
||||
|
||||
return VirtualAllocExNuma(GetCurrentProcess(), NULL, size, MEM_RESERVE | MEM_COMMIT,
|
||||
PAGE_READWRITE, (DWORD)numa_id);
|
||||
}
|
||||
|
||||
NFN_API void NFN_DECL nfn_numa_free(void *ptr, size_t size, int numa_id)
|
||||
{
|
||||
(void)size;
|
||||
(void)numa_id;
|
||||
|
||||
if (ptr != NULL)
|
||||
VirtualFree(ptr, 0, MEM_RELEASE);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -1,7 +1,7 @@
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_internal.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_internal.h"
|
||||
|
||||
INTERNAL void NIF_DECL f32_normalize(VEC_SEGMENT(float) *restrict source, VEC_SEGMENT(float) *restrict target)
|
||||
INTERNAL void NFN_DECL f32_normalize(VEC_SEGMENT(float) *restrict source, VEC_SEGMENT(float) *restrict target)
|
||||
{
|
||||
register __m128 s = source->f32_x[0];
|
||||
register __m128 a_length_sq = _mm_mul_ps(s, s);
|
||||
@@ -1,9 +1,9 @@
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_internal.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_internal.h"
|
||||
|
||||
static const int avxdot_indices[] = { 0, 4, 1, 5, 2, 3, 6, 7 };
|
||||
|
||||
INTERNAL void FORCEINLINE NIF_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
|
||||
INTERNAL void FORCEINLINE NFN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
|
||||
__m128 const *vector, __m128 *ans)
|
||||
{
|
||||
*ans = _mm_dp_ps(transpose->x[0], *vector, 0B11110001);
|
||||
@@ -15,7 +15,7 @@ INTERNAL void FORCEINLINE NIF_DECL dot_41_sse(MAT_KERNEL(float) const *transpose
|
||||
SSE_INSERT_SIMPLE(1, 4));
|
||||
}
|
||||
|
||||
INTERNAL void NIF_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
INTERNAL void NFN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans)
|
||||
{
|
||||
__m256 _double = _mm256_set_m128(vector[0], vector[0]);
|
||||
@@ -26,7 +26,7 @@ INTERNAL void NIF_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 con
|
||||
_mm256_permutevar8x32_ps(tmp, _mm256_loadu_epi32(avxdot_indices)));
|
||||
}
|
||||
|
||||
INTERNAL void NIF_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
INTERNAL void NFN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans)
|
||||
{
|
||||
ans[0] = _mm_dp_ps(transpose->x[0], vector[0], 0B11110001);
|
||||
@@ -46,7 +46,7 @@ INTERNAL void NIF_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 con
|
||||
SSE_INSERT_SIMPLE(1, 4));
|
||||
}
|
||||
|
||||
INTERNAL void NIF_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
INTERNAL void NFN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans)
|
||||
{
|
||||
__m256 temp = transpose->y[0];
|
||||
+9
-9
@@ -1,8 +1,8 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#define NIF_MK_FMA_F32_512_CYCLE(i) \
|
||||
#define NFN_MK_FMA_F32_512_CYCLE(i) \
|
||||
do { \
|
||||
c0 = _mm512_fmadd_ps(c0, a0, _mm512_permute_ps(b0, _FMA_SELECT(i))); \
|
||||
c1 = _mm512_fmadd_ps(c1, a1, _mm512_permute_ps(b1, _FMA_SELECT(i))); \
|
||||
@@ -10,7 +10,7 @@
|
||||
c3 = _mm512_fmadd_ps(c3, a3, _mm512_permute_ps(b3, _FMA_SELECT(i))); \
|
||||
} while (0)
|
||||
|
||||
#define NIF_MK_FMA_F32_AVX2_CYCLE(i) \
|
||||
#define NFN_MK_FMA_F32_AVX2_CYCLE(i) \
|
||||
do { \
|
||||
c0 = _mm256_fmadd_ps(c0, a0, _mm256_permute_ps(b0, _FMA_SELECT(i))); \
|
||||
c1 = _mm256_fmadd_ps(c1, a1, _mm256_permute_ps(b1, _FMA_SELECT(i))); \
|
||||
@@ -18,7 +18,7 @@
|
||||
c3 = _mm256_fmadd_ps(c3, a3, _mm256_permute_ps(b3, _FMA_SELECT(i))); \
|
||||
} while (0)
|
||||
|
||||
#define NIF_MK_FMA_F64_AVX2_CYCLE(i) \
|
||||
#define NFN_MK_FMA_F64_AVX2_CYCLE(i) \
|
||||
do { \
|
||||
c0 = _mm512_fmadd_ps(c0, a0, _mm512_permutex_pd(b0, _FMA_SELECT(i))); \
|
||||
c1 = _mm512_fmadd_ps(c1, a1, _mm512_permutex_pd(b1, _FMA_SELECT(i))); \
|
||||
@@ -26,7 +26,7 @@
|
||||
c3 = _mm512_fmadd_ps(c3, a3, _mm512_permutex_pd(b3, _FMA_SELECT(i))); \
|
||||
} while (0)
|
||||
|
||||
#define NIF_FMA_F32_512_PACK_COLUMN(col, target) \
|
||||
#define NFN_FMA_F32_512_PACK_COLUMN(col, target) \
|
||||
do { \
|
||||
register __m256 low = _mm512_castps512_ps256(col); \
|
||||
register __m256 high = _mm512_extractf32x8_ps(col, 1); \
|
||||
@@ -37,7 +37,7 @@
|
||||
_mm_store_ps(target, half128); \
|
||||
} while (0)
|
||||
|
||||
#define NIF_FMA_F32_AVX2_PACK_COLUMN(col, target) \
|
||||
#define NFN_FMA_F32_AVX2_PACK_COLUMN(col, target) \
|
||||
do { \
|
||||
register __m128 high128 = _mm256_extractf32x4_ps(col, 1); \
|
||||
register __m128 low128 = _mm256_castps256_ps128(col); \
|
||||
@@ -45,7 +45,7 @@
|
||||
_mm_store_ps(target, half128); \
|
||||
} while (0)
|
||||
|
||||
#define NIF_FMA_F64_512_PACK_COLUMN(col, target) \
|
||||
#define NFN_FMA_F64_512_PACK_COLUMN(col, target) \
|
||||
do { \
|
||||
register __m128 high128 = _mm256_extractf32x4_ps(col, 1); \
|
||||
register __m128 low128 = _mm256_castps256_ps128(col); \
|
||||
@@ -54,7 +54,7 @@
|
||||
} while (0)
|
||||
|
||||
// double
|
||||
#define NIF_MK_FMA_F64_SSE_PHASE(i) \
|
||||
#define NFN_MK_FMA_F64_SSE_PHASE(i) \
|
||||
do { \
|
||||
/* k = 0 */ \
|
||||
register __m128d b_lo = _mm_load_pd(right_values + (2 * i + 0)); \
|
||||
@@ -1,35 +0,0 @@
|
||||
#ifndef NIF_NATIVE_H
|
||||
#define NIF_NATIVE_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
#error "This header is not meant to be included in C++ code. Nitload.Native is a C library and should be used with C linkage."
|
||||
#endif
|
||||
|
||||
#include "nif_internal.h"
|
||||
#include "nif_target.h"
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <assert.h>
|
||||
#include <glfw_function.h>
|
||||
|
||||
#define byte char
|
||||
#define cs_char uint16_t
|
||||
#define nif_null_assert(ptr) assert(ptr != NULL)
|
||||
|
||||
#ifndef NIF_
|
||||
#endif
|
||||
|
||||
NIF_API void NIF_DECL nif_load_glfw_functions(GLFWFunctionTable *table);
|
||||
NIF_API void NIF_DECL nif_load_intrinsic_function(void *intrinsic_context, uint64_t id);
|
||||
NIF_API SIMDID NIF_DECL nif_get_simd_level_global();
|
||||
/* Per-core variant: always re-executes CPUID, never caches.
|
||||
* Must be called on the target core (after affinity is set). */
|
||||
NIF_API SIMDID NIF_DECL nif_get_simd_level_local();
|
||||
NIF_API SIMDID NIF_DECL nif_get_unite_simd(SIMDID left, SIMDID right);
|
||||
NIF_API int NIF_DECL nif_compare_simd(SIMDID left, SIMDID right);
|
||||
NIF_API void NIF_DECL nif_set_thread_affinity(int core_id, int numa_id);
|
||||
NIF_API void* NIF_DECL nif_numa_alloc(size_t size, int numa_id);
|
||||
NIF_API void NIF_DECL nif_numa_free(void *ptr, size_t size, int numa_id);
|
||||
NIF_API int NIF_DECL nif_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global);
|
||||
NIF_API int NIF_DECL nif_get_numa_overall(int *out_numa_count, int *count_of_each_numa);
|
||||
#endif
|
||||
@@ -1,54 +0,0 @@
|
||||
#ifndef NIF_MATKERNEL
|
||||
#define NIF_MATKERNEL
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "nif_internal.h"
|
||||
|
||||
/* Kernel storage unions for a fixed 4x4 block. */
|
||||
#define ELEMENTSIZE(T) sizeof(T)
|
||||
#define XMMCOUNT(T) (sizeof(T))
|
||||
#define YMMCOUNT(T) (sizeof(T) / 2)
|
||||
#define ZMMCOUNT(T) (sizeof(T) / 4)
|
||||
|
||||
#define NIF_MAT_KERNEL_MEMBER(T) \
|
||||
{ \
|
||||
__m128 x[XMMCOUNT(T)]; \
|
||||
__m256 y[YMMCOUNT(T)]; \
|
||||
__m512 z[ZMMCOUNT(T)]; \
|
||||
T m[4][4]; \
|
||||
}
|
||||
|
||||
#define NIF_MAT_KERNEL_DEFINE(T) \
|
||||
typedef union NIF_MAT_KERNEL_TYPE_##T NIF_MAT_KERNEL_MEMBER(T) NIF_MAT_KERNEL_TYPE_##T
|
||||
|
||||
#define MAT_KERNEL(T) NIF_MAT_KERNEL_TYPE(T)
|
||||
#define NIF_MAT_KERNEL_TYPE(T) NIF_MAT_KERNEL_TYPE_##T
|
||||
|
||||
NIF_MAT_KERNEL_DEFINE(float);
|
||||
NIF_MAT_KERNEL_DEFINE(double);
|
||||
NIF_MAT_KERNEL_DEFINE(uint64_t);
|
||||
|
||||
|
||||
#define NIF_VEC_SEGMENT_MEMBER(T) \
|
||||
{ \
|
||||
__m128 f32_x[XMMCOUNT(T) / 4]; \
|
||||
__m256 f32_y[YMMCOUNT(T) / 4]; \
|
||||
__m512 f32_z[ZMMCOUNT(T) / 4]; \
|
||||
T v[4]; \
|
||||
}
|
||||
|
||||
#define NIF_VEC_SEGMENT_DEFINE(T) \
|
||||
typedef union NIF_VEC_SEGMENT_TYPE_##T NIF_VEC_SEGMENT_MEMBER(T) NIF_VEC_SEGMENT_TYPE_##T
|
||||
|
||||
#define VEC_SEGMENT(T) NIF_VEC_SEGMENT_TYPE(T)
|
||||
#define NIF_VEC_SEGMENT_TYPE(T) NIF_VEC_SEGMENT_TYPE_##T
|
||||
|
||||
NIF_VEC_SEGMENT_DEFINE(float);
|
||||
NIF_VEC_SEGMENT_DEFINE(double);
|
||||
|
||||
/* Global zero kernel declaration (defined in dllmain.c), aligned to 64 bytes */
|
||||
extern __declspec(align(64)) MAT_KERNEL(u64) NIF_ZERO_KERNEL;
|
||||
|
||||
#endif /* NIF_MATKERNEL */
|
||||
@@ -1,101 +0,0 @@
|
||||
#ifndef NIF_INTRINSIC_CALLCONV_STD
|
||||
#define NIF_INTRINSIC_CALLCONV_STD
|
||||
|
||||
#include "nif_internal.h"
|
||||
#include "nif_intrinsic.h"
|
||||
|
||||
#ifndef static_assert
|
||||
#define static_assert _Static_assert
|
||||
#endif
|
||||
|
||||
#define NIF_STD_TARGET_ATTR(T_arch, T_feature) NIF_STD_TARGET_ATTR_IMPL(T_arch, T_feature)
|
||||
|
||||
#pragma region mk_proc attributes
|
||||
#define MK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
|
||||
T_type##_multi_kernel_proc_##T_op##_##T_arch##_##T_feature
|
||||
#define GET_MK_PROC_STD(T_type, T_arch, T_feature, T_op) \
|
||||
((UNI_MK_PROC)MK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op))
|
||||
#define NIF_MK_PROC_DECL(T_type, P_name) INTERNAL void NIF_DECL P_name(MAT_TASK(T_type) const *task)
|
||||
#define NIF_MK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
|
||||
INTERNAL NOINLINE NIF_STD_TARGET_ATTR(T_arch, T_feature) void NIF_DECL MK_PROC_NAME_STD( \
|
||||
T_type, T_arch, T_feature, T_op)(MAT_TASK(T_type) const *task)
|
||||
#pragma endregion
|
||||
|
||||
#pragma region sk_proc attributes
|
||||
#define SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
|
||||
T_type##_single_kernel_proc_##T_op##_##T_arch##_##T_feature
|
||||
#define GET_SK_PROC_STD(T_type, T_arch, T_feature, T_op) \
|
||||
((UNI_SK_PROC)SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op))
|
||||
#define NIF_SK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
|
||||
INTERNAL NOINLINE NIF_STD_TARGET_ATTR(T_arch, T_feature) void NIF_DECL SK_PROC_NAME_STD( \
|
||||
T_type, T_arch, T_feature, \
|
||||
T_op)(MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
|
||||
MAT_KERNEL(T_type) *restrict ans, uint64_t scalar)
|
||||
#pragma endregion
|
||||
|
||||
#define _SN_STD__PROBE_0 0
|
||||
#define _SN_STD__PROBE_1 1
|
||||
#define NIF_STD__PROBE_PRINT(x) _SN_STD__PROBE_##x
|
||||
#define NIF_STD__PROBE(x) NIF_STD__PROBE_PRINT(x)
|
||||
|
||||
#pragma region param styles
|
||||
|
||||
#define _SN_MATPROC_LEFT_RIGHT_ANS 1
|
||||
#define _SN_MATPROC_RIGHT_ANS 1
|
||||
#define _SN_MATPROC_LEFT_RIGHT_ANS_SCALAR 1
|
||||
#define _SN_MATPROC_RIGHT_ANS_SCALAR 1
|
||||
#define _SN_MATPROC_ANS_SCALAR 1
|
||||
|
||||
#define _SN_MK_EXPAND_BUFFER_SIZE(i)\
|
||||
int offset_##i = task->offset_##i;\
|
||||
int count_##i = task->count_##i;\
|
||||
|
||||
#define _SN_MK_EXPAND_RIGHT_ANS \
|
||||
NIF_MAT_KERNEL_LOCAL const *right = (NIF_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans = (NIF_MAT_KERNEL_LOCAL *)task->buffer_2.ptr;\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(1); \
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS \
|
||||
NIF_MAT_KERNEL_LOCAL const *left = (NIF_MAT_KERNEL_LOCAL const *)task->buffer_0.ptr; \
|
||||
NIF_MAT_KERNEL_LOCAL const *right = (NIF_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans = (NIF_MAT_KERNEL_LOCAL *)task->buffer_2.ptr;\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(0); \
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(1);\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _SN_MK_EXPAND_RIGHT_ANS_SCALAR \
|
||||
NIF_MAT_KERNEL_LOCAL const *right = (NIF_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans = (NIF_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
|
||||
M128 scalar = task->scalar_pack;\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(1);\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS_SCALAR \
|
||||
NIF_MAT_KERNEL_LOCAL const *left = (NIF_MAT_KERNEL_LOCAL const *)task->buffer_0.ptr; \
|
||||
NIF_MAT_KERNEL_LOCAL const *right = (NIF_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans = (NIF_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
|
||||
M128 scalar = task->scalar_pack;\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(0);\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(1);\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _SN_MK_EXPAND_ANS_SCALAR \
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans = (NIF_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
|
||||
M128 scalar = task->scalar_pack;\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _SN_PARAM_STYLE(T) _SN_MATPROC_##T
|
||||
#define _SN_MK_EXPAND_PARAM(style) _SN_MK_EXPAND_##style
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#define __mk_param_std(param, T) \
|
||||
typedef MAT_KERNEL(T) NIF_MAT_KERNEL_LOCAL; \
|
||||
matrix_task const *_ = (_SN_PARAM_STYLE(param), task); \
|
||||
_SN_MK_EXPAND_PARAM(param)
|
||||
|
||||
#define __sk_param_std(T) \
|
||||
typedef MAT_KERNEL(T) NIF_MAT_KERNEL_LOCAL; \
|
||||
(void)scalar
|
||||
#endif // !NIF_INTRINSIC_PARAM_EXPAND
|
||||
@@ -1,94 +0,0 @@
|
||||
#ifndef NIF_TARGET
|
||||
#define NIF_TARGET
|
||||
|
||||
#include "nif_internal.h"
|
||||
#include <stdint.h>
|
||||
|
||||
#define NIF_OS_WINDOWS 0
|
||||
#define NIF_OS_LINUX 1
|
||||
#define NIF_OS_MACOS 2
|
||||
#define NIF_OS_IOS 3
|
||||
#define NIF_OS_ANDROID 4
|
||||
#define NIF_OS_UNKNOWN 5
|
||||
|
||||
#define NIF_ARCH_X86_64 0
|
||||
#define NIF_ARCH_ARM64 1
|
||||
#define NIF_ARCH_RISCV64 2
|
||||
#define NIF_ARCH_PPC64le 3
|
||||
|
||||
#if defined(_M_X64) || defined(_M_AMD64) || defined(__x86_64__) || defined(__amd64__)
|
||||
#define NIF_CURRENT_ARCH NIF_ARCH_X86_64
|
||||
|
||||
#elif defined(_M_ARM64) || defined(__aarch64__)
|
||||
#define NIF_CURRENT_ARCH NIF_ARCH_ARM64
|
||||
|
||||
#elif defined(_M_ARM) || defined(__arm__)
|
||||
#define NIF_CURRENT_ARCH NIF_ARCH_ARM32
|
||||
|
||||
#elif defined(_M_IX86) || defined(__i386__)
|
||||
#define NIF_CURRENT_ARCH NIF_ARCH_X86_32
|
||||
|
||||
#elif defined(__powerpc64__) && defined(__LITTLE_ENDIAN__)
|
||||
#define NIF_CURRENT_ARCH NIF_ARCH_PPC64LE
|
||||
|
||||
#elif defined(__riscv) && (__riscv_xlen == 64)
|
||||
#define NIF_CURRENT_ARCH NIF_ARCH_RISCV64
|
||||
|
||||
#else
|
||||
#error "SN: Unsupported target architecture"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Target selection policy:
|
||||
* - Default (no NIF_EXTERN_TARGET): fixed Windows + x86_64.
|
||||
* - External (NIF_EXTERN_TARGET): build system must provide NIF_OS and NIF_ARCH.
|
||||
*/
|
||||
#ifndef NIF_EXTERN_TARGET
|
||||
|
||||
#define NIF_OS_TARGET NIF_OS_WINDOWS
|
||||
#define NIF_ARCH_TARGET NIF_ARCH_X86_64
|
||||
|
||||
#else
|
||||
|
||||
#ifndef NIF_OS_TARGET
|
||||
#error "NIF_EXTERN_TARGET is defined but NIF_OS is not provided."
|
||||
#endif
|
||||
|
||||
#ifndef NIF_ARCH_TARGET
|
||||
#error "NIF_EXTERN_TARGET is defined but NIF_ARCH is not provided."
|
||||
#endif
|
||||
|
||||
/* Validate externally supplied values are within known enum constants. */
|
||||
#if (NIF_OS_TARGET != NIF_OS_WINDOWS) && (NIF_OS_TARGET != NIF_OS_LINUX) && \
|
||||
(NIF_OS_TARGET != NIF_OS_MACOS) && (NIF_OS_TARGET != NIF_OS_IOS) && \
|
||||
(NIF_OS_TARGET != NIF_OS_ANDROID) && (NIF_OS_TARGET != NIF_OS_UNKNOWN)
|
||||
#error "NIF_OS has an invalid value. Use NIF_OS_<...> enum constants."
|
||||
#endif
|
||||
|
||||
#if (NIF_ARCH_TARGET != NIF_ARCH_X86_64) && (NIF_ARCH_TARGET != NIF_ARCH_ARM64) && \
|
||||
(NIF_ARCH_TARGET != NIF_ARCH_RISCV64) && (NIF_ARCH_TARGET != NIF_ARCH_PPC64le)
|
||||
#error "NIF_ARCH has an invalid value. Use NIF_ARCH_<...> enum constants."
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#define NIF_IS_OS(os) ((NIF_OS_TARGET) == (os))
|
||||
|
||||
#define NIF_IS_ARCH(arch) ((NIF_ARCH_TARGET) == (arch))
|
||||
|
||||
/*
|
||||
* Per-function ISA selection is required for runtime-dispatched kernels that
|
||||
* mix multiple x86 feature levels in one build target.
|
||||
*/
|
||||
#if defined(__clang__) && ((NIF_ARCH_TARGET) == NIF_ARCH_X86_64)
|
||||
#define NIF_CLANG_TARGET_ATTR(options) __attribute__((target(options)))
|
||||
#else
|
||||
#define NIF_CLANG_TARGET_ATTR(options)
|
||||
#endif
|
||||
|
||||
typedef union SIMDID {
|
||||
uint64_t mask;
|
||||
char make_byte[8];
|
||||
} SIMDID;
|
||||
|
||||
#endif // NIF_TARGET
|
||||
@@ -1,95 +0,0 @@
|
||||
#include "nif_target.h"
|
||||
#include "NitloadFrameWork.h"
|
||||
|
||||
#if NIF_IS_OS(NIF_OS_WINDOWS)
|
||||
|
||||
/* Pin the calling thread to a single logical processor.
|
||||
* core_id >= 0 : bind to that logical processor (global index, across groups).
|
||||
* core_id < 0 : clear affinity constraint within the current group. */
|
||||
NIF_API void NIF_DECL nif_set_thread_affinity(int core_id, int numa_id)
|
||||
{
|
||||
if (core_id < 0 || numa_id < 0) {
|
||||
/* Query which group the thread currently belongs to */
|
||||
GROUP_AFFINITY current;
|
||||
GetThreadGroupAffinity(GetCurrentThread(), ¤t);
|
||||
/* Build a full mask for that group's actual core count */
|
||||
DWORD count = GetActiveProcessorCount(current.Group);
|
||||
GROUP_AFFINITY ga;
|
||||
memset(&ga, 0, sizeof(ga));
|
||||
ga.Group = current.Group;
|
||||
ga.Mask = (count >= 64) ? ~(KAFFINITY)0 : ((KAFFINITY)1 << count) - 1;
|
||||
SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL);
|
||||
return;
|
||||
}
|
||||
int index = core_id % 64;
|
||||
DWORD count = GetActiveProcessorCount(numa_id);
|
||||
if ((DWORD)index >= count)
|
||||
return;
|
||||
GROUP_AFFINITY ga;
|
||||
memset(&ga, 0, sizeof(ga));
|
||||
ga.Group = (WORD)numa_id;
|
||||
ga.Mask = (KAFFINITY)1 << index;
|
||||
SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL);
|
||||
}
|
||||
|
||||
NIF_API int NIF_DECL nif_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
|
||||
{
|
||||
ULONG highest;
|
||||
if (!GetNumaHighestNodeNumber(&highest))
|
||||
return 0;
|
||||
*out_numa_count = (int)highest + 1;
|
||||
|
||||
for (UCHAR n = 0; n <= (UCHAR)highest; n++) {
|
||||
ULONGLONG mask = 0;
|
||||
GetNumaNodeProcessorMask(n, &mask);
|
||||
count_of_each_numa[n] = (int)__popcnt64(mask);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
NIF_API int NIF_DECL nif_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global)
|
||||
{
|
||||
ULONGLONG mask = 0;
|
||||
if (!GetNumaNodeProcessorMask((UCHAR)numa_id, &mask))
|
||||
return 0;
|
||||
|
||||
int intra = 0;
|
||||
while (mask) {
|
||||
DWORD idx;
|
||||
_BitScanForward64(&idx, mask);
|
||||
if (internal != NULL)
|
||||
internal[intra] = (uint64_t)intra;
|
||||
global[intra] = (uint32_t)idx;
|
||||
intra++;
|
||||
mask &= mask - 1;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
NIF_API void *NIF_DECL nif_numa_alloc(size_t size, int numa_id)
|
||||
{
|
||||
if (size == 0)
|
||||
return NULL;
|
||||
|
||||
if (numa_id < 0)
|
||||
return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
|
||||
|
||||
ULONG highest = 0;
|
||||
if (!GetNumaHighestNodeNumber(&highest) || (ULONG)numa_id > highest)
|
||||
return NULL;
|
||||
|
||||
return VirtualAllocExNuma(GetCurrentProcess(), NULL, size, MEM_RESERVE | MEM_COMMIT,
|
||||
PAGE_READWRITE, (DWORD)numa_id);
|
||||
}
|
||||
|
||||
NIF_API void NIF_DECL nif_numa_free(void *ptr, size_t size, int numa_id)
|
||||
{
|
||||
(void)size;
|
||||
(void)numa_id;
|
||||
|
||||
if (ptr != NULL)
|
||||
VirtualFree(ptr, 0, MEM_RELEASE);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,49 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="Mix"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.RegularAnalysis.Abstraction
|
||||
{
|
||||
public class Mix<TElement, TValue, TEvaluator>(
|
||||
TEvaluator evaluator
|
||||
) where TEvaluator : IMixEvaluator<TElement, TValue>
|
||||
{
|
||||
|
||||
public TValue Weight { get; private set; } = default!;
|
||||
|
||||
private TEvaluator Evaluator { get; init; } = evaluator;
|
||||
|
||||
public TValue Evaluate(
|
||||
Span<Relation<TElement, TValue>> relations
|
||||
)
|
||||
{
|
||||
Weight = Evaluator.Evaluate(relations);
|
||||
return Weight;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public sealed class Mix<TElement, TValue>(
|
||||
IMixEvaluator<TElement, TValue> evaluator
|
||||
) : Mix<TElement, TValue, IMixEvaluator<TElement, TValue>>(evaluator) { }
|
||||
|
||||
public interface IMixEvaluator<TElement, TValue>
|
||||
{
|
||||
|
||||
public Relation<TElement, TValue, IRelationEvaluator<TElement, TValue>>[] DownstreamPorts { get; }
|
||||
|
||||
TValue Evaluate(
|
||||
Span<Relation<TElement, TValue>> relations
|
||||
);
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="Relation"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.RegularAnalysis.Abstraction
|
||||
{
|
||||
public class Relation<TElement, TValue, TEvaluator>(
|
||||
TEvaluator evaluator
|
||||
) where TEvaluator : IRelationEvaluator<TElement, TValue>
|
||||
{
|
||||
|
||||
public TEvaluator Evaluator { get; init; } = evaluator;
|
||||
|
||||
public TValue Weight { get; private set; } = default!;
|
||||
|
||||
public TValue Evaluate(
|
||||
ReadOnlySpan<TElement> sequence
|
||||
)
|
||||
{
|
||||
Weight = Evaluate(sequence);
|
||||
return Weight;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public class Relation<TElement, TValue>(
|
||||
IRelationEvaluator<TElement, TValue> evaluator
|
||||
) : Relation<TElement, TValue, IRelationEvaluator<TElement, TValue>>(evaluator) { }
|
||||
|
||||
public interface IRelationEvaluator<TElement, TValue>
|
||||
{
|
||||
|
||||
TValue Evaluate(
|
||||
ReadOnlySpan<TElement> sequence
|
||||
);
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
# NGRA-Tensor 设计讨论记忆
|
||||
|
||||
更新日期：2026-09-18
|
||||
目标项目：`D:\platform-dev\src\Nitload.RegularAnalysis`
|
||||
来源：关于正则分析、NFA、Transformer、GPU 执行与 NGRA 双算子设计的讨论。
|
||||
|
||||
> 本文件记录设计意图与讨论结果，不代表项目现有代码已经实现这些功能。本次仅检查了目标目录及祖先目录的 AGENTS.md，没有审计源码。下文区分用户已提出的设计决定、辅助建议与待验证问题。
|
||||
|
||||
最新设计补充（2026-09-18，详见第 10 节）：NGRA 使用持久状态上的增量流输入/输出；以连接占用作为主要容量指标的候选方案；容量不足时丢弃低激活连接。**字典本身是结构的一部分，结构是记忆的一部分。** 字典与运行时实例的区分属于身份与生命周期管理，不表示字典位于记忆系统之外。
|
||||
|
||||
## 1. 项目目标与讨论方式
|
||||
|
||||
NGRA-Tensor 试图结合结构化关系、模糊特征识别、多候选解释及模式预测。出发点包含正则模式分析与 NFA，但不应预设必须逐节点模拟 NFA 的执行方式。
|
||||
|
||||
核心工程矛盾：细粒度、动态、自组织的关系网络表达灵活，却容易产生大量小操作、串行依赖、零碎访存和复杂调度，影响 GPU 利用率与训练。
|
||||
|
||||
用户希望学习“如何构造能运行且高效的系统”，而不只是阅读既有模型的前向公式或事后解释。后续讨论应说明：需求 → 设计选择 → 数学表达 → 执行方式 → 代价 → 验证方法。不要用 Transformer 也存在串行步骤，来回避 NGRA 的节点过小与调度问题。
|
||||
|
||||
“Tr”在不影响理解时作为 Transformer 的简称。
|
||||
|
||||
## 2. 数学符号约定
|
||||
|
||||
| 符号 | 含义 |
|
||||
| --- | --- |
|
||||
| `Scal` | 标量 |
|
||||
| `Vec_d` | d 维向量，本文使用列形式表示分量 |
|
||||
| `Covec_d` | 对向量进行线性求值的余向量 |
|
||||
| `Mat_x_y` | 每行 x 个元素、每列 y 个元素，即 y 行、x 列；列数在前 |
|
||||
| `Tens_u_v` | 接收 u 个 Vec 和 v 个 Covec、输出 Scal 的多重线性对象 |
|
||||
|
||||
注意：
|
||||
|
||||
- 数组的存储维数不等于 `Tens_u_v` 的槽位数。ONNX 等工具中的 tensor 通常只是多维数组的名称。
|
||||
- 线性映射 L 可通过 `T(x, φ) = φ(Lx)` 对应 `Tens_1_1`；输入向量与输出余向量可以属于不同维度的空间。
|
||||
- 点积所对应的度量可视为 `Tens_2_0`。将向量与余向量识别需要相应的度量约定，不能只凭行列形状混为一谈。
|
||||
- 带偏置映射一般为仿射映射；softmax、ReLU、LayerNorm 等不是多重线性运算。完整神经网络不能直接按上述定义称为一个 `Tens_u_v`。
|
||||
|
||||
## 3. 用户已提出的核心架构决定
|
||||
|
||||
### 3.1 两种组合算子
|
||||
|
||||
**relation：评估一定数量的 notation 按顺序串接形成的关系。**
|
||||
|
||||
```text
|
||||
R = Relation(N1, N2, ..., Np)
|
||||
```
|
||||
|
||||
port 表达有序关系，交换 port 输入通常会改变含义。此前使用过 `R_6` 表示 64 口 relation 的约定；64 指串行关系中的输入数量，不是特征向量维数。早期等级与端口数使用过 `2^L` 的联系，但它与新架构传播层级的准确对应仍须正式定义。
|
||||
|
||||
**notation（亦讨论为 idea）：组织若干并列 relation。**
|
||||
|
||||
```text
|
||||
N = Notation{R1, R2, ..., Rk}
|
||||
```
|
||||
|
||||
这为多种可能结构提供显式位置，避免把并列候选强行塞进 relation 的串行 port。
|
||||
|
||||
- 两者底层数据结构可以形态一致，但必须保留类型及不同的运算语义。
|
||||
- 并列候选不自动等于语义独立，也不自动等于互斥。互斥解释与可以同时成立的特征，需要不同的组合规则。
|
||||
- 候选评估、候选权重、候选混合是可区分的操作；不必一生成候选就压成一个 Vec。
|
||||
|
||||
### 3.2 双向分层与反复传播
|
||||
|
||||
用户提出的方向：
|
||||
|
||||
```text
|
||||
relation 方向：L0 → L1 → ... → L6
|
||||
notation 方向：N6 → N5 → ... → N0
|
||||
一次推理可能经历多轮上述传播。
|
||||
```
|
||||
|
||||
这是当前设计意图，尚不是完整的状态更新规范。L/N 等级的含义、跨级连接是否合法、上下两个方向如何连接，都需要进一步明确。
|
||||
|
||||
讨论中提出的一种解释（待落实）：
|
||||
|
||||
- 上行：由局部证据评估更大的组合与解释。
|
||||
- 下行：较高层的解释、约束和关注影响局部候选的保留与活跃程度。
|
||||
- notation 的候选评估仍须汇集，才能为高层提供依据；“notation 向下传播”不能遗漏候选证据如何上传。
|
||||
- 区分来自输入的证据与来自上层的反馈，防止反馈被反复当成独立证据累加，产生自我强化。
|
||||
|
||||
### 3.3 统一机器字典与索引空间
|
||||
|
||||
用户决定让 token、relation、notation 位于同一个字典和索引空间。“字”本身也是一种原子理念。
|
||||
|
||||
- token 使用预存字典提供初始表示。
|
||||
- relation、notation 的实例“词义”通过节点计算特征向量得到。
|
||||
- 字典可以保存结构定义、参数及引用；运行时计算得到本次语境中的实例特征。
|
||||
- 预存 token 特征不必永久固定为运行时最终特征，可以接受上下文更新。
|
||||
|
||||
建议明确区分两层身份：
|
||||
|
||||
| 层面 | 内容 |
|
||||
| --- | --- |
|
||||
| 字典条目 | token/关系/概念的定义、类型、结构与参数 |
|
||||
| 运行时实例 | 引用哪个条目、出现在何处、连接哪些实例、本轮状态与得分 |
|
||||
|
||||
同一个字在文本中出现两次，可以共享字典 ID，但不能因此共享同一份位置相关状态。可采用“实例 ID + 字典引用”的实现，不要求推翻统一字典。
|
||||
|
||||
## 4. 每轮输出：面向下一轮的规范形
|
||||
|
||||
用户提出：一次“小 node → 大 node → 小 node”的传播后，输出一种较工整、供下一轮继续处理的状态。
|
||||
|
||||
示例：
|
||||
|
||||
```text
|
||||
输入：吃饭了吗，您？
|
||||
初步规范表达：我，提问，你，吃饭，状态
|
||||
```
|
||||
|
||||
各状态同时具有特征向量，用于表达语气等细节。
|
||||
|
||||
这里建议把“规范形”理解为结构统一的语义中间表示，暂不要求第一轮得到唯一、最终正确的解释。
|
||||
|
||||
### 4.1 必须保留角色绑定
|
||||
|
||||
仅有上述扁平序列，不能明确区分“我问你是否吃饭”和“我问你我是否吃饭”。一个可参考的结构为：
|
||||
|
||||
```text
|
||||
提问
|
||||
├─ 发起者：当前说话者（来自会话上下文）
|
||||
├─ 接收者：您所指的人
|
||||
└─ 内容：进食事件是否已经发生
|
||||
├─ 行为者：您所指的人
|
||||
├─ 行为：吃饭
|
||||
└─ 发生状态：待确认
|
||||
```
|
||||
|
||||
这是讨论中的建议示例，不是已决定的 IR 格式。
|
||||
|
||||
- “我”没有直接出现在原句，应记录其上下文来源。
|
||||
- 被询问的事件状态不能在规范化时变成已成立的事实。
|
||||
- 敬称、语气等细节可通过特征表示；关键角色关系应有明确结构。
|
||||
- 返回小节点时，单位可以是实体引用、行为、状态变量或关系实例，不必恢复成原始字符。
|
||||
|
||||
### 4.2 保留候选与来源
|
||||
|
||||
“询问进食情况”和“以此寒暄”可能是候选解释，也可能兼有。notation 应能保留候选身份、得分与结构，而不是强制提前选唯一结果。
|
||||
|
||||
建议保留输入片段或上下文依据，区分：显式内容、上下文补足、模型推断、待确认状态。
|
||||
|
||||
### 4.3 轮次与稳定性
|
||||
|
||||
可用以下关系描述一次传播，但具体更新函数尚未确定：
|
||||
|
||||
```text
|
||||
S(t) → 组合与解释评估 → H(t) → 角色绑定、反馈与整理 → S(t+1)
|
||||
```
|
||||
|
||||
建议每轮读取上一轮状态并集中形成下一轮状态，避免执行顺序无意改变语义。最终应定义最大轮数、变化阈值或其他停止条件。
|
||||
|
||||
期望已稳定的表示在没有新证据时满足 `F(S*) ≈ S*`；这只是验证目标，不代表目前已证明收敛。
|
||||
|
||||
## 5. 构建、剪枝与训练：尚未完成的设计
|
||||
|
||||
用户明确指出，当前“葫芦形”传播还不够明确，需要剪枝和额外构建规则。
|
||||
|
||||
优先需要确定：
|
||||
|
||||
1. relation 各 port 的角色、输入类型、顺序与允许的连接范围。
|
||||
2. notation 中哪些候选回答同一个解释问题，是否覆盖相同区间或具有相容的输出类型。
|
||||
3. 候选是互斥、可并存还是具有其他依赖；采用何种打分与归一化。
|
||||
4. 构建网络、选择路径与更新特征，哪些是离散决策，哪些可微、哪些需要额外训练机制。
|
||||
5. 如何限制候选数量，并测量剪枝造成的正确解释丢失。
|
||||
6. 如何防止重复证据、自我支持、循环振荡及无上限的关系扩张。
|
||||
7. 如何保留原始词序、实体身份、否定、疑问、范围及来源。
|
||||
|
||||
简单 regex/NFA 的布尔关系可以用向量、矩阵及相应的逻辑运算表达；转成浮点矩阵后，必须重新定义多路径的组合含义。布尔可达性、概率、模糊兼容度与特征混合不可直接互换。
|
||||
|
||||
目前没有得到“任意一组 NFA 能以常数总计算量执行”的结论。并行深度、总工作量、状态规模和存储量必须分别分析。
|
||||
|
||||
## 6. GPU 执行路线
|
||||
|
||||
### 6.1 语义粒度与执行粒度分离
|
||||
|
||||
讨论中形成的总体判断：
|
||||
|
||||
- 常规稠密 Tr 从规则的大计算出发，再为硬件分块、融合与安排数据复用。
|
||||
- NGRA 从细粒度的关系与候选出发，再把已经就绪、形状相容的操作合并为批次。
|
||||
|
||||
不必让每个小 relation 对应一次 GPU 调用，也不必为批处理把语义上的多个 relation 合并成一个大 relation。
|
||||
|
||||
| 方法 | 收益与限制 |
|
||||
| --- | --- |
|
||||
| 批处理独立实例 | 可保持实例语义，同时提高单次执行规模 |
|
||||
| 融合有依赖的操作 | 可减少启动及中间搬运；不会自动消除数据依赖 |
|
||||
| 按层级、轮次、形状分组 | 可以简化调度；具体规则尚待验证 |
|
||||
| 稳定的数据布局与参数分组 | 有机会提高复用并减少碎片化读取 |
|
||||
|
||||
早期低阶、小 port 关系的串行活动密集，是重点瓶颈。把 L0/L1 放在 CPU、把短模式预筛编译成矩阵运算等，均只是候选方向，尚未通过基准测试决定。
|
||||
|
||||
### 6.2 按需加载模型内容
|
||||
|
||||
用户提出的潜在优势：根据激活情况，只加载本次需要的 relation/notation 参数，降低显存工作集。
|
||||
|
||||
成立条件与待验证因素：
|
||||
|
||||
- 用廉价、常驻的索引或路由信息判断加载对象，避免必须先读完完整参数才能决定不用它。
|
||||
- 测量当前活跃集合、跨轮复用、加载粒度、预取能力及批次内激活集合的并集。
|
||||
- 同时核算参数、活跃状态、候选记录、运行缓冲与传输缓冲。
|
||||
- 省显存不自动等于提速；零碎且依赖刚完成计算的加载可能使 GPU 等待。
|
||||
- 区分 GPU 内部缓存未命中与数据不驻留显存、需要从主存迁入。
|
||||
|
||||
Tr 也能参数卸载、逐层加载或使用 MoE，故不能把“必须所有权重常驻显存”当作 Tr 的数学要求。NGRA 的优势仍需在质量相当时实测。
|
||||
|
||||
## 7. Tr 对照：已经澄清的关键概念
|
||||
|
||||
### 7.1 block、上下文与执行分块不是一回事
|
||||
|
||||
- Tr block 通常指深度方向的一层计算，不是固定 4096 口的文本单元。
|
||||
- 4096 是本讨论的示例长度，也可能是某个模型的上下文配置；不是 Tr 固有 I/O 宽度。
|
||||
- 同一层的投影权重作用于特征轴，通常不含序列长度 n 这个参数维度。
|
||||
- 支持长上下文还涉及位置机制、训练与资源，不能据此宣称任意短上下文模型可直接可靠处理 1M。
|
||||
- 彼此隔离的文本窗口不会自动互通。共享模型权重不等于保留上一段输入。
|
||||
|
||||
### 7.2 输入与输出
|
||||
|
||||
以 512 维、8 个头、每头 64 维的示例来说：
|
||||
|
||||
```text
|
||||
X: Mat_n_512 → 一个完整 Tr block → Y: Mat_n_512
|
||||
```
|
||||
|
||||
每个输出列仍对应一个位置，但携带上下文整合后的特征。输出通常不是显式 relation 表，也不是 vocabulary 概率。只有第一层直接接收 embedding 与初始位置表示，后续层接收前层输出。
|
||||
|
||||
token embedding 提供初始向量，不能把后续位置向量理解成始终只有原始 token 的静态词义。
|
||||
|
||||
### 7.3 本文列向量约定下的主要运算
|
||||
|
||||
| 运算 | 形状或说明 |
|
||||
| --- | --- |
|
||||
| `Q = WQ X`, `K = WK X`, `V = WV X` | W 为 `Mat_512_64`；Q/K/V 为 `Mat_n_64` |
|
||||
| `S = QᵀK / sqrt(64) + M` | `Mat_n_n`；行是查询，列是候选 |
|
||||
| `A = softmax_rows(S)` | 对每个查询的允许候选统一归一化 |
|
||||
| `H = V Aᵀ` | `Mat_n_64`，按位置混合内容 |
|
||||
| 8 个 H 沿特征轴拼接 | `Mat_n_512` |
|
||||
| 输出投影、残差、Norm、FFN、残差、Norm | 最终仍为 `Mat_n_512`；此顺序是原论文 post-norm 示例 |
|
||||
|
||||
参数 WQ/WK/WV 通常分层、分头拥有，并在所有位置复用。ONNX 的图级 initializer 存储位置不代表各层共享同一组参数。Q/K/V/A 是本次输入产生的中间结果，不是查表得到的固定权重。
|
||||
|
||||
### 7.4 混合不等于重排
|
||||
|
||||
- `W X`：特征投影，不移动 token 列。
|
||||
- `V Aᵀ`：根据输入计算权重，允许多个输出读取同一来源。
|
||||
- `X Π`：用置换矩阵重排位置。
|
||||
- 标准注意力不默认执行“聚类相关 token → 打包 → 分组路由”。若需要这种行为，须明确设计路由过程。
|
||||
- 两个局部组的输出可以按 `[B00h B01h]`、`[B00l B01l]` 重组后交给下一层。若 h/l 表示 token 子集，这属于改变窗口分组；若表示特征通道子集，则是另一种操作。
|
||||
- 重组本身不做语义融合，下游计算才融合；需要保留位置身份与因果约束。
|
||||
|
||||
### 7.5 1M 上下文与跨计算块通信
|
||||
|
||||
全局注意力可以逻辑上覆盖整个上下文，执行时按块读取 K/V。对于两个分块 A/B，跨块关系直接来自 `QAᵀ KB` 与 `QBᵀ KA`；因果掩码限制允许的方向。
|
||||
|
||||
一次处理 4096 个查询，不代表这些查询只能访问 4096 个候选。它们可以分批访问全部允许的 K/V，并维护统一的 softmax 累加状态。
|
||||
|
||||
不能对各候选块独立 softmax 后简单平均。FlashAttention 使用分块与在线归一化执行精确注意力；跨设备方案如 Ring Attention 可以传递 K/V 块完成跨分片交互。
|
||||
|
||||
这不要求先分配完整的巨大注意力矩阵再把它切开。
|
||||
|
||||
### 7.6 成本与能力边界
|
||||
|
||||
- 全局注意力的配对计算为 `O(n²d)`；序列长度翻倍，该部分工作量约四倍，不代表端到端推理时间必然四倍。
|
||||
- 固定窗口宽度 w 的局部注意力为 `O(nwd)`；固定层数的间接信息覆盖范围有限。
|
||||
- 使用 KV cache 生成一个新 token 时，该 token 对历史的注意力计算随历史长度线性增长，而非重新计算完整历史的所有配对。
|
||||
- KV cache 不会自动把上下文容量变成无限，且需要区分缓存、压缩记忆与原始历史可访问性。
|
||||
- 普通前馈网络与关闭随机操作的 Tr 都可确定性执行。“DNN”通常表示深度神经网络，不能用来与 Tr 作“确定/非确定”分类。
|
||||
- 普通网络通常训练权重与偏置，而非修改固定 ReLU 等激活函数本身。
|
||||
- 注意力混合不自动保留完整候选结构，也不等于显式逻辑关系或可解释的置信度。
|
||||
|
||||
## 8. 下一阶段建议产物（尚未执行）
|
||||
|
||||
1. 明确统一字典条目与运行时实例的最小 schema。
|
||||
2. 明确 relation/notation 的输入输出类型、候选组合与角色绑定规则。
|
||||
3. 写出一个完整传播轮次的状态读取、写入与反馈语义。
|
||||
4. 用“吃饭了吗，您？”制作可逐节点检查的示例，包括歧义保留与下一轮稳定性。
|
||||
5. 定义候选数量上限、剪枝指标、最大轮数与停止条件。
|
||||
6. 先建立语义正确的参考执行，再比较逐节点、批处理、融合和按需加载的性能。
|
||||
7. 训练目标、路由学习方式、收敛性质以及相对 Tr 的性能优势均保持待验证状态。
|
||||
|
||||
## 9. 讨论中核对过的参考资料
|
||||
|
||||
- [Attention Is All You Need](https://arxiv.org/html/1706.03762v7)：架构、注意力、复杂度与参数作用。
|
||||
- [Neural Machine Translation by Jointly Learning to Align and Translate](https://arxiv.org/abs/1409.0473)：按当前需要读取输入位置的早期注意力设计。
|
||||
- [FlashAttention](https://arxiv.org/abs/2205.14135)：分块、IO 优化与精确注意力。
|
||||
- [Ring Attention](https://arxiv.org/abs/2310.01889)：跨设备传递 K/V 块与长上下文。
|
||||
- [Swin Transformer](https://arxiv.org/abs/2103.14030)：局部窗口之间通过改变划分建立联系的实例。
|
||||
- [Switch Transformers](https://arxiv.org/abs/2101.03961)：选择性激活专家参数。
|
||||
- [ZeRO-Inference](https://www.deepspeed.ai/2022/09/09/zero-inference.html)：参数卸载、逐层加载与预取。
|
||||
- [ONNX Concepts](https://onnx.ai/onnx/intro/concepts.html)：计算图、节点与 initializer。
|
||||
|
||||
上述资料作为对照，不意味着 NGRA 已被证明等价于其中某种架构。
|
||||
|
||||
## 10. 2026-09-18：持久结构、增量流与连接容量
|
||||
|
||||
### 10.1 用户明确提出的设计方向
|
||||
|
||||
- 从正则引擎的流式增量输入思路演进：输入、输出都在上一次运行结果所保留的状态上继续进行。
|
||||
- 不以每次重新输入完整历史文本作为基本执行语义；历史对当前的影响由已保留的结构及其状态承接。
|
||||
- 容量很可能按连接数量或连接占用评估，而不是按 token 数量评估；具体度量尚未定稿。
|
||||
- 容量不足时，通过丢弃低激活连接释放容量。这是已提出的方向，评分、阈值、删除语义与正确性规则尚未确定。
|
||||
- 字典本身也是结构，结构本身也是记忆。记忆不只包含特征向量数值，还包括对象身份、连接、组合方式、候选与参数化定义。
|
||||
|
||||
### 10.2 建议的抽象接口（非已实现 API）
|
||||
|
||||
```text
|
||||
(S[t+1], ΔO[t]) = Step(S[t], ΔI[t])
|
||||
```
|
||||
|
||||
- `S[t]`：上次保留的结构、字典相关状态、实例特征、候选、待执行依赖及流位置等。
|
||||
- `ΔI[t]`：新增输入；是否包含撤回、修改和外部反馈，需进一步定义。
|
||||
- `ΔO[t]`：本次新增输出；若允许修订先前解释，需要明确输出更新或撤回协议。
|
||||
- `S[t+1]`：完成必要传播与容量管理后保留的状态。
|
||||
|
||||
输入流步 t 与一次输入触发的内部上下传播轮次应分开编号；一次增量输入可能触发多轮关系更新。
|
||||
|
||||
### 10.3 增量失效规则
|
||||
|
||||
Roslyn 增量管线与因果 Tr 的缓存复用提供了讨论类比：依赖没有改变的结果可以复用。但 NGRA 必须按自己的依赖图定义失效，而不能只检查原始文字是否变化。
|
||||
|
||||
- 下游输入、上层反馈、候选集合、相关字典结构或参数的变化，都可能使实例状态失效。
|
||||
- 新输入可能激活此前尚未选择的连接，因此只记录已经活跃的依赖可能不够；候选发现与再激活机制待设计。
|
||||
- 双向反馈可以扩大受影响区域，增量输入不保证只需要少量增量计算。
|
||||
- 纯函数结果复用、依赖版本与参数版本是实现候选，不是目前已经确定的数据结构。
|
||||
|
||||
### 10.4 连接预算与遗忘：需要进一步决定的内容
|
||||
|
||||
1. 预算计算活跃连接、全部保留连接、驻留显存的连接，还是三者分别统计？
|
||||
2. 不同连接携带的特征、参数、元数据大小可能不同；是否采用连接数量加权或真实字节数作为物理预算？
|
||||
3. “丢弃”表示暂停执行、移出显存、删除当前实例边，还是删除可持久化的字典结构？这些行为对记忆的影响不同。
|
||||
4. 当前激活低不必然意味着未来无用或结构可删除；角色绑定、串行 port、待完成模式等所需连接应有明确的完整性规则。
|
||||
5. 连接移除后，哪些实例需要重新评估？候选如何重新发现？还保留哪些历史依据？
|
||||
6. 遗忘顺序不必等同于时间顺序；连接重要性的估计方法和低激活持续时间阈值尚待实验。
|
||||
|
||||
这些是辅助建议与待决问题，不覆盖用户已提出的“容量不足时优先丢弃低激活连接”方向。
|
||||
|
||||
### 10.5 容量与智能的关系：待验证假设
|
||||
|
||||
用户提出“越智能的模型可能上下文更快耗尽”：在相同连接预算下，若模型为同一输入识别并保留更多关系、候选和依赖，容量可能更快用尽。
|
||||
|
||||
这不是已证明的必然关系。更好的抽象、共享、合并与剪枝也可能降低连接需求。后续应测量单位输入的新增连接、保留连接、复用率、候选召回与任务质量，而不是直接把智能程度等同于连接消耗。
|
||||
|
||||
补充参考：[Roslyn Incremental Generators Cookbook](https://github.com/dotnet/roslyn/blob/main/docs/features/incremental-generators.cookbook.md#pipeline-model-design)。该资料只支持增量管线类比，不证明 NGRA 的依赖追踪或遗忘方案正确。
|
||||
@@ -0,0 +1,25 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="PropertyEvaluator"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Nitload.RegularAnalysis.Abstraction;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.RegularAnalysis.PropertyRegex
|
||||
{
|
||||
internal abstract class PropertyEvaluator<T> : IRelationEvaluator<T, PropertyMatchState>
|
||||
{
|
||||
|
||||
public abstract PropertyMatchState Evaluate(
|
||||
ReadOnlySpan<T> sequence
|
||||
);
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="PropertyMatchEngine"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Nitload.RegularAnalysis.Abstraction;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.RegularAnalysis.PropertyRegex
|
||||
{
|
||||
public class PropertyMatchEngine<T>
|
||||
{
|
||||
|
||||
private readonly List<Mix<T, PropertyMatchState>> _nodes = [];
|
||||
private readonly List<Relation<T, PropertyMatchState, PropertyEvaluator<T>>> _relations = [];
|
||||
|
||||
}
|
||||
|
||||
public class FrozenPropertyMatchEngine<T>
|
||||
{
|
||||
|
||||
private readonly Mix<T, PropertyMatchState>[] _nodes;
|
||||
private readonly Relation<T, PropertyMatchState, PropertyEvaluator<T>>[] _relations;
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="PropertyMatchState"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.RegularAnalysis.PropertyRegex
|
||||
{
|
||||
internal class PropertyMatchState { }
|
||||
}
|
||||
@@ -14,11 +14,385 @@ left boarder-> match group start //push stack, add group index
|
||||
-> match group end //pop stack, set group end, build sub sequence
|
||||
|
||||
|
||||
---
|
||||
|
||||
# NGRA / NGRA-Tensor 设计讨论纪要
|
||||
|
||||
记录日期：2026-09-18。本文压缩本次讨论，作为研究与设计记录；未检查现有实现是否符合这些描述。以下区分设计意图、候选实现和待验证问题，不将讨论建议视为已经确定或验证的架构。无歧义时以 `tr` 指 Transformer。
|
||||
|
||||
## 1. NGRA 的背景与当前重点
|
||||
|
||||
- NGRA 的基础方向是把正则从字符序列推广到一般符号及结构化属性序列。
|
||||
- 基本有限正则 DLR（Deterministic Limited Regex）：面向定长输入元素，重复具有上限，不采用断言及前后瞻，处理选择、重复、组合。
|
||||
- DLR-CP：加入 case 优先级，可通过后处理器或节点信息处理。case 排序与模糊匹配是不同问题。
|
||||
- 属性正则（讨论中称 ALR）：输入是含有限个属性的 Struct，通过属性布尔判定 λ 进行匹配。有限属性个数不自动意味着有限取值空间。
|
||||
- 编译器前端可采用增量管线：字符 → token → 属性处理 → 更高层序列/region → AST；通过递归提取“岛屿”region，再交给下级 matcher。
|
||||
- 当前重点是 NGRA-Tensor：以关联程度处理模糊序列，研究如何根据新输入建立、调整和组合 relation。模糊包括关键特征之间存在低关联内容，不仅是 case 优先级或字符编辑距离。
|
||||
- DFA 自发现与这一方向有联系，但布尔自动机学习不能未经修改就用于任意加权、上下文相关的关系网络。
|
||||
|
||||
## 2. 统一词汇与理念系统：核心设计意图
|
||||
|
||||
字、词、短语、语义理念在计算接口上平等，均可存在于统一字典中，并通过向量表示；序列允许混合不同粒度的节点。
|
||||
|
||||
示例：
|
||||
|
||||
```text
|
||||
原句：吃饭了么，您？
|
||||
基本词序列：吃饭，了，么，逗号，您，问号
|
||||
候选语义标准型：我，询问，你的，吃饭，状态
|
||||
```
|
||||
|
||||
标准型是设计示例，不是已确定的规范语法或唯一正确解释。理念的可读标签不必直接等同于自然语言词义。需要能表达角色、顺序、指代、时间、否定与上下文条件；不能把理念集合当作无结构的词袋。
|
||||
|
||||
应区分字典条目与本次出现的实例：相同词条可以在不同位置、事件和角色中出现。统一字典不要求把这些实例状态合并。
|
||||
|
||||
需要区分三件事：
|
||||
|
||||
1. 词条/理念的长期向量和参数。
|
||||
2. 本次出现的上下文状态、绑定与激活程度。
|
||||
3. 本轮计算后创建的新实例，以及长期学习时登记的新词条。
|
||||
|
||||
创建下一波次实例不必扩充词典。将稳定、反复复用的组合固化为新理念，属于独立的结构学习问题。
|
||||
|
||||
## 3. relation 与 mixer 交替执行
|
||||
|
||||
用户澄清：relation 判定器与 mixer 都有很多；mixer 是分布式、可学习的局部计算单元，不是一个集中式全局裁决器。
|
||||
|
||||
```text
|
||||
有序理念实例
|
||||
→ 多个 relation 计算关系响应
|
||||
→ 多个 mixer 选择、融合、抑制、重表示
|
||||
→ 新的有序理念实例
|
||||
→ 下一计算波次
|
||||
```
|
||||
|
||||
- mixer 可保留输入、融合成抽象节点、降低响应或使其不再进入下一波。
|
||||
- 关系响应除数值外必须有对象身份或结构对应关系；裸分数不能唯一确定输出含义。这些对应可以由参数及数据布局实现，不要求全部采用显式符号提案。
|
||||
- relation 可以独立作为计算结构，不必属于词汇表；这是可选架构方向，尚未最终确定。
|
||||
- 表面词可退出传播，功能由抽象节点接替。例如“吗”可转为疑问功能；但将“了吗”全部只压成疑问可能丢失“是否已发生”的区别。
|
||||
- 大量单元提供容量；训练目标仍须约束应保留哪些区别、合并哪些等价表达。
|
||||
- 确定的执行算法不等于正确或唯一的语义推理。可以确定地保留多个解释；不能通过强制选择来宣称消除了歧义。
|
||||
|
||||
之前讨论过“relation 输出带证据提案、mixer 做约束组装”的方案，它可作局部实现参考，但不是用户最终确认的集中式架构。
|
||||
|
||||
## 4. 波次调度、依赖与候选发现
|
||||
|
||||
既有思路：下游节点编译后知道依赖及调用约束，通过 GPU 加载并激活。动态架构下，需要补充运行时激活发现。
|
||||
|
||||
区分：
|
||||
|
||||
- 依赖拉取：已经决定执行某个 relation，需要哪些输入与参数。
|
||||
- 激活发现：当前结果出现后，哪些下游 relation 值得成为候选。
|
||||
- 就绪判定：候选是否已绑定足够、相容的输入。
|
||||
|
||||
编译期可以保留 relation 模板、输入槽位、参数需求与先后约束；运行时决定实例绑定、候选组合与执行波次。反向依赖索引可通知受影响的候选，relation 自身的依赖描述负责取数，两者可以并存。
|
||||
|
||||
调度候选时需区分实例/region，避免跨语境错误组合；上游激活不代表下游立即执行。预算应同时约束活跃连接、候选展开和部分匹配状态，而不只限制最终执行数。
|
||||
|
||||
GPU 原型可按运算类型批处理任务数组，批量构造下一波任务；逻辑节点激活不必对应独立 kernel 启动。稀疏算法需要计入索引、去重、同步和随机访存成本。
|
||||
|
||||
## 5. 记忆与预测性淘汰
|
||||
|
||||
目标是持续更新语义状态，以活跃节点/连接控制工作规模，而不是一直保存全部原始 token。
|
||||
|
||||
三类操作应分开：
|
||||
|
||||
1. 已由高层表示承载含义的中间状态：可以释放。
|
||||
2. 当前低激活但仍可能有用的信息：可移入持久记忆，未来召回。
|
||||
3. 永久删除：接受由此失去某些未来回答能力。
|
||||
|
||||
用户明确接受遗忘：不要求无限保留，缺少信息时可以承认无法确认。系统需避免以相似候选替代缺失事实；完全遗忘后未必知道自己曾知道，不宜总声称“我忘了”。
|
||||
|
||||
淘汰策略可以借鉴预测缓存，综合估计未来使用概率、遗漏损失、保留成本、召回成本与成功率。当前激活低不代表未来价值低。
|
||||
|
||||
持续抽象可能优于临时文本摘要，但摘要本身也可以承载抽象；关键区别是状态能否局部更新、按需召回、保存关系与来源。tr 也存在内部状态记忆/压缩变体，不能把文本摘要当成所有 tr 的唯一记忆方式。
|
||||
|
||||
## 6. 小型 tr 作为局部计算模块：候选方向
|
||||
|
||||
考虑由 NGRA 管理字典、记忆、候选和波次，小型注意力模块计算局部 relation/mix。多个实例可共享参数，通过任务向量和角色信息区分，不必每条关系配一套网络。
|
||||
|
||||
```text
|
||||
当前状态 → 路由/召回候选 → 配对的 K/V → 局部计算 → 输出映射 → 下一波
|
||||
```
|
||||
|
||||
潜在强链接可提前组织候选，但不能仅允许历史强关系进入，否则新关系难以被发现。候选组织是独立难点，局部 attention 不会自动解决。
|
||||
|
||||
若每轮有 m 个包、每包 k 个实例、维度 d，局部稠密注意力部分约为 O(m k² d)；总成本还包括投影、前馈网络、路由、访存、包间通信和波次数。
|
||||
|
||||
必须明确局部 V 是输入证据，还是候选输出理念。前者混合后仍需输出映射；后者已把输出候选发现交给上游。
|
||||
|
||||
## 7. 已澄清的 Transformer 计算原理
|
||||
|
||||
标准自注意力并不把整个字典放入 V，也不要求上下文窗口大于词表。
|
||||
|
||||
设词表大小 N、上下文长度 L、隐藏维度 d：
|
||||
|
||||
| 对象 | 形状/作用 |
|
||||
|---|---|
|
||||
| 词嵌入表 E | N × d，所有词条的基础表示 |
|
||||
| 上下文状态 H | L × d，各位置的上下文表示 |
|
||||
| Q、K、V | 由上下文状态投影；自注意力中行对应位置 |
|
||||
| 注意力权重 A | 每头在数学上为 L × L，受掩码约束 |
|
||||
| 最终预测状态 h | d 维，生成时通常使用最后位置 |
|
||||
| 输出矩阵 W_out | N × d，覆盖全词表 |
|
||||
|
||||
```text
|
||||
A = softmax(Q Kᵀ / sqrt(d_k)) # 决定从各位置读取多少
|
||||
O = A V # 已经实际读取并混合信息
|
||||
多层注意力、前馈、残差等处理 → h
|
||||
logits = W_out h + b # 对整个词表打分
|
||||
p = softmax(logits) # 下一 token 概率
|
||||
```
|
||||
|
||||
两个 softmax 作用不同：注意力 softmax 面向可访问的上下文位置；输出 softmax 面向词表。注意力不是仅安排“以后如何注意”，AV 当场执行读取。
|
||||
|
||||
上下文未出现的词仍在 W_out 中有对应行，所以可以获得分数。训练同时调整前面的网络与输出映射，使合适上下文提高正确输出的概率。输入、输出嵌入可以共享权重，但不意味着输出层等于注意力 V。
|
||||
|
||||
上下文是生成条件，不是随机种子；随机采样种子影响如何从分布选择。新 token 加入上下文后，后续步骤再利用它。
|
||||
|
||||
有限字典不意味着无限长度序列的集合有限；有限属性个数不意味着各属性取值有限。生成未出现符号的能力来自可学习输出映射，不依赖“窗口覆盖字典”或“符号空间封闭”的解释。
|
||||
|
||||
注意力权重是当前层/头的相对读取权重，不宜直接视为永久关系强度或事实可信度。模型能力也分布在前馈、投影等参数中。上下文状态、KV 缓存与注意力激活通常不是可训练参数。
|
||||
|
||||
注意力不会保证全面、充分、正确地理解全部联系。多个 block 通常具有各自参数，不能无条件视为同一计算的循环。
|
||||
|
||||
## 8. 训练、生成与事实约束
|
||||
|
||||
从“相关”到“可推导”需要关系类型、方向、角色和组合规则。强边不自动具有传递性；向量检索适合召回候选，不能独自证明结论。
|
||||
|
||||
用户的事实定义：训练/前文中被接受为真，并经逻辑检验支持的信息。实现时仍需区分前提来源、适用条件与推导；内部一致不等于外部真实。
|
||||
|
||||
生成不是理解过程的简单求逆。可探索：问题与记忆 → 回答标准型/内容计划 → 混合理念与词的逐步展开 → 自然语言。需要跟踪角色、时态、否定、覆盖与依据，避免风格润色新增事实。
|
||||
|
||||
内容规划、语义到文本生成、忠实性检查是可分开训练和评估的任务。这是候选方案，不要求必须使用符号规划器。回译标准型可检查一致性，但共享错误使其不能成为唯一验证。
|
||||
|
||||
训练至少涉及：
|
||||
|
||||
- 表示与预测：下一节点、解释序列或其他明确任务。
|
||||
- 参数学习：relation/mixer 的匹配和组合行为。
|
||||
- 结构学习：新增、拆分、合并、删除关系及理念。
|
||||
- 路由学习：有限预算下召回有用候选，并为新关系保留发现机会。
|
||||
- 记忆学习：保留、休眠、淘汰与召回。
|
||||
- 反例训练：错配人物、日期、否定、确定程度以及缺乏依据的组合。
|
||||
|
||||
tr 的进步同时依赖数据质量与规模、容量、后训练、可验证奖励、推理计算和工具；不能将连贯性或事实性提升仅归因于 QKV 或语料量。流畅、自洽与事实正确需要分开评价。
|
||||
|
||||
## 9. 当前判断与待验证事项
|
||||
|
||||
NGRA-Tensor 的组成思想有研究基础，但完整架构尚未被证明能训练通用 LLM。也不能据此断言它只适合典型模式，或必然缺少长程理解与自然交互能力。
|
||||
|
||||
优先做小型闭环：输入 → relation/mixer → 输出理念/序列 → 损失 → 更新；验证未见组合泛化、候选遗漏、信息遗忘与实际效率。
|
||||
|
||||
关键开放问题：
|
||||
|
||||
1. relation 与 mixer 的精确定义、参数共享和输出接口。
|
||||
2. 多个 mixer 输出的顺序、实例绑定、冲突及重复处理。
|
||||
3. 活跃集合外的新关系如何获得训练机会。
|
||||
4. 向量如何映射到已有理念，何时登记新词条。
|
||||
5. 离散激活/淘汰/结构变更的训练方式。
|
||||
6. 弱证据如何积累，休眠信息如何被后续线索唤起。
|
||||
7. 在同等质量下，端到端速度、显存和训练成本是否优于基线。
|
||||
|
||||
## 10. 讨论中相关文献
|
||||
|
||||
以下是可借鉴的组成部分，不表示这些系统已实现完整 NGRA-Tensor：
|
||||
|
||||
- [Transformer 原论文](https://arxiv.org/html/1706.03762v7)：QKV、前馈网络与词表输出层。
|
||||
- [SoPa](https://aclanthology.org/P18-1028/)：可学习软序列模式；原始模式拓扑受限。
|
||||
- [Rational Recurrences](https://aclanthology.org/D18-1152/)：加权自动机与循环状态更新、语言建模。
|
||||
- [流式概率自动机学习](https://borjaballe.github.io/papers/preprint-bcg13.pdf)：增量状态发现与合并。
|
||||
- [TensorLog](https://arxiv.org/abs/1605.06523)、[Neural LP](https://arxiv.org/abs/1702.08367)：可微关系推理与规则学习。
|
||||
- [NBFNet](https://arxiv.org/abs/2106.06935)：路径上的关系组合和多路径汇总。
|
||||
- [RetoMaton](https://arxiv.org/abs/2201.12431)：通过自动机连接辅助语言模型检索。
|
||||
- [Step-by-Step](https://aclanthology.org/N19-1236/)：内容规划与语言实现分离。
|
||||
- [DataTuner](https://aclanthology.org/2020.coling-main.218/)：生成与语义忠实性重排序。
|
||||
- [Transformer-XL](https://aclanthology.org/P19-1285/)、[Compressive Transformer](https://arxiv.org/abs/1911.05507)：跨片段状态与内部记忆压缩。
|
||||
- [Mamba](https://arxiv.org/abs/2312.00752)、[RWKV](https://arxiv.org/abs/2305.13048)：其他可训练序列架构。
|
||||
- [FlashAttention](https://arxiv.org/abs/2205.14135)：精确注意力的访存优化，与减少候选关系不同。
|
||||
|
||||
|
||||
## 11. 自动机理论：从 DLR 到可学习关系计算
|
||||
|
||||
本节整理讨论中的理论联系，并补齐理解公式所需的定义；不把这些形式化直接认作 NGRA 当前实现。
|
||||
|
||||
### 11.1 有限语言、正则语言与有限状态不是同一个概念
|
||||
|
||||
设有限字母表为 Σ，普通 DFA 的转移为 δ: Q × Σ → Q，识别语言 L ⊆ Σ*。DFA 状态有限，但识别的语言可以包含无限多个串，例如 a*。
|
||||
|
||||
若 DLR 同时满足有限输入字母表、所有重复有上限，且没有额外引入无界递归，则允许的串长具有有限上限 B，语言也是有限集：
|
||||
|
||||
```text
|
||||
L ⊆ Σ^{≤B}
|
||||
|Σ^{≤B}| = Σ_{k=0}^{B} |Σ|^k
|
||||
```
|
||||
|
||||
这不代表状态数或学习成本必然小。递归 region 是另一层机制：局部有限匹配器配合可增长的栈/递归结构后，整体不能不加说明地继续视为一个固定有限状态机。
|
||||
|
||||
属性匹配可把字母表推广为 Struct，并用谓词检查输入。有限个字段不意味着有限个可能的 Struct；若要归约到普通有限字母表，需要有限取值或可用的有限等价分类。UTF-16 code unit 是定长单元，但不等于每个 Unicode 字符或字素都只占一个单元。
|
||||
|
||||
原文件顶部的 lookaround 草稿被保留作历史内容；它与本次 DLR“不采用断言/前后瞻”的约束不应未经核实就合并为一份最终规范。
|
||||
|
||||
### 11.2 自动机自发现的关键是区分后续行为
|
||||
|
||||
对于语言 L，两个历史前缀 u、v 在所有后缀上具有相同接受行为时，可以属于同一右同余类：
|
||||
|
||||
```text
|
||||
u ~_L v ⇔ 对所有 z∈Σ*，uz∈L 当且仅当 vz∈L
|
||||
```
|
||||
|
||||
这提供了“何时需要不同状态”的理论视角。样本学习只能观察有限后缀，因此是否合并还依赖样本覆盖、查询能力、统计假设或归纳偏好。
|
||||
|
||||
- RPNI 类方法：利用正反例，尝试合并前缀树状态，并避免与已知标签冲突。
|
||||
- ALERGIA/流式 PDFA 学习：比较后续概率行为的统计兼容性，决定区分或合并。
|
||||
- 从有限样本得到一个一致模型，不代表已经唯一恢复真实规则，也不代表所有未知组合都可靠。
|
||||
|
||||
对 NGRA 的启示：新序列不是必然要求新增 relation；需要判断现有结构能否表达新的行为，以及扩展结构是否改善泛化而非只记住样例。
|
||||
|
||||
### 11.3 加权计算需要定义路径组合与路径汇总
|
||||
|
||||
一个一般的加权路径计算可以示意为：
|
||||
|
||||
```text
|
||||
f(x) = ⊕_{π 为消费 x 的接受路径} [初始权重 ⊗ 路径各边权重 ⊗ 终止权重]
|
||||
```
|
||||
|
||||
其中 ⊗ 表示同一路径中如何组合，⊕ 表示不同路径之间如何汇总。不同运算意味着不同语义：
|
||||
|
||||
| 组合与汇总 | 含义示例 |
|
||||
|---|---|
|
||||
| AND / OR | 是否存在接受路径 |
|
||||
| 乘法 / 求和 | 汇总路径权重；满足概率约束时可解释为概率计算 |
|
||||
| 乘法 / max | 最强路径，常见于非负权重下的最佳路径计算 |
|
||||
| 加法 / max | 累积路径得分，再选择最佳路径 |
|
||||
| min / max | 一种模糊隶属度组合方式 |
|
||||
|
||||
半环等代数条件使动态规划与路径汇总能够一致地组合，但任意设计的 relation/mixer 并不自动满足这些条件。数值在 [0,1] 内，也不自动是概率。
|
||||
|
||||
用户的“case 优先级加权”只是加权模型的一种用途；文献中的 weighted automaton 还包括路径得分、代价及概率。模糊匹配也不必只保留最高分路径。
|
||||
|
||||
### 11.4 实数加权自动机与谱学习
|
||||
|
||||
不含 ε 转移的实数加权自动机可写成：
|
||||
|
||||
```text
|
||||
f(x₁…x_T) = αᵀ A_{x₁} … A_{x_T} ω
|
||||
H[u,v] = f(uv) # Hankel 矩阵
|
||||
```
|
||||
|
||||
在实数域的相应线性表示理论中，有限 Hankel 秩对应有限维加权自动机表示，最小表示维数由该秩刻画。谱学习通过有限子矩阵的估计和分解恢复模型，仍依赖数据与可识别性条件。
|
||||
|
||||
潜在维度不保证是可命名的语义关系，实数分解也不自动产生合法概率或隶属度。该理论不能直接移植到任意非线性、动态增删结构或任意 max/min 计算。
|
||||
|
||||
参考：[模糊正则到模糊自动机](https://arxiv.org/abs/1105.6190)、[实数加权自动机谱学习](https://cs.nyu.edu/~mohri/pub/swa.pdf)、[符号加权自动机查询学习](https://proceedings.mlr.press/v153/suzuki21a.html)。
|
||||
|
||||
## 12. Transformer 数学补充：为什么能输出上下文外的词
|
||||
|
||||
### 12.1 两次打分具有不同的候选范围
|
||||
|
||||
以列向量 h 表示最终位置状态，一个标准预测过程为：
|
||||
|
||||
```text
|
||||
输入 token ID → 词嵌入与位置信息 → 多个 block → 最终状态 h
|
||||
|
||||
block 中：
|
||||
Q = H W_Q，K = H W_K，V = H W_V
|
||||
A = softmax(QKᵀ / √d_k + mask) # 沿允许的 key 位置归一化
|
||||
O = A V # 混合上下文信息
|
||||
随后还有输出投影、残差、归一化、前馈网络等；顺序取决于变体
|
||||
|
||||
词表输出：
|
||||
s_i = w_iᵀ h + b_i # i 遍历整个词表
|
||||
p_i = exp(s_i) / Σ_j exp(s_j)
|
||||
```
|
||||
|
||||
前一个 softmax 回答“读哪些位置、各读多少”；后一个回答“输出哪个词表符号”。标准 V 的行数由被读取序列长度决定，不由词表大小决定。交叉注意力也可读取另一段序列，但仍不能把它默认等同于全词表。
|
||||
|
||||
### 12.2 本次讨论中的数值例子
|
||||
|
||||
忽略偏置，设 h=[2,1,-1]，输出表有四行：
|
||||
|
||||
| token | 输出权重 w_i | 点积得分 |
|
||||
|---|---|---:|
|
||||
| 我 | [0,1,0] | 1 |
|
||||
| 吃饭 | [1,0,0] | 2 |
|
||||
| 灯塔 | [1,1,-1] | 4 |
|
||||
| 苹果 | [-1,0,1] | -3 |
|
||||
|
||||
即使“灯塔”没有出现在输入里，它对应的输出参数行仍参与点积。此例仅说明计算过程，向量坐标不是人为规定的真实语义属性。
|
||||
|
||||
训练时，目标 token 为 y，交叉熵为：
|
||||
|
||||
```text
|
||||
loss = -log p_y
|
||||
∂loss/∂s_i = p_i - 1[i=y] # 标准全词表 softmax 下
|
||||
```
|
||||
|
||||
因此目标输出与其他输出的得分共同参与训练，梯度也会通过 h 传回前面的网络。模型能输出未在当前上下文出现的词，来自这个训练过的映射，不需要先将目标词放进注意力 V。
|
||||
|
||||
### 12.3 条件、参数与随机采样
|
||||
|
||||
```text
|
||||
Pθ(x₁…x_T) = ∏_t Pθ(x_t | x_{<t})
|
||||
```
|
||||
|
||||
上下文提供条件，θ 保存训练形成的计算规律，解码策略从概率分布选择输出。随机种子只涉及随机选择，不是上下文本身；贪心选择可以不使用随机抽样。
|
||||
|
||||
网络可以组合训练所得规律生成未见过的整段文本，不代表该段文本作为完整记录储存在参数中，也不代表生成内容真实。
|
||||
|
||||
注意力行权重之和为 1，属于当前候选之间的相对分配：候选变化会影响权重。它不是可直接跨上下文比较的绝对事实置信度。强注意力也不自动证明某项输入对最终答案具有唯一因果作用。
|
||||
|
||||
### 12.4 参数、激活与复杂度
|
||||
|
||||
- 词表 N、上下文 L、隐藏维度 d 是不同轴，无须满足 L>N。
|
||||
- 数学上的注意力得分矩阵为每头 L×L；高效实现不一定完整物化它。
|
||||
- 稠密注意力的主要位置交互计算约为 O(L²d)；投影、前馈网络及全词表输出另有成本。
|
||||
- 给单个位置做标准全词表输出投影约为 O(Nd)，不因输入里只有少数词而自动缩小。
|
||||
- 增长的 KV 缓存等是运行时状态，不等于新增训练参数。
|
||||
|
||||
对 NGRA：输入候选路由与输出理念选择是两个问题。小型 mixer 产生 h 后仍可映射到大字典；若也要避免全字典输出计算，需增加输出候选检索/路由，而非要求字典都进入局部 V。
|
||||
|
||||
## 13. 数学与研究判断的边界
|
||||
|
||||
### 13.1 必须分别验证的性质
|
||||
|
||||
| 性质 | 问题 |
|
||||
|---|---|
|
||||
| 表达能力 | 是否存在能表达目标行为的参数/结构？ |
|
||||
| 可学习性 | 给定数据与反馈，能否找到这种结构？ |
|
||||
| 执行确定性 | 固定输入、状态及选择规则，是否输出一致？ |
|
||||
| 逻辑可靠性 | 结论是否由有效规则和适用前提支持？ |
|
||||
| 唯一解释 | 信息是否足以排除其他候选？ |
|
||||
| 工程效率 | 把路由、访存、同步算进去后是否实际更快？ |
|
||||
|
||||
不能从“确定地取最大值”推出语义无歧义，也不能从“存在理论表示”推出可高效训练。减少活跃张量不自动减少全局参数，更不自动提高 GPU 吞吐。
|
||||
|
||||
### 13.2 关系强度不等于关系组合规则
|
||||
|
||||
“位于某城市”与“城市位于某国家”可在适当语义下组合；“喜欢”不能因两条边都强就自动传递。需要保留关系类型、方向、角色和条件。不同证据路径还可能重复使用同一事实，简单累加可能制造虚假支持。
|
||||
|
||||
同时，长距离不等于弱关系；当前低分不等于未来无用；稀疏不等于必然缺少长程能力。潜在难点是低显著度线索被过早删除，或语义压缩丢失后来需要的区别。
|
||||
|
||||
### 13.3 联合学习与预测淘汰的候选形式
|
||||
|
||||
讨论中的结构/参数学习可概括为研究目标：
|
||||
|
||||
```text
|
||||
min_{G,θ} Σ_(x,y) loss(f_{G,θ}(x),y) + λ C(G)
|
||||
```
|
||||
|
||||
G 为关系结构，θ 为数值参数，C(G) 惩罚结构复杂度。这只是问题定义，不提供自动增删关系或反向传播穿过离散操作的算法。
|
||||
|
||||
预测淘汰可用启发式价值表达：
|
||||
|
||||
```text
|
||||
保留价值 ≈ 未来使用概率 × 遗忘损失 − 保留成本
|
||||
```
|
||||
|
||||
实际还应计入召回成功率、召回成本和时间范围；这些量需要学习或明确估计，不能把当前激活值直接代入所有项。
|
||||
|
||||
### 13.4 关于“封闭”的最终澄清
|
||||
|
||||
固定有限词表 D 的长度至多 L 的序列集合有限；D* 在非空字表、长度无上限时无限。有限维实向量空间在数学上无限，机器有限精度不使实际学习与枚举问题自动变简单。
|
||||
|
||||
输入和输出共享词表，只说明可在相同符号系统中表达，不证明推理正确，也不构成预测未出现符号的充分解释。真正需要明确的是输入状态如何经可学习函数映射为输出候选及其得分。
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the ¡°Software¡±), to deal in the Software without restriction,
|
||||
// files (the ��Software��), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
@@ -17,7 +17,7 @@
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED ¡°AS IS¡±,
|
||||
// THE SOFTWARE IS PROVIDED ��AS IS��,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="TestSwiss"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Nitload.Common.Collections;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload
|
||||
{
|
||||
internal static partial class Program
|
||||
{
|
||||
|
||||
public static void TestSwiss()
|
||||
{
|
||||
SwissTable swiss = SwissTable.Create();
|
||||
for (nuint i = 0; i < 5000; i++)
|
||||
{
|
||||
Console.Write($"Set {i}: ");
|
||||
_ = swiss.TryAddOrSet(i, i * 2, out _);
|
||||
Console.WriteLine($"Set a kvp of ({i},{i * 2})");
|
||||
}
|
||||
Random rand = new Random();
|
||||
for (int i = 0; i < 100; i++)
|
||||
{
|
||||
Console.Write($"Get {i}: ");
|
||||
nuint key = (nuint)rand.Next(4000, 5000);
|
||||
if (swiss.TryGet(key, out nuint value))
|
||||
{
|
||||
Console.WriteLine($"Success in get a kvp of ({key},{value})");
|
||||
} else
|
||||
{
|
||||
Console.WriteLine($"Failed to get a kvp of ({key},?)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,215 +0,0 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[switch] $Apply
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$taskWorkspaceRoot = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..'))
|
||||
$nativeProjectRoot = Join-Path $taskWorkspaceRoot 'src\Nitload.Native'
|
||||
$textRoots = @(
|
||||
$nativeProjectRoot
|
||||
(Join-Path $taskWorkspaceRoot 'src\Nitload.Common')
|
||||
(Join-Path $taskWorkspaceRoot 'src\Nitload.Mathematics')
|
||||
(Join-Path $taskWorkspaceRoot 'src\Nitload.Graphics')
|
||||
)
|
||||
|
||||
$textExtensions = [Collections.Generic.HashSet[string]]::new(
|
||||
[StringComparer]::OrdinalIgnoreCase
|
||||
)
|
||||
@(
|
||||
'.c', '.h', '.inl', '.inc', '.ipp',
|
||||
'.cs', '.cmake', '.txt', '.md',
|
||||
'.csproj', '.vcxproj', '.filters',
|
||||
'.json', '.props', '.targets'
|
||||
) | ForEach-Object { [void] $textExtensions.Add($_) }
|
||||
|
||||
$replacementRules = @(
|
||||
@{ Pattern = '\bSTGSHARP_'; Replacement = 'NIF_' }
|
||||
@{ Pattern = '\bSN_'; Replacement = 'NIF_' }
|
||||
@{ Pattern = '(?<![A-Za-z0-9])sn_'; Replacement = 'nif_' }
|
||||
@{ Pattern = '\bload_glfw_functions\b'; Replacement = 'nif_load_glfw_functions' }
|
||||
@{ Pattern = '\bload_intrinsic_function\b'; Replacement = 'nif_load_intrinsic_function' }
|
||||
@{ Pattern = '\bStgSharpNative\b'; Replacement = 'NitloadFrameWork' }
|
||||
@{ Pattern = '\bNitloadInnovationFramework\b'; Replacement = 'NitloadFrameWork' }
|
||||
@{ Pattern = '\bNitloadFramework\b'; Replacement = 'NitloadFrameWork' }
|
||||
@{ Pattern = '\bSNative\b'; Replacement = 'NIF_NATIVE_H' }
|
||||
@{ Pattern = 'sn-head'; Replacement = 'nif-head' }
|
||||
@{ Pattern = 'sn-src'; Replacement = 'nif-src' }
|
||||
@{ Pattern = 'sn-template'; Replacement = 'nif-template' }
|
||||
)
|
||||
|
||||
function Test-TaskPath {
|
||||
param([Parameter(Mandatory)][string] $Path)
|
||||
|
||||
$resolvedPath = [IO.Path]::GetFullPath($Path)
|
||||
if (-not $resolvedPath.StartsWith(
|
||||
$taskWorkspaceRoot + [IO.Path]::DirectorySeparatorChar,
|
||||
[StringComparison]::OrdinalIgnoreCase
|
||||
)) {
|
||||
throw "Path escapes workspace: $resolvedPath"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-TaskTextFile {
|
||||
foreach ($textRoot in $textRoots) {
|
||||
Get-ChildItem -LiteralPath $textRoot -Recurse -File | Where-Object {
|
||||
$relativePath = [IO.Path]::GetRelativePath($taskWorkspaceRoot, $_.FullName)
|
||||
if ($relativePath -match '(^|[\\/])(bin|obj|\.discard)([\\/]|$)') {
|
||||
return $false
|
||||
}
|
||||
|
||||
$nativeLibRoot = Join-Path $nativeProjectRoot 'lib'
|
||||
if ($_.FullName.StartsWith(
|
||||
$nativeLibRoot + [IO.Path]::DirectorySeparatorChar,
|
||||
[StringComparison]::OrdinalIgnoreCase
|
||||
) -and $_.Name -ne 'CMakeLists.txt') {
|
||||
return $false
|
||||
}
|
||||
|
||||
return $textExtensions.Contains($_.Extension)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Read-TaskText {
|
||||
param([Parameter(Mandatory)][string] $Path)
|
||||
|
||||
$bytes = [IO.File]::ReadAllBytes($Path)
|
||||
$encoding = [Text.UTF8Encoding]::new($false, $true)
|
||||
$preambleLength = 0
|
||||
$preambleBytes = [byte[]]::new(0)
|
||||
|
||||
if ($bytes.Length -ge 3 -and
|
||||
$bytes[0] -eq 0xEF -and $bytes[1] -eq 0xBB -and $bytes[2] -eq 0xBF) {
|
||||
$preambleLength = 3
|
||||
}
|
||||
elseif ($bytes.Length -ge 2 -and $bytes[0] -eq 0xFF -and $bytes[1] -eq 0xFE) {
|
||||
$encoding = [Text.UnicodeEncoding]::new($false, $false, $true)
|
||||
$preambleLength = 2
|
||||
}
|
||||
elseif ($bytes.Length -ge 2 -and $bytes[0] -eq 0xFE -and $bytes[1] -eq 0xFF) {
|
||||
$encoding = [Text.UnicodeEncoding]::new($true, $false, $true)
|
||||
$preambleLength = 2
|
||||
}
|
||||
|
||||
if ($preambleLength -ne 0) {
|
||||
$preambleBytes = [byte[]] $bytes[0..($preambleLength - 1)]
|
||||
}
|
||||
|
||||
[PSCustomObject]@{
|
||||
Text = $encoding.GetString($bytes, $preambleLength, $bytes.Length - $preambleLength)
|
||||
Encoding = $encoding
|
||||
Preamble = $preambleBytes
|
||||
}
|
||||
}
|
||||
|
||||
function Write-TaskText {
|
||||
param(
|
||||
[Parameter(Mandatory)][string] $Path,
|
||||
[Parameter(Mandatory)][string] $Text,
|
||||
[Parameter(Mandatory)] $Encoding,
|
||||
[Parameter(Mandatory)][AllowEmptyCollection()][byte[]] $Preamble
|
||||
)
|
||||
|
||||
$body = $Encoding.GetBytes($Text)
|
||||
$output = [byte[]]::new($Preamble.Length + $body.Length)
|
||||
if ($Preamble.Length -ne 0) {
|
||||
[Buffer]::BlockCopy($Preamble, 0, $output, 0, $Preamble.Length)
|
||||
}
|
||||
[Buffer]::BlockCopy($body, 0, $output, $Preamble.Length, $body.Length)
|
||||
[IO.File]::WriteAllBytes($Path, $output)
|
||||
}
|
||||
|
||||
$changedTextFiles = [Collections.Generic.List[string]]::new()
|
||||
foreach ($textFile in Get-TaskTextFile) {
|
||||
$fileContent = Read-TaskText -Path $textFile.FullName
|
||||
$updatedText = $fileContent.Text
|
||||
foreach ($rule in $replacementRules) {
|
||||
$updatedText = [Text.RegularExpressions.Regex]::Replace(
|
||||
$updatedText,
|
||||
$rule.Pattern,
|
||||
$rule.Replacement,
|
||||
[Text.RegularExpressions.RegexOptions]::CultureInvariant
|
||||
)
|
||||
}
|
||||
|
||||
if ($updatedText -ceq $fileContent.Text) {
|
||||
continue
|
||||
}
|
||||
|
||||
$relativePath = [IO.Path]::GetRelativePath($taskWorkspaceRoot, $textFile.FullName)
|
||||
$changedTextFiles.Add($relativePath)
|
||||
if ($Apply) {
|
||||
Write-TaskText -Path $textFile.FullName `
|
||||
-Text $updatedText `
|
||||
-Encoding $fileContent.Encoding `
|
||||
-Preamble $fileContent.Preamble
|
||||
}
|
||||
}
|
||||
|
||||
$pathMoves = [Collections.Generic.List[object]]::new()
|
||||
Get-ChildItem -LiteralPath $nativeProjectRoot -Recurse -File |
|
||||
Where-Object {
|
||||
$_.Name -match '^sn_' -or
|
||||
$_.Name -cin @(
|
||||
'StgSharpNative.h',
|
||||
'NitloadInnovationFramework.h',
|
||||
'NitloadFramework.h'
|
||||
)
|
||||
} |
|
||||
ForEach-Object {
|
||||
$newName = if ($_.Name -cin @(
|
||||
'StgSharpNative.h',
|
||||
'NitloadInnovationFramework.h',
|
||||
'NitloadFramework.h'
|
||||
)) {
|
||||
'NitloadFrameWork.h'
|
||||
}
|
||||
else {
|
||||
$_.Name -replace '^sn_', 'nif_'
|
||||
}
|
||||
$pathMoves.Add([PSCustomObject]@{
|
||||
Source = $_.FullName
|
||||
Target = Join-Path $_.DirectoryName $newName
|
||||
})
|
||||
}
|
||||
|
||||
Get-ChildItem -LiteralPath $nativeProjectRoot -Recurse -Directory |
|
||||
Where-Object { $_.Name -in @('sn-head', 'sn-src', 'sn-template') } |
|
||||
Sort-Object { $_.FullName.Length } -Descending |
|
||||
ForEach-Object {
|
||||
$pathMoves.Add([PSCustomObject]@{
|
||||
Source = $_.FullName
|
||||
Target = Join-Path $_.Parent.FullName ($_.Name -replace '^sn-', 'nif-')
|
||||
})
|
||||
}
|
||||
|
||||
foreach ($pathMove in $pathMoves) {
|
||||
Test-TaskPath -Path $pathMove.Source
|
||||
Test-TaskPath -Path $pathMove.Target
|
||||
if (Test-Path -LiteralPath $pathMove.Target) {
|
||||
throw "Rename target already exists: $($pathMove.Target)"
|
||||
}
|
||||
}
|
||||
|
||||
"Text files to update: $($changedTextFiles.Count)"
|
||||
$changedTextFiles | ForEach-Object { " TEXT $_" }
|
||||
"Paths to rename: $($pathMoves.Count)"
|
||||
$pathMoves | ForEach-Object {
|
||||
$source = [IO.Path]::GetRelativePath($taskWorkspaceRoot, $_.Source)
|
||||
$target = [IO.Path]::GetRelativePath($taskWorkspaceRoot, $_.Target)
|
||||
" MOVE $source -> $target"
|
||||
}
|
||||
|
||||
if (-not $Apply) {
|
||||
'Dry run only. Re-run with -Apply to perform the migration.'
|
||||
exit 0
|
||||
}
|
||||
|
||||
foreach ($pathMove in $pathMoves) {
|
||||
Move-Item -LiteralPath $pathMove.Source -Destination $pathMove.Target
|
||||
}
|
||||
|
||||
'NIF prefix migration completed.'
|
||||
@@ -0,0 +1,77 @@
|
||||
[CmdletBinding()]
|
||||
param([switch] $Apply)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$workspaceRoot = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..'))
|
||||
$nativeRoot = Join-Path $workspaceRoot 'src/Nitload.Native'
|
||||
$textRoots = @(
|
||||
(Join-Path $workspaceRoot 'Directory.Build.targets'),
|
||||
$nativeRoot,
|
||||
(Join-Path $workspaceRoot 'src/Nitload.Common'),
|
||||
(Join-Path $workspaceRoot 'src/Nitload.Mathematics'),
|
||||
(Join-Path $workspaceRoot 'src/Nitload.Graphics')
|
||||
)
|
||||
$textExtensions = @('.c', '.h', '.cs', '.cmake', '.txt', '.md', '.vcxproj', '.filters', '.targets')
|
||||
$utf8 = [Text.UTF8Encoding]::new($false, $true)
|
||||
|
||||
$files = foreach ($root in $textRoots) {
|
||||
if (Test-Path -LiteralPath $root -PathType Leaf) {
|
||||
Get-Item -LiteralPath $root
|
||||
} else {
|
||||
Get-ChildItem -LiteralPath $root -Recurse -File | Where-Object {
|
||||
$_.Extension -in $textExtensions -and
|
||||
$_.FullName -notmatch '[\\/](bin|obj|cmake_build)[\\/]'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$changes = [Collections.Generic.List[string]]::new()
|
||||
foreach ($file in $files) {
|
||||
$bytes = [IO.File]::ReadAllBytes($file.FullName)
|
||||
try { $content = $utf8.GetString($bytes) }
|
||||
catch [Text.DecoderFallbackException] { continue }
|
||||
if ($file.FullName -eq (Join-Path $nativeRoot 'README.md') -and
|
||||
$content.Contains('The C implementation uses `nfn`')) { continue }
|
||||
|
||||
$updated = $content.Replace('NIF_', 'NFN_').Replace('nif_', 'nfn_').Replace('nif-', 'nfn-')
|
||||
$updated = $updated.Replace('NF_', 'NFN_').Replace('nf_', 'nfn_').Replace('nf-', 'nfn-')
|
||||
# The previous migration left private SN macros in this header.
|
||||
if ($file.Name -in @('nif_intrinsic.std.h', 'nf_intrinsic.std.h')) {
|
||||
$updated = $updated.Replace('_SN_', '_NFN_')
|
||||
}
|
||||
if ($updated -ceq $content) { continue }
|
||||
|
||||
$changes.Add([IO.Path]::GetRelativePath($workspaceRoot, $file.FullName))
|
||||
if ($Apply) { [IO.File]::WriteAllText($file.FullName, $updated, $utf8) }
|
||||
}
|
||||
|
||||
$moves = [Collections.Generic.List[object]]::new()
|
||||
foreach ($directory in @('nif-head', 'nif-src', 'nif-template', 'nf-head', 'nf-src', 'nf-template')) {
|
||||
$sourceDir = Join-Path $nativeRoot $directory
|
||||
if (-not (Test-Path -LiteralPath $sourceDir)) { continue }
|
||||
foreach ($file in Get-ChildItem -LiteralPath $sourceDir -File | Where-Object { $_.Name -cmatch '^(nif|nf)_' }) {
|
||||
$moves.Add(@($file.FullName, (Join-Path $sourceDir ($file.Name -creplace '^(nif|nf)_', 'nfn_'))))
|
||||
}
|
||||
$moves.Add(@($sourceDir, (Join-Path $nativeRoot ($directory -creplace '^(nif|nf)-', 'nfn-'))))
|
||||
}
|
||||
|
||||
foreach ($move in $moves) {
|
||||
if (Test-Path -LiteralPath $move[1]) { throw "Rename target already exists: $($move[1])" }
|
||||
}
|
||||
|
||||
"Text files to update: $($changes.Count)"
|
||||
$changes | ForEach-Object { " TEXT $_" }
|
||||
"Paths to rename: $($moves.Count)"
|
||||
$moves | ForEach-Object {
|
||||
" MOVE $([IO.Path]::GetRelativePath($workspaceRoot, $_[0])) -> $([IO.Path]::GetRelativePath($workspaceRoot, $_[1]))"
|
||||
}
|
||||
|
||||
if (-not $Apply) {
|
||||
'Dry run only. Re-run with -Apply to perform the migration.'
|
||||
exit 0
|
||||
}
|
||||
|
||||
foreach ($move in $moves) { Move-Item -LiteralPath $move[0] -Destination $move[1] }
|
||||
'NFN prefix migration completed.'
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user