Compare commits
22
Commits
2e7c3040f4
...
giga
| Author | SHA1 | Date | |
|---|---|---|---|
 Youmu Liu
|
91cf6a43cd | ||
|
Youmu Liu
|
9867e1f262 | ||
|
Youmu Liu
|
c64f7ae9f6 | ||
|
Youmu Liu
|
6bc7224d4c | ||
 youmuliu
|
b7f1d3d2ba
|
||
|
youmuliu
|
76a9dfa5af
|
||
|
youmuliu
|
2bef14493f
|
||
|
Youmu Liu
|
0cef5aa517 | ||
|
Youmu Liu
|
172f64c6f9 | ||
|
Youmu Liu
|
3a2b123478 | ||
|
Nitload
|
ddd83b1b05 | ||
|
youmuliu
|
e24efd9f9c
|
||
|
youmuliu
|
9fc23c29fb
|
||
|
youmuliu
|
3d1d475e25
|
||
|
youmuliu
|
7a820261cf
|
||
|
youmuliu
|
fa1186cd25
|
||
|
youmuliu
|
e49b14f037
|
||
|
youmuliu
|
052afa1dd4
|
||
|
youmuliu
|
1d3d3c10dd
|
||
|
youmuliu
|
902be141d0
|
||
|
youmuliu
|
b5b7c75b5e
|
||
|
copilot-swe-agent[bot]
|
aa7394503e |
@@ -0,0 +1,5 @@
|
||||
root = true
|
||||
|
||||
[*.cs]
|
||||
dotnet_diagnostic.IDE0032.severity = none
|
||||
dotnet_diagnostic.IDE0055.severity = none
|
||||
+1
-1
@@ -29,7 +29,7 @@ build.ninja
|
||||
BenchmarkDotNet.Artifacts/
|
||||
|
||||
# Khronos OpenGL Registry input is supplied locally; keep only its instructions.
|
||||
/tools/Nitload.GenerateGL/xml/*
|
||||
/tools/*GenerateGL/xml/*
|
||||
!/tools/Nitload.GenerateGL/xml/README.md
|
||||
|
||||
/src/Nitload.Native/lib/*
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
<NativeCMakeBuildDir Condition="'$(NativeCMakeBuildDir)'==''">$(SolutionRoot)cmake_build/$(NativeCMakePreset)</NativeCMakeBuildDir>
|
||||
<_NativeCMakeOutputDir>$([System.String]::Copy('$(NativeBuildDir)').Replace('\', '/'))</_NativeCMakeOutputDir>
|
||||
</PropertyGroup>
|
||||
<Exec Command="cmake --preset "$(NativeCMakePreset)" -S "$(SolutionRoot)src/Nitload.Native" -B "$(NativeCMakeBuildDir)" "-DNIF_BUILD_OUTPUT_DIR:PATH=$(_NativeCMakeOutputDir)" "-DNIF_EXPECTED_RID:STRING=$(BuildRid)"" />
|
||||
<Exec Command="cmake --preset "$(NativeCMakePreset)" -S "$(SolutionRoot)src/Nitload.Native" -B "$(NativeCMakeBuildDir)" "-DNFN_BUILD_OUTPUT_DIR:PATH=$(_NativeCMakeOutputDir)" "-DNFN_EXPECTED_RID:STRING=$(BuildRid)"" />
|
||||
<Exec Command="cmake --build "$(NativeCMakeBuildDir)" --target "$(NativeLibName)"" />
|
||||
<Error Condition="!Exists('$(NativeBuildDir)$(NativeFileName)')"
|
||||
Text="CMake did not produce the expected native library: $(NativeBuildDir)$(NativeFileName). Check the target OS, architecture and output name." />
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Nitload StgSharp Team
|
||||
Copyright (c) 2024 Nitload
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -20,9 +20,9 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
StgSharp program uses the following open source projects,
|
||||
I'd like to refer to them here and show my sincere appreciation,
|
||||
while the names are sorted in no particular order
|
||||
The nFramework project uses the following open-source projects.
|
||||
Their respective license notices are reproduced below,
|
||||
and the projects are listed in no particular order.
|
||||
|
||||
1. GLFW
|
||||
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
<projectUrl>https://github.com/Nitload-NSI/StgSharp</projectUrl>
|
||||
<description>No des right now</description>
|
||||
<releaseNotes>Summary of changes made in this release of the package.</releaseNotes>
|
||||
<copyright>Nitload reserve</copyright>
|
||||
<copyright>Copyright (c) 2024 Nitload</copyright>
|
||||
<tags>OpenGL C#</tags>
|
||||
</metadata>
|
||||
<files>
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
# Nitload
|
||||
|
||||
|
||||
|
||||
# nFramework
|
||||
|
||||
Nitload (formerly StgSharp) is an experimental .NET framework for graphics, numerical computing, language analysis, and other performance-sensitive applications. It began as the foundation of a next-generation STG (shoot 'em up) engine, but the current repository focuses on reusable low-level infrastructure rather than a complete, ready-to-use game engine.
|
||||
|
||||
The managed libraries now use the `Nitload.*` namespace and project prefix. Repository URLs, the logo filename, and some legacy test artifacts still retain the StgSharp name.
|
||||
nFramework (formerly StgSharp, now also nitload.Framework) is an experimental framework built on .NET for graphics, numerical computing, language analysis, and other performance-sensitive applications. It combines managed C# libraries with a C native runtime, targets .NET 8, and provides Windows and Linux native implementations.
|
||||
|
||||
## Overview
|
||||
|
||||
The framework combines managed C# APIs with a C native library. Its current work includes OpenGL bindings and rendering infrastructure, generic matrix and vector types, SIMD-accelerated native kernels, custom allocators and collections, regular-language analysis, and state-oriented UI infrastructure. Nitload targets .NET 8; platform and architecture coverage varies by module, with Windows and Linux as the principal native targets.
|
||||
nFramework combines managed C# APIs with a C native library. Its current work includes OpenGL bindings and rendering infrastructure, generic matrix and vector types, SIMD-accelerated native kernels, custom allocators and collections, regular-language analysis, and state-oriented UI infrastructure. nFramework targets .NET 8; platform and architecture coverage varies by module, with Windows and Linux as the principal native targets.
|
||||
|
||||
## Core Modules
|
||||
|
||||
@@ -139,7 +137,7 @@ The benchmark reports normalized time per allocation/free operation. Each iterat
|
||||
|
||||
## Installation
|
||||
|
||||
Nitload is currently in development and requires compilation from source. **NuGet packages are not yet available** - you must build the project from source code.
|
||||
nFramework is currently in development and requires compilation from source. **NuGet packages are not yet available** - you must build the project from source code.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
@@ -245,7 +243,7 @@ allocator.Free(handle);
|
||||
|
||||
## Contributing
|
||||
|
||||
Nitload is developed by Nitload Space. Contributions are welcome, particularly in:
|
||||
nFramework is developed by Nitload Space under the Nitload brand. Contributions are welcome, particularly in:
|
||||
|
||||
- Performance optimizations
|
||||
- Cross-platform compatibility
|
||||
@@ -254,7 +252,7 @@ Nitload is developed by Nitload Space. Contributions are welcome, particularly i
|
||||
|
||||
## License
|
||||
|
||||
Nitload is licensed under the MIT License. See [LICENSE](LICENSE) for details.
|
||||
nFramework is licensed under the MIT License. See [LICENSE](LICENSE) for details.
|
||||
|
||||
## Future Plans
|
||||
|
||||
|
||||
@@ -1,762 +0,0 @@
|
||||
# L4 Software Cache Design Specification
|
||||
|
||||
> Project: StgSharp
|
||||
> Author: Nitload
|
||||
> Version: Draft v0.3
|
||||
> Date: 2026-03-10
|
||||
|
||||
---
|
||||
|
||||
## 1. Purpose
|
||||
|
||||
### 1.1 Problem Background
|
||||
|
||||
StgSharp's matrix computation system is migrating from a "single global context table" to a
|
||||
"per-thread context" architecture to adapt to the scheduling characteristics of hybrid-architecture
|
||||
CPUs (e.g., Intel P-Core / E-Core). After the migration, each thread independently holds its own
|
||||
computation context, but matrix data may still be allocated by the HLSF allocator or global
|
||||
`malloc` on a remote NUMA node, resulting in significantly increased cross-node access latency.
|
||||
|
||||
At the same time, in matrix multiplication (FMA), the core row data of matrix A in the
|
||||
checkerboard-shaped kernel grid layout must be repacked (Packed) into a micro-kernel-friendly
|
||||
contiguous arrangement. In libraries such as OpenBLAS, this Pack operation is re-executed on
|
||||
every tile switch, whereas L4 can cache the Pack results to avoid repeated reordering.
|
||||
|
||||
Additionally, the segmented accumulation (segment) of matrix multiplication requires storing
|
||||
intermediate partial sums locally to avoid cross-NUMA writes to the target matrix on every
|
||||
segment. L4 can serve as a local accumulation buffer for partial sums, writing them back in a
|
||||
single operation after all segments complete locally.
|
||||
|
||||
### 1.2 Solution
|
||||
|
||||
The L4 cache is a purely software-implemented **data caching layer** logically positioned
|
||||
between the CPU hardware caches (L1/L2/L3) and main memory. Its core responsibilities:
|
||||
|
||||
1. **Pack result caching**: Cache the repack results of FMA core rows in matrix A to avoid repeated Pack operations
|
||||
2. **NUMA migration cache**: Migrate data from remote NUMA nodes to local memory and cache it, eliminating cross-node latency
|
||||
3. **Segment accumulation buffer**: Temporarily store partial sums from FMA segmented computation to reduce remote write operations
|
||||
4. **PLU intermediate data buffer**: Provide a local contiguous workspace for PLU decomposition
|
||||
|
||||
### 1.3 Design Goals
|
||||
|
||||
- Provide NUMA-friendly local data access for per-thread matrix computation contexts
|
||||
- Fixed capacity (64 MB), no runtime expansion, deterministic latency
|
||||
- Transparent pointer redirection: upper-layer code queries via original pointers; L4 returns local cache pointers
|
||||
- Cache eviction policy based on **RRPV mask + windowed OPT prediction**
|
||||
- Support both **StreamTransparent** and **CacheBuffer** working modes
|
||||
- Support **writable cache lines** that invoke registered callback functions for pre-processing and write-back upon eviction or explicit flush
|
||||
|
||||
---
|
||||
|
||||
## 2. Overall Architecture
|
||||
|
||||
### 2.1 Three-Component Model
|
||||
|
||||
```
|
||||
┌���─────────────────────────────────────────────────────────────────┐
|
||||
│ L4 │
|
||||
│ │
|
||||
│ ┌──────────────┐ ┌────────────────────┐ ┌──────────────────┐ │
|
||||
│ │ CacheLine │ │ L4 Core │ │ Predictor │ │
|
||||
│ │ Handle │ │ Cache │ │ Prefetch │ │
|
||||
│ │ │ │ │ │ Predictor │ │
|
||||
│ │ • local addr │ │ • Swiss Table │ │ • 2048-entry buf │ │
|
||||
│ │ • ref count │ │ • Slab Allocator │ │ • threshold 1024 │ │
|
||||
│ │ • Dispose │ │ • RRPV mask │ │ • windowed OPT │ │
|
||||
│ │ drops ref │ │ • SIMD eviction │ │ • active key set │ │
|
||||
│ │ │ │ • writeback mgmt │ │ │ │
|
||||
│ └──────────────┘ └────────────────────┘ └──────────────────┘ │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 2.2 Data Access Path
|
||||
|
||||
```
|
||||
Compute thread accessing data:
|
||||
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ Read-only path: L4.Map(originKey) │
|
||||
│ ├─ StreamTransparent + no repack needed? │
|
||||
│ │ └─ Yes → return original pointer directly │
|
||||
│ │ (zero overhead) │
|
||||
│ │ │
|
||||
│ └─ CacheBuffer mode │
|
||||
│ ├─ Swiss Table lookup (2.6ns, lock-free) │
|
||||
│ ├─ Hit → update RRPV=0, return CacheLine handle │
|
||||
│ └─ Miss → SpinWait for Predictor prefetch │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ Writable path: L4.MapWritable(originKey, preprocess, writeback) │
|
||||
│ ├─ Hit → return existing partial sum cache line │
|
||||
│ └─ Miss → allocate zero-initialized cache line, │
|
||||
│ register writeback callbacks │
|
||||
│ cache line marked dirty (dirty=1) │
|
||||
│ eviction invokes preprocess + writeback │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ After computation: │
|
||||
│ 3. CacheLine.Dispose() (refCount--) │
|
||||
│ 4. L4.StepComplete() │
|
||||
│ ├─ Predictor.AdvanceExecutionPoint() │
|
||||
│ └─ Predictor.NeedsPrefetch? │
|
||||
│ └─ Yes → CAS-acquire lock, execute prefetch │
|
||||
└─────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 2.3 Key Parameters
|
||||
|
||||
| Parameter | Value | Description |
|
||||
|------------------------|--------------------------|------------------------------------------------------|
|
||||
| Total cache capacity | 64 MB | Per-NUMA-node contiguous memory |
|
||||
| Cache line size | 8192 Bytes | = 64 × `double MAT_KERNEL` |
|
||||
| Cache line count | 8192 | 64MB / 8192B |
|
||||
| Mapping table type | Swiss Table | 1024 buckets × 16 slots, SIMD-accelerated |
|
||||
| Cache line pool | Slab Allocator | Non-concurrent, fixed capacity, no auto-expansion |
|
||||
| Eviction policy | RRPV mask + windowed OPT | SoA layout + SSE2 batch scan |
|
||||
| Prefetch policy | Predictor windowed OPT | 2048-entry buffer, 1024-step lookahead |
|
||||
| Writeback policy | Callback-driven | Invokes registered callbacks on dirty eviction/flush |
|
||||
|
||||
---
|
||||
|
||||
## 3. Key Components
|
||||
|
||||
### 3.1 Swiss Table (Pointer Mapping Table)
|
||||
|
||||
**Responsibility**: Maintains the `original pointer → cache pointer` mapping.
|
||||
|
||||
- Key: original pointer `nint`, pointing to data allocated by HLSF/malloc on a remote node
|
||||
- Value: cache pointer `nint`, pointing to locally Slab-allocated cache line
|
||||
- Hash: hardware `CRC32C` instruction on the 64-bit pointer
|
||||
- Low 10 bits → bucket index (1024 buckets)
|
||||
- Bits 10-16 → H2 control byte (used for SIMD pre-filtering)
|
||||
- Scan: SSE2 `pmovmskb` instruction for parallel comparison of 16 control bytes
|
||||
- Tombstone mechanism: deleted entries set ctrl to `0xFE`, preserving probe chain integrity
|
||||
- Capacity: 1024 × 16 = 16,384 mappings, significantly larger than total cache line count (8192 lines),
|
||||
ensuring extremely low hash collision rate
|
||||
|
||||
**Measured Performance** (Ryzen 7 6800H, .NET 8, BenchmarkDotNet):
|
||||
|
||||
| Operation | Swiss Table | Dictionary | Speedup |
|
||||
|--------------------------|-------------|-------------|---------|
|
||||
| Zipfian Lookup (8192×) | 20.0 μs | 76.1 μs | 3.8× |
|
||||
| Uniform Lookup (8192×) | 18.6 μs | 50.6 μs | 2.7× |
|
||||
| Insert (4096 entries) | 18.5 μs | 46.5 μs | 2.5× |
|
||||
| SteadyState (4096 ops) | 25.3 μs | 68.3 μs | 2.7× |
|
||||
|
||||
Single lookup latency ≈ 2.6 ns, GC allocation = 0.
|
||||
|
||||
### 3.2 Slab Allocator (Cache Line Pool)
|
||||
|
||||
**Responsibility**: Manages allocation and reclamation of fixed-size (8192B) cache lines.
|
||||
|
||||
- Mode: **Non-concurrent, fixed capacity** (`FixedSlabAllocator<T>` mode),
|
||||
reusing the existing `SlabAllocator.FromBuffer<T>()` interface
|
||||
- No auto-expansion: capacity is fixed at initialization (64MB / 8192B = 8192 lines);
|
||||
allocation failure triggers eviction rather than expansion
|
||||
- Allocation complexity: O(1), pops from free stack
|
||||
- Reclamation complexity: O(1), pushes back to free stack
|
||||
- Alignment: all cache lines are 64B aligned, ensuring no cross-hardware-cache-line boundaries
|
||||
|
||||
### 3.3 Eviction Metadata (SoA Layout + RRPV Mask + Writeback Callbacks)
|
||||
|
||||
**Responsibility**: Tracks per-cache-line reference state, heat, and writeback information to determine eviction priority.
|
||||
|
||||
Replaces the four-level circular linked list from v0.1 with parallel arrays in Structure-of-Arrays layout:
|
||||
|
||||
```
|
||||
SoA Layout — Parallel Arrays
|
||||
|
||||
_refFlags[i] : byte Reference count snapshot (0=free, non-zero=held)
|
||||
_evictionFlags[i] : byte dirty(1bit) + RRPV(2bit) + costClass(1bit)
|
||||
_originKeys[i] : nuint Original pointer (reverse-deregisters Swiss Table entry on eviction)
|
||||
_preprocess[i] : nint Writeback pre-processing function pointer (called before dirty eviction)
|
||||
_writeback[i] : nint Writeback function pointer (called during dirty eviction)
|
||||
_callbackCtx[i] : nuint Callback context (passed to preprocess/writeback)
|
||||
```
|
||||
|
||||
**Eviction Flag Bit Layout** (`_evictionFlags[i]`):
|
||||
|
||||
The RRPV array uses byte storage for each cache line's usage state; the low 2 bits encode the RRPV value.
|
||||
|
||||
**RRPV Semantics**:
|
||||
|
||||
| RRPV | Meaning | Eviction Priority |
|
||||
|------|---------------------------------|---------------------------|
|
||||
| 0 | Will be re-accessed very soon | Lowest (do not evict) |
|
||||
| 1 | Will be accessed shortly | Low |
|
||||
| 2 | Will be accessed in a while | Medium |
|
||||
| 3 | May never be accessed again | Highest (evict first) |
|
||||
|
||||
**RRPV Update Events**:
|
||||
|
||||
| Event | Operation | Description |
|
||||
|--------------------|---------------------------------|------------------------------------------------------|
|
||||
| Load (insert) | RRPV = 0 | Freshly loaded data will be used soon |
|
||||
| Hit | `_evictionFlags[i] = 0` | Bits 1-2 cleared, RRPV reset to 0 |
|
||||
| Eviction scan | Find lines with RRPV=3 to evict | If none found, increment all RRPV +1 (aging) |
|
||||
|
||||
**Advantages over the v0.1 four-level circular linked list**:
|
||||
|
||||
| Comparison | v0.1 Circular List | v0.3 RRPV Mask + Callbacks |
|
||||
|----------------------|-------------------------------------|--------------------------------------|
|
||||
| Data structure | 4 doubly-linked lists + next ptrs | Byte array + function pointer arrays |
|
||||
| Warm-up operation | Unlink + relink (4 pointer writes) | `+= 1` (1-byte write) |
|
||||
| Aging operation | Per-node list migration | SIMD batch +1 (16 entries/cycle) |
|
||||
| Eviction scan | Iterate list head (pointer chasing) | SIMD pattern matching (16/cycle) |
|
||||
| Cache friendliness | Poor (pointer chasing) | Good (contiguous memory scan) |
|
||||
| Reference protection | None | Lines with refCount > 0 not evicted |
|
||||
| Writeback support | None | Callback-function driven |
|
||||
|
||||
### 3.4 Writeback Callback Mechanism
|
||||
|
||||
**Responsibility**: Provides flexible writeback strategies for writable cache lines, using native function pointers to avoid managed overhead.
|
||||
|
||||
**Design Principle**: All writable cache lines are treated as dirty by default. On eviction or explicit flush,
|
||||
L4 itself does not manage the writeback strategy; instead, the bound predictor is responsible for implementing it.
|
||||
|
||||
The predictor is defined via the following interface:
|
||||
|
||||
```
|
||||
public interface IPrefetchPredict
|
||||
{
|
||||
|
||||
int Predict(
|
||||
Span<CacheLinePrediction> node
|
||||
);
|
||||
|
||||
bool Prefetch(
|
||||
nuint origin,
|
||||
ulong policy,
|
||||
nuint cache
|
||||
);
|
||||
|
||||
void WriteBack(
|
||||
nuint origin,
|
||||
ulong policy,
|
||||
nuint cache
|
||||
);
|
||||
|
||||
}
|
||||
```
|
||||
|
||||
**Typical Callback Implementations**:
|
||||
|
||||
```
|
||||
Scenario 1: Segment accumulation (FMA partial sum)
|
||||
preprocess = null (no pre-processing needed; data is already a double array)
|
||||
writeback = AccumulateAdd (element-wise dst[i] += src[i], AVX2-vectorized)
|
||||
|
||||
Scenario 2: Writeback of packed intermediate results
|
||||
preprocess = UnpackKernelRow (reverse-repack from Pack format to row-major order)
|
||||
writeback = AccumulateAdd
|
||||
|
||||
Scenario 3: PLU decomposition intermediate data
|
||||
preprocess = null
|
||||
writeback = DirectCopy (direct overwrite memcpy)
|
||||
```
|
||||
|
||||
### 3.5 CacheLine (Cache Line Handle)
|
||||
|
||||
**Responsibility**: A cache line access handle held by the compute thread; prevents in-use lines from being evicted via reference counting.
|
||||
|
||||
```csharp
|
||||
public ref struct CacheLine : IDisposable
|
||||
{
|
||||
private ref int _activeCount; // L4 global active count
|
||||
private ref int _refCount; // Per-line reference count
|
||||
private nuint _handle; // Local cache data address
|
||||
|
||||
// Construction: Interlocked.Increment(ref _activeCount)
|
||||
// Interlocked.Increment(ref _refCount)
|
||||
// Dispose: Interlocked.Decrement on both counters
|
||||
}
|
||||
```
|
||||
|
||||
- `ref struct` ensures it cannot escape to the heap
|
||||
- Reference counts use `Interlocked` atomic operations for thread safety
|
||||
- Lines with `_refCount > 0` are skipped during eviction scans
|
||||
- StreamTransparent mode returns a CacheLine that does not occupy a cache slot
|
||||
- Both read-only and writable paths return the same CacheLine type,
|
||||
callers decide read/write behavior via `.Address`
|
||||
|
||||
### 3.6 Predictor (Prefetch Predictor)
|
||||
|
||||
**Responsibility**: Provides deterministic prefetch predictions for L4 based on the windowed OPT strategy.
|
||||
|
||||
**Core Principle**: Matrix computation task sequences are deterministic — the scheduler pre-allocates
|
||||
task blocks `(kr, kc, zz)` for each thread, allowing precise computation of the cache line keys
|
||||
required at each future step. This gives L4 a predictive capability that hardware caches cannot
|
||||
possess: **direct visibility into the future**, rather than inferring the future from history as
|
||||
strategies like Hawkeye do.
|
||||
|
||||
```
|
||||
Predictor Sliding Window:
|
||||
|
||||
Task sequence: ... [T_now] [T+1] [T+2] ... [T+1023] ... [T+2047]
|
||||
↑ execution point ↑ buffer tail
|
||||
|
||||
Window capacity = 2048 prediction entries
|
||||
Prefetch trigger threshold = trigger refill when prefetch frontier
|
||||
leads execution point by fewer than 1024 steps
|
||||
Refill batch size = 64 entries (avoids excessive single-refill latency)
|
||||
```
|
||||
|
||||
**数据结构**：
|
||||
|
||||
```csharp
|
||||
internal class Predictor
|
||||
{
|
||||
private const int WindowCapacity = 2048;
|
||||
private const int RefillThreshold = 1024;
|
||||
|
||||
private PrefetchEntry[] _buffer; // Circular prediction buffer
|
||||
private int _head; // Prefetch frontier (prefetched up to here)
|
||||
private int _tail; // Production position (scheduler fills up to here)
|
||||
private int _executed; // Execution point (compute thread has completed up to here)
|
||||
|
||||
private HashSet<nuint> _activeKeys; // All active keys within the window (deduplicated)
|
||||
}
|
||||
|
||||
struct PrefetchEntry
|
||||
{
|
||||
public nuint Key; // Cache line identifier
|
||||
public nuint SourceAddress; // Source data address
|
||||
public bool NeedsRepack; // Whether non-contiguous repacking is needed
|
||||
}
|
||||
```
|
||||
|
||||
**Cooperation with Eviction Policy**:
|
||||
|
||||
```
|
||||
Eviction priority (from highest to lowest priority):
|
||||
|
||||
1. refCount==0, not in window, RRPV==3, dirty==0 ← First (zero-cost eviction)
|
||||
2. refCount==0, not in window, RRPV==3, dirty==1 ← Second (needs writeback but cold)
|
||||
3. refCount==0, not in window, RRPV<3, dirty==0 ← Ordered by RRPV
|
||||
4. refCount==0, not in window, RRPV<3, dirty==1 ← Ordered by RRPV
|
||||
5. refCount==0, in window ← Avoid evicting if possible
|
||||
6. refCount>0 ← Never evict
|
||||
```
|
||||
|
||||
Windowed OPT means most eviction decisions do not depend on RRPV — "not in window" alone is
|
||||
sufficient to determine evictability. RRPV provides fine-grained differentiation among
|
||||
out-of-window lines; the dirty bit preferentially evicts clean lines at equal RRPV to avoid writeback overhead.
|
||||
|
||||
### 3.7 Concurrency Model
|
||||
|
||||
L4 uses a **no-extra-threads** design; all operations are performed by compute threads themselves:
|
||||
|
||||
| Operation | Executor | Concurrency |
|
||||
|------------------------------------|---------------------------------|--------------------------------------|
|
||||
| Lookup (Swiss Table read) | Any compute thread | ✅ Fully parallel, lock-free |
|
||||
| HIT RRPV update | Any compute thread | ✅ Single-byte write, no sync needed |
|
||||
| refCount increment/decrement | Any compute thread | ✅ Interlocked atomic operations |
|
||||
| Prefetch refresh (eviction + load) | CAS-winner thread | Serial (only one thread enters) |
|
||||
| Dirty writeback | CAS-winner thread | Serial (within eviction flow) |
|
||||
| FlushAllDirty | Single thread after computation | Serial (no concurrency) |
|
||||
|
||||
**Prefetch Refresh CAS Lock**:
|
||||
|
||||
```csharp
|
||||
private int _refreshLock; // 0=free, 1=locked
|
||||
|
||||
private void TryRefreshPrefetch()
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _refreshLock, 1, 0) != 0)
|
||||
return; // Another thread is doing it; current thread continues computing
|
||||
try { /* eviction + prefetch */ }
|
||||
finally { Volatile.Write(ref _refreshLock, 0); }
|
||||
}
|
||||
```
|
||||
|
||||
**Swiss Table Concurrent Read Safety Guarantee**: The prefetch refresh thread follows strict write ordering
|
||||
on insertion: write Slab data first, then write eviction metadata and callback pointers, `sfence`, then
|
||||
write the Swiss Table control byte and key/value, ensuring that when a compute thread reads the control
|
||||
byte, the data is already fully visible.
|
||||
|
||||
---
|
||||
|
||||
## 4. Operation Flows
|
||||
|
||||
### 4.1 Cache Lookup (Read-Only Hit)
|
||||
|
||||
1. Compute CRC32C hash of the original pointer
|
||||
2. Swiss Table lookup: SIMD scan ctrl → compare key → return cache pointer
|
||||
3. Update RRPV: `_evictionFlags[slot] &= 0xF9` (RRPV zeroed, dirty bit unchanged)
|
||||
4. Construct CacheLine handle (refCount++)
|
||||
5. Return CacheLine; upper-layer code accesses local data via `.Address`
|
||||
|
||||
### 4.2 Cache Lookup (Writable Hit)
|
||||
|
||||
1. Swiss Table lookup hits
|
||||
2. Update RRPV: `_evictionFlags[slot] &= 0xF9`
|
||||
3. Construct CacheLine handle (refCount++)
|
||||
4. Return CacheLine; upper-layer code reads/writes the cache line via `.Address` (e.g., partial sum +=)
|
||||
|
||||
### 4.3 Cache Miss (Read-Only)
|
||||
|
||||
1. Swiss Table lookup misses
|
||||
2. Compute thread enters `Thread.Sleep(0)` spin loop
|
||||
3. Wait for Predictor-driven prefetch refresh to load the line into L4
|
||||
4. Exit wait loop when Swiss Table shows a hit
|
||||
5. Construct CacheLine handle and return
|
||||
|
||||
Under normal operation, the Predictor always leads the execution point by 1024+ steps,
|
||||
misses occur only during cold start or extreme bursts.
|
||||
|
||||
### 4.4 Cache Miss (Writable)
|
||||
|
||||
1. Swiss Table lookup misses
|
||||
2. Allocate a cache line from Slab
|
||||
3. If Slab has no free slots → trigger eviction (see 4.6)
|
||||
4. Zero-initialize the cache line (`Unsafe.InitBlock(dest, 0, 8192)`)
|
||||
5. Register writeback callbacks: `_preprocess[slot]`, `_writeback[slot]`, `_callbackCtx[slot]`
|
||||
6. Set eviction flags: `dirty=1, RRPV=0`
|
||||
7. Write `_originKeys[slot]`
|
||||
8. `sfence`
|
||||
9. Write Swiss Table mapping
|
||||
10. Construct CacheLine handle and return
|
||||
|
||||
### 4.5 Prefetch Refresh
|
||||
|
||||
Triggered by compute threads in `StepComplete()`; CAS lock ensures single-thread execution:
|
||||
|
||||
1. Synchronize refFlags snapshot (`_refCounts[i] > 0 → _refFlags[i] = 1`)
|
||||
2. Retrieve pending prefetch entries from Predictor (up to 64 at a time)
|
||||
3. For each entry:
|
||||
- Swiss Table hit → mark RRPV=0 (hot-line protection)
|
||||
- Swiss Table miss → execute load:
|
||||
a. Allocate cache line from Slab
|
||||
b. If Slab has no free slots → trigger eviction (see 4.6)
|
||||
c. Execute Pack reorder or memcpy based on `NeedsRepack` flag
|
||||
d. Write eviction metadata (originKey, RRPV=0, dirty=0, callbacks=null)
|
||||
e. `sfence`
|
||||
f. Write Swiss Table mapping
|
||||
|
||||
### 4.6 Cache Eviction (SIMD Scan + Writeback)
|
||||
|
||||
```
|
||||
Eviction scan pseudocode:
|
||||
|
||||
Pass 1: Find refCount==0, RRPV==3, dirty==0 (zero-cost eviction)
|
||||
for i in range(scanCursor, CacheLineCount), step=16:
|
||||
refChunk = SSE2.Load(refFlags + i)
|
||||
refFreeMask = SSE2.MoveMask(SSE2.CompareEqual(refChunk, zero))
|
||||
|
||||
evChunk = SSE2.Load(evictionFlags + i)
|
||||
// RRPV==3 and dirty==0 → byte & 0x07 == 0x06
|
||||
masked = SSE2.And(evChunk, 0x07)
|
||||
coldCleanMask = SSE2.MoveMask(SSE2.CompareEqual(masked, 0x06))
|
||||
|
||||
candidates = refFreeMask & coldCleanMask
|
||||
while candidates != 0:
|
||||
bit = TrailingZeroCount(candidates)
|
||||
idx = i + bit
|
||||
if originKeys[idx] not in Predictor.ActiveKeys:
|
||||
evict(idx) // No writeback, discard directly
|
||||
return
|
||||
candidates &= candidates - 1
|
||||
|
||||
Pass 2: Find refCount==0, RRPV==3, dirty==1 (needs writeback)
|
||||
Same as above, but match byte & 0x07 == 0x07
|
||||
evict triggers writeback callbacks
|
||||
|
||||
Pass 3: Increment all RRPV+1 (aging), retry
|
||||
Up to 3 rounds
|
||||
```
|
||||
|
||||
**逐出执行**：
|
||||
|
||||
```csharp
|
||||
private void Evict(int slot)
|
||||
{
|
||||
byte flags = _evictionFlags[slot];
|
||||
|
||||
// Dirty writeback
|
||||
if ((flags & 0x01) != 0)
|
||||
{
|
||||
nuint origin = _originKeys[slot];
|
||||
nuint cached = SlotAddressFromIndex(slot);
|
||||
nuint ctx = _callbackCtx[slot];
|
||||
|
||||
// 1. Pre-processing (optional)
|
||||
var preprocess = (delegate* unmanaged<nuint, nuint, void>)_preprocess[slot];
|
||||
if (preprocess != null)
|
||||
preprocess(cached, ctx);
|
||||
|
||||
// 2. Writeback
|
||||
var writeback = (delegate* unmanaged<nuint, nuint, nuint, void>)_writeback[slot];
|
||||
if (writeback != null)
|
||||
writeback(origin, cached, ctx);
|
||||
}
|
||||
|
||||
// Standard eviction
|
||||
_map.TryRemove(_originKeys[slot], out _);
|
||||
_slab.Free(SlotAddressFromIndex(slot));
|
||||
_evictionFlags[slot] = 0;
|
||||
_originKeys[slot] = 0;
|
||||
_preprocess[slot] = 0;
|
||||
_writeback[slot] = 0;
|
||||
_callbackCtx[slot] = 0;
|
||||
}
|
||||
```
|
||||
|
||||
### 4.7 显式刷新
|
||||
|
||||
```csharp
|
||||
/// <summary>
|
||||
/// Explicitly writes back the specified dirty cache line. Called after all segments complete.
|
||||
/// </summary>
|
||||
public void FlushDirty(nuint originKey)
|
||||
{
|
||||
if (_map.TryGet(originKey, out nuint addr))
|
||||
{
|
||||
int slot = SlotIndex(addr);
|
||||
if ((_evictionFlags[slot] & 0x01) != 0)
|
||||
{
|
||||
// Execute pre-processing + writeback
|
||||
InvokeWriteBack(slot);
|
||||
_evictionFlags[slot] &= 0xFE; // Clear dirty bit
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes back all dirty cache lines in bulk. Called after entire matrix computation completes.
|
||||
/// </summary>
|
||||
public void FlushAllDirty()
|
||||
{
|
||||
for (int i = 0; i < CacheLineCount; i++)
|
||||
{
|
||||
if ((_evictionFlags[i] & 0x01) != 0 && _originKeys[i] != 0)
|
||||
{
|
||||
InvokeWriteBack(i);
|
||||
_evictionFlags[i] &= 0xFE;
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 4.8 Working Mode Switch
|
||||
|
||||
```csharp
|
||||
public enum L4Mode
|
||||
{
|
||||
/// <summary>
|
||||
/// Pass-through mode: Map() returns the original pointer directly; L4 does not intervene.
|
||||
/// Suitable for single-NUMA + no-repack scenarios (e.g., vector addition, scalar multiplication).
|
||||
/// </summary>
|
||||
StreamTransparent,
|
||||
|
||||
/// <summary>
|
||||
/// Buffer mode: full lookup/Pack/cache/eviction pipeline.
|
||||
/// Suitable for FMA scenarios requiring repacking or multi-NUMA scenarios.
|
||||
/// </summary>
|
||||
CacheBuffer
|
||||
}
|
||||
```
|
||||
|
||||
Mode is determined at construction time and does not change at runtime. Decision logic:
|
||||
|
||||
- Requires Pack reordering (FMA core rows of matrix A) → CacheBuffer
|
||||
- Multiple NUMA nodes → CacheBuffer
|
||||
- Requires segment accumulation → CacheBuffer
|
||||
- Single NUMA + no reordering + no segment → StreamTransparent
|
||||
|
||||
---
|
||||
|
||||
## 5. Segment Accumulation Scenario
|
||||
|
||||
### 5.1 Problem
|
||||
|
||||
For matrix multiplication `C = A × B` when the K dimension is large, segmented accumulation is required:
|
||||
|
||||
```
|
||||
segment 0: C_partial = A[kr, 0..KC] × B[0..KC, kc]
|
||||
segment 1: C_partial += A[kr, KC..2KC] × B[KC..2KC, kc]
|
||||
...
|
||||
segment N: C_partial += A[kr, N*KC..K] × B[N*KC..K, kc]
|
||||
|
||||
Final: C[kr, kc] = Σ all segment partial sums
|
||||
```
|
||||
|
||||
If each segment's result is `+=` directly to the C matrix,
|
||||
and C resides on a remote NUMA node, every segment incurs cross-node write latency.
|
||||
|
||||
### 5.2 L4 Solution
|
||||
|
||||
Temporarily store C's partial sums in L4's writable cache lines:
|
||||
|
||||
```
|
||||
segment 0: c = l4.MapWritable(C_key, null, &AccumulateAdd)
|
||||
// First miss → allocate zero-initialized cache line
|
||||
FMA(c.Address, a.Address, b.Address) // c += a*b
|
||||
c.Dispose()
|
||||
|
||||
segment 1: c = l4.MapWritable(C_key, null, &AccumulateAdd)
|
||||
// Hit! Returns existing partial sum cache line
|
||||
FMA(c.Address, a.Address, b.Address) // c += a*b
|
||||
c.Dispose()
|
||||
|
||||
... after all segments complete ...
|
||||
|
||||
l4.FlushDirty(C_key)
|
||||
// Calls AccumulateAdd(C_origin, cacheline, ctx)
|
||||
// → Adds local partial sum back to remote C matrix in one shot
|
||||
```
|
||||
|
||||
### 5.3 Benefits
|
||||
|
||||
```
|
||||
Without segment buffering (OpenBLAS approach):
|
||||
Each segment writes C matrix across NUMA
|
||||
Remote writes per output tile = Zk times
|
||||
Per-write latency = ~200-300ns (cross-node)
|
||||
|
||||
With L4 segment buffering:
|
||||
All segment += operations complete locally in L4 cache lines
|
||||
Partial sums likely remain in L1/L2 (8KB << L1 capacity)
|
||||
Remote writes per output tile = 1 (final FlushDirty)
|
||||
Writeback uses AVX2-vectorized stores: 8192B = 32 × 256-bit stores
|
||||
|
||||
Reduction ratio = Zk : 1
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Uniqueness and Advantages
|
||||
|
||||
### 6.1 Deterministic Prefetch (Windowed OPT)
|
||||
|
||||
Essential difference from hardware cache replacement policies (LRU, RRIP, Hawkeye):
|
||||
|
||||
| Policy | Information Source | Prediction Method | Accuracy |
|
||||
|---------------------|-------------------------|-------------------------------|---------------------------|
|
||||
| LRU | Past access order | Least recently used | Moderate |
|
||||
| RRIP | Insert/hit markers | Predict reuse distance | Moderate-High |
|
||||
| Hawkeye | OPTgen retrospection | Learn past to infer future | High |
|
||||
| **L4 Predictor** | **Future task queue** | **Direct future visibility** | **Perfect** (within window) |
|
||||
|
||||
L4's Predictor is a **deterministic Oracle** — it does not predict; it knows.
|
||||
Because matrix computation task sequences are fully determined at scheduler allocation time.
|
||||
|
||||
### 6.2 NUMA Adaptability
|
||||
|
||||
The same code automatically gains NUMA benefits on different hardware configurations:
|
||||
|
||||
| Hardware Configuration | L4 Behavior |
|
||||
|-------------------------|-------------------------------------------------------------|
|
||||
| Single-socket CPU | Pack cache + segment accumulation buffer |
|
||||
| Dual-socket CPU | Pack cache + NUMA migration cache + segment accum. |
|
||||
| Hybrid-architecture CPU | Pack cache + cross-domain migration cache + segment accum. |
|
||||
|
||||
### 6.3 Callback-Driven Writeback Strategy
|
||||
|
||||
Using native function pointers for writeback avoids:
|
||||
|
||||
- Virtual method call overhead
|
||||
- Interface dispatch overhead
|
||||
- Delegate GC pressure
|
||||
- Hard-coding specific writeback logic
|
||||
|
||||
Callers register `preprocess` and `writeback` callbacks at `MapWritable` time,
|
||||
L4 invokes them directly through function pointers on eviction or Flush,
|
||||
**zero managed overhead, zero virtual dispatch, performance equivalent to C function calls**.
|
||||
|
||||
### 6.4 Reference Count Protection
|
||||
|
||||
Through `CacheLine` handle reference counting, cache lines actively in use by compute threads are
|
||||
absolutely never evicted (including writeback not being triggered). The SIMD eviction scan skips
|
||||
lines with `refCount > 0` immediately via the `_refFlags` array, without checking linked lists or acquiring locks.
|
||||
|
||||
### 6.5 Cache Line Size Aligned to Matrix Kernel
|
||||
|
||||
The 8192B cache line size is not arbitrary:
|
||||
|
||||
- 1 `double MAT_KERNEL` (4×4 double matrix) = 128 Bytes
|
||||
- 1 cache line = 64 `MAT_KERNEL` units
|
||||
- Matrix multiplication micro-kernels operate on data sizes perfectly aligned to cache lines
|
||||
- Compute kernel functions operate on cache lines as units; no cross-cache-line accesses occur
|
||||
|
||||
### 6.6 Fixed-Capacity Determinism
|
||||
|
||||
- Slab allocator never expands → memory footprint upper bound is known (64 MB)
|
||||
- SIMD eviction scan → latency upper bound is known (at most 3 aging rounds × 512 SIMD comparisons)
|
||||
- Swiss Table fixed 16,384 slots → lookup latency upper bound is known
|
||||
- Writeback callback latency is controlled by the caller → predictable
|
||||
- Worst-case L4 latency = lookup + eviction (incl. writeback) + allocation + memcpy, all statically computable
|
||||
|
||||
### 6.7 Reuse of Existing StgSharp Infrastructure
|
||||
|
||||
| L4 Component | Reused StgSharp Infrastructure |
|
||||
|-----------------------|-------------------------------------------------------|
|
||||
| Mapping table | `SwissTable` (implemented and benchmark-validated) |
|
||||
| Mapping table SIMD | `IBucketScanner` + `SIMDID` architecture detection |
|
||||
| Cache line alloc | `SlabAllocator.FromBuffer<T>()` |
|
||||
| Hash computation | Hardware `CRC32C` instruction |
|
||||
| Memory allocation | `NativeMemory.AlignedAlloc` |
|
||||
| Future multi-platform | `IBucketScanner` + NRGA compilation pipeline |
|
||||
|
||||
### 6.8 Originality Analysis
|
||||
|
||||
After surveying existing mainstream numerical computation libraries, L4's design has
|
||||
**no direct precedent or equivalent in known public implementations**.
|
||||
|
||||
#### NUMA Strategy Comparison with Existing Libraries
|
||||
|
||||
| Library / Framework | NUMA Strategy | Software Cache Layer | Mapping Table | Eviction Policy | Dirty Writeback |
|
||||
|---------------------|-------------------------------------|---------------------|-----------------|-------------------------|--------------------|
|
||||
| Intel MKL | Thread pinning + `numactl` | None | None | None | None |
|
||||
| OpenBLAS | Thread affinity + block alignment | None | None | None | None |
|
||||
| BLIS | Multi-level block alignment L1/L2/L3| None | None | None | None |
|
||||
| Eigen | No explicit NUMA handling | None | None | None | None |
|
||||
| cuBLAS / rocBLAS | GPU memory (hardware-solved) | None | None | None | None |
|
||||
| **StgSharp L4** | **Software-managed thread-local cache** | **Yes** | **Swiss Table** | **RRPV + windowed OPT** | **Callback-driven** |
|
||||
|
||||
---
|
||||
|
||||
## 7. 从 v0.1 到 v0.3 的变更摘要
|
||||
|
||||
| 项目 | v0.1 | v0.3 |
|
||||
|-----------------|-------------------------------|---------------------------------------|
|
||||
| 逐出数据结构 | 4 级环形双向链表 | RRPV 2-bit 掩码 + 8KB SoA 数组 |
|
||||
| 逐出扫描方式 | 链表头指针遍历 | SSE2 批量模式匹配（16 行/周期） |
|
||||
| 热���更新 | 链表节点摘插（4 指针写） | 单字节位运算 `&= 0x03` |
|
||||
| 老化操作 | 逐节点移链表 | SIMD 批量 +1 |
|
||||
| 引用保护 | 无 | refCount > 0 时 SIMD 扫描跳过 |
|
||||
| 预取策略 | 无 | Predictor 窗口 OPT（2048 条缓冲） |
|
||||
| 线程模型 | L4 服务线程 + Command Queue | 无额外线程，CAS 抢锁 |
|
||||
| 命令队列 | 链式 CommandCache 分段 | 不需要（预取刷新替代命令驱动） |
|
||||
| 缓存行元数据 | Slab 分配的 CacheLineHead 结构 | SoA 平行数组 + 函数指针数组 |
|
||||
| 工作模式 | 单一模式 | StreamTransparent / CacheBuffer 切换 |
|
||||
| 写回策略 | 无（只读缓存） | dirty 位 + preprocess/writeback 回调 |
|
||||
| 写回机制 | 无 | 本机函数指针，零托管开销 |
|
||||
|
||||
---
|
||||
|
||||
## 8. 未来工作
|
||||
|
||||
- [ ] L4 根类型重构（组合 Swiss Table、Slab、SoA 元数据、Predictor）
|
||||
- [ ] Predictor 与矩阵乘法调度器的集成
|
||||
- [ ] Segment 累加写回的 AVX2 向量化实现
|
||||
- [ ] SIMD 逐出扫描的 AVX2 升级（32 行/周期）
|
||||
- [ ] Swiss Table ARM NEON 实现
|
||||
- [ ] Swiss Table AVX-512 升级（64 槽扫描从 16 次降至 4 次）
|
||||
- [ ] 墓碑累积监控与自动 rehash 策略
|
||||
- [ ] 性能基准测试（NUMA 本地 vs 远端访问延迟对比）
|
||||
- [ ] Segment 累加 vs 直接写回的 benchmark 对比
|
||||
- [ ] 与逐线程矩阵上下文的集成验证
|
||||
- [ ] 双路 6242R 环境下的 NUMA 搬运性能验证
|
||||
- [ ] 远期：PPC64LE 平台的 VSX 原生 Swiss Table 扫描支持
|
||||
|
||||
---
|
||||
|
||||
## 9. 参考
|
||||
|
||||
- Google Abseil Swiss Tables Design Notes
|
||||
- HP Labs RRIP (Re-Reference Interval Prediction) 论文
|
||||
- Hawkeye: Efficient Cache Replacement via OPTgen (ISCA 2017)
|
||||
- StgSharp HLSF Allocator 设计文档
|
||||
- StgSharp SIMD Architecture (`SIMDID` 体系)
|
||||
- StgSharp NRGA Roadmap
|
||||
- StgSharp `SlabAllocator<T>` 实现
|
||||
- StgSharp `SwissTable` 实现及 Benchmark
|
||||
@@ -0,0 +1,11 @@
|
||||
# nFramework Icon
|
||||
|
||||
`nFramework-icon.svg` is the icon-only version of the nFramework mark.
|
||||
|
||||
Copyright (c) 2026 Nitload.
|
||||
|
||||
The artwork in this directory is licensed under the [Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International License (CC BY-NC-ND 4.0)](https://creativecommons.org/licenses/by-nc-nd/4.0/).
|
||||
|
||||
You may copy and redistribute the unmodified artwork for noncommercial purposes when you give appropriate credit and link to the license. This license does not permit distributing adaptations, and it does not grant permission for commercial use. Contact the copyright holder for other permissions.
|
||||
|
||||
The repository's root MIT License applies to software, not the artwork in this directory. This copyright license does not grant trademark rights in the Nitload or nFramework names or marks, including the former StgSharp name, or imply endorsement.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 28 KiB |
@@ -0,0 +1,6 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="616" height="104" viewBox="0 0 616 104" role="img" aria-labelledby="title desc">
|
||||
<title id="title">nFramework</title>
|
||||
<desc id="desc">Horizontal nFramework logo with the upright punctuation icon and an IBM Plex Mono Medium wordmark.</desc>
|
||||
<g fill="#59B2DA" transform="translate(13.6 13.6) scale(0.8)"><polygon points="8,8 31,8 31,20 20,20 20,76 31,76 31,88 8,88" /><polygon points="97,8 120,8 120,88 97,88 97,76 108,76 108,20 97,20" /><polygon points="40,16 53,16 53,45 40,45" /><circle cx="80" cy="76" r="8" /></g>
|
||||
<g fill="#E9EEF5"><path d="M146.384 78.524V39.308H154.668V45.996H155.048Q155.656 44.476 156.606 43.07Q157.556 41.664 158.924 40.638Q160.292 39.612 162.116 39.004Q163.94 38.396 166.372 38.396Q172.3 38.396 175.948 42.272Q179.596 46.148 179.596 53.292V78.524H171.312V54.508Q171.312 45.16 163.18 45.16Q161.584 45.16 160.064 45.578Q158.544 45.996 157.328 46.832Q156.112 47.668 155.39 48.96Q154.668 50.252 154.668 51.924V78.524Z"/><path d="M192.136 78.524V25.476H226.26V32.62H200.648V48.048H223.904V55.192H200.648V78.524Z"/><path d="M236.976 71.836H247.388V45.996H236.976V39.308H255.672V49.72H256.128Q256.66 47.668 257.648 45.768Q258.636 43.868 260.156 42.462Q261.676 41.056 263.88 40.182Q266.084 39.308 269.048 39.308H273.76V47.212H266.008Q261.448 47.212 258.56 49.91Q255.672 52.608 255.672 57.168V71.836H269.884V78.524H236.976Z"/><path d="M314.192 78.524Q310.62 78.524 308.834 76.7Q307.048 74.876 306.668 71.912H306.288Q305.148 75.484 302.146 77.46Q299.144 79.436 294.508 79.436Q288.504 79.436 284.932 76.282Q281.36 73.128 281.36 67.58Q281.36 61.88 285.578 58.878Q289.796 55.876 298.536 55.876H306.06V52.836Q306.06 44.856 297.624 44.856Q293.824 44.856 291.468 46.338Q289.112 47.82 287.516 50.252L282.576 46.224Q284.248 43.032 288.2 40.714Q292.152 38.396 298.384 38.396Q305.832 38.396 310.088 41.968Q314.344 45.54 314.344 52.304V71.988H319.284V78.524ZM296.864 73.356Q300.892 73.356 303.476 71.532Q306.06 69.708 306.06 66.82V61.12H298.688Q289.72 61.12 289.72 66.516V68.036Q289.72 70.696 291.62 72.026Q293.52 73.356 296.864 73.356Z"/><path d="M325.744 78.524V39.308H332.812V44.4H333.192Q334.104 41.892 335.814 40.144Q337.524 38.396 340.716 38.396Q343.832 38.396 345.542 40.106Q347.252 41.816 347.708 44.78H348.012Q349.076 42.12 350.976 40.258Q352.876 38.396 356.448 38.396Q361.008 38.396 362.794 41.588Q364.58 44.78 364.58 50.936V78.524H357.512V52Q357.512 47.896 356.524 46.338Q355.536 44.78 353.332 44.78Q351.356 44.78 350.026 46.11Q348.696 47.44 348.696 50.1V78.524H341.628V52Q341.628 47.896 340.678 46.338Q339.728 44.78 337.524 44.78Q335.472 44.78 334.142 46.11Q332.812 47.44 332.812 50.1V78.524Z"/><path d="M391.636 79.436Q387.152 79.436 383.58 77.992Q380.008 76.548 377.576 73.888Q375.144 71.228 373.852 67.466Q372.56 63.704 372.56 58.992Q372.56 54.28 373.89 50.48Q375.22 46.68 377.614 43.982Q380.008 41.284 383.428 39.84Q386.848 38.396 391.028 38.396Q395.132 38.396 398.476 39.802Q401.82 41.208 404.176 43.792Q406.532 46.376 407.786 49.948Q409.04 53.52 409.04 57.852V60.968H380.92V62.26Q380.92 66.896 383.808 69.822Q386.696 72.748 391.788 72.748Q395.588 72.748 398.286 71.114Q400.984 69.48 402.656 66.744L407.824 71.304Q405.848 74.724 401.782 77.08Q397.716 79.436 391.636 79.436ZM391.028 44.704Q388.824 44.704 386.962 45.464Q385.1 46.224 383.77 47.63Q382.44 49.036 381.68 50.936Q380.92 52.836 380.92 55.116V55.648H400.604V54.888Q400.604 50.252 397.982 47.478Q395.36 44.704 391.028 44.704Z"/><path d="M415.12 39.308H422.188L425.988 72.292H426.672L432.524 39.308H440.58L446.356 72.292H447.04L450.916 39.308H457.68L452.208 78.524H442.024L436.78 48.124H436.096L430.776 78.524H420.592Z"/><path d="M482 79.436Q477.668 79.436 474.21 77.992Q470.752 76.548 468.358 73.888Q465.964 71.228 464.672 67.428Q463.38 63.628 463.38 58.916Q463.38 54.204 464.672 50.404Q465.964 46.604 468.358 43.944Q470.752 41.284 474.21 39.84Q477.668 38.396 482 38.396Q486.332 38.396 489.79 39.84Q493.248 41.284 495.642 43.944Q498.036 46.604 499.328 50.404Q500.62 54.204 500.62 58.916Q500.62 63.628 499.328 67.428Q498.036 71.228 495.642 73.888Q493.248 76.548 489.79 77.992Q486.332 79.436 482 79.436ZM482 72.9Q486.56 72.9 489.258 70.126Q491.956 67.352 491.956 61.88V55.952Q491.956 50.48 489.258 47.706Q486.56 44.932 482 44.932Q477.44 44.932 474.742 47.706Q472.044 50.48 472.044 55.952V61.88Q472.044 67.352 474.742 70.126Q477.44 72.9 482 72.9Z"/><path d="M510.576 71.836H520.988V45.996H510.576V39.308H529.272V49.72H529.728Q530.26 47.668 531.248 45.768Q532.236 43.868 533.756 42.462Q535.276 41.056 537.48 40.182Q539.684 39.308 542.648 39.308H547.36V47.212H539.608Q535.048 47.212 532.16 49.91Q529.272 52.608 529.272 57.168V71.836H543.484V78.524H510.576Z"/><path d="M557.088 22.284H565.372V56.256H565.752L571.908 49.948L582.928 39.308H592.732L576.848 54.736L594.48 78.524H584.524L570.844 59.448L565.372 64.54V78.524H557.088Z"/></g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 4.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 28 KiB |
@@ -0,0 +1,6 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="616" height="104" viewBox="0 0 616 104" role="img" aria-labelledby="title desc">
|
||||
<title id="title">nFramework</title>
|
||||
<desc id="desc">Horizontal nFramework logo with the upright punctuation icon and an IBM Plex Mono Medium wordmark.</desc>
|
||||
<g fill="#59B2DA" transform="translate(13.6 13.6) scale(0.8)"><polygon points="8,8 31,8 31,20 20,20 20,76 31,76 31,88 8,88" /><polygon points="97,8 120,8 120,88 97,88 97,76 108,76 108,20 97,20" /><polygon points="40,16 53,16 53,45 40,45" /><circle cx="80" cy="76" r="8" /></g>
|
||||
<g fill="#14202D"><path d="M146.384 78.524V39.308H154.668V45.996H155.048Q155.656 44.476 156.606 43.07Q157.556 41.664 158.924 40.638Q160.292 39.612 162.116 39.004Q163.94 38.396 166.372 38.396Q172.3 38.396 175.948 42.272Q179.596 46.148 179.596 53.292V78.524H171.312V54.508Q171.312 45.16 163.18 45.16Q161.584 45.16 160.064 45.578Q158.544 45.996 157.328 46.832Q156.112 47.668 155.39 48.96Q154.668 50.252 154.668 51.924V78.524Z"/><path d="M192.136 78.524V25.476H226.26V32.62H200.648V48.048H223.904V55.192H200.648V78.524Z"/><path d="M236.976 71.836H247.388V45.996H236.976V39.308H255.672V49.72H256.128Q256.66 47.668 257.648 45.768Q258.636 43.868 260.156 42.462Q261.676 41.056 263.88 40.182Q266.084 39.308 269.048 39.308H273.76V47.212H266.008Q261.448 47.212 258.56 49.91Q255.672 52.608 255.672 57.168V71.836H269.884V78.524H236.976Z"/><path d="M314.192 78.524Q310.62 78.524 308.834 76.7Q307.048 74.876 306.668 71.912H306.288Q305.148 75.484 302.146 77.46Q299.144 79.436 294.508 79.436Q288.504 79.436 284.932 76.282Q281.36 73.128 281.36 67.58Q281.36 61.88 285.578 58.878Q289.796 55.876 298.536 55.876H306.06V52.836Q306.06 44.856 297.624 44.856Q293.824 44.856 291.468 46.338Q289.112 47.82 287.516 50.252L282.576 46.224Q284.248 43.032 288.2 40.714Q292.152 38.396 298.384 38.396Q305.832 38.396 310.088 41.968Q314.344 45.54 314.344 52.304V71.988H319.284V78.524ZM296.864 73.356Q300.892 73.356 303.476 71.532Q306.06 69.708 306.06 66.82V61.12H298.688Q289.72 61.12 289.72 66.516V68.036Q289.72 70.696 291.62 72.026Q293.52 73.356 296.864 73.356Z"/><path d="M325.744 78.524V39.308H332.812V44.4H333.192Q334.104 41.892 335.814 40.144Q337.524 38.396 340.716 38.396Q343.832 38.396 345.542 40.106Q347.252 41.816 347.708 44.78H348.012Q349.076 42.12 350.976 40.258Q352.876 38.396 356.448 38.396Q361.008 38.396 362.794 41.588Q364.58 44.78 364.58 50.936V78.524H357.512V52Q357.512 47.896 356.524 46.338Q355.536 44.78 353.332 44.78Q351.356 44.78 350.026 46.11Q348.696 47.44 348.696 50.1V78.524H341.628V52Q341.628 47.896 340.678 46.338Q339.728 44.78 337.524 44.78Q335.472 44.78 334.142 46.11Q332.812 47.44 332.812 50.1V78.524Z"/><path d="M391.636 79.436Q387.152 79.436 383.58 77.992Q380.008 76.548 377.576 73.888Q375.144 71.228 373.852 67.466Q372.56 63.704 372.56 58.992Q372.56 54.28 373.89 50.48Q375.22 46.68 377.614 43.982Q380.008 41.284 383.428 39.84Q386.848 38.396 391.028 38.396Q395.132 38.396 398.476 39.802Q401.82 41.208 404.176 43.792Q406.532 46.376 407.786 49.948Q409.04 53.52 409.04 57.852V60.968H380.92V62.26Q380.92 66.896 383.808 69.822Q386.696 72.748 391.788 72.748Q395.588 72.748 398.286 71.114Q400.984 69.48 402.656 66.744L407.824 71.304Q405.848 74.724 401.782 77.08Q397.716 79.436 391.636 79.436ZM391.028 44.704Q388.824 44.704 386.962 45.464Q385.1 46.224 383.77 47.63Q382.44 49.036 381.68 50.936Q380.92 52.836 380.92 55.116V55.648H400.604V54.888Q400.604 50.252 397.982 47.478Q395.36 44.704 391.028 44.704Z"/><path d="M415.12 39.308H422.188L425.988 72.292H426.672L432.524 39.308H440.58L446.356 72.292H447.04L450.916 39.308H457.68L452.208 78.524H442.024L436.78 48.124H436.096L430.776 78.524H420.592Z"/><path d="M482 79.436Q477.668 79.436 474.21 77.992Q470.752 76.548 468.358 73.888Q465.964 71.228 464.672 67.428Q463.38 63.628 463.38 58.916Q463.38 54.204 464.672 50.404Q465.964 46.604 468.358 43.944Q470.752 41.284 474.21 39.84Q477.668 38.396 482 38.396Q486.332 38.396 489.79 39.84Q493.248 41.284 495.642 43.944Q498.036 46.604 499.328 50.404Q500.62 54.204 500.62 58.916Q500.62 63.628 499.328 67.428Q498.036 71.228 495.642 73.888Q493.248 76.548 489.79 77.992Q486.332 79.436 482 79.436ZM482 72.9Q486.56 72.9 489.258 70.126Q491.956 67.352 491.956 61.88V55.952Q491.956 50.48 489.258 47.706Q486.56 44.932 482 44.932Q477.44 44.932 474.742 47.706Q472.044 50.48 472.044 55.952V61.88Q472.044 67.352 474.742 70.126Q477.44 72.9 482 72.9Z"/><path d="M510.576 71.836H520.988V45.996H510.576V39.308H529.272V49.72H529.728Q530.26 47.668 531.248 45.768Q532.236 43.868 533.756 42.462Q535.276 41.056 537.48 40.182Q539.684 39.308 542.648 39.308H547.36V47.212H539.608Q535.048 47.212 532.16 49.91Q529.272 52.608 529.272 57.168V71.836H543.484V78.524H510.576Z"/><path d="M557.088 22.284H565.372V56.256H565.752L571.908 49.948L582.928 39.308H592.732L576.848 54.736L594.48 78.524H584.524L570.844 59.448L565.372 64.54V78.524H557.088Z"/></g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 4.8 KiB |
@@ -0,0 +1,11 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="128" height="104" viewBox="0 0 128 104" role="img" aria-labelledby="title desc">
|
||||
<title id="title">nFramework icon</title>
|
||||
<desc id="desc">The geometric nFramework mark, with a diagonal slash suggesting an N.</desc>
|
||||
<metadata>Copyright (c) 2026 Nitload. Licensed under CC BY-NC-ND 4.0: https://creativecommons.org/licenses/by-nc-nd/4.0/</metadata>
|
||||
<g fill="#59B2DA" transform="translate(13.6 13.6) scale(0.8)">
|
||||
<polygon points="8,8 31,8 31,20 20,20 20,76 31,76 31,88 8,88" />
|
||||
<polygon points="97,8 120,8 120,88 97,88 97,76 108,76 108,20 97,20" />
|
||||
<polygon points="40,16 53,16 53,45 40,45" />
|
||||
<circle cx="80" cy="76" r="8" />
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 709 B |
@@ -9,16 +9,23 @@ using BenchmarkDotNet.Configs;
|
||||
using BenchmarkDotNet.Jobs;
|
||||
using BenchmarkDotNet.Loggers;
|
||||
using BenchmarkDotNet.Running;
|
||||
using BenchmarkDotNet.Toolchains.InProcess.Emit;
|
||||
using Nitload.Benchmark;
|
||||
|
||||
if (args.Length > 0 && args[0] == "--disasm-swiss")
|
||||
{
|
||||
IConfig disassemblyConfig = ManualConfig.Create(DefaultConfig.Instance)
|
||||
.AddLogger(ConsoleLogger.Default)
|
||||
.AddJob(Job.ShortRun.WithWarmupCount(6));
|
||||
BenchmarkRunner.Run<SwissX86DisassemblyBench>(disassemblyConfig);
|
||||
return;
|
||||
}
|
||||
|
||||
IConfig config = ManualConfig.Create(DefaultConfig.Instance)
|
||||
.AddLogger(ConsoleLogger.Default)
|
||||
.AddJob(Job.Default
|
||||
.WithToolchain(InProcessEmitToolchain.Instance)
|
||||
.WithLaunchCount(1)
|
||||
.WithWarmupCount(6)
|
||||
.WithIterationCount(15)
|
||||
.WithInvocationCount(3)
|
||||
.WithUnrollFactor(1));
|
||||
_ = BenchmarkRunner.Run<TlsfAllocatorBench>(config);
|
||||
BenchmarkSwitcher.FromAssembly(typeof(TlsfAllocatorBench).Assembly).Run(args, config);
|
||||
|
||||
@@ -1,485 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SwissBenchmark"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using BenchmarkDotNet.Attributes;
|
||||
using Nitload.Common.Collections;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace Nitload.Benchmark
|
||||
{
|
||||
/// <summary>
|
||||
/// Benchmark comparing SwissTable against <see cref="Dictionary{TKey, TValue}" /> under
|
||||
/// realistic L4 software cache address-mapping workloads. <para>/// L4 scenario
|
||||
/// characteristics: /// - Keys are HLSF/malloc pointer addresses (large, 1024-byte aligned)
|
||||
/// /// - Access pattern is Zipfian-skewed (20% hot cache lines receive 80% of lookups) ///
|
||||
/// - Steady state: high hit-rate lookups dominate, with rare evict+refill on miss /// -
|
||||
/// Working set: 512–4096 active cache line mappings /// </para>
|
||||
/// </summary>
|
||||
[MemoryDiagnoser]
|
||||
public class SwissTableBench
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// L4 cache line size in bytes (per L4_design.md).
|
||||
/// </summary>
|
||||
private const int CacheLineSize = 1024;
|
||||
|
||||
/// <summary>
|
||||
/// Simulated heap base address for pointer-like keys.
|
||||
/// </summary>
|
||||
private const long HeapBase = 0x7FFF_0000_0000L;
|
||||
|
||||
/// <summary>
|
||||
/// Number of lookup operations per Lookup benchmark call.
|
||||
/// </summary>
|
||||
private const int LookupIterations = 8192;
|
||||
|
||||
/// <summary>
|
||||
/// Fraction of steady-state ops that are cache misses.
|
||||
/// </summary>
|
||||
private const double MissRate = 0.05;
|
||||
|
||||
/// <summary>
|
||||
/// Number of steady-state operations per call. Each op is either a lookup (hit) or an
|
||||
/// evict+insert+lookup (miss).
|
||||
/// </summary>
|
||||
private const int SteadyStateOps = 4096;
|
||||
|
||||
/// <summary>
|
||||
/// Indices into _residentKeys for eviction targets (cold entries, tail region).
|
||||
/// </summary>
|
||||
private int[] _evictOrder = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Additional keys not in the table, for triggering misses.
|
||||
/// </summary>
|
||||
private nuint[] _externalKeys = null!;
|
||||
|
||||
// ── Pre-generated key arrays (pointer-like, 1024-byte aligned) ──
|
||||
|
||||
/// <summary>
|
||||
/// Keys for initial table fill (simulates cached addresses).
|
||||
/// </summary>
|
||||
private nuint[] _residentKeys = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Pre-built steady-state operation script. Each entry: positive index → lookup
|
||||
/// _residentKeys[idx], negative value → cache miss trigger (evict + insert from
|
||||
/// _externalKeys).
|
||||
/// </summary>
|
||||
private int[] _steadyStateScript = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Uniform random lookup sequence (baseline comparison).
|
||||
/// </summary>
|
||||
private nuint[] _uniformLookups = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Zipfian-skewed lookup sequence (hot 20% accessed 80% of time).
|
||||
/// </summary>
|
||||
private nuint[] _zipfianLookups = null!;
|
||||
private Dictionary<nuint, nuint> _dictLookup = null!;
|
||||
private Dictionary<nuint, nuint> _freshDict = null!;
|
||||
|
||||
// ── Fresh instances for insert/mixed (rebuilt each iteration) ──
|
||||
|
||||
private SwissTable _freshSwiss = null!;
|
||||
|
||||
// ── Pre-filled lookup targets ──
|
||||
|
||||
private SwissTable _swissLookup = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Active cache line count. Matches L4 capacity range: 5 MB / 1024B ≈ 5120, 10 MB /
|
||||
/// 1024B ≈ 10240. 512 and 1024 test low-occupancy; 4096 tests near-capacity.
|
||||
/// </summary>
|
||||
[Params(512, 1024, 4096)]
|
||||
public int KeyCount;
|
||||
|
||||
[GlobalCleanup]
|
||||
public void Cleanup()
|
||||
{
|
||||
_dictLookup.Clear();
|
||||
_dictLookup.TrimExcess();
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════
|
||||
// Insert — bulk fill with scattered pointer addresses
|
||||
// Simulates L4 initial cache warm-up phase.
|
||||
// ════════════════════════════════════════════════════════════
|
||||
|
||||
[Benchmark]
|
||||
public void Dictionary_Insert()
|
||||
{
|
||||
Dictionary<nuint, nuint> dict = _freshDict;
|
||||
nuint[] keys = _residentKeys;
|
||||
for (int i = 0; i < keys.Length; i++) {
|
||||
dict[keys[i]] = keys[i] ^ unchecked((nuint)0xDEAD_BEEF);
|
||||
}
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════
|
||||
// Lookup — Uniform (baseline comparison)
|
||||
// ════════════════════════════════════════════════════════════
|
||||
|
||||
[Benchmark]
|
||||
public nuint Dictionary_Lookup_Uniform()
|
||||
{
|
||||
nuint sum = 0;
|
||||
nuint[] keys = _uniformLookups;
|
||||
Dictionary<nuint, nuint> dict = _dictLookup;
|
||||
for (int i = 0; i < keys.Length; i++)
|
||||
{
|
||||
if (dict.TryGetValue(keys[i], out nuint value)) {
|
||||
sum += value;
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════
|
||||
// Lookup — Zipfian (L4 hot path, skewed access)
|
||||
// 20% of cache lines receive 80% of accesses.
|
||||
// This is the most representative L4 benchmark.
|
||||
// ════════════════════════════════════════════════════════════
|
||||
|
||||
[Benchmark(Baseline = true)]
|
||||
public nuint Dictionary_Lookup_Zipfian()
|
||||
{
|
||||
nuint sum = 0;
|
||||
nuint[] keys = _zipfianLookups;
|
||||
Dictionary<nuint, nuint> dict = _dictLookup;
|
||||
for (int i = 0; i < keys.Length; i++)
|
||||
{
|
||||
if (dict.TryGetValue(keys[i], out nuint value)) {
|
||||
sum += value;
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════
|
||||
// SteadyState — the KEY L4 benchmark
|
||||
// Pre-filled table at capacity, then 4096 operations:
|
||||
// ~95% are lookups (cache hits)
|
||||
// ~5% are miss → evict cold entry → insert new entry → lookup new
|
||||
// This directly simulates L4's main runtime loop.
|
||||
// ════════════════════════════════════════════════════════════
|
||||
|
||||
[Benchmark]
|
||||
public nuint Dictionary_SteadyState()
|
||||
{
|
||||
nuint sum = 0;
|
||||
Dictionary<nuint, nuint> dict = _freshDict;
|
||||
nuint[] resident = _residentKeys;
|
||||
nuint[] external = _externalKeys;
|
||||
int[] script = _steadyStateScript;
|
||||
int[] evict = _evictOrder;
|
||||
int evictCursor = 0;
|
||||
int externalCursor = 0;
|
||||
|
||||
for (int i = 0; i < script.Length; i++)
|
||||
{
|
||||
int op = script[i];
|
||||
if (op >= 0)
|
||||
{
|
||||
// Cache hit: lookup existing entry
|
||||
if (dict.TryGetValue(resident[op], out nuint value))
|
||||
{
|
||||
sum += value;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// Cache miss: evict a cold entry, insert a new one, then lookup
|
||||
int evictIdx = evict[evictCursor % evict.Length];
|
||||
evictCursor++;
|
||||
dict.Remove(resident[evictIdx]);
|
||||
|
||||
int extIdx = externalCursor % external.Length;
|
||||
externalCursor++;
|
||||
nuint newKey = external[extIdx];
|
||||
nuint newVal = newKey ^ unchecked((nuint)0xDEAD_BEEF);
|
||||
dict[newKey] = newVal;
|
||||
|
||||
if (dict.TryGetValue(newKey, out nuint v)) {
|
||||
sum += v;
|
||||
}
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
[IterationSetup(Targets = [
|
||||
nameof(SwissTable_Insert),
|
||||
nameof(SwissTable_SteadyState),
|
||||
nameof(Dictionary_Insert),
|
||||
nameof(Dictionary_SteadyState)
|
||||
])]
|
||||
public void IterationSetup()
|
||||
{
|
||||
_freshSwiss = SwissTable.Create();
|
||||
_freshDict = new Dictionary<nuint, nuint>(KeyCount);
|
||||
|
||||
// Pre-fill for steady-state benchmarks
|
||||
for (int i = 0; i < KeyCount; i++)
|
||||
{
|
||||
nuint key = _residentKeys[i];
|
||||
nuint val = key ^ unchecked((nuint)0xDEAD_BEEF);
|
||||
SwissTable.Writer.TrySet(_freshSwiss, key, val);
|
||||
_freshDict[key] = val;
|
||||
}
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════
|
||||
// Setup / Teardown
|
||||
// ════════════════════════════════════════════════════════════
|
||||
|
||||
[GlobalSetup]
|
||||
public void Setup()
|
||||
{
|
||||
Random rng = new Random(42);
|
||||
|
||||
// ── Generate pointer-like keys (HeapBase + random_offset * 1024) ──
|
||||
_residentKeys = GeneratePointerKeys(KeyCount, rng);
|
||||
_externalKeys = GeneratePointerKeys(KeyCount, rng); // disjoint set for miss simulation
|
||||
|
||||
// ── Build lookup sequences ──
|
||||
_zipfianLookups = GenerateZipfianLookups(_residentKeys, LookupIterations, rng);
|
||||
_uniformLookups = GenerateUniformLookups(_residentKeys, LookupIterations, rng);
|
||||
|
||||
// ── Build steady-state operation script ──
|
||||
_steadyStateScript = GenerateSteadyStateScript(KeyCount, SteadyStateOps, MissRate, rng);
|
||||
|
||||
// ── Build eviction order (cold tail entries first) ──
|
||||
_evictOrder = new int[KeyCount];
|
||||
for (int i = 0; i < KeyCount; i++) {
|
||||
_evictOrder[i] = i;
|
||||
}
|
||||
|
||||
// Shuffle so eviction isn't just sequential
|
||||
Shuffle(_evictOrder, rng);
|
||||
|
||||
// ── Pre-fill lookup tables ──
|
||||
_swissLookup = SwissTable.Create();
|
||||
_dictLookup = new Dictionary<nuint, nuint>(KeyCount);
|
||||
|
||||
for (int i = 0; i < KeyCount; i++)
|
||||
{
|
||||
nuint key = _residentKeys[i];
|
||||
nuint val = key ^ unchecked((nuint)0xDEAD_BEEF); // simulated cache pointer
|
||||
SwissTable.Writer.TrySet(_swissLookup, key, val);
|
||||
_dictLookup[key] = val;
|
||||
}
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public void SwissTable_Insert()
|
||||
{
|
||||
SwissTable swiss = _freshSwiss;
|
||||
nuint[] keys = _residentKeys;
|
||||
for (int i = 0; i < keys.Length; i++) {
|
||||
SwissTable.Writer.TrySet(swiss, keys[i], keys[i] ^ unchecked((nuint)0xDEAD_BEEF));
|
||||
}
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public nuint SwissTable_Lookup_Uniform()
|
||||
{
|
||||
nuint sum = 0;
|
||||
nuint[] keys = _uniformLookups;
|
||||
SwissTable swiss = _swissLookup;
|
||||
for (int i = 0; i < keys.Length; i++)
|
||||
{
|
||||
if (SwissTable.Reader.TryGet(swiss, keys[i], out nuint value)) {
|
||||
sum += value;
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public nuint SwissTable_Lookup_Zipfian()
|
||||
{
|
||||
nuint sum = 0;
|
||||
nuint[] keys = _zipfianLookups;
|
||||
SwissTable swiss = _swissLookup;
|
||||
for (int i = 0; i < keys.Length; i++)
|
||||
{
|
||||
if (SwissTable.Reader.TryGet(swiss, keys[i], out nuint value)) {
|
||||
sum += value;
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public nuint SwissTable_SteadyState()
|
||||
{
|
||||
nuint sum = 0;
|
||||
SwissTable swiss = _freshSwiss;
|
||||
nuint[] resident = _residentKeys;
|
||||
nuint[] external = _externalKeys;
|
||||
int[] script = _steadyStateScript;
|
||||
int[] evict = _evictOrder;
|
||||
int evictCursor = 0;
|
||||
int externalCursor = 0;
|
||||
|
||||
for (int i = 0; i < script.Length; i++)
|
||||
{
|
||||
int op = script[i];
|
||||
if (op >= 0)
|
||||
{
|
||||
// Cache hit: lookup existing entry
|
||||
if (SwissTable.Reader.TryGet(swiss, resident[op], out nuint value))
|
||||
{
|
||||
sum += value;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// Cache miss: evict a cold entry, insert a new one, then lookup
|
||||
int evictIdx = evict[evictCursor % evict.Length];
|
||||
evictCursor++;
|
||||
SwissTable.Writer.TryRemove(swiss, resident[evictIdx], out _);
|
||||
|
||||
int extIdx = externalCursor % external.Length;
|
||||
externalCursor++;
|
||||
nuint newKey = external[extIdx];
|
||||
nuint newVal = newKey ^ unchecked((nuint)0xDEAD_BEEF);
|
||||
SwissTable.Writer.TryAdd(swiss, newKey, newVal);
|
||||
|
||||
if (SwissTable.Reader.TryGet(swiss, newKey, out nuint v)) {
|
||||
sum += v;
|
||||
}
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════
|
||||
// Data generation helpers
|
||||
// ════════════════════════════════════════════════════════════
|
||||
|
||||
/// <summary>
|
||||
/// Generate <paramref name="count" /> unique pointer-like keys. Each key = HeapBase +
|
||||
/// randomOffset * CacheLineSize, simulating HLSF-allocated 1024-byte aligned addresses
|
||||
/// scattered across a ~1 GB virtual address range.
|
||||
/// </summary>
|
||||
private static nuint[] GeneratePointerKeys(
|
||||
int count,
|
||||
Random rng
|
||||
)
|
||||
{
|
||||
nuint[] keys = new nuint[count];
|
||||
HashSet<long> used = new HashSet<long>(count);
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
long offset;
|
||||
do
|
||||
{
|
||||
// Random offset within ~1M cache lines (1 GB range)
|
||||
offset = (long)rng.Next(0, 1 << 20) * CacheLineSize;
|
||||
} while (!used.Add(offset));
|
||||
keys[i] = (nuint)(HeapBase + offset);
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generate a steady-state operation script. Each entry: non-negative value = index into
|
||||
/// _residentKeys (lookup/hit), negative value (-1) = cache miss trigger (evict → insert →
|
||||
/// lookup). Lookup indices are Zipfian-skewed.
|
||||
/// </summary>
|
||||
private static int[] GenerateSteadyStateScript(
|
||||
int keyCount,
|
||||
int opCount,
|
||||
double missRate,
|
||||
Random rng
|
||||
)
|
||||
{
|
||||
int hotSetSize = Math.Max(1, keyCount / 5);
|
||||
int[] script = new int[opCount];
|
||||
for (int i = 0; i < opCount; i++)
|
||||
{
|
||||
if (rng.NextDouble() < missRate)
|
||||
{
|
||||
script[i] = -1; // miss
|
||||
} else
|
||||
{
|
||||
// Zipfian hit
|
||||
if (rng.NextDouble() < 0.8)
|
||||
{
|
||||
script[i] = rng.Next(0, hotSetSize);
|
||||
} else
|
||||
{
|
||||
script[i] = rng.Next(hotSetSize, keyCount);
|
||||
}
|
||||
}
|
||||
}
|
||||
return script;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generate uniform random lookup sequence (baseline for comparison with Zipfian).
|
||||
/// </summary>
|
||||
private static nuint[] GenerateUniformLookups(
|
||||
nuint[] sourceKeys,
|
||||
int count,
|
||||
Random rng
|
||||
)
|
||||
{
|
||||
nuint[] lookups = new nuint[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
lookups[i] = sourceKeys[rng.Next(0, sourceKeys.Length)];
|
||||
}
|
||||
return lookups;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generate Zipfian-skewed lookup sequence: 80% of accesses hit the hottest 20% of keys
|
||||
/// (sorted by index as "hotness proxy"), 20% of accesses hit the remaining 80%. This
|
||||
/// models matrix computation where a small panel working set is reused heavily.
|
||||
/// </summary>
|
||||
private static nuint[] GenerateZipfianLookups(
|
||||
nuint[] sourceKeys,
|
||||
int count,
|
||||
Random rng
|
||||
)
|
||||
{
|
||||
int hotSetSize = Math.Max(1, sourceKeys.Length / 5);
|
||||
nuint[] lookups = new nuint[count];
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
if (rng.NextDouble() < 0.8)
|
||||
{
|
||||
lookups[i] = sourceKeys[rng.Next(0, hotSetSize)];
|
||||
} else
|
||||
{
|
||||
lookups[i] = sourceKeys[rng.Next(hotSetSize, sourceKeys.Length)];
|
||||
}
|
||||
}
|
||||
return lookups;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fisher-Yates shuffle.
|
||||
/// </summary>
|
||||
private static void Shuffle(
|
||||
int[] array,
|
||||
Random rng
|
||||
)
|
||||
{
|
||||
for (int i = array.Length - 1; i > 0; i--)
|
||||
{
|
||||
int j = rng.Next(0, i + 1);
|
||||
(array[i], array[j]) = (array[j], array[i]);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SwissX86Bench"
|
||||
// Project: nFramework
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using BenchmarkDotNet.Attributes;
|
||||
using BenchmarkDotNet.Configs;
|
||||
using Nitload.Common.HighPerformance.Memory;
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.Benchmark
|
||||
{
|
||||
[MemoryDiagnoser]
|
||||
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
|
||||
[CategoriesColumn]
|
||||
public unsafe class SwissX86Bench
|
||||
{
|
||||
|
||||
private const int QueryCount = 1 << 21;
|
||||
private const int ConcurrentQueryCount = QueryCount * 6;
|
||||
private const int MaximumResidentCount = 8192;
|
||||
|
||||
private byte* _buffer;
|
||||
private L4.SwissTable _swiss;
|
||||
private Dictionary<L4.AddressSource, ulong> _dictionary = null!;
|
||||
private ConcurrentDictionary<L4.AddressSource, ulong> _concurrentDictionary = null!;
|
||||
private L4.AddressSource[] _hitQueries = [];
|
||||
private L4.AddressSource[] _missQueries = [];
|
||||
private L4.AddressSource[] _writerQueries = [];
|
||||
private ulong[] _writerValues = [];
|
||||
private byte* _readBuffer;
|
||||
private byte* _writeBuffer;
|
||||
private ParallelOptions _parallelOptions = null!;
|
||||
private Func<ulong> _parallelLocalInit = null!;
|
||||
private Func<int, ParallelLoopState, ulong, ulong> _swissParallelBody = null!;
|
||||
private Func<int, ParallelLoopState, ulong, ulong> _concurrentDictionaryParallelBody = null!;
|
||||
private Action<ulong> _parallelLocalFinally = null!;
|
||||
private long _parallelChecksum;
|
||||
|
||||
[Params(25, 50, 100)]
|
||||
public int LoadPercent { get; set; }
|
||||
|
||||
private int ResidentCount => MaximumResidentCount * LoadPercent / 100;
|
||||
|
||||
[GlobalSetup]
|
||||
public void Setup()
|
||||
{
|
||||
if (!Sse42.X64.IsSupported)
|
||||
throw new PlatformNotSupportedException("The Swiss X86 benchmark requires x64 SSE4.2.");
|
||||
|
||||
_buffer = (byte*)NativeMemory.AlignedAlloc((nuint)L4.BufferSize, 64);
|
||||
if (_buffer == null)
|
||||
throw new OutOfMemoryException("Could not allocate Swiss benchmark tables.");
|
||||
|
||||
_swiss = L4.SwissTable.Create(_buffer);
|
||||
_dictionary = new Dictionary<L4.AddressSource, ulong>(ResidentCount);
|
||||
_concurrentDictionary = new ConcurrentDictionary<L4.AddressSource, ulong>(
|
||||
Math.Max(1, Math.Min(Environment.ProcessorCount, 8)),
|
||||
ResidentCount);
|
||||
|
||||
L4.AddressSource[] residentKeys = new L4.AddressSource[ResidentCount];
|
||||
for (int i = 0; i < ResidentCount; i++)
|
||||
{
|
||||
L4.AddressSource key = MakeKey(i);
|
||||
ulong value = (ulong)i + 1;
|
||||
residentKeys[i] = key;
|
||||
if (!_swiss.TryAddOrSet(key, value))
|
||||
throw new InvalidOperationException($"Swiss insertion failed while seeding key {i}.");
|
||||
_dictionary.Add(key, value);
|
||||
_concurrentDictionary.TryAdd(key, value);
|
||||
}
|
||||
|
||||
_swiss.SwapBuffers();
|
||||
for (int i = 0; i < ResidentCount; i++)
|
||||
{
|
||||
if (!_swiss.TryAddOrSet(residentKeys[i], (ulong)i + 1))
|
||||
throw new InvalidOperationException($"Swiss writer-buffer seeding failed for key {i}.");
|
||||
}
|
||||
|
||||
_readBuffer = _swiss.ReadBufferForLockFreeBenchmark;
|
||||
_writeBuffer = _swiss.WriteBufferForLockFreeBenchmark;
|
||||
|
||||
_hitQueries = new L4.AddressSource[QueryCount];
|
||||
_missQueries = new L4.AddressSource[QueryCount];
|
||||
_writerQueries = new L4.AddressSource[QueryCount];
|
||||
_writerValues = new ulong[QueryCount];
|
||||
Random random = new(0x51_15_2026);
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
{
|
||||
_hitQueries[i] = residentKeys[random.Next(ResidentCount)];
|
||||
_missQueries[i] = MakeKey(MaximumResidentCount + i);
|
||||
_writerQueries[i] = residentKeys[random.Next(ResidentCount)];
|
||||
_writerValues[i] = (ulong)i + 0x10000;
|
||||
}
|
||||
|
||||
_parallelOptions = new ParallelOptions
|
||||
{
|
||||
MaxDegreeOfParallelism = Math.Max(2, Math.Min(Environment.ProcessorCount, 8))
|
||||
};
|
||||
_parallelLocalInit = static() => 0UL;
|
||||
_swissParallelBody = ReadSwissParallel;
|
||||
_concurrentDictionaryParallelBody = ReadConcurrentDictionaryParallel;
|
||||
_parallelLocalFinally = MergeParallelChecksum;
|
||||
|
||||
ValidateLookupResults();
|
||||
}
|
||||
|
||||
[GlobalCleanup]
|
||||
public void Cleanup()
|
||||
{
|
||||
_swiss.Dispose();
|
||||
NativeMemory.AlignedFree(_buffer);
|
||||
_buffer = null;
|
||||
}
|
||||
|
||||
[Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Hit lookup")]
|
||||
public ulong DictionaryHitLookup()
|
||||
{
|
||||
ulong checksum = 0;
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
checksum += _dictionary[_hitQueries[i]];
|
||||
return checksum;
|
||||
}
|
||||
|
||||
[Benchmark(OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Hit lookup")]
|
||||
public ulong SwissLockFreeHitLookup()
|
||||
{
|
||||
ulong checksum = 0;
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
{
|
||||
if (!L4.SwissReaderX86.TryGet(_readBuffer, _hitQueries[i], out ulong value))
|
||||
throw new InvalidOperationException("Swiss table missed a seeded hit key.");
|
||||
checksum += value;
|
||||
}
|
||||
return checksum;
|
||||
}
|
||||
|
||||
[Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Miss lookup")]
|
||||
public int DictionaryMissLookup()
|
||||
{
|
||||
int misses = 0;
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
misses += _dictionary.TryGetValue(_missQueries[i], out _) ? 0 : 1;
|
||||
return misses;
|
||||
}
|
||||
|
||||
[Benchmark(OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Miss lookup")]
|
||||
public int SwissLockFreeMissLookup()
|
||||
{
|
||||
int misses = 0;
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
misses += L4.SwissReaderX86.TryGet(_readBuffer, _missQueries[i], out _) ? 0 : 1;
|
||||
return misses;
|
||||
}
|
||||
|
||||
[Benchmark(Baseline = true, OperationsPerInvoke = ConcurrentQueryCount)]
|
||||
[BenchmarkCategory("Concurrent hit lookup")]
|
||||
public ulong ConcurrentDictionaryConcurrentHitLookup()
|
||||
{
|
||||
Interlocked.Exchange(ref _parallelChecksum, 0);
|
||||
Parallel.For(
|
||||
0,
|
||||
ConcurrentQueryCount,
|
||||
_parallelOptions,
|
||||
_parallelLocalInit,
|
||||
_concurrentDictionaryParallelBody,
|
||||
_parallelLocalFinally);
|
||||
return unchecked((ulong)Interlocked.Read(ref _parallelChecksum));
|
||||
}
|
||||
|
||||
[Benchmark(OperationsPerInvoke = ConcurrentQueryCount)]
|
||||
[BenchmarkCategory("Concurrent hit lookup")]
|
||||
public ulong SwissLockFreeConcurrentHitLookup()
|
||||
{
|
||||
Interlocked.Exchange(ref _parallelChecksum, 0);
|
||||
Parallel.For(
|
||||
0,
|
||||
ConcurrentQueryCount,
|
||||
_parallelOptions,
|
||||
_parallelLocalInit,
|
||||
_swissParallelBody,
|
||||
_parallelLocalFinally);
|
||||
return unchecked((ulong)Interlocked.Read(ref _parallelChecksum));
|
||||
}
|
||||
|
||||
[Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Upsert existing")]
|
||||
public ulong DictionaryUpsertExisting()
|
||||
{
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
_dictionary[_writerQueries[i]] = _writerValues[i];
|
||||
return _dictionary[_writerQueries[^1]];
|
||||
}
|
||||
|
||||
[Benchmark(OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Upsert existing")]
|
||||
public ulong SwissUpsertExisting()
|
||||
{
|
||||
ulong failures = 0;
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
failures += L4.SwissWriterX86.TryAddOrSet(_writeBuffer, _writerQueries[i], _writerValues[i]) ? 0UL : 1UL;
|
||||
if (failures != 0 || !L4.SwissReaderX86.TryGet(_writeBuffer, _writerQueries[^1], out ulong value))
|
||||
throw new InvalidOperationException("Swiss upsert failed to preserve the final value.");
|
||||
return value;
|
||||
}
|
||||
|
||||
[Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Remove and reinsert")]
|
||||
public ulong DictionaryRemoveAndReinsert()
|
||||
{
|
||||
ulong checksum = 0;
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
{
|
||||
L4.AddressSource key = _writerQueries[i];
|
||||
ulong value = _dictionary[key];
|
||||
_dictionary.Remove(key);
|
||||
_dictionary.Add(key, value + 1);
|
||||
checksum += value;
|
||||
}
|
||||
return checksum;
|
||||
}
|
||||
|
||||
[Benchmark(OperationsPerInvoke = QueryCount)]
|
||||
[BenchmarkCategory("Remove and reinsert")]
|
||||
public ulong SwissRemoveAndReinsert()
|
||||
{
|
||||
ulong checksum = 0;
|
||||
for (int i = 0; i < QueryCount; i++)
|
||||
{
|
||||
L4.AddressSource key = _writerQueries[i];
|
||||
if (!L4.SwissWriterX86.TryRemove(_writeBuffer, key, out ulong value) ||
|
||||
!L4.SwissWriterX86.TryAddOrSet(_writeBuffer, key, value + 1)) {
|
||||
throw new InvalidOperationException("Swiss remove/reinsert failed.");
|
||||
}
|
||||
checksum += value;
|
||||
}
|
||||
return checksum;
|
||||
}
|
||||
|
||||
private static L4.AddressSource MakeKey(int index)
|
||||
{
|
||||
return new L4.AddressSource
|
||||
{
|
||||
Address = 0x1000_0000UL + ((ulong)index * 64UL),
|
||||
Description = ((ulong)index * 0x9e37_79b9UL) ^ 0xd1b5_4a32_d192_ed03UL
|
||||
};
|
||||
}
|
||||
|
||||
private ulong ReadSwissParallel(int index, ParallelLoopState state, ulong checksum)
|
||||
{
|
||||
int queryIndex = index & (QueryCount - 1);
|
||||
if (!L4.SwissReaderX86.TryGet(_readBuffer, _hitQueries[queryIndex], out ulong value))
|
||||
throw new InvalidOperationException("Swiss table missed a seeded concurrent hit key.");
|
||||
return checksum + value;
|
||||
}
|
||||
|
||||
private ulong ReadConcurrentDictionaryParallel(int index, ParallelLoopState state, ulong checksum)
|
||||
{
|
||||
int queryIndex = index & (QueryCount - 1);
|
||||
if (!_concurrentDictionary.TryGetValue(_hitQueries[queryIndex], out ulong value))
|
||||
throw new InvalidOperationException("ConcurrentDictionary missed a seeded hit key.");
|
||||
return checksum + value;
|
||||
}
|
||||
|
||||
private void MergeParallelChecksum(ulong checksum)
|
||||
{
|
||||
_ = Interlocked.Add(ref _parallelChecksum, unchecked((long)checksum));
|
||||
}
|
||||
|
||||
private void ValidateLookupResults()
|
||||
{
|
||||
for (int i = 0; i < _hitQueries.Length; i++)
|
||||
{
|
||||
bool dictionaryFound = _dictionary.TryGetValue(_hitQueries[i], out ulong dictionaryValue);
|
||||
bool swissFound = L4.SwissReaderX86.TryGet(_readBuffer, _hitQueries[i], out ulong swissValue);
|
||||
if (!dictionaryFound || !swissFound || dictionaryValue != swissValue)
|
||||
throw new InvalidOperationException("Swiss and Dictionary hit results differ.");
|
||||
|
||||
if (!_concurrentDictionary.TryGetValue(_hitQueries[i], out ulong concurrentValue) ||
|
||||
concurrentValue != dictionaryValue)
|
||||
throw new InvalidOperationException("ConcurrentDictionary and Dictionary hit results differ.");
|
||||
}
|
||||
|
||||
for (int i = 0; i < _missQueries.Length; i++)
|
||||
{
|
||||
bool dictionaryFound = _dictionary.TryGetValue(_missQueries[i], out _);
|
||||
bool swissFound = L4.SwissReaderX86.TryGet(_readBuffer, _missQueries[i], out _);
|
||||
if (dictionaryFound || swissFound)
|
||||
throw new InvalidOperationException("Swiss and Dictionary miss results differ.");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SwissX86DisassemblyBench"
|
||||
// Project: nFramework
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using BenchmarkDotNet.Attributes;
|
||||
using BenchmarkDotNet.Diagnosers;
|
||||
using Nitload.Common.HighPerformance.Memory;
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
|
||||
namespace Nitload.Benchmark
|
||||
{
|
||||
[DisassemblyDiagnoser(maxDepth: 3, exportHtml: true)]
|
||||
public unsafe class SwissX86DisassemblyBench
|
||||
{
|
||||
|
||||
private const int ResidentCount = 8192;
|
||||
|
||||
private byte* _buffer;
|
||||
private L4.SwissTable _table;
|
||||
private byte* _readBuffer;
|
||||
private byte* _writeBuffer;
|
||||
private L4.AddressSource _hitKey;
|
||||
private L4.AddressSource _missKey;
|
||||
private ulong _value;
|
||||
|
||||
[GlobalSetup]
|
||||
public void Setup()
|
||||
{
|
||||
if (!Sse42.X64.IsSupported)
|
||||
throw new PlatformNotSupportedException("The Swiss X86 disassembly probe requires x64 SSE4.2.");
|
||||
|
||||
_buffer = (byte*)NativeMemory.AlignedAlloc((nuint)L4.BufferSize, 64);
|
||||
if (_buffer == null)
|
||||
throw new OutOfMemoryException("Could not allocate Swiss disassembly buffers.");
|
||||
|
||||
_table = L4.SwissTable.Create(_buffer);
|
||||
for (int index = 0; index < ResidentCount; index++)
|
||||
{
|
||||
if (!L4.SwissWriterX86.TryAddOrSet(_table.WriteBufferForLockFreeBenchmark,
|
||||
MakeKey(index), (ulong)index + 1))
|
||||
throw new InvalidOperationException($"Failed to seed Swiss key {index}.");
|
||||
}
|
||||
|
||||
_table.SwapBuffers();
|
||||
_readBuffer = _table.ReadBufferForLockFreeBenchmark;
|
||||
_writeBuffer = _table.WriteBufferForLockFreeBenchmark;
|
||||
for (int index = 0; index < ResidentCount; index++)
|
||||
{
|
||||
if (!L4.SwissWriterX86.TryAddOrSet(_writeBuffer, MakeKey(index), (ulong)index + 1))
|
||||
throw new InvalidOperationException($"Failed to seed inactive Swiss key {index}.");
|
||||
}
|
||||
|
||||
_hitKey = MakeKey(ResidentCount / 2);
|
||||
_missKey = MakeKey(ResidentCount + 1);
|
||||
_value = 0x1234_5678;
|
||||
|
||||
if (!L4.SwissReaderX86.TryGet(_readBuffer, _hitKey, out _) ||
|
||||
L4.SwissReaderX86.TryGet(_readBuffer, _missKey, out _))
|
||||
throw new InvalidOperationException("Swiss reader setup validation failed.");
|
||||
}
|
||||
|
||||
[GlobalCleanup]
|
||||
public void Cleanup()
|
||||
{
|
||||
_table.Dispose();
|
||||
NativeMemory.AlignedFree(_buffer);
|
||||
_buffer = null;
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool RawReaderHit()
|
||||
{
|
||||
return L4.SwissReaderX86.TryGet(_readBuffer, _hitKey, out _value);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool RawReaderMiss()
|
||||
{
|
||||
return L4.SwissReaderX86.TryGet(_readBuffer, _missKey, out _value);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool RawWriterUpsertExisting()
|
||||
{
|
||||
return L4.SwissWriterX86.TryAddOrSet(_writeBuffer, _hitKey, _value);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool RawWriterRemoveAndReinsert()
|
||||
{
|
||||
return L4.SwissWriterX86.TryRemove(_writeBuffer, _hitKey, out ulong previousValue) &&
|
||||
L4.SwissWriterX86.TryAddOrSet(_writeBuffer, _hitKey, previousValue);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool TableReaderHit()
|
||||
{
|
||||
return _table.TryGet(_hitKey, out _value);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool TableReaderMiss()
|
||||
{
|
||||
return _table.TryGet(_missKey, out _value);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool TableWriterUpsertExisting()
|
||||
{
|
||||
return _table.TryAddOrSet(_hitKey, _value);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public bool TableWriterRemoveAndReinsert()
|
||||
{
|
||||
return _table.TryRemove(_hitKey, out ulong previousValue) &&
|
||||
_table.TryAddOrSet(_hitKey, previousValue);
|
||||
}
|
||||
|
||||
private static L4.AddressSource MakeKey(int index)
|
||||
{
|
||||
return new L4.AddressSource
|
||||
{
|
||||
Address = 0x1000_0000UL + ((ulong)index * 64UL),
|
||||
Description = ((ulong)index * 0x9e37_79b9UL) ^ 0xd1b5_4a32_d192_ed03UL
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SwissTable"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Nitload.Common.HighPerformance.ProcessorAbstraction;
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace Nitload.Common.Collections
|
||||
{
|
||||
public sealed unsafe partial class SwissTable
|
||||
{
|
||||
|
||||
private const int BucketSize = 16;
|
||||
private const uint CrcMagic = 0xffff9961; // Magic number for CRC32C, used in hash computation
|
||||
private const int EntryMaskWidth = 10; // 1024 entries total, 16 per bucket
|
||||
|
||||
private const int GroupCount = 1 << EntryMaskWidth;
|
||||
public const int MaxCapacity = GroupCount * BucketSize; // 256 entries per bucket, 1024 buckets
|
||||
|
||||
public static readonly nuint RequestedNativeMemorySize = (nuint)Unsafe.SizeOf<byte>() + (nuint)Unsafe.SizeOf<M128>();
|
||||
|
||||
private byte* _controlBytes;
|
||||
|
||||
private SwissTable(
|
||||
byte* buffer
|
||||
)
|
||||
{
|
||||
_controlBytes = buffer;
|
||||
}
|
||||
|
||||
private M128* KvpArray => (M128*)(_controlBytes + MaxCapacity);
|
||||
|
||||
private static void Initialize(
|
||||
byte* buffer
|
||||
)
|
||||
{
|
||||
Unsafe.InitBlock(buffer, 0b10000000, BucketSize);
|
||||
}
|
||||
|
||||
private unsafe struct Bucket
|
||||
{
|
||||
|
||||
public BucketControl Control;
|
||||
public BucketKeyStorage Keys;
|
||||
public BucketValueStorage Values;
|
||||
|
||||
}
|
||||
|
||||
public static class Reader
|
||||
{
|
||||
|
||||
public static bool TryGet(
|
||||
SwissTable table,
|
||||
nuint key,
|
||||
out nuint value
|
||||
)
|
||||
{
|
||||
throw new NotImplementedException("SwissTable reader is awaiting a redesigned storage and lookup implementation.");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static class Writer
|
||||
{
|
||||
|
||||
public static bool TryAdd(
|
||||
SwissTable table,
|
||||
nuint key,
|
||||
nuint value
|
||||
)
|
||||
{
|
||||
throw new NotImplementedException("SwissTable writer is awaiting a redesigned storage and insertion implementation.");
|
||||
}
|
||||
|
||||
public static bool TryRemove(
|
||||
SwissTable table,
|
||||
nuint key,
|
||||
out nuint value
|
||||
)
|
||||
{
|
||||
throw new NotImplementedException("SwissTable writer is awaiting a redesigned storage and removal implementation.");
|
||||
}
|
||||
|
||||
public static bool TrySet(
|
||||
SwissTable table,
|
||||
nuint key,
|
||||
nuint value
|
||||
)
|
||||
{
|
||||
throw new NotImplementedException("SwissTable writer is awaiting a redesigned storage and update implementation.");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
[InlineArray(BucketSize)]
|
||||
private struct BucketControl
|
||||
{
|
||||
|
||||
public byte Control;
|
||||
|
||||
}
|
||||
|
||||
[InlineArray(BucketSize)]
|
||||
private struct BucketKeyStorage
|
||||
{
|
||||
|
||||
public nuint value;
|
||||
|
||||
}
|
||||
|
||||
[InlineArray(BucketSize)]
|
||||
private struct BucketValueStorage
|
||||
{
|
||||
|
||||
public nuint value;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -15,4 +15,5 @@ global using System.Threading;
|
||||
[assembly: InternalsVisibleTo("Nitload.RegularAnalysis")]
|
||||
[assembly: InternalsVisibleTo("Nitload.Graphics")]
|
||||
[assembly: InternalsVisibleTo("Nitload.Mathematics")]
|
||||
[assembly: InternalsVisibleTo("Nitload.Benchmark")]
|
||||
[assembly: InternalsVisibleTo("StgSharpDebug")]
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Nitload.Common.HighPerformance.ProcessorAbstraction;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
@@ -23,14 +22,12 @@ namespace Nitload.Common.HighPerformance.Memory
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Explicit, Size = 32, Pack = 32)]
|
||||
private unsafe struct CacheLineMetadata
|
||||
private struct CacheLineMetadata
|
||||
{
|
||||
|
||||
internal const uint MaxBufferSize = 0xFFFFFFFF - 64U;
|
||||
|
||||
[FieldOffset(0)] private readonly M256 FillingMask;
|
||||
|
||||
[FieldOffset(24)] internal nuint Origin;
|
||||
[FieldOffset(16)] internal AllocationState State;
|
||||
|
||||
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
|
||||
|
||||
@@ -48,33 +45,25 @@ namespace Nitload.Common.HighPerformance.Memory
|
||||
[FieldOffset(8)] public uint PositionMask;
|
||||
[FieldOffset(12)] public uint SizeMask;
|
||||
|
||||
#endregion
|
||||
|
||||
#region mask
|
||||
|
||||
[FieldOffset(16)] internal AllocationState State;
|
||||
[FieldOffset(17)] internal byte ReversedProfile;
|
||||
[FieldOffset(18)] internal ushort CustomizeProfile;
|
||||
[FieldOffset(20)] internal int RefCount;
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
#pragma warning disable IDE0032
|
||||
[StructLayout(LayoutKind.Explicit)]
|
||||
[StructLayout(LayoutKind.Explicit, Size = 16)]
|
||||
public unsafe struct Handle : IDisposable
|
||||
{
|
||||
|
||||
[FieldOffset(8)] private readonly CacheLineMetadata* _head;
|
||||
[FieldOffset(0)] private nuint _handle;
|
||||
|
||||
private Handle(
|
||||
nuint handle,
|
||||
CacheLineMetadata* cacheRefCount
|
||||
[FieldOffset(8)] private nuint _reserved;
|
||||
|
||||
internal Handle(
|
||||
nuint handle,
|
||||
nuint reserved
|
||||
)
|
||||
{
|
||||
_handle = handle;
|
||||
_head = cacheRefCount;
|
||||
_reserved = reserved;
|
||||
}
|
||||
|
||||
[Obsolete("You should not create a CacheLineHandle instance outside a L4", true)]
|
||||
@@ -88,7 +77,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
||||
return;
|
||||
}
|
||||
_handle = 0;
|
||||
_ = Interlocked.Decrement(ref _head->RefCount);
|
||||
_reserved = 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Debug"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Nitload.Common.Collections;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
/**/
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
public void DebugAgeAll()
|
||||
{
|
||||
// AgeAll();
|
||||
}
|
||||
|
||||
public bool DebugContains(
|
||||
nuint origin
|
||||
)
|
||||
{
|
||||
return SwissTable.Reader.TryGet(_map, origin, out _);
|
||||
}
|
||||
|
||||
public int DebugEvict(
|
||||
int count = 1
|
||||
)
|
||||
{
|
||||
return Evict(count);
|
||||
}
|
||||
|
||||
public unsafe DebugSnapshot GetDebugSnapshot()
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
/*
|
||||
int maxId = _maxId;
|
||||
byte[] rrpv = new byte[maxId];
|
||||
byte* p = (byte*)_eviction;
|
||||
for (int i = 0; i < maxId; i++) {
|
||||
rrpv[i] = p[i];
|
||||
}
|
||||
return new DebugSnapshot(_activeCount, _aheadCount, maxId, rrpv);
|
||||
/**/
|
||||
}
|
||||
|
||||
public readonly struct DebugSnapshot
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// RRPV value for each line（0~3）, length == MaxId
|
||||
/// </summary>
|
||||
public readonly byte[] RrpvSnapshot;
|
||||
/// <summary>
|
||||
/// Count of activated cache line handle
|
||||
/// </summary>
|
||||
public readonly int ActiveCount;
|
||||
|
||||
/// <summary>
|
||||
/// Count of predictor ahead of current map position（aheadCount）
|
||||
/// </summary>
|
||||
public readonly int AheadCount;
|
||||
|
||||
/// <summary>
|
||||
/// Max ID of the line being using or once used（maxId）
|
||||
/// </summary>
|
||||
public readonly int MaxId;
|
||||
|
||||
internal DebugSnapshot(
|
||||
int activeCount,
|
||||
int aheadCount,
|
||||
int maxId,
|
||||
byte[] rrpv
|
||||
)
|
||||
{
|
||||
ActiveCount = activeCount;
|
||||
AheadCount = aheadCount;
|
||||
MaxId = maxId;
|
||||
RrpvSnapshot = rrpv;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
/**/
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
# L4 运行模型
|
||||
|
||||
> 工作稿，2026-10-14。本文替代旧版 L4 设计草案，区分当前代码、已讨论方向和仍待确认的问题。
|
||||
|
||||
## 1. 范围与职责
|
||||
|
||||
每个 `L4` 实例绑定一个 NUMA 节点，拥有本地 arena，其中放置 TLSF 分配器状态和 L4 Swiss 映射表的双缓冲区。
|
||||
|
||||
- **L4** 负责本地分配、映射、缓冲区发布和阶段协调。
|
||||
- **Swiss** 保存并查询 L4 使用的映射/预测条目，不解释 source layout。
|
||||
- **Predictor** 解释自己的 source spec，生成 packed 数据，并负责格式相关的软件写回或撤销。
|
||||
- **PredictorMonitor** 计划负责预测器注册、稳定 ID、执行顺序和生命周期；它与 Swiss 中的预测任务/映射状态是两回事。
|
||||
|
||||
本文不假设自动写回。若 packed 数据需要写回，必须能识别其所属预测器并显式调用对应逻辑。
|
||||
|
||||
## 2. AddressSource 与缓存行数据
|
||||
|
||||
`AddressSource` 是 16 字节 key：
|
||||
|
||||
- `Address`：源地址或 source identity。
|
||||
- `Description`：64 位 source/layout spec，由所属预测器解释。
|
||||
|
||||
当前 Swiss 条目使用该 key 和一个 `ulong` value。源码中的 `CacheLineMetadata` 是 32 字节 TLSF metadata；`L4.Handle` 仍是 16 字节 ABI 占位。引用计数、驱逐保护和写回归属尚未端到端实现。
|
||||
|
||||
### 当前与候选 KVP 布局
|
||||
|
||||
- 设计讨论中的 `ushort` 是缓存行计数，不是预测器数量或预测器注册表容量。
|
||||
- 当前 Swiss bucket 将 key 与 `ulong` value 分开组织；现有的 8-KVP 交叉存储描述的是数据存储布局，不是 H1/hash 探测分组。
|
||||
- 一个候选布局是把 value 扩成 16 字节：保留现有 `ulong` cache-line metadata，再增加 8 字节存放预测计数等状态。与 16 字节 `AddressSource` 组成 32 字节 KVP，每条 64 字节 cache line 放两个 KVP。
|
||||
- 该方案旨在减少当前 8-KVP 交叉存储造成的 cache-line 分散/冲突。它只调整 KVP 的存储编排，不意味着 H1 探测组从当前 16 槽改成 8 槽。此方案尚未实现，需实测对查找和替换扫描的影响。
|
||||
|
||||
## 3. Swiss 映射
|
||||
|
||||
当前表有 1024 组，每组 16 个槽位。控制字节扫描使用 SSE2；x64 路径使用 CRC32C 混合两个 64 位 key 字段。两个表缓冲区都分配在 L4 arena 内。
|
||||
|
||||
published buffer 地址以 `ulong` 保存：
|
||||
|
||||
- `TryGet` 用一次 `Volatile.Read` 捕获 published pointer，并在整个查找中使用该快照。
|
||||
- `SwapBuffers` 用 `Interlocked.Exchange` 发布下一缓冲区，返回的旧 published pointer 成为下一轮 inactive write buffer。
|
||||
- 原子发布指针不代表旧 buffer 可以立即复用。Map/swap 阶段协议必须先确保读者不再访问旧 buffer，writer 才能清空或覆盖它。
|
||||
|
||||
当前只实现 x64/SSE4.2 后端，其他 ISA 后端尚未实现。
|
||||
|
||||
## 4. 预测器接口与阶段
|
||||
|
||||
当前接口骨架位于 `L4Predictor.cs`：
|
||||
|
||||
- `IL4Predictor` 是非泛型管理接口，暴露预测器 ID 和同步的 `PredictMission(in Span<AddressSource>)`。
|
||||
- `IL4Predictor<TMission>` 为预测器实现提供强类型 mission API。
|
||||
- `PredictMission` 填充调用方提供的 span，并返回有效预测数。span 可以由调用方 `stackalloc`，不得逃逸出本次同步调用。
|
||||
- `PredictorMonitor` 目前是空占位；预测器注册和生命周期尚未实现。
|
||||
|
||||
当前讨论中的 L4 生命周期分为不同阶段：
|
||||
|
||||
1. **注册预测器**：monitor 按稳定顺序保存异构预测器。固定容量数组（讨论过 256）是候选，其容量与缓存行计数无关。
|
||||
2. **Prefetch**：预测器按依赖顺序执行。每个 L4 同时只有一个活动 prefetch worker，但 prefetch 可与 Map 重叠，也可能跨 NUMA 节点访问目标数据。
|
||||
3. **发布**：完成后的 inactive Swiss buffer 被原子发布。Swiss 保存预测/映射任务状态；预测器注册表不是预测任务队列。
|
||||
4. **Map**：调用方查询已发布的 `AddressSource` 并取得 L4 handle。
|
||||
5. **撤销/写回**：所属预测器解释 packed 数据，并按需执行软件写回或撤销。
|
||||
|
||||
预测器接口本身是同步的。若预测结果需要在 `PredictMission` 返回后或另一线程继续使用，必须先复制到生命周期足够长的自有存储中。异步边界应位于拥有的 batch/work item 周围，而不能让 stack span 逃逸。
|
||||
|
||||
## 5. 预测窗口与 Map 同步
|
||||
|
||||
两个阈值含义不同：
|
||||
|
||||
- `_low_predict = 512`：预测触发/补充阈值。
|
||||
- `_predict_slow_path_count = 2 * LogicalCoreCount`：按 L4 绑定 NUMA 节点逻辑处理器数量计算的提前慢路径窗口。
|
||||
|
||||
当前 Map 的分流意图如下：
|
||||
|
||||
| 剩余预测数 | Map 行为 |
|
||||
|---|---|
|
||||
| 小于 512 | 尝试成为预测 writer；预测可能完成并发布新 buffer。竞争失败的调用走带读者计数的查找路径。 |
|
||||
| 512 到 `512 + _predict_slow_path_count` 之间 | 走带计数的慢查找路径，以便 swap 等待这些读者退出。 |
|
||||
| 大于或等于 `512 + _predict_slow_path_count` | 走无计数的直接查找快路径。 |
|
||||
|
||||
计数读者协议先等待 swap gate 打开，再增加 reader count 并复查 gate。如果 gate 在两步之间关闭，则撤销计数并重试。swap 方关闭 gate、等待已计数读者归零、交换 buffer、发布新的剩余预测数，最后重新打开 gate。
|
||||
|
||||
无计数快路径依赖预测提前量和阶段时序。这是性能启发式，本身不能证明一个被抢占的 reader 不会在旧 buffer 复用时仍继续访问。无计数 reader 与 buffer 生命周期之间的不变量，以及快慢路径边界，都仍需验证。
|
||||
|
||||
当前 `Predict()` 返回 `int`，但实现仍是返回 `0` 的空 stub，因此预测数生命周期尚未端到端运行。
|
||||
|
||||
## 6. 预测器注册、任务状态与在途缓冲
|
||||
|
||||
预测器调度和预测任务执行是两条相关但不同的生命周期，不能把“预测器已预测完”当成“它产生的工作已完成”。
|
||||
|
||||
- **待调度预测器（多个）**：保存在 PredictorMonitor 的有序管理数组/队列中，等待轮到自己。固定槽位容量（曾讨论 256）表示预测器注册上限，与缓存行计数无关。
|
||||
- **活动预测器（最多一个）**：当前 L4 正在执行其同步 `PredictMission`。预测器之间按依赖顺序切换，不需要多 lane 或 thread-local lane 归属。
|
||||
- **在途预测批次（可以多个）**：某预测器已经产出候选，但候选对应的实际任务尚未全部执行。此时要保留稳定的 predictor ID/代次、mission 或写回上下文；该记录属于任务 bag，不等同于预测器注册槽位。`PredictMission` 按预测器依赖顺序串行调用；前一批次的实际任务可以仍在途，是否与后续批次重叠由任务依赖决定。
|
||||
- **已完成批次**：对应实际工作全部完成，且所需软件写回或显式丢弃已完成。此时释放在途记录及其引用资源，不再由 L4 保存该批次。预测器本身是否注销是另一项生命周期操作。
|
||||
|
||||
Swiss 负责预测任务/映射条目的查找状态；Monitor 负责预测器注册和调度；在途预测器缓冲负责尚未完成的预测工作及其归属。三者不可混为同一个队列或数组。
|
||||
|
||||
`PredictMission(in Span<AddressSource>)` 是同步接口，L4 可传入 stackalloc span。若返回后仍需跨线程或跨 NUMA 处理其内容，必须将内容复制到拥有明确生命周期的缓冲区；不得让 stack span 逃逸。由于预测器顺序有依赖，待处理预测必须按既定顺序交接。若采用异步 worker，队列应有界且不阻塞 Map 快路径。
|
||||
|
||||
## 7. 驱逐与写回策略
|
||||
|
||||
L4 不得推断或解码预测器的 packed 格式。若丢弃一个尚未写回、且对应实际工作仍在途的预测条目会丢失必要状态，就不能静默驱逐它；必须由缓存行保护/引用状态阻止替换，或由 Monitor/预测器先完成写回、撤销或显式释放。相关状态可放在 KVP metadata 或 sidecar，布局尚未定稿。
|
||||
|
||||
owner ID 可用于路由显式写回；但若多个预测器能独立声明同一条目，单一 owner 字段不够。引用计数可保护共享条目，而写回路径仍需获得预测器 identity/source spec。
|
||||
|
||||
## 8. 当前状态与待决问题
|
||||
|
||||
**源码中已实现：**
|
||||
|
||||
- NUMA 绑定的 L4 arena 和 TLSF 包装。
|
||||
- 双缓冲 Swiss 表及原子 published pointer 交换。
|
||||
- x64 CRC32C hash 和 SSE2 控制字节扫描。
|
||||
- 固定预测触发阈值，以及按 NUMA 节点逻辑处理器数量计算的提前慢路径窗口。
|
||||
- 慢路径的 Map reader gate/counter。
|
||||
- 同步的泛型/非泛型预测器接口骨架。
|
||||
|
||||
**尚未实现或尚未验证：**
|
||||
|
||||
- PredictorMonitor 注册表容量、注册/注销和生命周期状态转换。
|
||||
- 实际预测算法与有效预测条目的生成。
|
||||
- 端到端撤销/写回归属和缓存行驱逐保护。
|
||||
- Prefetch 与 Map 跨 NUMA 重叠时的自有 queue/batch 交接方式。
|
||||
- 无计数 Map 快路径不会与 published buffer 复用重叠的证明/测试。
|
||||
- 覆盖预测器发布、Map、撤销和 Dispose 的并发压力测试。
|
||||
|
||||
## 9. L4 的四个核心操作
|
||||
|
||||
### 预测（Prefetch）
|
||||
|
||||
Monitor 按注册顺序选择预测器，调用其同步预测接口；预测器解释自己的 source spec，将预测项写入调用方提供的 span。Swiss 管理预测/映射条目。若预测项对应的实际工作尚未完成，则放入在途预测器缓冲，保留后续完成、撤销或写回所需的归属信息。预测期间若没有可用的 cache-line 槽位，替换/保护策略由后续设计决定，不能隐式丢弃仍在途的条目。
|
||||
|
||||
### 映射（Map）
|
||||
|
||||
计算 worker 按预测器调度获取 `AddressSource`，通过 Swiss 查询 NUMA 本地的 packed 数据并得到 L4 handle。Map 不解释 `Description`，也不负责预测器注册或写回。
|
||||
|
||||
### 注册预测器
|
||||
|
||||
将预测器放入 `PredictorMonitor` 的管理槽位，分配稳定 ID 和执行顺序，并纳入生命周期监视。注册表槽位与 Swiss 预测条目、在途预测工作是不同资源。
|
||||
|
||||
### 移除预测器
|
||||
|
||||
先停止向该预测器分派新 mission，再等待其在途预测对应的实际工作完成或被撤销；需要的软件写回完成后，才能释放在途状态并回收注册槽位。具体等待与写回协议尚待设计。
|
||||
@@ -1,66 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Evict"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Nitload.Common.Collections;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
///
|
||||
/// </summary>
|
||||
/// <param name="count">
|
||||
///
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// Count of lines not evicted
|
||||
/// </returns>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private unsafe int Evict(
|
||||
int count = 1
|
||||
)
|
||||
{
|
||||
count = (count < 0) ? 0 : count;
|
||||
int* pPtr = _predictCount, mPtr = _mapCount;
|
||||
|
||||
|
||||
int remain = count;
|
||||
for (int i = 0; i < CacheLineCount; i++)
|
||||
{
|
||||
if (pPtr[i] > mPtr[i])
|
||||
{
|
||||
continue;
|
||||
}
|
||||
remain -= TryEvictAt(i) ? 1 : 0;
|
||||
}
|
||||
return remain;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private unsafe bool TryEvictAt(
|
||||
int index
|
||||
)
|
||||
{
|
||||
CacheLineMetadata* head = &_head[index];
|
||||
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
|
||||
{
|
||||
// EvictLine(i + index);
|
||||
_ = SwissTable.Writer.TryRemove(_map, head->Origin, out _);
|
||||
IL4Predict predict = CurrentPredict;
|
||||
predict.WriteBack(head->Origin, head->CustomizeProfile, new ReadOnlySpan<byte>((byte*)(ulong)(head->PositionMask + _baseAddress), (int)(ulong)head->SizeMask));
|
||||
_bufferAllocator.Free(index);
|
||||
_mapCount[index] = 0;
|
||||
_predictCount[index] = int.MaxValue;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Exceptions"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
#pragma warning disable CA1032 // Implement standard exception constructors
|
||||
|
||||
public sealed class L4PredictUnregisteredException : Exception
|
||||
{
|
||||
|
||||
public L4PredictUnregisteredException(
|
||||
int id
|
||||
)
|
||||
: base($"L4 predictor with id {id} is null. Maybe it has been unregistered.") { }
|
||||
|
||||
}
|
||||
|
||||
public sealed class L4PredictConflictException : Exception
|
||||
{
|
||||
|
||||
public L4PredictConflictException(
|
||||
nuint basePtr,
|
||||
ushort policy,
|
||||
int current
|
||||
)
|
||||
: base($"L4 predictor with id {current} has already registered a cache line with base pointer {basePtr} and policy {policy}.") { }
|
||||
|
||||
}
|
||||
#pragma warning restore CA1032 // Implement standard exception constructors
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Map"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class L4
|
||||
{
|
||||
|
||||
private const int _low_predict = 512;
|
||||
|
||||
private int _predict_count;
|
||||
private readonly int _predict_slow_path_count;
|
||||
private int _swiss_reader_counter;
|
||||
private int _swiss_swap_lock;
|
||||
private int _swiss_writer_lock;
|
||||
|
||||
public partial Handle Map(
|
||||
AddressSource source
|
||||
)
|
||||
{
|
||||
ulong handle;
|
||||
int predictCount = Volatile.Read(ref _predict_count);
|
||||
if (predictCount < _low_predict)
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _swiss_writer_lock, 1, 0) == 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
int effectivePredictionCount = Predict();
|
||||
if (effectivePredictionCount > 0)
|
||||
{
|
||||
_ = Interlocked.Exchange(ref _swiss_swap_lock, 1);
|
||||
while (Volatile.Read(ref _swiss_reader_counter) != 0) { }
|
||||
|
||||
_addressMap.SwapBuffers();
|
||||
Volatile.Write(ref _predict_count, effectivePredictionCount);
|
||||
_addressMap.TryRehash();
|
||||
}
|
||||
|
||||
if (_addressMap.TryGet(source, out handle)) {
|
||||
throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}].");
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_ = Interlocked.Exchange(ref _swiss_swap_lock, 0);
|
||||
_ = Interlocked.Exchange(ref _swiss_writer_lock, 0);
|
||||
}
|
||||
} else if (!TryGetAddress(source, out handle)) {
|
||||
throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}].");
|
||||
}
|
||||
} else if (predictCount < _low_predict + _predict_slow_path_count)
|
||||
{
|
||||
if (!TryGetAddress(source, out handle)) {
|
||||
throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}].");
|
||||
}
|
||||
} else if (!_addressMap.TryGet(source, out handle)) {
|
||||
throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}].");
|
||||
}
|
||||
|
||||
CacheLineMetadata* metadata = (CacheLineMetadata*)handle;
|
||||
|
||||
// process metadata, unfinished
|
||||
uint offset = metadata->PositionMask;
|
||||
nuint address = /*base ptr + */ offset;
|
||||
_ = Interlocked.Decrement(ref _predict_count);
|
||||
|
||||
return new Handle(address, (nuint)handle);
|
||||
}
|
||||
|
||||
private bool TryGetAddress(
|
||||
AddressSource source,
|
||||
out ulong handle
|
||||
)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
while (Volatile.Read(ref _swiss_swap_lock) != 0) { }
|
||||
|
||||
_ = Interlocked.Increment(ref _swiss_reader_counter);
|
||||
if (Volatile.Read(ref _swiss_swap_lock) == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
_ = Interlocked.Decrement(ref _swiss_reader_counter);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return _addressMap.TryGet(source, out handle);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_ = Interlocked.Decrement(ref _swiss_reader_counter);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -5,10 +5,7 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Nitload.Common.Collections;
|
||||
using Nitload.Common.HighPerformance.Memory;
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
@@ -16,22 +13,12 @@ using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
public unsafe partial class L4
|
||||
{
|
||||
|
||||
private IL4Predict CurrentPredict { get; set; }
|
||||
|
||||
public void ResetPredict(
|
||||
IL4Predict predict
|
||||
)
|
||||
internal partial int Predict()
|
||||
{
|
||||
if (CurrentPredict is null)
|
||||
{
|
||||
CurrentPredict = predict;
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: write back all and register new predictor
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Predictor"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
[Flags]
|
||||
public enum PredictResult
|
||||
{
|
||||
|
||||
Success = 1,
|
||||
Exhausted = 2,
|
||||
|
||||
}
|
||||
|
||||
public interface IL4Predict
|
||||
{
|
||||
|
||||
PredictResult Predict(
|
||||
Span<CacheLineDescription> node,
|
||||
out int count
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Materializes the cache line identified by <paramref name="origin" /> into <paramref ///
|
||||
/// name="cache" />.
|
||||
/// </summary>
|
||||
/// <param name="origin">
|
||||
/// Source address associated with the cache line.
|
||||
/// </param>
|
||||
/// <param name="policy">
|
||||
/// Mapping policy associated with the cache line.
|
||||
/// </param>
|
||||
/// <param name="cache">
|
||||
/// Destination cache line buffer.
|
||||
/// </param>
|
||||
void Prefetch(
|
||||
nuint origin,
|
||||
uint policy,
|
||||
Span<byte> cache
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Writes the cache line in <paramref name="cache" /> back to <paramref name="origin" />.
|
||||
/// </summary>
|
||||
/// <param name="origin">
|
||||
/// Source address associated with the cache line.
|
||||
/// </param>
|
||||
/// <param name="policy">
|
||||
/// Mapping policy associated with the cache line.
|
||||
/// </param>
|
||||
/// <param name="cache">
|
||||
/// Source cache line buffer.
|
||||
/// </param>
|
||||
void WriteBack(
|
||||
nuint origin,
|
||||
uint policy,
|
||||
ReadOnlySpan<byte> cache
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Explicit)]
|
||||
public struct CacheLineDescription : IEquatable<CacheLineDescription>
|
||||
{
|
||||
|
||||
public override readonly bool Equals(
|
||||
object? obj
|
||||
)
|
||||
{
|
||||
return (obj is CacheLineDescription prediction) &&
|
||||
Address.Equals(prediction.Address) &&
|
||||
(MapPolicy == prediction.MapPolicy);
|
||||
}
|
||||
|
||||
public readonly bool Equals(
|
||||
CacheLineDescription other
|
||||
)
|
||||
{
|
||||
return
|
||||
(Address == other.Address) && (MapPolicy == other.MapPolicy);
|
||||
}
|
||||
|
||||
public override readonly int GetHashCode()
|
||||
{
|
||||
return (int)ComposePredictionKey64(Address, MapPolicy);
|
||||
}
|
||||
|
||||
public readonly nuint GetHashCodeLong()
|
||||
{
|
||||
return ComposePredictionKey64(Address, MapPolicy);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static nuint ComposePredictionKey64(
|
||||
nuint address,
|
||||
ushort mapPolicy
|
||||
)
|
||||
{
|
||||
nuint x = address;
|
||||
nuint y = mapPolicy;
|
||||
unchecked
|
||||
{
|
||||
nuint mixed = x ^ (y * ((nuint)0x9e3779b97f4a7c15UL));
|
||||
mixed ^= mixed >> 30;
|
||||
mixed *= (nuint)0xbf58476d1ce4e5b9UL;
|
||||
mixed ^= mixed >> 27;
|
||||
mixed *= (nuint)0x94d049bb133111ebUL;
|
||||
mixed ^= mixed >> 31;
|
||||
return mixed;
|
||||
}
|
||||
}
|
||||
|
||||
public static bool operator !=(
|
||||
CacheLineDescription left,
|
||||
CacheLineDescription right
|
||||
)
|
||||
{
|
||||
return !(left == right);
|
||||
}
|
||||
public static bool operator ==(
|
||||
CacheLineDescription left,
|
||||
CacheLineDescription right
|
||||
)
|
||||
{
|
||||
return left.Equals(right);
|
||||
}
|
||||
|
||||
#pragma warning disable CA1051
|
||||
[FieldOffset(0)]public nuint Address;
|
||||
[FieldOffset(8)]public uint Size;
|
||||
[FieldOffset(12)]public ushort MapPolicy;
|
||||
[FieldOffset(14)]internal short Predictor;
|
||||
#pragma warning restore CA1051
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.PredictorQueue"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
private readonly IL4Predictor[] _ring = new IL4Predictor[256];
|
||||
private volatile int _consume;
|
||||
private volatile int _enter;
|
||||
private volatile int _predict;
|
||||
|
||||
private int _queue_sync;
|
||||
|
||||
internal void DropPredictor(
|
||||
IL4Predictor predictor
|
||||
)
|
||||
{
|
||||
if (predictor is null) {
|
||||
return;
|
||||
}
|
||||
SpinWait sp = new();
|
||||
while (Interlocked.CompareExchange(ref _queue_sync, 1, 0) != 0) {
|
||||
sp.SpinOnce();
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
unchecked
|
||||
{
|
||||
for (int i = _consume; i < _predict; i++)
|
||||
{
|
||||
byte idx = (byte)i;
|
||||
if (ReferenceEquals(predictor, _ring[idx]))
|
||||
{
|
||||
int end = _predict;
|
||||
for (int move = i; move < end - 1; move++)
|
||||
{
|
||||
_ring[(byte)move] = _ring[(byte)(move + 1)];
|
||||
}
|
||||
_ring[(byte)(end - 1)] = default!;
|
||||
_predict = end - 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
Interlocked.Exchange(ref _queue_sync, 0);
|
||||
}
|
||||
}
|
||||
|
||||
internal void RegistPredictor(
|
||||
IL4Predictor predictor
|
||||
)
|
||||
{
|
||||
if (predictor is null) {
|
||||
return;
|
||||
}
|
||||
SpinWait sp = new();
|
||||
while (Interlocked.CompareExchange(ref _queue_sync, 1, 0) != 0) {
|
||||
sp.SpinOnce();
|
||||
}
|
||||
|
||||
WaitForSpare();
|
||||
|
||||
_consume++;
|
||||
|
||||
byte idx = (byte)_consume;
|
||||
_ring[idx] = predictor;
|
||||
}
|
||||
|
||||
private void WaitForSpare()
|
||||
{
|
||||
int spin_count = 0;
|
||||
|
||||
// wait until the queue has space
|
||||
while (_enter - _consume >= 256)
|
||||
{
|
||||
switch (spin_count)
|
||||
{
|
||||
case < 1024:
|
||||
spin_count++;
|
||||
break;
|
||||
case < 65536:
|
||||
spin_count++;
|
||||
Thread.SpinWait(1);
|
||||
break;
|
||||
default:
|
||||
Thread.Sleep(1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Prefetch"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Nitload.Common.Collections;
|
||||
using System.ComponentModel;
|
||||
using System.Numerics;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class L4
|
||||
{
|
||||
|
||||
private partial void Prefetch()
|
||||
{
|
||||
if (CurrentPredict is null) {
|
||||
return;
|
||||
}
|
||||
IL4Predict predict = CurrentPredict;
|
||||
Span<CacheLineDescription> span = new(_prediction, PredictionCount);
|
||||
PredictResult result = predict.Predict(span, out int actualCount);
|
||||
_exhausted = (result & PredictResult.Exhausted) != 0;
|
||||
if (actualCount <= 0)
|
||||
{
|
||||
_exhausted = true;
|
||||
return;
|
||||
}
|
||||
int missIndex = 0;
|
||||
for (int i = 0; i < actualCount; i++)
|
||||
{
|
||||
ref CacheLineDescription handle = ref span[i];
|
||||
nuint address = handle.Address;
|
||||
if (!SwissTable.Reader.TryGet(_map, address, out _))
|
||||
{
|
||||
// miss and record
|
||||
span[missIndex] = handle;
|
||||
missIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
// now prefetch the misses
|
||||
for (int i = 0; i < missIndex; i++)
|
||||
{
|
||||
CacheLineMetadata* head;
|
||||
ref CacheLineDescription pHandle = ref span[i];
|
||||
nuint address = pHandle.Address;
|
||||
int index;
|
||||
if (SwissTable.Reader.TryGet(_map, address, out nuint handle))
|
||||
{
|
||||
head = (CacheLineMetadata*)handle;
|
||||
index = (int)(head - _head);
|
||||
_predictCount[index] = 0; // reset eviction counter
|
||||
|
||||
#region update predictor belonging
|
||||
|
||||
if (_predictCount[index] > short.MaxValue / 2)
|
||||
{
|
||||
int predictCount = Volatile.Read(ref _predictCount[index]);
|
||||
int mapCount = Volatile.Read(ref _mapCount[index]);
|
||||
int decrement = (int.Min(predictCount, mapCount) + 1) / 2;
|
||||
_ = Interlocked.Add(ref _predictCount[index], -decrement);
|
||||
_ = Interlocked.Add(ref _mapCount[index], -decrement);
|
||||
}
|
||||
_predictCount[index]++;
|
||||
|
||||
#endregion
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
int retryCount = 0, remain = missIndex - i;
|
||||
const int maxRetry = 1024;
|
||||
while (!_bufferAllocator.TryAlloc(pHandle.Size, out index))
|
||||
{
|
||||
remain = Evict(remain);
|
||||
retryCount++;
|
||||
if (retryCount > maxRetry) {
|
||||
throw new OverflowException($"Failed to evict cache line after {maxRetry} retries.");
|
||||
}
|
||||
if (retryCount > maxRetry / 2) {
|
||||
Thread.Sleep(1);
|
||||
}
|
||||
}
|
||||
head = _head + index;
|
||||
head->Origin = address;
|
||||
head->CustomizeProfile = pHandle.MapPolicy;
|
||||
head->RefCount = 0;
|
||||
|
||||
_predictCount[index] = 1;
|
||||
_mapCount[index] = 1;
|
||||
|
||||
// pre fetch them here
|
||||
predict.Prefetch(head->Origin, head->CustomizeProfile, new Span<byte>((byte*)(head->PositionMask + _baseAddress), (int)head->SizeMask));
|
||||
_ = SwissTable.Writer.TrySet(_map, address, (nuint)head);
|
||||
}
|
||||
_ = Interlocked.Add(ref _aheadCount, actualCount);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.SwissTable"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Buffers;
|
||||
using System.Numerics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Intrinsics;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public sealed unsafe partial class L4
|
||||
{
|
||||
|
||||
private const int BucketCapacity = 16;
|
||||
private const int GroupCount = 1 << GroupCountLog;
|
||||
private const int GroupCountLog = 10;
|
||||
private const int IdealCount = GroupCount * BucketCapacity;//16384
|
||||
private const int OffsetOfBucket = IdealCount;
|
||||
|
||||
private static readonly int TableBufferSize =
|
||||
IdealCount + (GroupCount * Unsafe.SizeOf<SwissBucket>());
|
||||
internal static readonly int BufferSize = TableBufferSize * 2;
|
||||
|
||||
private enum SwissControl : byte
|
||||
{
|
||||
|
||||
Empty = 0b10000000,
|
||||
Deleted = 0b10000001
|
||||
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct AddressSource : IEquatable<AddressSource>
|
||||
{
|
||||
|
||||
public ulong Address;
|
||||
public ulong Description;
|
||||
|
||||
public readonly bool Equals(
|
||||
AddressSource other
|
||||
)
|
||||
{
|
||||
return this == other;
|
||||
}
|
||||
|
||||
public override readonly bool Equals(
|
||||
object? obj
|
||||
)
|
||||
{
|
||||
return obj is AddressSource source && Equals(source);
|
||||
}
|
||||
|
||||
public override readonly int GetHashCode()
|
||||
{
|
||||
return HashCode.Combine(Address, Description);
|
||||
}
|
||||
|
||||
public static bool operator !=(
|
||||
AddressSource left,
|
||||
AddressSource right
|
||||
)
|
||||
{
|
||||
return !(left == right);
|
||||
}
|
||||
|
||||
public static bool operator ==(
|
||||
AddressSource left,
|
||||
AddressSource right
|
||||
)
|
||||
{
|
||||
return left.Address == right.Address && left.Description == right.Description;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
[InlineArray(BucketCapacity)]
|
||||
internal struct KeyArray
|
||||
{
|
||||
|
||||
private AddressSource _element0;
|
||||
|
||||
}
|
||||
|
||||
internal struct SwissBucket
|
||||
{
|
||||
|
||||
public KeyArray Keys;
|
||||
public fixed ulong Values[BucketCapacity];
|
||||
|
||||
}
|
||||
|
||||
internal unsafe struct SwissEnumerator(
|
||||
byte* buffer
|
||||
)
|
||||
{
|
||||
|
||||
private readonly SwissBucket* _buckets = (SwissBucket*)(buffer + IdealCount);
|
||||
private readonly Vector128<byte>* _control_bytes = (Vector128<byte>*)buffer;
|
||||
private int _current_group;
|
||||
private int _current_mask;
|
||||
private int _current_offset;
|
||||
private bool _group_loaded;
|
||||
|
||||
public (AddressSource key, ulong value) Current
|
||||
{
|
||||
get
|
||||
{
|
||||
SwissBucket* bucket = &_buckets[_current_group];
|
||||
AddressSource key = bucket->Keys[_current_offset];
|
||||
ulong value = bucket->Values[_current_offset];
|
||||
return (key, value);
|
||||
}
|
||||
}
|
||||
|
||||
public bool MoveNext()
|
||||
{
|
||||
while (_current_group < GroupCount)
|
||||
{
|
||||
if (!_group_loaded)
|
||||
{
|
||||
if (Sse42.IsSupported)
|
||||
{
|
||||
_current_mask = SwissReaderX86.CompareOccupiedMask(*(_control_bytes + _current_group));
|
||||
} else
|
||||
{
|
||||
throw new PlatformNotSupportedException();
|
||||
}
|
||||
_group_loaded = true;
|
||||
}
|
||||
|
||||
if (_current_mask != 0)
|
||||
{
|
||||
_current_offset = BitOperations.TrailingZeroCount(_current_mask);
|
||||
_current_mask &= _current_mask - 1;
|
||||
return true;
|
||||
}
|
||||
|
||||
_current_group++;
|
||||
_group_loaded = false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal unsafe struct SwissTable : IDisposable
|
||||
{
|
||||
|
||||
private byte* _buffer_write;
|
||||
private int _delete_count;
|
||||
private ulong _published_buffer;
|
||||
|
||||
private SwissTable(
|
||||
byte* buffers
|
||||
)
|
||||
{
|
||||
_published_buffer = (ulong)buffers;
|
||||
_buffer_write = buffers + TableBufferSize;
|
||||
InitializeBuffer(buffers);
|
||||
InitializeBuffer(_buffer_write);
|
||||
}
|
||||
|
||||
internal byte* ReadBufferForLockFreeBenchmark => (byte*)Volatile.Read(ref _published_buffer);
|
||||
|
||||
internal readonly byte* WriteBufferForLockFreeBenchmark => _buffer_write;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_ = Interlocked.Exchange(ref _published_buffer, 0);
|
||||
_buffer_write = null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Publishes the write buffer. The caller must provide the phase barrier around this
|
||||
/// operation.
|
||||
/// </summary>
|
||||
public void SwapBuffers()
|
||||
{
|
||||
ulong previousBuffer = Interlocked.Exchange(ref _published_buffer, (ulong)_buffer_write);
|
||||
_buffer_write = (byte*)previousBuffer;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes to the inactive buffer. The caller must serialize all writes.
|
||||
/// </summary>
|
||||
public bool TryAddOrSet(
|
||||
AddressSource key,
|
||||
ulong value
|
||||
)
|
||||
{
|
||||
if (Sse42.IsSupported) {
|
||||
return SwissWriterX86.TryAddOrSet(_buffer_write, key, value, ref _delete_count);
|
||||
}
|
||||
|
||||
throw new NotImplementedException("The Swiss backend is not implemented for this ISA.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the published buffer. The caller must prevent swaps until all reads finish.
|
||||
/// </summary>
|
||||
public bool TryGet(
|
||||
AddressSource key,
|
||||
out ulong value
|
||||
)
|
||||
{
|
||||
byte* publishedBuffer = (byte*)Volatile.Read(ref _published_buffer);
|
||||
if (Sse42.IsSupported) {
|
||||
return SwissReaderX86.TryGet(publishedBuffer, key, out value);
|
||||
}
|
||||
|
||||
throw new NotImplementedException("The Swiss backend is not implemented for this ISA.");
|
||||
}
|
||||
|
||||
public void TryRehash()
|
||||
{
|
||||
if (_delete_count < 2048)
|
||||
{
|
||||
NativeMemory.Copy(_buffer_write, (byte*)_published_buffer, (nuint)TableBufferSize);
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
SwissEnumerator e = new((byte*)_published_buffer);
|
||||
InitializeBuffer(_buffer_write);
|
||||
if (Sse42.IsSupported)
|
||||
{
|
||||
while (e.MoveNext())
|
||||
{
|
||||
(AddressSource key, ulong value) = e.Current;
|
||||
if (!SwissWriterX86.TryAddOrSet(_buffer_write, key, value, ref _delete_count))
|
||||
{
|
||||
throw new InvalidOperationException("The Swiss table could not fit all active entries during rehash.");
|
||||
}
|
||||
}
|
||||
_delete_count = 0;
|
||||
} else
|
||||
{
|
||||
throw new PlatformNotSupportedException();
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
NativeMemory.Copy(_buffer_write, (byte*)_published_buffer, (nuint)TableBufferSize);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes from the inactive buffer. The caller must serialize all writes.
|
||||
/// </summary>
|
||||
public bool TryRemove(
|
||||
AddressSource key,
|
||||
out ulong value
|
||||
)
|
||||
{
|
||||
if (Sse42.IsSupported)
|
||||
{
|
||||
return SwissWriterX86.TryRemove(_buffer_write, key, out value, ref _delete_count);
|
||||
} else
|
||||
{
|
||||
throw new NotImplementedException("The Swiss backend is not implemented for this ISA.");
|
||||
}
|
||||
}
|
||||
|
||||
internal static SwissTable Create(
|
||||
byte* buffers
|
||||
)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(buffers);
|
||||
return new SwissTable(buffers);
|
||||
}
|
||||
|
||||
private static void InitializeBuffer(
|
||||
byte* buffer
|
||||
)
|
||||
{
|
||||
Unsafe.InitBlock(buffer, (byte)SwissControl.Empty, IdealCount);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Swiss_X86"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Numerics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.Intrinsics;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
private const uint CrcMagic = 0x7f1c9961;
|
||||
|
||||
internal static class SwissHashX64
|
||||
{
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static uint Compute(
|
||||
AddressSource key
|
||||
)
|
||||
{
|
||||
ulong hash = CrcMagic;
|
||||
hash = Sse42.X64.Crc32(hash, key.Address);
|
||||
hash = Sse42.X64.Crc32(hash, key.Description);
|
||||
return (uint)hash;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal readonly unsafe struct SwissWriterX86
|
||||
{
|
||||
|
||||
public static bool TryAddOrSet(
|
||||
byte* buffer,
|
||||
AddressSource key,
|
||||
ulong value,
|
||||
ref int delete_count
|
||||
)
|
||||
{
|
||||
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
|
||||
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
|
||||
|
||||
uint hash = SwissHashX64.Compute(key);
|
||||
uint begin = hash % GroupCount;
|
||||
|
||||
uint first_tomb_group = 0;
|
||||
int first_tomb_idx = -1;
|
||||
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
|
||||
|
||||
for (int i = 0; i < GroupCount; i++)
|
||||
{
|
||||
Vector128<byte>* control_bytes_ptr = control_bytes + begin;
|
||||
Vector128<byte> control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)control_bytes_ptr);
|
||||
|
||||
Vector128<byte> target_hash_v128 = Vector128.Create(target_hash);
|
||||
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
|
||||
Vector128<byte> target_tomb_v128 = Vector128.Create((byte)SwissControl.Deleted);
|
||||
|
||||
Vector128<byte> cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128);
|
||||
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
|
||||
Vector128<byte> cmp_tomb = Sse2.CompareEqual(control_bytes_v128, target_tomb_v128);
|
||||
|
||||
int hash_mask = Sse2.MoveMask(cmp_hash);
|
||||
int empty_mask = Sse2.MoveMask(cmp_empty);
|
||||
int tomb_mask = Sse2.MoveMask(cmp_tomb);
|
||||
|
||||
if (tomb_mask != 0 && first_tomb_idx < 0)
|
||||
{
|
||||
int index = BitOperations.TrailingZeroCount(tomb_mask);
|
||||
first_tomb_group = begin;
|
||||
first_tomb_idx = index;
|
||||
}
|
||||
while (hash_mask != 0)
|
||||
{
|
||||
int index = BitOperations.TrailingZeroCount(hash_mask);
|
||||
SwissBucket* bucket = &buckets[begin];
|
||||
if (bucket->Keys[index] == key)
|
||||
{
|
||||
bucket->Values[index] = value;
|
||||
return true;
|
||||
}
|
||||
hash_mask &= ~(1 << index);
|
||||
}
|
||||
while (empty_mask != 0)
|
||||
{
|
||||
int index = BitOperations.TrailingZeroCount(empty_mask);
|
||||
if (first_tomb_idx < 0)
|
||||
{
|
||||
SwissBucket* bucket = &buckets[begin];
|
||||
bucket->Keys[index] = key;
|
||||
bucket->Values[index] = value;
|
||||
((byte*)control_bytes_ptr)[index] = target_hash;
|
||||
} else
|
||||
{
|
||||
SwissBucket* bucket = &buckets[first_tomb_group];
|
||||
bucket->Keys[first_tomb_idx] = key;
|
||||
bucket->Values[first_tomb_idx] = value;
|
||||
((byte*)(control_bytes + first_tomb_group))[first_tomb_idx] = target_hash;
|
||||
delete_count--;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
begin = (begin + 1) & (GroupCount - 1);
|
||||
}
|
||||
if (first_tomb_idx >= 0)
|
||||
{
|
||||
SwissBucket* bucket = &buckets[first_tomb_group];
|
||||
bucket->Keys[first_tomb_idx] = key;
|
||||
bucket->Values[first_tomb_idx] = value;
|
||||
((byte*)(control_bytes + first_tomb_group))[first_tomb_idx] = target_hash;
|
||||
delete_count--;
|
||||
return true;
|
||||
}
|
||||
value = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
public static bool TryRemove(
|
||||
byte* buffer,
|
||||
AddressSource key,
|
||||
out ulong value,
|
||||
ref int delete_count
|
||||
)
|
||||
{
|
||||
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
|
||||
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
|
||||
uint hash = SwissHashX64.Compute(key);
|
||||
uint begin = hash % GroupCount;
|
||||
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
|
||||
|
||||
for (int i = 0; i < GroupCount; i++)
|
||||
{
|
||||
Vector128<byte> control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)(control_bytes + begin));
|
||||
|
||||
Vector128<byte> target_hash_v128 = Vector128.Create(target_hash);
|
||||
Vector128<byte> cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128);
|
||||
|
||||
int mask = Sse2.MoveMask(cmp_hash);
|
||||
|
||||
while (mask != 0)
|
||||
{
|
||||
int index = BitOperations.TrailingZeroCount(mask);
|
||||
SwissBucket* bucket = &buckets[begin];
|
||||
if (bucket->Keys[index] == key)
|
||||
{
|
||||
((byte*)(control_bytes + begin))[index] = (byte)SwissControl.Deleted;
|
||||
value = bucket->Values[index];
|
||||
delete_count++;
|
||||
return true;
|
||||
}
|
||||
mask &= ~(1 << index);
|
||||
}
|
||||
|
||||
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
|
||||
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
|
||||
int empty_mask = Sse2.MoveMask(cmp_empty);
|
||||
if (empty_mask != 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
begin = (begin + 1) & (GroupCount - 1);
|
||||
}
|
||||
value = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal readonly unsafe struct SwissReaderX86
|
||||
{
|
||||
|
||||
private static readonly Vector128<byte> _empty_mask = Vector128.Create((byte)SwissControl.Empty);
|
||||
private static readonly Vector128<byte> _tomb_mask = Vector128.Create((byte)SwissControl.Deleted);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static int CompareOccupiedMask(
|
||||
Vector128<byte> control_bytes_v128
|
||||
)
|
||||
{
|
||||
Vector128<byte> cmp_eql = Sse2.CompareEqual(control_bytes_v128, _empty_mask);
|
||||
Vector128<byte> cmp_del = Sse2.CompareEqual(control_bytes_v128, _tomb_mask);
|
||||
int specialMask = Sse2.MoveMask(Sse2.Or(cmp_eql, cmp_del));
|
||||
return (ushort)~(ushort)specialMask;
|
||||
}
|
||||
|
||||
public static bool TryGet(
|
||||
byte* buffer,
|
||||
AddressSource key,
|
||||
out ulong value
|
||||
)
|
||||
{
|
||||
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
|
||||
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
|
||||
|
||||
uint hash = SwissHashX64.Compute(key);
|
||||
uint begin = hash % GroupCount;
|
||||
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
|
||||
|
||||
for (int i = 0; i < GroupCount; i++)
|
||||
{
|
||||
Vector128<byte> control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)(control_bytes + begin));
|
||||
Vector128<byte> target_hash_v128 = Vector128.Create(target_hash);
|
||||
Vector128<byte> cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128);
|
||||
int mask = Sse2.MoveMask(cmp_hash);
|
||||
while (mask != 0)
|
||||
{
|
||||
int index = BitOperations.TrailingZeroCount(mask);
|
||||
SwissBucket* bucket = &buckets[begin];
|
||||
if (bucket->Keys[index] == key)
|
||||
{
|
||||
value = bucket->Values[index];
|
||||
return true;
|
||||
}
|
||||
mask &= ~(1 << index);
|
||||
}
|
||||
|
||||
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
|
||||
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
|
||||
if (Sse2.MoveMask(cmp_empty) != 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
begin = (begin + 1) & (GroupCount - 1);
|
||||
}
|
||||
value = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -19,20 +19,21 @@ namespace Nitload.Common.HighPerformance.Memory
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
private unsafe class TLSF
|
||||
private unsafe struct TLSF
|
||||
{
|
||||
|
||||
public const int MaxCacheLineCount = 8192;
|
||||
|
||||
public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * CacheLineMetadata.SizeOf;
|
||||
|
||||
private readonly short[] _bucket;
|
||||
public static readonly nuint RequestedAllocation =
|
||||
((nuint)MaxCacheLineCount + 1u) * CacheLineMetadata.SizeOf;
|
||||
private readonly CacheLineMetadata* _spareMemory;
|
||||
private readonly int _align;
|
||||
private readonly int _maxLevel;
|
||||
|
||||
private int _metadataEnd;
|
||||
private readonly Ptr64 _basePtr;
|
||||
|
||||
private fixed short _bucket[64];
|
||||
private readonly uint _maxAllocSize;
|
||||
private readonly uint _totalSize;
|
||||
|
||||
@@ -48,7 +49,9 @@ namespace Nitload.Common.HighPerformance.Memory
|
||||
_maxAllocSize = uint.Max(64, (~64U) & (uint.Min(maxBlockSize, CacheLineMetadata.MaxBufferSize) + 63U));
|
||||
_maxLevel = int.MaxValue;
|
||||
_maxLevel = GetLevel(_maxAllocSize);
|
||||
_bucket = new short[GetLevel(_maxAllocSize) + 2];
|
||||
|
||||
// _bucket = new short[GetLevel(_maxAllocSize) + 2];
|
||||
_bucket[63] = (short)(GetLevel(_maxAllocSize) + 2);
|
||||
_spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize);
|
||||
_metadataEnd = 0;
|
||||
|
||||
@@ -268,7 +271,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
||||
idx = candidate->NextLevel;
|
||||
}
|
||||
}
|
||||
for (int i = level + 1; i < _bucket.Length; i++)
|
||||
for (int i = level + 1; i < _bucket[63]; i++)
|
||||
{
|
||||
int idx = _bucket[i];
|
||||
if (idx != 0)
|
||||
|
||||
@@ -5,49 +5,20 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Nitload.Common.Collections;
|
||||
using Nitload.Common.HighPerformance.Memory;
|
||||
using Nitload.Common.HighPerformance.ProcessorAbstraction;
|
||||
using System.Data;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public sealed unsafe partial class L4 : IDisposable
|
||||
{
|
||||
|
||||
private const int CacheLineCount = 8192;
|
||||
private const nuint PageSize = 4096;
|
||||
private const int PredictionCount = 2048;
|
||||
private const int PredictionLeastAhead = 1024;
|
||||
|
||||
/// <summary>
|
||||
/// Array of metadata for each cache line, also used as metadata for tlsf allocator.
|
||||
/// </summary>
|
||||
private readonly CacheLineMetadata* _head;
|
||||
/// <summary>
|
||||
/// Count of cache lines mapped for each prediction, used for eviction policy.
|
||||
/// </summary>
|
||||
private readonly int* _mapCount;
|
||||
/// <summary>
|
||||
/// Count of times each cache line is predicted, used for eviction policy.
|
||||
/// </summary>
|
||||
private readonly int* _predictCount;
|
||||
/// <summary>
|
||||
/// Original memory view for prediction span.
|
||||
/// </summary>
|
||||
private readonly CacheLineDescription* _prediction;
|
||||
private bool _disposed;
|
||||
private bool _exhausted;
|
||||
|
||||
private int _aheadCount;
|
||||
|
||||
private readonly int _bindedNuma;
|
||||
private volatile int _prefetching;
|
||||
private readonly nuint _baseAddress;
|
||||
private readonly SwissTable _map;
|
||||
|
||||
private readonly TLSF _bufferAllocator;
|
||||
private SwissTable _addressMap;
|
||||
private TLSF _bufferAllocator;
|
||||
|
||||
public L4(
|
||||
int bindedNuma,
|
||||
@@ -59,130 +30,70 @@ namespace Nitload.Common.HighPerformance.Memory
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize);
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize);
|
||||
_bindedNuma = bindedNuma;
|
||||
|
||||
// init order is fallow by align requirement of each component.
|
||||
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineMetadata>()) * CacheLineCount; // 16 byte align
|
||||
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
|
||||
nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
|
||||
nuint predictionCountSize = sizeof(int) * CacheLineCount; // 16 byte align
|
||||
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
|
||||
_predict_slow_path_count = GetPredictSlowPathCount(bindedNuma);
|
||||
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1));
|
||||
ByteCapacity = normalizedBufferSize;
|
||||
|
||||
AllocSize = headSize + mapCountSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize;
|
||||
|
||||
// malloc all memory at once, and free at once in Dispose.
|
||||
// This is to avoid fragmentation and improve performance.
|
||||
nuint swissBufferOffset = normalizedBufferSize + TLSF.RequestedAllocation;
|
||||
AllocSize = swissBufferOffset + (nuint)BufferSize;
|
||||
_baseAddress = (nuint)Numa.Alloc(bindedNuma, AllocSize);
|
||||
nuint ptr = _baseAddress;
|
||||
|
||||
// init all components with the buffer, and move the buffer pointer forward.
|
||||
nuint _buffer = _baseAddress;
|
||||
_bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress);
|
||||
_buffer += normalizedBufferSize;
|
||||
_head = (CacheLineMetadata*)ptr;
|
||||
ptr += headSize;
|
||||
_prediction = (CacheLineDescription*)ptr;
|
||||
ptr += predictionSize;
|
||||
_map = SwissTable.Create(ptr, _ => { });
|
||||
ptr += mapCountSize;
|
||||
_predictCount = (int*)ptr;
|
||||
ptr += predictionCountSize;
|
||||
_mapCount = (int*)ptr;
|
||||
ptr += predictionCountSize;
|
||||
|
||||
M512* mPtr = (M512*)_head;
|
||||
for (uint i = 1; i < CacheLineCount; i++) {
|
||||
mPtr->BroadCastFrom<ulong>(0);
|
||||
}
|
||||
_addressMap = SwissTable.Create((byte*)(_baseAddress + swissBufferOffset));
|
||||
}
|
||||
|
||||
public nuint ByteCapacity { get; init; }
|
||||
|
||||
public nuint AllocSize { get; init; }
|
||||
|
||||
// public IL4Predict PrefetchPredict { get; set; }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ContinueOnce(
|
||||
Handle line
|
||||
)
|
||||
{
|
||||
if (line.Address > 0) {
|
||||
_ = Interlocked.Decrement(ref _aheadCount);
|
||||
}
|
||||
}
|
||||
|
||||
[System.Diagnostics.CodeAnalysis.SuppressMessage(
|
||||
"Design",
|
||||
"CA1063:Implement IDisposable correctly",
|
||||
Justification = """
|
||||
Only owns native memory via Numa.Free();
|
||||
no managed IDisposable fields.
|
||||
Dispose(bool) pattern is unnecessary.
|
||||
""")]
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed) {
|
||||
return;
|
||||
}
|
||||
_disposed = true;
|
||||
_map.Dispose();
|
||||
#pragma warning disable CA1816
|
||||
GC.SuppressFinalize(_bufferAllocator);
|
||||
#pragma warning restore CA1816
|
||||
|
||||
|
||||
// init order is fallow by align requirement of each component.
|
||||
_addressMap.Dispose();
|
||||
Numa.Free((void*)_baseAddress, _bindedNuma, AllocSize);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public Handle Map(
|
||||
ref CacheLineDescription prediction
|
||||
)
|
||||
public partial Handle Map(
|
||||
AddressSource source
|
||||
);
|
||||
|
||||
internal void Collect()
|
||||
{
|
||||
if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (_aheadCount < PredictionLeastAhead)
|
||||
{
|
||||
// do prefetch in work load now, sync
|
||||
Prefetch();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_prefetching = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
while (true)
|
||||
{
|
||||
if (SwissTable.Reader.TryGet(_map, prediction.GetHashCodeLong(), out nuint entryHandle))
|
||||
{
|
||||
// cache hit
|
||||
CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle;
|
||||
if (Interlocked.Increment(ref entry->RefCount) > 0)
|
||||
{
|
||||
_predictCount[entry - _head] = 0;
|
||||
_ = Interlocked.Decrement(ref _aheadCount);
|
||||
_ = Interlocked.Decrement(ref _mapCount[entry - _head]);
|
||||
nuint* hPtr = stackalloc nuint[2] { entry->PositionMask + _baseAddress, (nuint)entry };
|
||||
return *(Handle*)hPtr;
|
||||
} else
|
||||
{
|
||||
_ = Interlocked.Decrement(ref entry->RefCount);
|
||||
}
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
_bufferAllocator.Collect();
|
||||
}
|
||||
|
||||
private partial void Prefetch();
|
||||
internal void Free(
|
||||
int handle
|
||||
)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
_bufferAllocator.Free(handle);
|
||||
}
|
||||
|
||||
internal bool TryAllocate(
|
||||
uint size,
|
||||
out int handle
|
||||
)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
return _bufferAllocator.TryAlloc(size, out handle);
|
||||
}
|
||||
|
||||
internal partial int Predict();
|
||||
|
||||
private static int GetPredictSlowPathCount(int bindedNuma)
|
||||
{
|
||||
foreach (NumaNode node in Numa.AllNumaNode)
|
||||
{
|
||||
if (node.NumaNodeId == (uint)bindedNuma)
|
||||
return checked(node.LogicalCoreCount * 2);
|
||||
}
|
||||
|
||||
throw new ArgumentOutOfRangeException(nameof(bindedNuma), "No topology information exists for the bound NUMA node.");
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4Predictor"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload.Common.HighPerformance.Memory
|
||||
{
|
||||
public interface IL4Predictor
|
||||
{
|
||||
|
||||
int Id { get; set; }
|
||||
|
||||
int PredictMission(
|
||||
in Span<L4.AddressSource> RequiredBuffer
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
public interface IL4Predictor<TMission> : IL4Predictor
|
||||
{
|
||||
|
||||
bool TryGetNextMission(
|
||||
out TMission mission
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
internal class PredictorMonitor { }
|
||||
}
|
||||
@@ -75,7 +75,7 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
|
||||
NumaFreeNative(address, size, numaId);
|
||||
}
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_get_numa_each")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_get_numa_each")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
public static partial int GetNumaEach(
|
||||
@@ -86,7 +86,7 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
|
||||
Span<uint> numaNodeIds
|
||||
);
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_get_numa_overall")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_get_numa_overall")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
public static partial int GetNumaOverall(
|
||||
@@ -136,7 +136,7 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
|
||||
}
|
||||
}
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_set_thread_affinity")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_set_thread_affinity")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
internal static partial void SetThreadAffinity(
|
||||
@@ -170,7 +170,7 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
|
||||
return false;
|
||||
}
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_numa_alloc")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_numa_alloc")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static unsafe partial void* NumaAllocNative(
|
||||
@@ -178,7 +178,7 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
|
||||
int numaId
|
||||
);
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_numa_free")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_numa_free")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static unsafe partial void NumaFreeNative(
|
||||
|
||||
@@ -51,7 +51,7 @@ namespace Nitload.Threading
|
||||
|
||||
#pragma warning disable CA5393
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_set_thread_affinity")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_set_thread_affinity")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static partial void SetThreadAffinity(
|
||||
|
||||
@@ -43,7 +43,7 @@ namespace Nitload.Graphics
|
||||
|
||||
public static GLFWFunction glfwContext => _glfw;
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_load_glfw_functions")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_load_glfw_functions")]
|
||||
internal static unsafe partial void glfwLoadLibrary(
|
||||
GLFWFunction* context
|
||||
);
|
||||
|
||||
@@ -170,7 +170,7 @@ namespace Nitload.Mathematics.Numeric
|
||||
|
||||
#region C invoke
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_compare_simd")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_compare_simd")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static partial int CompareSIMDID(
|
||||
@@ -178,7 +178,7 @@ namespace Nitload.Mathematics.Numeric
|
||||
SIMDID right
|
||||
);
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_load_intrinsic_function")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_load_intrinsic_function")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static partial void LoadIntrinsicFunction(
|
||||
@@ -186,7 +186,7 @@ namespace Nitload.Mathematics.Numeric
|
||||
ulong id
|
||||
);
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_get_unite_simd")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_get_unite_simd")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static partial SIMDID UniteSIMDID(
|
||||
|
||||
@@ -155,12 +155,12 @@ namespace Nitload.Mathematics.Numeric
|
||||
}
|
||||
|
||||
#pragma warning disable CA5393
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_get_simd_level_local")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_get_simd_level_local")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
internal static partial ulong GetSimdId();
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nif_load_intrinsic_function")]
|
||||
[LibraryImport(Native.LibName, EntryPoint = "nfn_load_intrinsic_function")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static unsafe partial void LoadIntrinsicFunction(
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
CompileFlags:
|
||||
CompilationDatabase: ../..
|
||||
CompilationDatabase: .
|
||||
|
||||
@@ -3,91 +3,68 @@ project(Nitload.Native LANGUAGES C)
|
||||
|
||||
include(${CMAKE_CURRENT_LIST_DIR}/ExternalLibs.cmake OPTIONAL)
|
||||
|
||||
set(NIF_TARGET_OS "auto" CACHE STRING "Target OS identifier: auto, windows, linux, macos, ios, android, unknown")
|
||||
set_property(CACHE NIF_TARGET_OS PROPERTY STRINGS auto windows linux macos ios android unknown)
|
||||
set(NFN_TARGET_OS "auto" CACHE STRING "Target OS identifier: auto, windows, linux, macos, ios, android, unknown")
|
||||
set_property(CACHE NFN_TARGET_OS PROPERTY STRINGS auto windows linux macos ios android unknown)
|
||||
|
||||
set(NIF_TARGET_ARCH "auto" CACHE STRING "Target ISA identifier: auto, x86_64, arm64, riscv64, ppc64le")
|
||||
set_property(CACHE NIF_TARGET_ARCH PROPERTY STRINGS auto x86_64 arm64 riscv64 ppc64le)
|
||||
set(NFN_TARGET_ARCH "auto" CACHE STRING "Target ISA identifier: auto, x86_64, arm64, riscv64, ppc64le")
|
||||
set_property(CACHE NFN_TARGET_ARCH PROPERTY STRINGS auto x86_64 arm64 riscv64 ppc64le)
|
||||
|
||||
nif_resolve_target_identifiers(
|
||||
"${NIF_TARGET_OS}"
|
||||
"${NIF_TARGET_ARCH}"
|
||||
NIF_RESOLVED_OS_NAME
|
||||
NIF_RESOLVED_OS_DEFINE
|
||||
NIF_RESOLVED_ARCH_NAME
|
||||
NIF_RESOLVED_ARCH_DEFINE)
|
||||
nfn_resolve_target_identifiers(
|
||||
"${NFN_TARGET_OS}"
|
||||
"${NFN_TARGET_ARCH}"
|
||||
NFN_RESOLVED_OS_NAME
|
||||
NFN_RESOLVED_OS_DEFINE
|
||||
NFN_RESOLVED_ARCH_NAME
|
||||
NFN_RESOLVED_ARCH_DEFINE)
|
||||
|
||||
set(NIF_NATIVE_SOURCES
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/dllmain.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/load_glfw.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/load_intrinsic.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/m.bufop.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/m.solve.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/m.transpose.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/mk.fma.pack.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/mk.fma.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/mk.p_scan.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/s.factorial.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/s.hash.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/simdid_compare.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/simdid.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/sk.fma.x86.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/thread.linux.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/thread.windows.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/vector.c
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-src/vector_dot.c)
|
||||
# CONFIGURE_DEPENDS makes CMake reconfigure when files are added or removed,
|
||||
# so native sources, headers, and templates need no per-file CMake edits.
|
||||
file(GLOB_RECURSE NFN_NATIVE_SOURCES CONFIGURE_DEPENDS
|
||||
${CMAKE_CURRENT_LIST_DIR}/nfn-src/*.c)
|
||||
|
||||
set(NIF_NATIVE_HEADERS
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/glfw_function.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/nif_internal.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/nif_intrinsic.context.base.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/nif_intrinsic.context.x86.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/nif_intrinsic.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/nif_intrinsic.matkernel.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/nif_intrinsic.std.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/nif_target.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head/NitloadFrameWork.h)
|
||||
file(GLOB_RECURSE NFN_NATIVE_HEADERS CONFIGURE_DEPENDS
|
||||
${CMAKE_CURRENT_LIST_DIR}/nfn-head/*.h)
|
||||
|
||||
set(NIF_NATIVE_TEMPLATES
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-template/nif_template.fma.x86.h)
|
||||
file(GLOB_RECURSE NFN_NATIVE_TEMPLATES CONFIGURE_DEPENDS
|
||||
${CMAKE_CURRENT_LIST_DIR}/nfn-template/*.h)
|
||||
|
||||
# Nitload.Native now lives under <root>/src/Nitload.Native, so walk up two
|
||||
# levels (project dir -> src -> solution root) to reach the solution root.
|
||||
get_filename_component(NIF_NATIVE_SRC_ROOT ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
|
||||
get_filename_component(NIF_SOLUTION_ROOT ${NIF_NATIVE_SRC_ROOT} DIRECTORY)
|
||||
get_filename_component(NFN_NATIVE_SRC_ROOT ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
|
||||
get_filename_component(NFN_SOLUTION_ROOT ${NFN_NATIVE_SRC_ROOT} DIRECTORY)
|
||||
|
||||
if(NIF_RESOLVED_OS_NAME STREQUAL "windows")
|
||||
set(NIF_OS_TAG win)
|
||||
elseif(NIF_RESOLVED_OS_NAME STREQUAL "linux")
|
||||
set(NIF_OS_TAG linux)
|
||||
if(NFN_RESOLVED_OS_NAME STREQUAL "windows")
|
||||
set(NFN_OS_TAG win)
|
||||
elseif(NFN_RESOLVED_OS_NAME STREQUAL "linux")
|
||||
set(NFN_OS_TAG linux)
|
||||
else()
|
||||
set(NIF_OS_TAG unknown)
|
||||
set(NFN_OS_TAG unknown)
|
||||
endif()
|
||||
|
||||
if(NIF_RESOLVED_ARCH_NAME STREQUAL "x86_64")
|
||||
set(NIF_ARCH_TAG x64)
|
||||
if(NFN_RESOLVED_ARCH_NAME STREQUAL "x86_64")
|
||||
set(NFN_ARCH_TAG x64)
|
||||
else()
|
||||
set(NIF_ARCH_TAG ${NIF_RESOLVED_ARCH_NAME})
|
||||
set(NFN_ARCH_TAG ${NFN_RESOLVED_ARCH_NAME})
|
||||
endif()
|
||||
|
||||
set(NIF_EXPECTED_RID "" CACHE STRING "Optional managed target RID to validate before deployment")
|
||||
if(NIF_EXPECTED_RID AND NOT NIF_EXPECTED_RID STREQUAL "${NIF_OS_TAG}-${NIF_ARCH_TAG}")
|
||||
message(FATAL_ERROR "Native target ${NIF_OS_TAG}-${NIF_ARCH_TAG} does not match managed target ${NIF_EXPECTED_RID}. Use a matching compiler/toolchain and a separate build directory.")
|
||||
set(NFN_EXPECTED_RID "" CACHE STRING "Optional managed target RID to validate before deployment")
|
||||
if(NFN_EXPECTED_RID AND NOT NFN_EXPECTED_RID STREQUAL "${NFN_OS_TAG}-${NFN_ARCH_TAG}")
|
||||
message(FATAL_ERROR "Native target ${NFN_OS_TAG}-${NFN_ARCH_TAG} does not match managed target ${NFN_EXPECTED_RID}. Use a matching compiler/toolchain and a separate build directory.")
|
||||
endif()
|
||||
|
||||
set(NIF_BUILD_OUTPUT_DIR "${NIF_SOLUTION_ROOT}/bin/${NIF_OS_TAG}-${NIF_ARCH_TAG}"
|
||||
set(NFN_BUILD_OUTPUT_DIR "${NFN_SOLUTION_ROOT}/bin/${NFN_OS_TAG}-${NFN_ARCH_TAG}"
|
||||
CACHE PATH "Native artifact directory shared with managed build deployment")
|
||||
# Compile-time intermediates (native PDB / .obj side files) live inside the CMake
|
||||
# build tree (cmake_build/<preset>) so they get cleaned up with it.
|
||||
set(NIF_OBJECT_ROOT_DIR ${CMAKE_CURRENT_BINARY_DIR}/obj/${PROJECT_NAME})
|
||||
set(NIF_NATIVE_PROJECT_FILES
|
||||
${NIF_NATIVE_SOURCES}
|
||||
${NIF_NATIVE_HEADERS}
|
||||
${NIF_NATIVE_TEMPLATES})
|
||||
set(NFN_OBJECT_ROOT_DIR ${CMAKE_CURRENT_BINARY_DIR}/obj/${PROJECT_NAME})
|
||||
set(NFN_NATIVE_PROJECT_FILES
|
||||
${NFN_NATIVE_SOURCES}
|
||||
${NFN_NATIVE_HEADERS}
|
||||
${NFN_NATIVE_TEMPLATES})
|
||||
|
||||
add_library(${PROJECT_NAME} SHARED ${NIF_NATIVE_PROJECT_FILES})
|
||||
add_library(${PROJECT_NAME} SHARED ${NFN_NATIVE_PROJECT_FILES})
|
||||
|
||||
source_group(TREE ${CMAKE_CURRENT_LIST_DIR} FILES ${NIF_NATIVE_PROJECT_FILES})
|
||||
source_group(TREE ${CMAKE_CURRENT_LIST_DIR} FILES ${NFN_NATIVE_PROJECT_FILES})
|
||||
|
||||
set_target_properties(${PROJECT_NAME} PROPERTIES
|
||||
C_STANDARD 11
|
||||
@@ -98,10 +75,10 @@ set_target_properties(${PROJECT_NAME} PROPERTIES
|
||||
RELEASE_POSTFIX ""
|
||||
RELWITHDEBINFO_POSTFIX ""
|
||||
MINSIZEREL_POSTFIX ""
|
||||
RUNTIME_OUTPUT_DIRECTORY "${NIF_BUILD_OUTPUT_DIR}"
|
||||
LIBRARY_OUTPUT_DIRECTORY "${NIF_BUILD_OUTPUT_DIR}"
|
||||
ARCHIVE_OUTPUT_DIRECTORY "${NIF_BUILD_OUTPUT_DIR}"
|
||||
PDB_OUTPUT_DIRECTORY "${NIF_BUILD_OUTPUT_DIR}")
|
||||
RUNTIME_OUTPUT_DIRECTORY "${NFN_BUILD_OUTPUT_DIR}"
|
||||
LIBRARY_OUTPUT_DIRECTORY "${NFN_BUILD_OUTPUT_DIR}"
|
||||
ARCHIVE_OUTPUT_DIRECTORY "${NFN_BUILD_OUTPUT_DIR}"
|
||||
PDB_OUTPUT_DIRECTORY "${NFN_BUILD_OUTPUT_DIR}")
|
||||
|
||||
# P/Invoke loads Nitload.Native.dll on Windows, including GNU-style Clang toolchains.
|
||||
if(WIN32)
|
||||
@@ -111,22 +88,22 @@ endif()
|
||||
foreach(_config Debug Release RelWithDebInfo MinSizeRel)
|
||||
string(TOUPPER ${_config} _config_upper)
|
||||
set_target_properties(${PROJECT_NAME} PROPERTIES
|
||||
RUNTIME_OUTPUT_DIRECTORY_${_config_upper} ${NIF_BUILD_OUTPUT_DIR}
|
||||
LIBRARY_OUTPUT_DIRECTORY_${_config_upper} ${NIF_BUILD_OUTPUT_DIR}
|
||||
ARCHIVE_OUTPUT_DIRECTORY_${_config_upper} ${NIF_BUILD_OUTPUT_DIR}
|
||||
PDB_OUTPUT_DIRECTORY_${_config_upper} ${NIF_BUILD_OUTPUT_DIR}
|
||||
COMPILE_PDB_OUTPUT_DIRECTORY_${_config_upper} ${NIF_OBJECT_ROOT_DIR}/${_config}/${NIF_ARCH_TAG})
|
||||
RUNTIME_OUTPUT_DIRECTORY_${_config_upper} ${NFN_BUILD_OUTPUT_DIR}
|
||||
LIBRARY_OUTPUT_DIRECTORY_${_config_upper} ${NFN_BUILD_OUTPUT_DIR}
|
||||
ARCHIVE_OUTPUT_DIRECTORY_${_config_upper} ${NFN_BUILD_OUTPUT_DIR}
|
||||
PDB_OUTPUT_DIRECTORY_${_config_upper} ${NFN_BUILD_OUTPUT_DIR}
|
||||
COMPILE_PDB_OUTPUT_DIRECTORY_${_config_upper} ${NFN_OBJECT_ROOT_DIR}/${_config}/${NFN_ARCH_TAG})
|
||||
endforeach()
|
||||
|
||||
set_target_properties(${PROJECT_NAME} PROPERTIES
|
||||
COMPILE_PDB_OUTPUT_DIRECTORY ${NIF_OBJECT_ROOT_DIR}/$<CONFIG>/${NIF_ARCH_TAG})
|
||||
COMPILE_PDB_OUTPUT_DIRECTORY ${NFN_OBJECT_ROOT_DIR}/$<CONFIG>/${NFN_ARCH_TAG})
|
||||
|
||||
if(MSVC OR (CMAKE_C_COMPILER_ID STREQUAL "Clang" AND CMAKE_C_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC"))
|
||||
target_compile_options(${PROJECT_NAME} PRIVATE
|
||||
/Fd${NIF_OBJECT_ROOT_DIR}/$<CONFIG>/${NIF_ARCH_TAG}/)
|
||||
/Fd${NFN_OBJECT_ROOT_DIR}/$<CONFIG>/${NFN_ARCH_TAG}/)
|
||||
endif()
|
||||
|
||||
if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND NIF_RESOLVED_ARCH_NAME STREQUAL "x86_64")
|
||||
if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND NFN_RESOLVED_ARCH_NAME STREQUAL "x86_64")
|
||||
# Keep the intrinsic type system and builtin declarations visible to all TUs.
|
||||
# The runtime dispatch contract still follows T_arch/T_feature at the function level.
|
||||
if(CMAKE_C_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC")
|
||||
@@ -156,19 +133,19 @@ endif()
|
||||
|
||||
target_include_directories(${PROJECT_NAME}
|
||||
PRIVATE
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-head
|
||||
${CMAKE_CURRENT_LIST_DIR}/nif-template
|
||||
${CMAKE_CURRENT_LIST_DIR}/nfn-head
|
||||
${CMAKE_CURRENT_LIST_DIR}/nfn-template
|
||||
${CMAKE_CURRENT_LIST_DIR}/lib)
|
||||
|
||||
target_compile_definitions(${PROJECT_NAME}
|
||||
PRIVATE
|
||||
NIF_EXTERN_TARGET
|
||||
NIF_OS_TARGET=${NIF_RESOLVED_OS_DEFINE}
|
||||
NIF_ARCH_TARGET=${NIF_RESOLVED_ARCH_DEFINE})
|
||||
NFN_EXTERN_TARGET
|
||||
NFN_OS_TARGET=${NFN_RESOLVED_OS_DEFINE}
|
||||
NFN_ARCH_TARGET=${NFN_RESOLVED_ARCH_DEFINE})
|
||||
|
||||
set(NIF_NATIVE_EXTERNAL_TARGET ${PROJECT_NAME})
|
||||
set(NFN_NATIVE_EXTERNAL_TARGET ${PROJECT_NAME})
|
||||
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/lib)
|
||||
|
||||
message(STATUS "Nitload.Native target OS: ${NIF_RESOLVED_OS_NAME} (${NIF_RESOLVED_OS_DEFINE})")
|
||||
message(STATUS "Nitload.Native target ISA: ${NIF_RESOLVED_ARCH_NAME} (${NIF_RESOLVED_ARCH_DEFINE})")
|
||||
message(STATUS "Nitload.Native output dir: ${NIF_BUILD_OUTPUT_DIR}")
|
||||
message(STATUS "Nitload.Native target OS: ${NFN_RESOLVED_OS_NAME} (${NFN_RESOLVED_OS_DEFINE})")
|
||||
message(STATUS "Nitload.Native target ISA: ${NFN_RESOLVED_ARCH_NAME} (${NFN_RESOLVED_ARCH_DEFINE})")
|
||||
message(STATUS "Nitload.Native output dir: ${NFN_BUILD_OUTPUT_DIR}")
|
||||
|
||||
@@ -9,8 +9,7 @@
|
||||
"cacheVariables": {
|
||||
"CMAKE_BUILD_TYPE": "Debug",
|
||||
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
|
||||
"CMAKE_C_COMPILER": "clang",
|
||||
"CMAKE_CXX_COMPILER": "clang++"
|
||||
"CMAKE_C_COMPILER": "clang"
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -21,8 +20,7 @@
|
||||
"cacheVariables": {
|
||||
"CMAKE_BUILD_TYPE": "Release",
|
||||
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
|
||||
"CMAKE_C_COMPILER": "clang",
|
||||
"CMAKE_CXX_COMPILER": "clang++"
|
||||
"CMAKE_C_COMPILER": "clang"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
||||
@@ -1,74 +1,74 @@
|
||||
function(nif_normalize_os_identifier INPUT_VALUE OUTPUT_NAME OUTPUT_DEFINE)
|
||||
string(TOLOWER "${INPUT_VALUE}" _nif_os)
|
||||
function(nfn_normalize_os_identifier INPUT_VALUE OUTPUT_NAME OUTPUT_DEFINE)
|
||||
string(TOLOWER "${INPUT_VALUE}" _nfn_os)
|
||||
|
||||
if(_nif_os STREQUAL "auto")
|
||||
string(TOLOWER "${CMAKE_SYSTEM_NAME}" _nif_system_name)
|
||||
if(_nif_system_name STREQUAL "windows")
|
||||
set(_nif_os "windows")
|
||||
elseif(_nif_system_name STREQUAL "linux")
|
||||
set(_nif_os "linux")
|
||||
if(_nfn_os STREQUAL "auto")
|
||||
string(TOLOWER "${CMAKE_SYSTEM_NAME}" _nfn_system_name)
|
||||
if(_nfn_system_name STREQUAL "windows")
|
||||
set(_nfn_os "windows")
|
||||
elseif(_nfn_system_name STREQUAL "linux")
|
||||
set(_nfn_os "linux")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(_nif_os STREQUAL "windows")
|
||||
set(_nif_define NIF_OS_WINDOWS)
|
||||
elseif(_nif_os STREQUAL "linux")
|
||||
set(_nif_define NIF_OS_LINUX)
|
||||
if(_nfn_os STREQUAL "windows")
|
||||
set(_nfn_define NFN_OS_WINDOWS)
|
||||
elseif(_nfn_os STREQUAL "linux")
|
||||
set(_nfn_define NFN_OS_LINUX)
|
||||
else()
|
||||
message(FATAL_ERROR "Unsupported NIF_TARGET_OS='${INPUT_VALUE}'.")
|
||||
message(FATAL_ERROR "Unsupported NFN_TARGET_OS='${INPUT_VALUE}'.")
|
||||
endif()
|
||||
|
||||
set(${OUTPUT_NAME} "${_nif_os}" PARENT_SCOPE)
|
||||
set(${OUTPUT_DEFINE} "${_nif_define}" PARENT_SCOPE)
|
||||
set(${OUTPUT_NAME} "${_nfn_os}" PARENT_SCOPE)
|
||||
set(${OUTPUT_DEFINE} "${_nfn_define}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
function(nif_normalize_arch_identifier INPUT_VALUE OUTPUT_NAME OUTPUT_DEFINE)
|
||||
string(TOLOWER "${INPUT_VALUE}" _nif_arch)
|
||||
function(nfn_normalize_arch_identifier INPUT_VALUE OUTPUT_NAME OUTPUT_DEFINE)
|
||||
string(TOLOWER "${INPUT_VALUE}" _nfn_arch)
|
||||
|
||||
if(_nif_arch STREQUAL "auto")
|
||||
string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _nif_processor)
|
||||
if(_nif_processor MATCHES "^(amd64|x86_64|x64)$")
|
||||
set(_nif_arch "x86_64")
|
||||
elseif(_nif_processor MATCHES "^(arm64|aarch64)$")
|
||||
set(_nif_arch "arm64")
|
||||
elseif(_nif_processor STREQUAL "riscv64")
|
||||
set(_nif_arch "riscv64")
|
||||
elseif(_nif_processor MATCHES "^(ppc64le|powerpc64le)$")
|
||||
set(_nif_arch "ppc64le")
|
||||
if(_nfn_arch STREQUAL "auto")
|
||||
string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _nfn_processor)
|
||||
if(_nfn_processor MATCHES "^(amd64|x86_64|x64)$")
|
||||
set(_nfn_arch "x86_64")
|
||||
elseif(_nfn_processor MATCHES "^(arm64|aarch64)$")
|
||||
set(_nfn_arch "arm64")
|
||||
elseif(_nfn_processor STREQUAL "riscv64")
|
||||
set(_nfn_arch "riscv64")
|
||||
elseif(_nfn_processor MATCHES "^(ppc64le|powerpc64le)$")
|
||||
set(_nfn_arch "ppc64le")
|
||||
else()
|
||||
message(FATAL_ERROR "Unable to infer ISA from CMAKE_SYSTEM_PROCESSOR='${CMAKE_SYSTEM_PROCESSOR}'. Set NIF_TARGET_ARCH explicitly.")
|
||||
message(FATAL_ERROR "Unable to infer ISA from CMAKE_SYSTEM_PROCESSOR='${CMAKE_SYSTEM_PROCESSOR}'. Set NFN_TARGET_ARCH explicitly.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(_nif_arch STREQUAL "x86_64")
|
||||
set(_nif_define NIF_ARCH_X86_64)
|
||||
elseif(_nif_arch STREQUAL "arm64")
|
||||
set(_nif_define NIF_ARCH_ARM64)
|
||||
elseif(_nif_arch STREQUAL "riscv64")
|
||||
set(_nif_define NIF_ARCH_RISCV64)
|
||||
elseif(_nif_arch STREQUAL "ppc64le")
|
||||
set(_nif_define NIF_ARCH_PPC64le)
|
||||
if(_nfn_arch STREQUAL "x86_64")
|
||||
set(_nfn_define NFN_ARCH_X86_64)
|
||||
elseif(_nfn_arch STREQUAL "arm64")
|
||||
set(_nfn_define NFN_ARCH_ARM64)
|
||||
elseif(_nfn_arch STREQUAL "riscv64")
|
||||
set(_nfn_define NFN_ARCH_RISCV64)
|
||||
elseif(_nfn_arch STREQUAL "ppc64le")
|
||||
set(_nfn_define NFN_ARCH_PPC64le)
|
||||
else()
|
||||
message(FATAL_ERROR "Unsupported NIF_TARGET_ARCH='${INPUT_VALUE}'.")
|
||||
message(FATAL_ERROR "Unsupported NFN_TARGET_ARCH='${INPUT_VALUE}'.")
|
||||
endif()
|
||||
|
||||
set(${OUTPUT_NAME} "${_nif_arch}" PARENT_SCOPE)
|
||||
set(${OUTPUT_DEFINE} "${_nif_define}" PARENT_SCOPE)
|
||||
set(${OUTPUT_NAME} "${_nfn_arch}" PARENT_SCOPE)
|
||||
set(${OUTPUT_DEFINE} "${_nfn_define}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
function(nif_resolve_target_identifiers INPUT_OS INPUT_ARCH OUTPUT_OS_NAME OUTPUT_OS_DEFINE OUTPUT_ARCH_NAME OUTPUT_ARCH_DEFINE)
|
||||
nif_normalize_os_identifier("${INPUT_OS}" _nif_os_name _nif_os_define)
|
||||
nif_normalize_arch_identifier("${INPUT_ARCH}" _nif_arch_name _nif_arch_define)
|
||||
function(nfn_resolve_target_identifiers INPUT_OS INPUT_ARCH OUTPUT_OS_NAME OUTPUT_OS_DEFINE OUTPUT_ARCH_NAME OUTPUT_ARCH_DEFINE)
|
||||
nfn_normalize_os_identifier("${INPUT_OS}" _nfn_os_name _nfn_os_define)
|
||||
nfn_normalize_arch_identifier("${INPUT_ARCH}" _nfn_arch_name _nfn_arch_define)
|
||||
|
||||
set(${OUTPUT_OS_NAME} "${_nif_os_name}" PARENT_SCOPE)
|
||||
set(${OUTPUT_OS_DEFINE} "${_nif_os_define}" PARENT_SCOPE)
|
||||
set(${OUTPUT_ARCH_NAME} "${_nif_arch_name}" PARENT_SCOPE)
|
||||
set(${OUTPUT_ARCH_DEFINE} "${_nif_arch_define}" PARENT_SCOPE)
|
||||
set(${OUTPUT_OS_NAME} "${_nfn_os_name}" PARENT_SCOPE)
|
||||
set(${OUTPUT_OS_DEFINE} "${_nfn_os_define}" PARENT_SCOPE)
|
||||
set(${OUTPUT_ARCH_NAME} "${_nfn_arch_name}" PARENT_SCOPE)
|
||||
set(${OUTPUT_ARCH_DEFINE} "${_nfn_arch_define}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
function(nif_register_external_package TARGET PACKAGE_NAME PACKAGE_DIR)
|
||||
function(nfn_register_external_package TARGET PACKAGE_NAME PACKAGE_DIR)
|
||||
if(NOT TARGET ${TARGET})
|
||||
message(FATAL_ERROR "nif_register_external_package: target '${TARGET}' does not exist.")
|
||||
message(FATAL_ERROR "nfn_register_external_package: target '${TARGET}' does not exist.")
|
||||
endif()
|
||||
|
||||
if(EXISTS "${PACKAGE_DIR}")
|
||||
@@ -79,7 +79,7 @@ function(nif_register_external_package TARGET PACKAGE_NAME PACKAGE_DIR)
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
function(nif_link_external_binary_if_exists TARGET LIBRARY_PATH)
|
||||
function(nfn_link_external_binary_if_exists TARGET LIBRARY_PATH)
|
||||
if(EXISTS "${LIBRARY_PATH}")
|
||||
target_link_libraries(${TARGET} PRIVATE "${LIBRARY_PATH}")
|
||||
message(STATUS "Linked external binary: ${LIBRARY_PATH}")
|
||||
|
||||
@@ -111,7 +111,7 @@
|
||||
<LanguageStandard_C>stdc11</LanguageStandard_C>
|
||||
<MultiProcessorCompilation>true</MultiProcessorCompilation>
|
||||
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
|
||||
<AdditionalIncludeDirectories>$(ProjectDir)lib;$(ProjectDir)nif-head;$(ProjectDir)nif-template;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<AdditionalIncludeDirectories>$(ProjectDir)lib;$(ProjectDir)nfn-head;$(ProjectDir)nfn-template;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<!-- DLL 应该用 Windows，不是 Console -->
|
||||
@@ -173,23 +173,23 @@
|
||||
<!-- source files -->
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<ItemGroup>
|
||||
<ClCompile Include="nif-src\dllmain.c" />
|
||||
<ClCompile Include="nif-src\m.bufop.x86.c" />
|
||||
<ClCompile Include="nif-src\m.solve.x86.c" />
|
||||
<ClCompile Include="nif-src\m.transpose.x86.c" />
|
||||
<ClCompile Include="nif-src\mk.fma.x86.c" />
|
||||
<ClCompile Include="nif-src\mk.p_scan.x86.c" />
|
||||
<ClCompile Include="nif-src\s.factorial.x86.c" />
|
||||
<ClCompile Include="nif-src\sk.fma.x86.c" />
|
||||
<ClCompile Include="nif-src\s.hash.x86.c" />
|
||||
<ClCompile Include="nif-src\load_intrinsic.x86.c" />
|
||||
<ClCompile Include="nif-src\load_glfw.c" />
|
||||
<ClCompile Include="nif-src\simdid.x86.c" />
|
||||
<ClCompile Include="nif-src\simdid_compare.x86.c" />
|
||||
<ClCompile Include="nif-src\thread.linux.c" />
|
||||
<ClCompile Include="nif-src\thread.windows.c" />
|
||||
<ClCompile Include="nif-src\vector.c" />
|
||||
<ClCompile Include="nif-src\vector_dot.c" />
|
||||
<ClCompile Include="nfn-src\dllmain.c" />
|
||||
<ClCompile Include="nfn-src\m.bufop.x86.c" />
|
||||
<ClCompile Include="nfn-src\m.solve.x86.c" />
|
||||
<ClCompile Include="nfn-src\m.transpose.x86.c" />
|
||||
<ClCompile Include="nfn-src\mk.fma.x86.c" />
|
||||
<ClCompile Include="nfn-src\mk.p_scan.x86.c" />
|
||||
<ClCompile Include="nfn-src\s.factorial.x86.c" />
|
||||
<ClCompile Include="nfn-src\sk.fma.x86.c" />
|
||||
<ClCompile Include="nfn-src\s.hash.x86.c" />
|
||||
<ClCompile Include="nfn-src\load_intrinsic.x86.c" />
|
||||
<ClCompile Include="nfn-src\load_glfw.c" />
|
||||
<ClCompile Include="nfn-src\simdid.x86.c" />
|
||||
<ClCompile Include="nfn-src\simdid_compare.x86.c" />
|
||||
<ClCompile Include="nfn-src\thread.linux.c" />
|
||||
<ClCompile Include="nfn-src\thread.windows.c" />
|
||||
<ClCompile Include="nfn-src\vector.c" />
|
||||
<ClCompile Include="nfn-src\vector_dot.c" />
|
||||
</ItemGroup>
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<!-- header files -->
|
||||
@@ -198,15 +198,15 @@
|
||||
<ClInclude Include="lib\glfw\glfw3.h" />
|
||||
<ClInclude Include="lib\glfw\glfw3native.h" />
|
||||
<ClInclude Include="lib\stbi\stb_image.h" />
|
||||
<ClInclude Include="nif-head\glfw_function.h" />
|
||||
<ClInclude Include="nif-head\nif_internal.h" />
|
||||
<ClInclude Include="nif-head\nif_intrinsic.context.x86.h" />
|
||||
<ClInclude Include="nif-head\nif_intrinsic.std.h" />
|
||||
<ClInclude Include="nif-head\nif_intrinsic.h" />
|
||||
<ClInclude Include="nif-head\nif_intrinsic.matkernel.h" />
|
||||
<ClInclude Include="nif-head\nif_target.h" />
|
||||
<ClInclude Include="nif-head\NitloadFrameWork.h" />
|
||||
<ClInclude Include="nif-template\nif_template.fma.x86.h" />
|
||||
<ClInclude Include="nfn-head\glfw_function.h" />
|
||||
<ClInclude Include="nfn-head\nfn_internal.h" />
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.context.x86.h" />
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.std.h" />
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.h" />
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.matkernel.h" />
|
||||
<ClInclude Include="nfn-head\nfn_target.h" />
|
||||
<ClInclude Include="nfn-head\NitloadFrameWork.h" />
|
||||
<ClInclude Include="nfn-template\nfn_template.fma.x86.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="naming.md" />
|
||||
|
||||
@@ -21,69 +21,69 @@
|
||||
</Filter>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="nif-src\dllmain.c">
|
||||
<ClCompile Include="nfn-src\dllmain.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\s.factorial.x86.c">
|
||||
<ClCompile Include="nfn-src\s.factorial.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\s.hash.x86.c">
|
||||
<ClCompile Include="nfn-src\s.hash.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\vector.c">
|
||||
<ClCompile Include="nfn-src\vector.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\vector_dot.c">
|
||||
<ClCompile Include="nfn-src\vector_dot.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\load_glfw.c">
|
||||
<ClCompile Include="nfn-src\load_glfw.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\sk.fma.x86.c">
|
||||
<ClCompile Include="nfn-src\sk.fma.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\load_intrinsic.x86.c">
|
||||
<ClCompile Include="nfn-src\load_intrinsic.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\m.bufop.x86.c">
|
||||
<ClCompile Include="nfn-src\m.bufop.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\m.solve.x86.c">
|
||||
<ClCompile Include="nfn-src\m.solve.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\simdid.x86.c">
|
||||
<ClCompile Include="nfn-src\simdid.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\mk.fma.x86.c">
|
||||
<ClCompile Include="nfn-src\mk.fma.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\simdid_compare.x86.c">
|
||||
<ClCompile Include="nfn-src\simdid_compare.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\thread.linux.c">
|
||||
<ClCompile Include="nfn-src\thread.linux.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\thread.windows.c">
|
||||
<ClCompile Include="nfn-src\thread.windows.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\m.transpose.x86.c">
|
||||
<ClCompile Include="nfn-src\m.transpose.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="nif-src\mk.p_scan.x86.c">
|
||||
<ClCompile Include="nfn-src\mk.p_scan.x86.c">
|
||||
<Filter>源文件</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="nif-head\nif_internal.h">
|
||||
<ClInclude Include="nfn-head\nfn_internal.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-head\NitloadFrameWork.h">
|
||||
<ClInclude Include="nfn-head\NitloadFrameWork.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-head\glfw_function.h">
|
||||
<ClInclude Include="nfn-head\glfw_function.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-head\nif_intrinsic.matkernel.h">
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.matkernel.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="lib\glfw\glfw3.h">
|
||||
@@ -95,19 +95,19 @@
|
||||
<ClInclude Include="lib\stbi\stb_image.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-head\nif_target.h">
|
||||
<ClInclude Include="nfn-head\nfn_target.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-head\nif_intrinsic.context.x86.h">
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.context.x86.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-head\nif_intrinsic.h">
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-head\nif_intrinsic.std.h">
|
||||
<ClInclude Include="nfn-head\nfn_intrinsic.std.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="nif-template\nif_template.fma.x86.h">
|
||||
<ClInclude Include="nfn-template\nfn_template.fma.x86.h">
|
||||
<Filter>模板文件</Filter>
|
||||
</ClInclude>
|
||||
</ItemGroup>
|
||||
|
||||
@@ -1,5 +1,18 @@
|
||||
# Nitload.Native
|
||||
|
||||
## Naming convention
|
||||
|
||||
The C implementation uses `nfn` for Nitload Framework Native: exported functions use
|
||||
`nfn_`, macros and CMake cache options use `NFN_`, and native source folders and
|
||||
headers use `nfn-` and `nfn_`. The shared library remains named `Nitload.Native`.
|
||||
The longer prefix also distinguishes these names from Linux Netfilter's `nf_` names.
|
||||
|
||||
This migration renames the exported C symbols from `nif_*` to `nfn_*`. Native
|
||||
binaries and managed assemblies must therefore be rebuilt and deployed together;
|
||||
pre-migration P/Invoke entry points are not compatible with a newly built library.
|
||||
The old `NIF_*` CMake cache options are no longer used. Reconfigure an existing
|
||||
build tree with the corresponding `NFN_*` options, or use a fresh build tree.
|
||||
|
||||
## COMPILE
|
||||
|
||||
Though this project do has a .vcxproj file,
|
||||
@@ -11,15 +24,21 @@ Currently you can only compile this lib via Clang and cmake.
|
||||
From the repository root:
|
||||
|
||||
```powershell
|
||||
# On Windows, add your LLVM bin directory if clang is not already on PATH.
|
||||
$env:PATH = 'C:\Program Files\LLVM\bin;' + $env:PATH
|
||||
cmake --preset clang-debug -S src/Nitload.Native
|
||||
cmake --build cmake_build/clang-debug --target Nitload.Native
|
||||
```
|
||||
|
||||
The presets require `clang` on `PATH`. If an existing CMake cache was configured
|
||||
with an absolute compiler path, make `clang` available on `PATH` before running
|
||||
the preset again. The `NFN_*` options replace the former `NIF_*` options.
|
||||
|
||||
CMake writes `Nitload.Native.dll` and its linker PDB to `bin/win-x64` on Windows x64
|
||||
(Linux uses `bin/linux-x64/libNitload.Native.so`). Debug and release presets currently
|
||||
share this platform directory; the most recent native build is the deployed version.
|
||||
`NIF_BUILD_OUTPUT_DIR` can override the destination. Managed builds pass their expected
|
||||
RID as `NIF_EXPECTED_RID` so an incompatible native target is rejected before deployment.
|
||||
`NFN_BUILD_OUTPUT_DIR` can override the destination. Managed builds pass their expected
|
||||
RID as `NFN_EXPECTED_RID` so an incompatible native target is rejected before deployment.
|
||||
|
||||
`StgSharpDebug.csproj` enables `BuildNativeBeforeManaged` and `RequireNativeLibrary`.
|
||||
Its build runs the incremental CMake build (`clang-debug` for Debug, `clang-release`
|
||||
@@ -32,6 +51,15 @@ Clang/Ninja environment and the local GLFW static library are required for nativ
|
||||
Cross-target builds require a matching toolchain/preset; changing a managed platform alone
|
||||
does not cross-compile the C library.
|
||||
|
||||
## clangd in VS Code
|
||||
|
||||
The `.clangd` file searches for `compile_commands.json` in `src/Nitload.Native`.
|
||||
When using CMake Tools, set `cmake.copyCompileCommands` to
|
||||
`${workspaceFolder}/src/Nitload.Native/compile_commands.json` so the active Debug or Release preset
|
||||
updates that file after configuration. For command-line configuration, copy
|
||||
`cmake_build/<preset>/compile_commands.json` to `src/Nitload.Native`. Restart
|
||||
clangd after changing the database or renaming headers.
|
||||
|
||||
## Code Layout
|
||||
|
||||
Conventions:
|
||||
|
||||
@@ -217,7 +217,7 @@ Detection:
|
||||
- **Unknown hybrid** (flag set but core type unrecognised): leave topology at 00.
|
||||
- **SIMD_DUAL**: set when any of the following: `AVX Feature DUAL=1` (glued YMM), or `AVX512_IMPL=DUAL`.
|
||||
|
||||
`nif_get_unite_simd` computes topology bits [1..0] with bitwise **AND** (conservative intersection — weaker topology wins) and SIMD_DUAL with bitwise **OR** (dual if either core is dual). `nif_compare_simd` compares topology value numerically as step 6 and SIMD_DUAL as the final (step 7) tie-break.
|
||||
`nfn_get_unite_simd` computes topology bits [1..0] with bitwise **AND** (conservative intersection — weaker topology wins) and SIMD_DUAL with bitwise **OR** (dual if either core is dual). `nfn_compare_simd` compares topology value numerically as step 6 and SIMD_DUAL as the final (step 7) tie-break.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
|
||||
if(NOT DEFINED NIF_NATIVE_EXTERNAL_TARGET)
|
||||
message(FATAL_ERROR "NIF_NATIVE_EXTERNAL_TARGET must be set before adding lib/.")
|
||||
if(NOT DEFINED NFN_NATIVE_EXTERNAL_TARGET)
|
||||
message(FATAL_ERROR "NFN_NATIVE_EXTERNAL_TARGET must be set before adding lib/.")
|
||||
endif()
|
||||
|
||||
include(${CMAKE_CURRENT_LIST_DIR}/../ExternalLibs.cmake OPTIONAL)
|
||||
@@ -17,6 +17,6 @@ foreach(_pkg ${LIB_PACKAGE_DIRS})
|
||||
if(EXISTS ${_pkg_dir}/CMakeLists.txt)
|
||||
add_subdirectory(${_pkg_dir})
|
||||
else()
|
||||
nif_register_external_package(${NIF_NATIVE_EXTERNAL_TARGET} ${_pkg} ${_pkg_dir})
|
||||
nfn_register_external_package(${NFN_NATIVE_EXTERNAL_TARGET} ${_pkg} ${_pkg_dir})
|
||||
endif()
|
||||
endforeach()
|
||||
@@ -1,11 +1,11 @@
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
|
||||
nif_register_external_package(${NIF_NATIVE_EXTERNAL_TARGET} glfw ${CMAKE_CURRENT_LIST_DIR})
|
||||
nfn_register_external_package(${NFN_NATIVE_EXTERNAL_TARGET} glfw ${CMAKE_CURRENT_LIST_DIR})
|
||||
|
||||
if(WIN32)
|
||||
if(NOT EXISTS ${CMAKE_CURRENT_LIST_DIR}/glfw3_mt.lib)
|
||||
message(FATAL_ERROR "Required GLFW static library not found: ${CMAKE_CURRENT_LIST_DIR}/glfw3_mt.lib")
|
||||
endif()
|
||||
|
||||
nif_link_external_binary_if_exists(${NIF_NATIVE_EXTERNAL_TARGET} ${CMAKE_CURRENT_LIST_DIR}/glfw3_mt.lib)
|
||||
nfn_link_external_binary_if_exists(${NFN_NATIVE_EXTERNAL_TARGET} ${CMAKE_CURRENT_LIST_DIR}/glfw3_mt.lib)
|
||||
endif()
|
||||
@@ -207,7 +207,7 @@ GLFWAPI const char *glfwGetWin32Monitor(GLFWmonitor *monitor);
|
||||
* GLFW_PLATFORM_UNAVAILABLE.
|
||||
*
|
||||
* @remark The `HDC` associated with the window can be queried with the
|
||||
* [GetDC](https://docs.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-getdc)
|
||||
* [GetDC](https://docs.microsoft.com/en-us/windows/win32/api/winuser/nfn-winuser-getdc)
|
||||
* function.
|
||||
* @code
|
||||
* HDC dc = GetDC(glfwGetWin32Window(window));
|
||||
@@ -234,7 +234,7 @@ GLFWAPI HWND glfwGetWin32Window(GLFWwindow *window);
|
||||
* GLFW_PLATFORM_UNAVAILABLE and @ref GLFW_NO_WINDOW_CONTEXT.
|
||||
*
|
||||
* @remark The `HDC` associated with the window can be queried with the
|
||||
* [GetDC](https://docs.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-getdc)
|
||||
* [GetDC](https://docs.microsoft.com/en-us/windows/win32/api/winuser/nfn-winuser-getdc)
|
||||
* function.
|
||||
* @code
|
||||
* HDC dc = GetDC(glfwGetWin32Window(window));
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
|
||||
nif_register_external_package(${NIF_NATIVE_EXTERNAL_TARGET} stbi ${CMAKE_CURRENT_LIST_DIR})
|
||||
nfn_register_external_package(${NFN_NATIVE_EXTERNAL_TARGET} stbi ${CMAKE_CURRENT_LIST_DIR})
|
||||
@@ -0,0 +1,37 @@
|
||||
#ifndef NFN_NATIVE_H
|
||||
#define NFN_NATIVE_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
#error "This header is not meant to be included in C++ code. Nitload.Native is a C library and should be used with C linkage."
|
||||
#endif
|
||||
|
||||
#include "nfn_internal.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <assert.h>
|
||||
#include <glfw_function.h>
|
||||
|
||||
#define byte char
|
||||
#define cs_char uint16_t
|
||||
#define nfn_null_assert(ptr) assert(ptr != NULL)
|
||||
|
||||
#ifndef NFN_
|
||||
#endif
|
||||
|
||||
NFN_API void NFN_DECL nfn_load_glfw_functions(GLFWFunctionTable *table);
|
||||
NFN_API void NFN_DECL nfn_load_intrinsic_function(void *intrinsic_context, uint64_t id);
|
||||
NFN_API SIMDID NFN_DECL nfn_get_simd_level_global();
|
||||
/* Per-core variant: always re-executes CPUID, never caches.
|
||||
* Must be called on the target core (after affinity is set). */
|
||||
NFN_API SIMDID NFN_DECL nfn_get_simd_level_local();
|
||||
NFN_API SIMDID NFN_DECL nfn_get_unite_simd(SIMDID left, SIMDID right);
|
||||
NFN_API int NFN_DECL nfn_compare_simd(SIMDID left, SIMDID right);
|
||||
NFN_API void NFN_DECL nfn_set_thread_affinity(int cpu_index, int processor_group_id);
|
||||
NFN_API void *NFN_DECL nfn_numa_alloc(size_t size, int numa_id);
|
||||
NFN_API void NFN_DECL nfn_numa_free(void *ptr, size_t size, int numa_id);
|
||||
NFN_API int NFN_DECL nfn_get_numa_each(int numa_id, uint32_t *processor_group_ids,
|
||||
uint32_t *socket_ids, uint32_t *cpu_indices,
|
||||
uint32_t *numa_node_ids);
|
||||
NFN_API int NFN_DECL nfn_get_numa_overall(int *out_numa_count, int *count_of_each_numa);
|
||||
#endif
|
||||
+14
-14
@@ -1,6 +1,6 @@
|
||||
|
||||
#ifndef NIF_INTERNAL
|
||||
#define NIF_INTERNAL
|
||||
#ifndef NFN_INTERNAL
|
||||
#define NFN_INTERNAL
|
||||
|
||||
#include <xmmintrin.h>
|
||||
|
||||
@@ -9,33 +9,33 @@
|
||||
#ifdef _MSC_VER
|
||||
#define _CRT_SECURE_NO_WARNINGS 1
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#define NIF_DECL __cdecl
|
||||
#define NIF_MAT_DECL __cdecl __forceinline
|
||||
#define NIF_API __declspec(dllexport)
|
||||
#define NFN_DECL __cdecl
|
||||
#define NFN_MAT_DECL __cdecl __forceinline
|
||||
#define NFN_API __declspec(dllexport)
|
||||
#define FORCEINLINE __forceinline
|
||||
#define NOINLINE __declspec(noinline)
|
||||
#include <windows.h>
|
||||
#elif defined(__clang__) || defined(__GNUC__)
|
||||
#define NIF_DECL __attribute__((cdecl))
|
||||
#define NIF_MAT_DECL __attribute__((cdecl)) __attribute__((always_inline))
|
||||
#define NIF_API __attribute__((dllexport))
|
||||
#define NFN_DECL __attribute__((cdecl))
|
||||
#define NFN_MAT_DECL __attribute__((cdecl)) __attribute__((always_inline))
|
||||
#define NFN_API __attribute__((dllexport))
|
||||
#define FORCEINLINE __attribute__((always_inline))
|
||||
#define NOINLINE __attribute__((noinline))
|
||||
#else
|
||||
#define NIF_API
|
||||
#define NIF_DECL
|
||||
#define NIF_MAT_DECL
|
||||
#define NFN_API
|
||||
#define NFN_DECL
|
||||
#define NFN_MAT_DECL
|
||||
#define FORCEINLINE
|
||||
#define NOINLINE
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#define NIF_ALIGNAS(n) __declspec(align(n))
|
||||
#define NFN_ALIGNAS(n) __declspec(align(n))
|
||||
#else
|
||||
#define NIF_ALIGNAS(n) __attribute__((aligned(n)))
|
||||
#define NFN_ALIGNAS(n) __attribute__((aligned(n)))
|
||||
#endif
|
||||
|
||||
#define NIF_APIDEF
|
||||
#define NFN_APIDEF
|
||||
|
||||
#define PRIVATE static
|
||||
#define INTERNAL extern
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
#ifndef NIF_INTRINSIC_CONTEXT_BASE
|
||||
#define NIF_INTRINSIC_CONTEXT_BASE
|
||||
#ifndef NFN_INTRINSIC_CONTEXT_BASE
|
||||
#define NFN_INTRINSIC_CONTEXT_BASE
|
||||
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
+107
-107
@@ -1,20 +1,20 @@
|
||||
#ifndef NIF_INTRINSIC_CONTEXT_X86_H
|
||||
#define NIF_INTRINSIC_CONTEXT_X86_H
|
||||
#ifndef NFN_INTRINSIC_CONTEXT_X86_H
|
||||
#define NFN_INTRINSIC_CONTEXT_X86_H
|
||||
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nif_intrinsic.context.base.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.context.base.h"
|
||||
|
||||
#ifndef NIF_X86_TARGET_ATTR_sse
|
||||
#define NIF_STD_TARGET_ATTR_IMPL(T_arch, T_feature) NIF_X86_TARGET_ATTR_##T_arch##T_feature
|
||||
#define NIF_X86_TARGET_ATTR_sse NIF_CLANG_TARGET_ATTR("sse4.2")
|
||||
#define NIF_X86_TARGET_ATTR_avx NIF_CLANG_TARGET_ATTR("avx")
|
||||
#define NIF_X86_TARGET_ATTR_avx_fma NIF_CLANG_TARGET_ATTR("avx,avx2,fma")
|
||||
#define NIF_X86_TARGET_ATTR_512 NIF_CLANG_TARGET_ATTR("avx512f,avx512vl")
|
||||
#ifndef NFN_X86_TARGET_ATTR_sse
|
||||
#define NFN_STD_TARGET_ATTR_IMPL(T_arch, T_feature) NFN_X86_TARGET_ATTR_##T_arch##T_feature
|
||||
#define NFN_X86_TARGET_ATTR_sse NFN_CLANG_TARGET_ATTR("sse4.2")
|
||||
#define NFN_X86_TARGET_ATTR_avx NFN_CLANG_TARGET_ATTR("avx")
|
||||
#define NFN_X86_TARGET_ATTR_avx_fma NFN_CLANG_TARGET_ATTR("avx,avx2,fma")
|
||||
#define NFN_X86_TARGET_ATTR_512 NFN_CLANG_TARGET_ATTR("avx512f,avx512vl")
|
||||
#endif
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
@@ -111,45 +111,45 @@
|
||||
|
||||
INTERNAL void f32_normalize(VEC_SEGMENT(float) * source, VEC_SEGMENT(float) * target);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , k_quality);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , k_quality);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , add);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , add);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , add);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , add);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , add);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , add);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , add);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , add);
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , add);
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , add);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , add);
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , add);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , sub);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , sub);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , sub);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , sub);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , sub);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , sub);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , sub);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , sub);
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , sub);
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , sub);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , sub);
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , sub);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , transpose);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , transpose);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , transpose);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , transpose);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , transpose);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , transpose);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , scalar_mul);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , scalar_mul);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , scalar_mul);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , scalar_mul);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , scalar_mul);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , scalar_mul);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , scalar_mul);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , scalar_mul);
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , scalar_mul);
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , scalar_mul);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , scalar_mul);
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , scalar_mul);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fill);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fill);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fill);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fill);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fill);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fill);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , fill);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , fill);
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , fill);
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , fill);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , fill);
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , fill);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , quality);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , quality);
|
||||
|
||||
/*
|
||||
* pack left matrix data to L4 buffer
|
||||
@@ -166,9 +166,9 @@ NIF_MK_PROC_DECL_STD(float, sse, , quality);
|
||||
* scalar pack carries column length and offset data:
|
||||
* [column length, empty, empty, empty]
|
||||
*/
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fma_pack_left);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fma_pack_left);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fma_pack_left);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fma_pack_left);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fma_pack_left);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fma_pack_left);
|
||||
|
||||
/*
|
||||
* pack right matrix data to L4 buffer
|
||||
@@ -185,104 +185,104 @@ NIF_MK_PROC_DECL_STD(float, 512, , fma_pack_left);
|
||||
* scalar pack carries column length and offset data:
|
||||
* [column length, empty, empty, empty]
|
||||
*/
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fma_pack_right);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fma_pack_right);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fma_pack_right);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fma_pack_right);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fma_pack_right);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fma_pack_right);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fma);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fma);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, _fma, fma);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fma);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fma);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fma);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, _fma, fma);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fma);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fma_unpack);
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fma_unpack);
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fma_unpack);
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fma_unpack);
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fma_unpack);
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fma_unpack);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , fma_pack_left);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , fma_pack_left);
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , fma_pack_left);
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , fma_pack_left);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , fma_pack_left);
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , fma_pack_left);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , fma);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , fma);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, _fma, fma);
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , fma);
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , fma);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , fma);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, _fma, fma);
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , fma);
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , fma_unpack);
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , fma_unpack);
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , fma_unpack);
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , fma_unpack);
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , fma_unpack);
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , fma_unpack);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, avx, , add);
|
||||
NIF_SK_PROC_DECL_STD(float, 512, , add);
|
||||
NFN_SK_PROC_DECL_STD(float, avx, , add);
|
||||
NFN_SK_PROC_DECL_STD(float, 512, , add);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, sse, , add);
|
||||
NIF_SK_PROC_DECL_STD(double, avx, , add);
|
||||
NIF_SK_PROC_DECL_STD(double, 512, , add);
|
||||
NFN_SK_PROC_DECL_STD(double, sse, , add);
|
||||
NFN_SK_PROC_DECL_STD(double, avx, , add);
|
||||
NFN_SK_PROC_DECL_STD(double, 512, , add);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, sse, , sub);
|
||||
NIF_SK_PROC_DECL_STD(float, avx, , sub);
|
||||
NIF_SK_PROC_DECL_STD(float, 512, , sub);
|
||||
NFN_SK_PROC_DECL_STD(float, sse, , sub);
|
||||
NFN_SK_PROC_DECL_STD(float, avx, , sub);
|
||||
NFN_SK_PROC_DECL_STD(float, 512, , sub);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, sse, , sub);
|
||||
NIF_SK_PROC_DECL_STD(double, avx, , sub);
|
||||
NIF_SK_PROC_DECL_STD(double, 512, , sub);
|
||||
NFN_SK_PROC_DECL_STD(double, sse, , sub);
|
||||
NFN_SK_PROC_DECL_STD(double, avx, , sub);
|
||||
NFN_SK_PROC_DECL_STD(double, 512, , sub);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, sse, , transpose);
|
||||
NIF_SK_PROC_DECL_STD(float, avx, , transpose);
|
||||
NIF_SK_PROC_DECL_STD(float, 512, , transpose);
|
||||
NFN_SK_PROC_DECL_STD(float, sse, , transpose);
|
||||
NFN_SK_PROC_DECL_STD(float, avx, , transpose);
|
||||
NFN_SK_PROC_DECL_STD(float, 512, , transpose);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, sse, , scalar_mul);
|
||||
NIF_SK_PROC_DECL_STD(float, avx, , scalar_mul);
|
||||
NIF_SK_PROC_DECL_STD(float, 512, , scalar_mul);
|
||||
NFN_SK_PROC_DECL_STD(float, sse, , scalar_mul);
|
||||
NFN_SK_PROC_DECL_STD(float, avx, , scalar_mul);
|
||||
NFN_SK_PROC_DECL_STD(float, 512, , scalar_mul);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, sse, , scalar_mul);
|
||||
NIF_SK_PROC_DECL_STD(double, avx, , scalar_mul);
|
||||
NIF_SK_PROC_DECL_STD(double, 512, , scalar_mul);
|
||||
NFN_SK_PROC_DECL_STD(double, sse, , scalar_mul);
|
||||
NFN_SK_PROC_DECL_STD(double, avx, , scalar_mul);
|
||||
NFN_SK_PROC_DECL_STD(double, 512, , scalar_mul);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, sse, , fill);
|
||||
NIF_SK_PROC_DECL_STD(float, avx, , fill);
|
||||
NIF_SK_PROC_DECL_STD(float, 512, , fill);
|
||||
NFN_SK_PROC_DECL_STD(float, sse, , fill);
|
||||
NFN_SK_PROC_DECL_STD(float, avx, , fill);
|
||||
NFN_SK_PROC_DECL_STD(float, 512, , fill);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, sse, , fill);
|
||||
NIF_SK_PROC_DECL_STD(double, avx, , fill);
|
||||
NIF_SK_PROC_DECL_STD(double, 512, , fill);
|
||||
NFN_SK_PROC_DECL_STD(double, sse, , fill);
|
||||
NFN_SK_PROC_DECL_STD(double, avx, , fill);
|
||||
NFN_SK_PROC_DECL_STD(double, 512, , fill);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, sse, , quality);
|
||||
NFN_SK_PROC_DECL_STD(float, sse, , quality);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, sse, , fma);
|
||||
NIF_SK_PROC_DECL_STD(float, avx, , fma);
|
||||
NIF_SK_PROC_DECL_STD(float, avx, _fma, fma);
|
||||
NIF_SK_PROC_DECL_STD(float, 512, , fma);
|
||||
NFN_SK_PROC_DECL_STD(float, sse, , fma);
|
||||
NFN_SK_PROC_DECL_STD(float, avx, , fma);
|
||||
NFN_SK_PROC_DECL_STD(float, avx, _fma, fma);
|
||||
NFN_SK_PROC_DECL_STD(float, 512, , fma);
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, sse, , fma);
|
||||
NIF_SK_PROC_DECL_STD(double, avx, , fma);
|
||||
NIF_SK_PROC_DECL_STD(double, avx, _fma, fma);
|
||||
NIF_SK_PROC_DECL_STD(double, 512, , fma);
|
||||
NFN_SK_PROC_DECL_STD(double, sse, , fma);
|
||||
NFN_SK_PROC_DECL_STD(double, avx, , fma);
|
||||
NFN_SK_PROC_DECL_STD(double, avx, _fma, fma);
|
||||
NFN_SK_PROC_DECL_STD(double, 512, , fma);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region scaler_simd
|
||||
|
||||
INTERNAL uint64_t NIF_DECL factorial_simd_sse(int n);
|
||||
INTERNAL uint64_t NFN_DECL factorial_simd_sse(int n);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region dot_product
|
||||
|
||||
INTERNAL void FORCEINLINE NIF_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
|
||||
INTERNAL void FORCEINLINE NFN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
|
||||
__m128 const *vector, __m128 *ans);
|
||||
INTERNAL void NIF_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
INTERNAL void NFN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans);
|
||||
INTERNAL void NIF_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
INTERNAL void NFN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans);
|
||||
INTERNAL void NIF_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
INTERNAL void NFN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region string
|
||||
|
||||
INTERNAL int NIF_DECL city_hash_simplify_sse(byte const *str, int length);
|
||||
INTERNAL int NIF_DECL index_pair_sse(short const *stram, uint32_t target, int length);
|
||||
INTERNAL int NFN_DECL city_hash_simplify_sse(byte const *str, int length);
|
||||
INTERNAL int NFN_DECL index_pair_sse(short const *stram, uint32_t target, int length);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
+24
-24
@@ -4,11 +4,11 @@
|
||||
#define SSC_INTRIN_HEAD
|
||||
|
||||
#include "NitloadFrameWork.h"
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
#include <xmmintrin.h>
|
||||
#include <immintrin.h>
|
||||
#include <smmintrin.h>
|
||||
#include "nif_intrinsic.matkernel.h"
|
||||
#include "nfn_intrinsic.matkernel.h"
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef _MSC_VER
|
||||
@@ -28,7 +28,7 @@
|
||||
((LASTTWO(s3) << 6) | (LASTTWO(s2) << 4) | (LASTTWO(s1) << 2) | (LASTTWO(s0)))
|
||||
#define BIT_CVRT(pack, index, T) *(T *)(pack->data + index)
|
||||
|
||||
#if NIF_IS_ARCH(x86_64)
|
||||
#if NFN_IS_ARCH(x86_64)
|
||||
|
||||
#define vec_128 __m128
|
||||
#define vec_256 __m256
|
||||
@@ -66,7 +66,7 @@ typedef union uint64_u {
|
||||
typedef struct mat_kernel mat_kernel;
|
||||
|
||||
typedef union M128{
|
||||
#if NIF_IS_ARCH(x86_64)
|
||||
#if NFN_IS_ARCH(x86_64)
|
||||
__m128 m128;
|
||||
#endif
|
||||
float f32[4];
|
||||
@@ -76,7 +76,7 @@ typedef union M128{
|
||||
}M128;
|
||||
|
||||
typedef union M256{
|
||||
#if NIF_IS_ARCH(x86_64)
|
||||
#if NFN_IS_ARCH(x86_64)
|
||||
__m256 m256;
|
||||
#endif
|
||||
float f32[8];
|
||||
@@ -86,7 +86,7 @@ typedef union M256{
|
||||
}M256;
|
||||
|
||||
typedef union M512{
|
||||
#if NIF_IS_ARCH(x86_64)
|
||||
#if NFN_IS_ARCH(x86_64)
|
||||
__m512 m512;
|
||||
#endif
|
||||
uint64_t u64[8];
|
||||
@@ -97,24 +97,24 @@ typedef union M512{
|
||||
int64_t i64[8];
|
||||
}M512;
|
||||
|
||||
typedef struct nif_L4_cache_head {
|
||||
typedef struct nfn_L4_cache_head {
|
||||
uint64_t reverse_mask_0;
|
||||
uint32_t position_mask;
|
||||
uint32_t size;
|
||||
uint64_t reverse_mask_1;
|
||||
uint64_t reverse_mask_2;
|
||||
} nif_L4_cache_head;
|
||||
} nfn_L4_cache_head;
|
||||
|
||||
typedef struct nif_L4_handle {
|
||||
typedef struct nfn_L4_handle {
|
||||
void *ptr;
|
||||
nif_L4_cache_head* head;
|
||||
} nif_L4_handle;
|
||||
nfn_L4_cache_head* head;
|
||||
} nfn_L4_handle;
|
||||
|
||||
typedef struct matrix_task {
|
||||
nif_L4_handle buffer_0;
|
||||
nif_L4_handle buffer_1;
|
||||
nif_L4_handle buffer_2;
|
||||
nif_L4_handle buffer_3;
|
||||
nfn_L4_handle buffer_0;
|
||||
nfn_L4_handle buffer_1;
|
||||
nfn_L4_handle buffer_2;
|
||||
nfn_L4_handle buffer_3;
|
||||
union {
|
||||
struct {
|
||||
int32_t reverse_0;
|
||||
@@ -144,18 +144,18 @@ static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mis
|
||||
static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mismatch");
|
||||
#endif
|
||||
|
||||
typedef void(NIF_DECL *UNI_MK_PROC)(matrix_task *task);
|
||||
typedef void(NIF_DECL *UNI_SK_PROC)(mat_kernel const *left, mat_kernel const *right,
|
||||
typedef void(NFN_DECL *UNI_MK_PROC)(matrix_task *task);
|
||||
typedef void(NFN_DECL *UNI_SK_PROC)(mat_kernel const *left, mat_kernel const *right,
|
||||
mat_kernel *restrict ans, uint64_t scalar);
|
||||
|
||||
typedef void(NIF_DECL *VECTORNORMALIZEPROC)(VEC_SEGMENT(float) *source, VEC_SEGMENT(float) *target);
|
||||
typedef void(NIF_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans);
|
||||
typedef uint64_t(NIF_DECL *FACTORIALROC)(int n);
|
||||
typedef void(NFN_DECL *VECTORNORMALIZEPROC)(VEC_SEGMENT(float) *source, VEC_SEGMENT(float) *target);
|
||||
typedef void(NFN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans);
|
||||
typedef uint64_t(NFN_DECL *FACTORIALROC)(int n);
|
||||
|
||||
#pragma region string
|
||||
|
||||
typedef int(NIF_DECL *HASH)(byte const *str, int length);
|
||||
typedef int(NIF_DECL *INDEX_PAIR)(short const *stram, uint32_t target, int length);
|
||||
typedef int(NFN_DECL *HASH)(byte const *str, int length);
|
||||
typedef int(NFN_DECL *INDEX_PAIR)(short const *stram, uint32_t target, int length);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
@@ -168,13 +168,13 @@ typedef struct mat_sk_intrinsic {
|
||||
} mat_sk_intrinsic;
|
||||
|
||||
// Mirrors C# IntrinsicContext field order so managed/unmanaged layouts stay in sync.
|
||||
typedef struct nif_intrinsic {
|
||||
typedef struct nfn_intrinsic {
|
||||
HASH city_hash_simplify;
|
||||
FACTORIALROC factorial_simd;
|
||||
VECTORNORMALIZEPROC vec_f32_normalize_3;
|
||||
mat_mk_intrinsic mat_mk[0x0a];
|
||||
mat_sk_intrinsic mat_sk[0x0a];
|
||||
} nif_intrinsic;
|
||||
} nfn_intrinsic;
|
||||
|
||||
|
||||
typedef enum mat_proc_index {
|
||||
@@ -0,0 +1,54 @@
|
||||
#ifndef NFN_MATKERNEL
|
||||
#define NFN_MATKERNEL
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "nfn_internal.h"
|
||||
|
||||
/* Kernel storage unions for a fixed 4x4 block. */
|
||||
#define ELEMENTSIZE(T) sizeof(T)
|
||||
#define XMMCOUNT(T) (sizeof(T))
|
||||
#define YMMCOUNT(T) (sizeof(T) / 2)
|
||||
#define ZMMCOUNT(T) (sizeof(T) / 4)
|
||||
|
||||
#define NFN_MAT_KERNEL_MEMBER(T) \
|
||||
{ \
|
||||
__m128 x[XMMCOUNT(T)]; \
|
||||
__m256 y[YMMCOUNT(T)]; \
|
||||
__m512 z[ZMMCOUNT(T)]; \
|
||||
T m[4][4]; \
|
||||
}
|
||||
|
||||
#define NFN_MAT_KERNEL_DEFINE(T) \
|
||||
typedef union NFN_MAT_KERNEL_TYPE_##T NFN_MAT_KERNEL_MEMBER(T) NFN_MAT_KERNEL_TYPE_##T
|
||||
|
||||
#define MAT_KERNEL(T) NFN_MAT_KERNEL_TYPE(T)
|
||||
#define NFN_MAT_KERNEL_TYPE(T) NFN_MAT_KERNEL_TYPE_##T
|
||||
|
||||
NFN_MAT_KERNEL_DEFINE(float);
|
||||
NFN_MAT_KERNEL_DEFINE(double);
|
||||
NFN_MAT_KERNEL_DEFINE(uint64_t);
|
||||
|
||||
|
||||
#define NFN_VEC_SEGMENT_MEMBER(T) \
|
||||
{ \
|
||||
__m128 f32_x[XMMCOUNT(T) / 4]; \
|
||||
__m256 f32_y[YMMCOUNT(T) / 4]; \
|
||||
__m512 f32_z[ZMMCOUNT(T) / 4]; \
|
||||
T v[4]; \
|
||||
}
|
||||
|
||||
#define NFN_VEC_SEGMENT_DEFINE(T) \
|
||||
typedef union NFN_VEC_SEGMENT_TYPE_##T NFN_VEC_SEGMENT_MEMBER(T) NFN_VEC_SEGMENT_TYPE_##T
|
||||
|
||||
#define VEC_SEGMENT(T) NFN_VEC_SEGMENT_TYPE(T)
|
||||
#define NFN_VEC_SEGMENT_TYPE(T) NFN_VEC_SEGMENT_TYPE_##T
|
||||
|
||||
NFN_VEC_SEGMENT_DEFINE(float);
|
||||
NFN_VEC_SEGMENT_DEFINE(double);
|
||||
|
||||
/* Global zero kernel declaration (defined in dllmain.c), aligned to 64 bytes */
|
||||
extern __declspec(align(64)) MAT_KERNEL(u64) NFN_ZERO_KERNEL;
|
||||
|
||||
#endif /* NFN_MATKERNEL */
|
||||
@@ -0,0 +1,101 @@
|
||||
#ifndef NFN_INTRINSIC_CALLCONV_STD
|
||||
#define NFN_INTRINSIC_CALLCONV_STD
|
||||
|
||||
#include "nfn_internal.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
|
||||
#ifndef static_assert
|
||||
#define static_assert _Static_assert
|
||||
#endif
|
||||
|
||||
#define NFN_STD_TARGET_ATTR(T_arch, T_feature) NFN_STD_TARGET_ATTR_IMPL(T_arch, T_feature)
|
||||
|
||||
#pragma region mk_proc attributes
|
||||
#define MK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
|
||||
T_type##_multi_kernel_proc_##T_op##_##T_arch##_##T_feature
|
||||
#define GET_MK_PROC_STD(T_type, T_arch, T_feature, T_op) \
|
||||
((UNI_MK_PROC)MK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op))
|
||||
#define NFN_MK_PROC_DECL(T_type, P_name) INTERNAL void NFN_DECL P_name(MAT_TASK(T_type) const *task)
|
||||
#define NFN_MK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
|
||||
INTERNAL NOINLINE NFN_STD_TARGET_ATTR(T_arch, T_feature) void NFN_DECL MK_PROC_NAME_STD( \
|
||||
T_type, T_arch, T_feature, T_op)(MAT_TASK(T_type) const *task)
|
||||
#pragma endregion
|
||||
|
||||
#pragma region sk_proc attributes
|
||||
#define SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
|
||||
T_type##_single_kernel_proc_##T_op##_##T_arch##_##T_feature
|
||||
#define GET_SK_PROC_STD(T_type, T_arch, T_feature, T_op) \
|
||||
((UNI_SK_PROC)SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op))
|
||||
#define NFN_SK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
|
||||
INTERNAL NOINLINE NFN_STD_TARGET_ATTR(T_arch, T_feature) void NFN_DECL SK_PROC_NAME_STD( \
|
||||
T_type, T_arch, T_feature, \
|
||||
T_op)(MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
|
||||
MAT_KERNEL(T_type) *restrict ans, uint64_t scalar)
|
||||
#pragma endregion
|
||||
|
||||
#define _NFN_STD__PROBE_0 0
|
||||
#define _NFN_STD__PROBE_1 1
|
||||
#define NFN_STD__PROBE_PRINT(x) _NFN_STD__PROBE_##x
|
||||
#define NFN_STD__PROBE(x) NFN_STD__PROBE_PRINT(x)
|
||||
|
||||
#pragma region param styles
|
||||
|
||||
#define _NFN_MATPROC_LEFT_RIGHT_ANS 1
|
||||
#define _NFN_MATPROC_RIGHT_ANS 1
|
||||
#define _NFN_MATPROC_LEFT_RIGHT_ANS_SCALAR 1
|
||||
#define _NFN_MATPROC_RIGHT_ANS_SCALAR 1
|
||||
#define _NFN_MATPROC_ANS_SCALAR 1
|
||||
|
||||
#define _NFN_MK_EXPAND_BUFFER_SIZE(i)\
|
||||
int offset_##i = task->offset_##i;\
|
||||
int count_##i = task->count_##i;\
|
||||
|
||||
#define _NFN_MK_EXPAND_RIGHT_ANS \
|
||||
NFN_MAT_KERNEL_LOCAL const *right = (NFN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans = (NFN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr;\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(1); \
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _NFN_MK_EXPAND_LEFT_RIGHT_ANS \
|
||||
NFN_MAT_KERNEL_LOCAL const *left = (NFN_MAT_KERNEL_LOCAL const *)task->buffer_0.ptr; \
|
||||
NFN_MAT_KERNEL_LOCAL const *right = (NFN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans = (NFN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr;\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(0); \
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(1);\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _NFN_MK_EXPAND_RIGHT_ANS_SCALAR \
|
||||
NFN_MAT_KERNEL_LOCAL const *right = (NFN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans = (NFN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
|
||||
M128 scalar = task->scalar_pack;\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(1);\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _NFN_MK_EXPAND_LEFT_RIGHT_ANS_SCALAR \
|
||||
NFN_MAT_KERNEL_LOCAL const *left = (NFN_MAT_KERNEL_LOCAL const *)task->buffer_0.ptr; \
|
||||
NFN_MAT_KERNEL_LOCAL const *right = (NFN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans = (NFN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
|
||||
M128 scalar = task->scalar_pack;\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(0);\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(1);\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _NFN_MK_EXPAND_ANS_SCALAR \
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans = (NFN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
|
||||
M128 scalar = task->scalar_pack;\
|
||||
_NFN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _NFN_PARAM_STYLE(T) _NFN_MATPROC_##T
|
||||
#define _NFN_MK_EXPAND_PARAM(style) _NFN_MK_EXPAND_##style
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#define __mk_param_std(param, T) \
|
||||
typedef MAT_KERNEL(T) NFN_MAT_KERNEL_LOCAL; \
|
||||
matrix_task const *_ = (_NFN_PARAM_STYLE(param), task); \
|
||||
_NFN_MK_EXPAND_PARAM(param)
|
||||
|
||||
#define __sk_param_std(T) \
|
||||
typedef MAT_KERNEL(T) NFN_MAT_KERNEL_LOCAL; \
|
||||
(void)scalar
|
||||
#endif // !NFN_INTRINSIC_PARAM_EXPAND
|
||||
@@ -0,0 +1,94 @@
|
||||
#ifndef NFN_TARGET
|
||||
#define NFN_TARGET
|
||||
|
||||
#include "nfn_internal.h"
|
||||
#include <stdint.h>
|
||||
|
||||
#define NFN_OS_WINDOWS 0
|
||||
#define NFN_OS_LINUX 1
|
||||
#define NFN_OS_MACOS 2
|
||||
#define NFN_OS_IOS 3
|
||||
#define NFN_OS_ANDROID 4
|
||||
#define NFN_OS_UNKNOWN 5
|
||||
|
||||
#define NFN_ARCH_X86_64 0
|
||||
#define NFN_ARCH_ARM64 1
|
||||
#define NFN_ARCH_RISCV64 2
|
||||
#define NFN_ARCH_PPC64le 3
|
||||
|
||||
#if defined(_M_X64) || defined(_M_AMD64) || defined(__x86_64__) || defined(__amd64__)
|
||||
#define NFN_CURRENT_ARCH NFN_ARCH_X86_64
|
||||
|
||||
#elif defined(_M_ARM64) || defined(__aarch64__)
|
||||
#define NFN_CURRENT_ARCH NFN_ARCH_ARM64
|
||||
|
||||
#elif defined(_M_ARM) || defined(__arm__)
|
||||
#define NFN_CURRENT_ARCH NFN_ARCH_ARM32
|
||||
|
||||
#elif defined(_M_IX86) || defined(__i386__)
|
||||
#define NFN_CURRENT_ARCH NFN_ARCH_X86_32
|
||||
|
||||
#elif defined(__powerpc64__) && defined(__LITTLE_ENDIAN__)
|
||||
#define NFN_CURRENT_ARCH NFN_ARCH_PPC64LE
|
||||
|
||||
#elif defined(__riscv) && (__riscv_xlen == 64)
|
||||
#define NFN_CURRENT_ARCH NFN_ARCH_RISCV64
|
||||
|
||||
#else
|
||||
#error "SN: Unsupported target architecture"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Target selection policy:
|
||||
* - Default (no NFN_EXTERN_TARGET): fixed Windows + x86_64.
|
||||
* - External (NFN_EXTERN_TARGET): build system must provide NFN_OS and NFN_ARCH.
|
||||
*/
|
||||
#ifndef NFN_EXTERN_TARGET
|
||||
|
||||
#define NFN_OS_TARGET NFN_OS_WINDOWS
|
||||
#define NFN_ARCH_TARGET NFN_ARCH_X86_64
|
||||
|
||||
#else
|
||||
|
||||
#ifndef NFN_OS_TARGET
|
||||
#error "NFN_EXTERN_TARGET is defined but NFN_OS is not provided."
|
||||
#endif
|
||||
|
||||
#ifndef NFN_ARCH_TARGET
|
||||
#error "NFN_EXTERN_TARGET is defined but NFN_ARCH is not provided."
|
||||
#endif
|
||||
|
||||
/* Validate externally supplied values are within known enum constants. */
|
||||
#if (NFN_OS_TARGET != NFN_OS_WINDOWS) && (NFN_OS_TARGET != NFN_OS_LINUX) && \
|
||||
(NFN_OS_TARGET != NFN_OS_MACOS) && (NFN_OS_TARGET != NFN_OS_IOS) && \
|
||||
(NFN_OS_TARGET != NFN_OS_ANDROID) && (NFN_OS_TARGET != NFN_OS_UNKNOWN)
|
||||
#error "NFN_OS has an invalid value. Use NFN_OS_<...> enum constants."
|
||||
#endif
|
||||
|
||||
#if (NFN_ARCH_TARGET != NFN_ARCH_X86_64) && (NFN_ARCH_TARGET != NFN_ARCH_ARM64) && \
|
||||
(NFN_ARCH_TARGET != NFN_ARCH_RISCV64) && (NFN_ARCH_TARGET != NFN_ARCH_PPC64le)
|
||||
#error "NFN_ARCH has an invalid value. Use NFN_ARCH_<...> enum constants."
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#define NFN_IS_OS(os) ((NFN_OS_TARGET) == (os))
|
||||
|
||||
#define NFN_IS_ARCH(arch) ((NFN_ARCH_TARGET) == (arch))
|
||||
|
||||
/*
|
||||
* Per-function ISA selection is required for runtime-dispatched kernels that
|
||||
* mix multiple x86 feature levels in one build target.
|
||||
*/
|
||||
#if defined(__clang__) && ((NFN_ARCH_TARGET) == NFN_ARCH_X86_64)
|
||||
#define NFN_CLANG_TARGET_ATTR(options) __attribute__((target(options)))
|
||||
#else
|
||||
#define NFN_CLANG_TARGET_ATTR(options)
|
||||
#endif
|
||||
|
||||
typedef union SIMDID {
|
||||
uint64_t mask;
|
||||
char make_byte[8];
|
||||
} SIMDID;
|
||||
|
||||
#endif // NFN_TARGET
|
||||
@@ -1,18 +1,18 @@
|
||||
#include "NitloadFrameWork.h"
|
||||
#include "nif_intrinsic.matkernel.h"
|
||||
#include "nfn_intrinsic.matkernel.h"
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
|
||||
/* Define and initialize a global zero kernel aligned to 64 bytes for universal ISA usage */
|
||||
|
||||
__declspec(align(64)) MAT_KERNEL(u64) NIF_ZERO_KERNEL;
|
||||
__declspec(align(64)) MAT_KERNEL(u64) NFN_ZERO_KERNEL;
|
||||
|
||||
static void init_zero_kernel(void)
|
||||
{
|
||||
for (int c = 0; c < 4; ++c) {
|
||||
for (int r = 0; r < 4; ++r) {
|
||||
NIF_ZERO_KERNEL.m[r][c] = 0;
|
||||
NFN_ZERO_KERNEL.m[r][c] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
#include "NitloadFrameWork.h"
|
||||
|
||||
NIF_API void NIF_DECL nif_load_glfw_functions(GLFWFunctionTable *table)
|
||||
NFN_API void NFN_DECL nfn_load_glfw_functions(GLFWFunctionTable *table)
|
||||
{
|
||||
table->glfwCreateCursor = glfwCreateCursor;
|
||||
table->glfwCreateStandardCursor = glfwCreateStandardCursor;
|
||||
+15
-15
@@ -1,28 +1,28 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "NitloadFrameWork.h"
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
|
||||
PRIVATE void load_intrinsic_sse(nif_intrinsic *context, SIMDID id);
|
||||
PRIVATE void load_intrinsic_avx(nif_intrinsic *context, SIMDID id);
|
||||
PRIVATE void load_intrinsic_512(nif_intrinsic *context, SIMDID id);
|
||||
PRIVATE void load_intrinsic_sse(nfn_intrinsic *context, SIMDID id);
|
||||
PRIVATE void load_intrinsic_avx(nfn_intrinsic *context, SIMDID id);
|
||||
PRIVATE void load_intrinsic_512(nfn_intrinsic *context, SIMDID id);
|
||||
|
||||
NIF_API void NIF_DECL nif_load_intrinsic_function(void *intrinsic_context, uint64_t id)
|
||||
NFN_API void NFN_DECL nfn_load_intrinsic_function(void *intrinsic_context, uint64_t id)
|
||||
{
|
||||
nif_null_assert(intrinsic_context);
|
||||
nfn_null_assert(intrinsic_context);
|
||||
|
||||
SIMDID simdid;
|
||||
if (id) {
|
||||
simdid.mask = id;
|
||||
} else {
|
||||
simdid = nif_get_simd_level_global();
|
||||
simdid = nfn_get_simd_level_global();
|
||||
}
|
||||
|
||||
nif_intrinsic *context = (nif_intrinsic *)intrinsic_context;
|
||||
nfn_intrinsic *context = (nfn_intrinsic *)intrinsic_context;
|
||||
|
||||
switch (simdid.make_byte[1]) {
|
||||
case SIMDID_MAIN_LVL_SSE:
|
||||
@@ -38,7 +38,7 @@ NIF_API void NIF_DECL nif_load_intrinsic_function(void *intrinsic_context, uint6
|
||||
}
|
||||
}
|
||||
|
||||
void load_intrinsic_sse(nif_intrinsic *context, SIMDID id)
|
||||
void load_intrinsic_sse(nfn_intrinsic *context, SIMDID id)
|
||||
{
|
||||
context->city_hash_simplify = city_hash_simplify_sse;
|
||||
context->factorial_simd = factorial_simd_sse;
|
||||
@@ -62,7 +62,7 @@ void load_intrinsic_sse(nif_intrinsic *context, SIMDID id)
|
||||
context->vec_f32_normalize_3 = f32_normalize;
|
||||
}
|
||||
|
||||
void load_intrinsic_avx(nif_intrinsic *context, SIMDID id)
|
||||
void load_intrinsic_avx(nfn_intrinsic *context, SIMDID id)
|
||||
{
|
||||
char avx_seg = id.make_byte[4];
|
||||
char is_fma = (avx_seg & SIMDID_AVX_FMA) != 0;
|
||||
@@ -91,7 +91,7 @@ void load_intrinsic_avx(nif_intrinsic *context, SIMDID id)
|
||||
context->vec_f32_normalize_3 = f32_normalize;
|
||||
}
|
||||
|
||||
void load_intrinsic_512(nif_intrinsic *context, SIMDID id)
|
||||
void load_intrinsic_512(nfn_intrinsic *context, SIMDID id)
|
||||
{
|
||||
context->city_hash_simplify = city_hash_simplify_sse;
|
||||
context->factorial_simd = factorial_simd_sse;
|
||||
+29
-29
@@ -1,13 +1,13 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include <immintrin.h>
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fill)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fill)
|
||||
{
|
||||
__mk_param_std(ANS_SCALAR, float);
|
||||
register __m128 vec = _mm_set_ps1(scalar.f32[0]);
|
||||
@@ -19,7 +19,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , fill)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fill)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fill)
|
||||
{
|
||||
__mk_param_std(ANS_SCALAR, float);
|
||||
register __m256 vec = _mm256_broadcast_ss(&scalar.f32[0]);
|
||||
@@ -29,7 +29,7 @@ NIF_MK_PROC_DECL_STD(float, avx, , fill)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fill)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fill)
|
||||
{
|
||||
__mk_param_std(ANS_SCALAR, float);
|
||||
register __m512 vec = _mm512_broadcastss_ps(scalar.m128);
|
||||
@@ -38,7 +38,7 @@ NIF_MK_PROC_DECL_STD(float, 512, , fill)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , fill)
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , fill)
|
||||
{
|
||||
__mk_param_std(ANS_SCALAR, double);
|
||||
register __m128d vec_d = _mm_set1_pd(scalar.f64[0]);
|
||||
@@ -55,7 +55,7 @@ NIF_MK_PROC_DECL_STD(double, sse, , fill)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , fill)
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , fill)
|
||||
{
|
||||
__mk_param_std(ANS_SCALAR, double);
|
||||
register __m256d vec = _mm256_broadcast_sd(&scalar.f64[0]);
|
||||
@@ -67,7 +67,7 @@ NIF_MK_PROC_DECL_STD(double, avx, , fill)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , fill)
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , fill)
|
||||
{
|
||||
__mk_param_std(ANS_SCALAR, double);
|
||||
register __m512d vec = _mm512_set1_pd(scalar.f64[0]);
|
||||
@@ -77,7 +77,7 @@ NIF_MK_PROC_DECL_STD(double, 512, , fill)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , add)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , add)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, float);
|
||||
|
||||
@@ -94,7 +94,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , add)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , add)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , add)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, float);
|
||||
|
||||
@@ -107,7 +107,7 @@ NIF_MK_PROC_DECL_STD(float, avx, , add)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , add)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , add)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, float);
|
||||
|
||||
@@ -116,7 +116,7 @@ NIF_MK_PROC_DECL_STD(float, 512, , add)
|
||||
ans[i].z[0] = c0;
|
||||
}
|
||||
}
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , add)
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , add)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, double);
|
||||
|
||||
@@ -149,7 +149,7 @@ NIF_MK_PROC_DECL_STD(double, sse, , add)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , add)
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , add)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, double);
|
||||
|
||||
@@ -166,7 +166,7 @@ NIF_MK_PROC_DECL_STD(double, avx, , add)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , add)
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , add)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, double);
|
||||
|
||||
@@ -178,7 +178,7 @@ NIF_MK_PROC_DECL_STD(double, 512, , add)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , sub)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , sub)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, float);
|
||||
|
||||
@@ -195,7 +195,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , sub)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , sub)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , sub)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, float);
|
||||
|
||||
@@ -207,7 +207,7 @@ NIF_MK_PROC_DECL_STD(float, avx, , sub)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , sub)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , sub)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, float);
|
||||
|
||||
@@ -217,7 +217,7 @@ NIF_MK_PROC_DECL_STD(float, 512, , sub)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , sub)
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , sub)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, double);
|
||||
|
||||
@@ -242,7 +242,7 @@ NIF_MK_PROC_DECL_STD(double, sse, , sub)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , sub)
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , sub)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, double);
|
||||
|
||||
@@ -254,7 +254,7 @@ NIF_MK_PROC_DECL_STD(double, avx, , sub)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , sub)
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , sub)
|
||||
{
|
||||
__mk_param_std( LEFT_RIGHT_ANS, double);
|
||||
|
||||
@@ -264,7 +264,7 @@ NIF_MK_PROC_DECL_STD(double, 512, , sub)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , scalar_mul)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , scalar_mul)
|
||||
{
|
||||
__mk_param_std( RIGHT_ANS_SCALAR, float);
|
||||
|
||||
@@ -282,7 +282,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , scalar_mul)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , scalar_mul)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , scalar_mul)
|
||||
{
|
||||
__mk_param_std( RIGHT_ANS_SCALAR, float);
|
||||
|
||||
@@ -296,7 +296,7 @@ NIF_MK_PROC_DECL_STD(float, avx, , scalar_mul)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , scalar_mul)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , scalar_mul)
|
||||
{
|
||||
__mk_param_std( RIGHT_ANS_SCALAR, float);
|
||||
|
||||
@@ -308,7 +308,7 @@ NIF_MK_PROC_DECL_STD(float, 512, , scalar_mul)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , scalar_mul)
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , scalar_mul)
|
||||
{
|
||||
__mk_param_std( RIGHT_ANS_SCALAR, double);
|
||||
|
||||
@@ -334,7 +334,7 @@ NIF_MK_PROC_DECL_STD(double, sse, , scalar_mul)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , scalar_mul)
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , scalar_mul)
|
||||
{
|
||||
__mk_param_std( RIGHT_ANS_SCALAR, double);
|
||||
|
||||
@@ -352,7 +352,7 @@ NIF_MK_PROC_DECL_STD(double, avx, , scalar_mul)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , scalar_mul)
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , scalar_mul)
|
||||
{
|
||||
__mk_param_std( RIGHT_ANS_SCALAR, double);
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
#include <math.h>
|
||||
|
||||
static const float quality_threshold = 1e-6f;
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , try_plu)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , try_plu)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS_SCALAR, float);
|
||||
|
||||
@@ -115,7 +115,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , try_plu)
|
||||
* layout of scalarpack:
|
||||
*
|
||||
*/
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , quality)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , quality)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS, float);
|
||||
|
||||
@@ -126,7 +126,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , quality)
|
||||
register __m128 col2 = _mm_load_ps((float *)&right->x[2]);
|
||||
register __m128 col3 = _mm_load_ps((float *)&right->x[3]);
|
||||
|
||||
NIF_ALIGNAS(16) float self_dot_src[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
|
||||
NFN_ALIGNAS(16) float self_dot_src[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
|
||||
// try in reigister mix later
|
||||
DOT_F32_SSE(col0, col0, self_dot_src[0]);
|
||||
DOT_F32_SSE(col1, col1, self_dot_src[1]);
|
||||
@@ -153,7 +153,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , quality)
|
||||
return;
|
||||
}
|
||||
|
||||
NIF_ALIGNAS(16) float cross_dot_src[2][4] = { 0 };
|
||||
NFN_ALIGNAS(16) float cross_dot_src[2][4] = { 0 };
|
||||
DOT_F32_SSE(col0, col1, cross_dot_src[0][0]);
|
||||
DOT_F32_SSE(col0, col2, cross_dot_src[0][1]);
|
||||
DOT_F32_SSE(col0, col3, cross_dot_src[0][2]);
|
||||
+8
-8
@@ -1,13 +1,13 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
|
||||
// Matrix transpose implementations
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , transpose)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , transpose)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS, float);
|
||||
|
||||
@@ -26,7 +26,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , transpose)
|
||||
ans->x[3] = _mm_movehl_ps(tmp3, tmp1);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , transpose)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , transpose)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS, float);
|
||||
register __m256 col_low = right->y[0];
|
||||
@@ -42,7 +42,7 @@ NIF_MK_PROC_DECL_STD(float, avx, , transpose)
|
||||
ans->y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , transpose)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , transpose)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS, float);
|
||||
register __m512 matrix = right->z[0];
|
||||
@@ -1,8 +1,8 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
|
||||
/*
|
||||
* pack left matrix data to L4 buffer
|
||||
@@ -19,14 +19,14 @@
|
||||
* scalar pack carries column length and offset data:
|
||||
* [column length, empty, empty, empty]
|
||||
*/
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fma_pack_left)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fma_pack_left)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS_SCALAR, float);
|
||||
// for SSE branch, no need for rounding, a direct copy is sufficient as long as the buffer is large enough to hold the packed data
|
||||
memcpy(ans, right + offset_1, sizeof(MAT_KERNEL(float)) * count_2);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fma_pack_left)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fma_pack_left)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS_SCALAR, float);
|
||||
size_t i = 0;
|
||||
@@ -61,7 +61,7 @@ NIF_MK_PROC_DECL_STD(float, avx, , fma_pack_left)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fma_pack_left)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fma_pack_left)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS_SCALAR, float);
|
||||
size_t i = 0;
|
||||
@@ -154,7 +154,7 @@ NIF_MK_PROC_DECL_STD(float, 512, , fma_pack_left)
|
||||
* scalar pack carries column length and offset data:
|
||||
* [column length, empty, empty, empty]
|
||||
*/
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fma_pack_right)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fma_pack_right)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS_SCALAR, float);
|
||||
int col_length = scalar.i32[0];
|
||||
@@ -163,7 +163,7 @@ NIF_MK_PROC_DECL_STD(float, sse, , fma_pack_right)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fma_pack_right)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fma_pack_right)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS_SCALAR, float);
|
||||
int col_length = scalar.i32[0];
|
||||
@@ -205,7 +205,7 @@ NIF_MK_PROC_DECL_STD(float, avx, , fma_pack_right)
|
||||
}
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fma_pack_right)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fma_pack_right)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS_SCALAR, float);
|
||||
int col_length = scalar.i32[0];
|
||||
@@ -1,22 +1,22 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
#include <immintrin.h>
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
// #include "nif_intrinsic.h"
|
||||
#include "nif_template.fma.x86.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
// #include "nfn_intrinsic.h"
|
||||
#include "nfn_template.fma.x86.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , fma)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, float);
|
||||
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
|
||||
register __m128 c0 = _mm_load_ps((float *)&ans_cur->x[0]);
|
||||
register __m128 c1 = _mm_load_ps((float *)&ans_cur->x[1]);
|
||||
@@ -25,8 +25,8 @@ NIF_MK_PROC_DECL_STD(float, sse, , fma)
|
||||
register __m128 bcol;
|
||||
|
||||
for (size_t z = 0; z < count_0; z++) {
|
||||
NIF_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NIF_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
NFN_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NFN_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
|
||||
const __m128 a0 = _mm_load_ps((float *)&left_base->x[0]);
|
||||
const __m128 a1 = _mm_load_ps((float *)&left_base->x[1]);
|
||||
@@ -68,18 +68,18 @@ NIF_MK_PROC_DECL_STD(float, sse, , fma)
|
||||
_mm_store_ps((float *)&ans_cur->x[3], c3);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, , fma)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, , fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, float);
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]);
|
||||
register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]);
|
||||
register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]);
|
||||
register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]);
|
||||
|
||||
for (size_t z = 0; z < count_0; z++) {
|
||||
NIF_MAT_KERNEL_LOCAL const *b_base = right;
|
||||
NIF_MAT_KERNEL_LOCAL const *a_base = left;
|
||||
NFN_MAT_KERNEL_LOCAL const *b_base = right;
|
||||
NFN_MAT_KERNEL_LOCAL const *a_base = left;
|
||||
|
||||
register __m256 b0 = _mm256_load_ps((float *)&b_base->y[0]);
|
||||
register __m256 b1 = _mm256_load_ps((float *)&b_base->y[1]);
|
||||
@@ -91,10 +91,10 @@ NIF_MK_PROC_DECL_STD(float, avx, , fma)
|
||||
register __m256 a2 = _mm256_load_ps((float *)&a_base->y[2]);
|
||||
register __m256 a3 = _mm256_load_ps((float *)&a_base->y[3]);
|
||||
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(0);
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(1);
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(2);
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(3);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(0);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(1);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(2);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(3);
|
||||
}
|
||||
_mm256_store_ps((float *)&(ans_cur->y[0]), c0);
|
||||
_mm256_store_ps((float *)&(ans_cur->y[1]), c1);
|
||||
@@ -102,18 +102,18 @@ NIF_MK_PROC_DECL_STD(float, avx, , fma)
|
||||
_mm256_store_ps((float *)&(ans_cur->y[3]), c3);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, avx, _fma, fma)
|
||||
NFN_MK_PROC_DECL_STD(float, avx, _fma, fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, float);
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]);
|
||||
register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]);
|
||||
register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]);
|
||||
register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]);
|
||||
|
||||
for (size_t z = 0; z < count_0; z += 2) {
|
||||
const NIF_MAT_KERNEL_LOCAL *a_base = left + z;
|
||||
const NIF_MAT_KERNEL_LOCAL *b_base = right + z;
|
||||
const NFN_MAT_KERNEL_LOCAL *a_base = left + z;
|
||||
const NFN_MAT_KERNEL_LOCAL *b_base = right + z;
|
||||
|
||||
register __m256 b0 = _mm256_load_ps((float *)&b_base->y[0]);
|
||||
register __m256 b1 = _mm256_load_ps((float *)&b_base->y[1]);
|
||||
@@ -125,10 +125,10 @@ NIF_MK_PROC_DECL_STD(float, avx, _fma, fma)
|
||||
register __m256 a2 = _mm256_load_ps((float *)&a_base->y[2]);
|
||||
register __m256 a3 = _mm256_load_ps((float *)&a_base->y[3]);
|
||||
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(0);
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(1);
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(2);
|
||||
NIF_MK_FMA_F32_AVX2_CYCLE(3);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(0);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(1);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(2);
|
||||
NFN_MK_FMA_F32_AVX2_CYCLE(3);
|
||||
}
|
||||
_mm256_store_ps((float *)&(ans_cur->y[0]), c0);
|
||||
_mm256_store_ps((float *)&(ans_cur->y[1]), c1);
|
||||
@@ -136,18 +136,18 @@ NIF_MK_PROC_DECL_STD(float, avx, _fma, fma)
|
||||
_mm256_store_ps((float *)&(ans_cur->y[3]), c3);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, 512, , fma)
|
||||
NFN_MK_PROC_DECL_STD(float, 512, , fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, float);
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
register __m512 c0 = _mm512_load_ps((float *)&ans_cur->z[0]);
|
||||
register __m512 c1 = _mm512_load_ps((float *)&ans_cur->z[0]);
|
||||
register __m512 c2 = _mm512_load_ps((float *)&ans_cur->z[0]);
|
||||
register __m512 c3 = _mm512_load_ps((float *)&ans_cur->z[0]);
|
||||
|
||||
for (size_t z = 0; z < count_0; z += 4) {
|
||||
const NIF_MAT_KERNEL_LOCAL *a_base = left + z;
|
||||
const NIF_MAT_KERNEL_LOCAL *b_base = right + z;
|
||||
const NFN_MAT_KERNEL_LOCAL *a_base = left + z;
|
||||
const NFN_MAT_KERNEL_LOCAL *b_base = right + z;
|
||||
|
||||
register __m512 b0 = _mm512_load_ps((float *)&b_base->z[0]);
|
||||
register __m512 b1 = _mm512_load_ps((float *)&b_base->z[1]);
|
||||
@@ -159,10 +159,10 @@ NIF_MK_PROC_DECL_STD(float, 512, , fma)
|
||||
register __m512 a2 = _mm512_load_ps((float *)&a_base->z[2]);
|
||||
register __m512 a3 = _mm512_load_ps((float *)&a_base->z[3]);
|
||||
|
||||
NIF_MK_FMA_F32_512_CYCLE(0);
|
||||
NIF_MK_FMA_F32_512_CYCLE(1);
|
||||
NIF_MK_FMA_F32_512_CYCLE(2);
|
||||
NIF_MK_FMA_F32_512_CYCLE(3);
|
||||
NFN_MK_FMA_F32_512_CYCLE(0);
|
||||
NFN_MK_FMA_F32_512_CYCLE(1);
|
||||
NFN_MK_FMA_F32_512_CYCLE(2);
|
||||
NFN_MK_FMA_F32_512_CYCLE(3);
|
||||
}
|
||||
_mm512_store_ps((float *)&ans_cur->z[0], c0);
|
||||
_mm512_store_ps((float *)&ans_cur->z[1], c1);
|
||||
@@ -170,11 +170,11 @@ NIF_MK_PROC_DECL_STD(float, 512, , fma)
|
||||
_mm512_store_ps((float *)&ans_cur->z[3], c3);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, sse, , fma)
|
||||
NFN_MK_PROC_DECL_STD(double, sse, , fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, double);
|
||||
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
|
||||
register __m128d c0_lo = ans_cur->x[0];
|
||||
register __m128d c0_hi = ans_cur->x[1];
|
||||
@@ -186,18 +186,18 @@ NIF_MK_PROC_DECL_STD(double, sse, , fma)
|
||||
register __m128d c3_hi = ans_cur->x[7];
|
||||
|
||||
for (size_t z = 0; z < count_0; z++) {
|
||||
NIF_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NIF_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
NFN_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NFN_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
const double *right_values = &right_base->m[0][0];
|
||||
const double *left_values = &left_base->m[0][0];
|
||||
|
||||
NIF_MK_FMA_F64_SSE_PHASE(0);
|
||||
NFN_MK_FMA_F64_SSE_PHASE(0);
|
||||
|
||||
NIF_MK_FMA_F64_SSE_PHASE(1);
|
||||
NFN_MK_FMA_F64_SSE_PHASE(1);
|
||||
|
||||
NIF_MK_FMA_F64_SSE_PHASE(2);
|
||||
NFN_MK_FMA_F64_SSE_PHASE(2);
|
||||
|
||||
NIF_MK_FMA_F64_SSE_PHASE(3);
|
||||
NFN_MK_FMA_F64_SSE_PHASE(3);
|
||||
}
|
||||
|
||||
ans_cur->x[0] = c0_lo;
|
||||
@@ -210,11 +210,11 @@ NIF_MK_PROC_DECL_STD(double, sse, , fma)
|
||||
ans_cur->x[7] = c3_hi;
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, avx, , fma)
|
||||
NFN_MK_PROC_DECL_STD(double, avx, , fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, double);
|
||||
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
|
||||
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->y[0]);
|
||||
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->y[1]);
|
||||
@@ -223,8 +223,8 @@ NIF_MK_PROC_DECL_STD(double, avx, , fma)
|
||||
register __m256d bcol;
|
||||
|
||||
for (size_t z = 0; z < count_0; z++) {
|
||||
NIF_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NIF_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
NFN_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NFN_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
|
||||
const __m256d a0 = _mm256_load_pd((double *)&left_base->y[0]);
|
||||
const __m256d a1 = _mm256_load_pd((double *)&left_base->y[1]);
|
||||
@@ -282,11 +282,11 @@ NIF_MK_PROC_DECL_STD(double, avx, , fma)
|
||||
_mm256_store_pd((double *)&ans_cur->y[3], c3);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, avx, _fma, fma)
|
||||
NFN_MK_PROC_DECL_STD(double, avx, _fma, fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, double);
|
||||
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
|
||||
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->y[0]);
|
||||
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->y[1]);
|
||||
@@ -295,8 +295,8 @@ NIF_MK_PROC_DECL_STD(double, avx, _fma, fma)
|
||||
register __m256d bcol;
|
||||
|
||||
for (size_t z = 0; z < count_0; z++) {
|
||||
NIF_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NIF_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
NFN_MAT_KERNEL_LOCAL const *right_base = right;
|
||||
NFN_MAT_KERNEL_LOCAL const *left_base = left;
|
||||
|
||||
const __m256d a0 = _mm256_load_pd((double *)&left_base->y[0]);
|
||||
const __m256d a1 = _mm256_load_pd((double *)&left_base->y[1]);
|
||||
@@ -338,10 +338,10 @@ NIF_MK_PROC_DECL_STD(double, avx, _fma, fma)
|
||||
_mm256_store_pd((double *)&ans_cur->y[3], c3);
|
||||
}
|
||||
|
||||
NIF_MK_PROC_DECL_STD(double, 512, , fma)
|
||||
NFN_MK_PROC_DECL_STD(double, 512, , fma)
|
||||
{
|
||||
__mk_param_std(LEFT_RIGHT_ANS, double);
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
NFN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
|
||||
|
||||
register __m512d c0 = _mm512_load_pd((double *)&ans_cur->z[0]);
|
||||
register __m512d c1 = _mm512_load_pd((double *)&ans_cur->z[1]);
|
||||
@@ -349,8 +349,8 @@ NIF_MK_PROC_DECL_STD(double, 512, , fma)
|
||||
register __m512d c3 = _mm512_load_pd((double *)&ans_cur->z[3]);
|
||||
|
||||
for (size_t z = 0; z < count_0; z += 2) {
|
||||
NIF_MAT_KERNEL_LOCAL const *right_base = right + z;
|
||||
NIF_MAT_KERNEL_LOCAL const *left_base = left + z;
|
||||
NFN_MAT_KERNEL_LOCAL const *right_base = right + z;
|
||||
NFN_MAT_KERNEL_LOCAL const *left_base = left + z;
|
||||
|
||||
register __m512d a0 = _mm512_load_pd((double *)&left_base->z[0]);
|
||||
register __m512d a1 = _mm512_load_pd((double *)&left_base->z[1]);
|
||||
@@ -362,10 +362,10 @@ NIF_MK_PROC_DECL_STD(double, 512, , fma)
|
||||
register __m512d b2 = _mm512_load_pd((double *)&right_base->z[2]);
|
||||
register __m512d b3 = _mm512_load_pd((double *)&right_base->z[3]);
|
||||
|
||||
NIF_MK_FMA_F64_AVX2_CYCLE(0);
|
||||
NIF_MK_FMA_F64_AVX2_CYCLE(1);
|
||||
NIF_MK_FMA_F64_AVX2_CYCLE(2);
|
||||
NIF_MK_FMA_F64_AVX2_CYCLE(3);
|
||||
NFN_MK_FMA_F64_AVX2_CYCLE(0);
|
||||
NFN_MK_FMA_F64_AVX2_CYCLE(1);
|
||||
NFN_MK_FMA_F64_AVX2_CYCLE(2);
|
||||
NFN_MK_FMA_F64_AVX2_CYCLE(3);
|
||||
}
|
||||
_mm512_store_pd((double *)&ans_cur->z[0], c0);
|
||||
_mm512_store_pd((double *)&ans_cur->z[1], c1);
|
||||
+10
-10
@@ -1,25 +1,25 @@
|
||||
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "nif_internal.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nif_intrinsic.matkernel.h"
|
||||
#include "nfn_internal.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.matkernel.h"
|
||||
#include <emmintrin.h>
|
||||
#include <immintrin.h>
|
||||
#include <pmmintrin.h>
|
||||
|
||||
NIF_MK_PROC_DECL_STD(float, sse, , k_quality)
|
||||
NFN_MK_PROC_DECL_STD(float, sse, , k_quality)
|
||||
{
|
||||
__mk_param_std(RIGHT_ANS_SCALAR, float);
|
||||
}
|
||||
|
||||
INTERNAL NIF_STD_TARGET_ATTR(sse, ) void kp_check_f32(mat_kernel *kernel)
|
||||
INTERNAL NFN_STD_TARGET_ATTR(sse, ) void kp_check_f32(mat_kernel *kernel)
|
||||
{
|
||||
typedef MAT_KERNEL(float) NIF_MAT;
|
||||
NIF_MAT const *k = (NIF_MAT const *)kernel;
|
||||
typedef MAT_KERNEL(float) NFN_MAT;
|
||||
NFN_MAT const *k = (NFN_MAT const *)kernel;
|
||||
float cross[4];
|
||||
cross[0] = k->m[0][0];
|
||||
cross[1] = k->m[1][1];
|
||||
+6
-6
@@ -1,14 +1,14 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "nif_internal.h"
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nfn_internal.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
|
||||
static __m128i facIncrement;
|
||||
static __m128i facBase;
|
||||
|
||||
INTERNAL uint64_t NIF_DECL factorial_simd_sse(int n)
|
||||
INTERNAL uint64_t NFN_DECL factorial_simd_sse(int n)
|
||||
{
|
||||
facIncrement = _mm_set1_epi32(4);
|
||||
facBase = _mm_setr_epi32(1, 2, 3, 4);
|
||||
@@ -20,7 +20,7 @@ INTERNAL uint64_t NIF_DECL factorial_simd_sse(int n)
|
||||
facBaseLocal = _mm_add_epi32(facBaseLocal, facIncrement);
|
||||
baseResult = _mm_mullo_epi32(baseResult, facBaseLocal);
|
||||
}
|
||||
NIF_ALIGNAS(16) int32_t lanes[4];
|
||||
NFN_ALIGNAS(16) int32_t lanes[4];
|
||||
_mm_store_si128((__m128i *)lanes, baseResult);
|
||||
uint64_t result = (uint64_t)lanes[0] * (uint64_t)lanes[1] * (uint64_t)lanes[2] *
|
||||
(uint64_t)lanes[3];
|
||||
@@ -1,10 +1,10 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "NitloadFrameWork.h"
|
||||
|
||||
INTERNAL int NIF_DECL city_hash_simplify_sse(byte const *str, int const length)
|
||||
INTERNAL int NFN_DECL city_hash_simplify_sse(byte const *str, int const length)
|
||||
{
|
||||
uint32_t hash = length, size = length << 1;
|
||||
byte const *end = str + size;
|
||||
@@ -1,13 +1,13 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
/* clang-format off */
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#include "nif_intrinsic.h"
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "NitloadFrameWork.h"
|
||||
static SIMDID global_simdid = { 0 };
|
||||
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <string.h>
|
||||
@@ -126,7 +126,7 @@ static int is_amd_glue_ymm(const char *vendor, int cpuid1_eax)
|
||||
return (family == 0x17) && (model < 0x30);
|
||||
}
|
||||
|
||||
NIF_API SIMDID NIF_DECL nif_get_simd_level_local()
|
||||
NFN_API SIMDID NFN_DECL nfn_get_simd_level_local()
|
||||
{
|
||||
SIMDID simdid = { 0 };
|
||||
|
||||
@@ -314,13 +314,13 @@ NIF_API SIMDID NIF_DECL nif_get_simd_level_local()
|
||||
return simdid;
|
||||
}
|
||||
|
||||
NIF_API SIMDID NIF_DECL nif_get_simd_level_global()
|
||||
NFN_API SIMDID NFN_DECL nfn_get_simd_level_global()
|
||||
{
|
||||
if (global_simdid.mask) {
|
||||
return global_simdid;
|
||||
}
|
||||
|
||||
global_simdid = nif_get_simd_level_local();
|
||||
global_simdid = nfn_get_simd_level_local();
|
||||
return global_simdid;
|
||||
}
|
||||
|
||||
+7
-7
@@ -1,9 +1,9 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "NitloadFrameWork.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
|
||||
/* -----------------------------------------------------------------------
|
||||
* Field extraction helpers.
|
||||
@@ -49,7 +49,7 @@ static int impl_strength(uint16_t avx512)
|
||||
}
|
||||
|
||||
/* -----------------------------------------------------------------------
|
||||
* nif_get_unite_simd
|
||||
* nfn_get_unite_simd
|
||||
*
|
||||
* Produce the conservative intersection of two SIMDIDs — the strongest
|
||||
* capability set that is safe to use on BOTH cores:
|
||||
@@ -60,7 +60,7 @@ static int impl_strength(uint16_t avx512)
|
||||
* - Root / Manu : keep left (must be same arch in practice)
|
||||
* - uArchHi : bitwise OR (HYBRID_P/E union; SIMD_DUAL set if either core is dual)
|
||||
* ----------------------------------------------------------------------- */
|
||||
NIF_API SIMDID NIF_DECL nif_get_unite_simd(SIMDID left, SIMDID right)
|
||||
NFN_API SIMDID NFN_DECL nfn_get_unite_simd(SIMDID left, SIMDID right)
|
||||
{
|
||||
SIMDID out;
|
||||
out.mask = 0;
|
||||
@@ -117,7 +117,7 @@ NIF_API SIMDID NIF_DECL nif_get_unite_simd(SIMDID left, SIMDID right)
|
||||
}
|
||||
|
||||
/* -----------------------------------------------------------------------
|
||||
* nif_compare_simd
|
||||
* nfn_compare_simd
|
||||
*
|
||||
* Compare two SIMDIDs by ISA strength. Returns:
|
||||
* +1 if left is strictly stronger than right
|
||||
@@ -154,7 +154,7 @@ NIF_API SIMDID NIF_DECL nif_get_unite_simd(SIMDID left, SIMDID right)
|
||||
* 7. uArchHi SIMD_DUAL (final tie-break)
|
||||
* Non-dual (SIMD_DUAL=0) > dual (SIMD_DUAL=1).
|
||||
* ----------------------------------------------------------------------- */
|
||||
NIF_API int NIF_DECL nif_compare_simd(SIMDID left, SIMDID right)
|
||||
NFN_API int NFN_DECL nfn_compare_simd(SIMDID left, SIMDID right)
|
||||
{
|
||||
/* 1. MainLevel */
|
||||
unsigned lm = FIELD_MAIN(left);
|
||||
@@ -1,10 +1,10 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_intrinsic.std.h"
|
||||
#include "nif_intrinsic.context.x86.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_intrinsic.std.h"
|
||||
#include "nfn_intrinsic.context.x86.h"
|
||||
|
||||
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
// float
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, sse, , fma)
|
||||
NFN_SK_PROC_DECL_STD(float, sse, , fma)
|
||||
{
|
||||
__sk_param_std(float);
|
||||
register __m128 c0 = _mm_load_ps((float *)&ans->x[0]);
|
||||
@@ -60,7 +60,7 @@ NIF_SK_PROC_DECL_STD(float, sse, , fma)
|
||||
_mm_store_ps((float *)&ans->x[3], c3);
|
||||
}
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, avx, , fma)
|
||||
NFN_SK_PROC_DECL_STD(float, avx, , fma)
|
||||
{
|
||||
__sk_param_std(float);
|
||||
register __m256 c01 = _mm256_load_ps((float *)&ans->y[0]);
|
||||
@@ -86,7 +86,7 @@ NIF_SK_PROC_DECL_STD(float, avx, , fma)
|
||||
_mm256_store_ps((float *)&ans->y[1], c23);
|
||||
}
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, avx, _fma, fma)
|
||||
NFN_SK_PROC_DECL_STD(float, avx, _fma, fma)
|
||||
{
|
||||
__sk_param_std(float);
|
||||
register __m256 c01 = _mm256_load_ps((float *)&ans->y[0]);
|
||||
@@ -112,7 +112,7 @@ NIF_SK_PROC_DECL_STD(float, avx, _fma, fma)
|
||||
_mm256_store_ps((float *)&ans->y[1], c23);
|
||||
}
|
||||
|
||||
NIF_SK_PROC_DECL_STD(float, 512, , fma)
|
||||
NFN_SK_PROC_DECL_STD(float, 512, , fma)
|
||||
{
|
||||
__sk_param_std(float);
|
||||
register __m512 c = _mm512_load_ps((float *)ans);
|
||||
@@ -173,7 +173,7 @@ NIF_SK_PROC_DECL_STD(float, 512, , fma)
|
||||
|
||||
// clang-format on
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, sse, , fma)
|
||||
NFN_SK_PROC_DECL_STD(double, sse, , fma)
|
||||
{
|
||||
__sk_param_std(double);
|
||||
const double *left_values = &left->m[0][0];
|
||||
@@ -204,7 +204,7 @@ NIF_SK_PROC_DECL_STD(double, sse, , fma)
|
||||
}
|
||||
|
||||
// clang-format off
|
||||
NIF_SK_PROC_DECL_STD(double, avx, , fma)
|
||||
NFN_SK_PROC_DECL_STD(double, avx, , fma)
|
||||
{
|
||||
__sk_param_std(double);
|
||||
register __m256d c0 = _mm256_load_pd((double *)&ans->y[0]);
|
||||
@@ -253,7 +253,7 @@ NIF_SK_PROC_DECL_STD(double, avx, , fma)
|
||||
}
|
||||
// clang-format on
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, avx, _fma, fma)
|
||||
NFN_SK_PROC_DECL_STD(double, avx, _fma, fma)
|
||||
{
|
||||
__sk_param_std(double);
|
||||
register __m256d c0 = _mm256_load_pd((double *)&ans->y[0]);
|
||||
@@ -301,7 +301,7 @@ NIF_SK_PROC_DECL_STD(double, avx, _fma, fma)
|
||||
_mm256_store_pd((double *)&ans->y[3], c3);
|
||||
}
|
||||
|
||||
NIF_SK_PROC_DECL_STD(double, 512, , fma)
|
||||
NFN_SK_PROC_DECL_STD(double, 512, , fma)
|
||||
{
|
||||
__sk_param_std(double);
|
||||
register __m512d c01 = _mm512_load_pd((double *)&ans->z[0]);
|
||||
+59
-59
@@ -1,8 +1,8 @@
|
||||
#define _GNU_SOURCE
|
||||
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_OS(NIF_OS_LINUX)
|
||||
#if NFN_IS_OS(NFN_OS_LINUX)
|
||||
#include <errno.h>
|
||||
#include <inttypes.h>
|
||||
#include <limits.h>
|
||||
@@ -16,30 +16,30 @@
|
||||
#include <sys/syscall.h>
|
||||
#include <unistd.h>
|
||||
|
||||
typedef int (*nif_linux_id_visitor)(int value, void *context);
|
||||
typedef int (*nfn_linux_id_visitor)(int value, void *context);
|
||||
|
||||
typedef struct nif_linux_counter_context {
|
||||
typedef struct nfn_linux_counter_context {
|
||||
int count;
|
||||
} nif_linux_counter_context;
|
||||
} nfn_linux_counter_context;
|
||||
|
||||
typedef struct nif_linux_max_context {
|
||||
typedef struct nfn_linux_max_context {
|
||||
int max_value;
|
||||
} nif_linux_max_context;
|
||||
} nfn_linux_max_context;
|
||||
|
||||
typedef struct nif_linux_fill_cpu_context {
|
||||
typedef struct nfn_linux_fill_cpu_context {
|
||||
uint32_t *processor_group_ids;
|
||||
uint32_t *socket_ids;
|
||||
uint32_t *cpu_indices;
|
||||
uint32_t *numa_node_ids;
|
||||
int numa_id;
|
||||
int index;
|
||||
} nif_linux_fill_cpu_context;
|
||||
} nfn_linux_fill_cpu_context;
|
||||
|
||||
typedef struct nif_linux_fill_node_count_context {
|
||||
typedef struct nfn_linux_fill_node_count_context {
|
||||
int *count_of_each_numa;
|
||||
} nif_linux_fill_node_count_context;
|
||||
} nfn_linux_fill_node_count_context;
|
||||
|
||||
static int nif_linux_read_first_line(char const *path, char *buffer, size_t buffer_size)
|
||||
static int nfn_linux_read_first_line(char const *path, char *buffer, size_t buffer_size)
|
||||
{
|
||||
FILE *file = fopen(path, "r");
|
||||
if (file == NULL)
|
||||
@@ -50,7 +50,7 @@ static int nif_linux_read_first_line(char const *path, char *buffer, size_t buff
|
||||
return line != NULL;
|
||||
}
|
||||
|
||||
static int nif_linux_parse_nonnegative_int(char const **cursor, int *value)
|
||||
static int nfn_linux_parse_nonnegative_int(char const **cursor, int *value)
|
||||
{
|
||||
char *end = NULL;
|
||||
uintmax_t parsed_value = 0;
|
||||
@@ -68,7 +68,7 @@ static int nif_linux_parse_nonnegative_int(char const **cursor, int *value)
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nif_linux_visit_id_list(char const *text, nif_linux_id_visitor visitor, void *context)
|
||||
static int nfn_linux_visit_id_list(char const *text, nfn_linux_id_visitor visitor, void *context)
|
||||
{
|
||||
char const *cursor = text;
|
||||
|
||||
@@ -80,14 +80,14 @@ static int nif_linux_visit_id_list(char const *text, nif_linux_id_visitor visito
|
||||
break;
|
||||
|
||||
int begin = 0;
|
||||
if (!nif_linux_parse_nonnegative_int(&cursor, &begin))
|
||||
if (!nfn_linux_parse_nonnegative_int(&cursor, &begin))
|
||||
return 0;
|
||||
|
||||
int finish = begin;
|
||||
|
||||
if (*cursor == '-') {
|
||||
cursor++;
|
||||
if (!nif_linux_parse_nonnegative_int(&cursor, &finish))
|
||||
if (!nfn_linux_parse_nonnegative_int(&cursor, &finish))
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -106,22 +106,22 @@ static int nif_linux_visit_id_list(char const *text, nif_linux_id_visitor visito
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nif_linux_count_id(int value, void *context)
|
||||
static int nfn_linux_count_id(int value, void *context)
|
||||
{
|
||||
(void)value;
|
||||
((nif_linux_counter_context *)context)->count++;
|
||||
((nfn_linux_counter_context *)context)->count++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nif_linux_track_max_id(int value, void *context)
|
||||
static int nfn_linux_track_max_id(int value, void *context)
|
||||
{
|
||||
nif_linux_max_context *max_context = (nif_linux_max_context *)context;
|
||||
nfn_linux_max_context *max_context = (nfn_linux_max_context *)context;
|
||||
if (value > max_context->max_value)
|
||||
max_context->max_value = value;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nif_linux_read_socket_id(int cpu_id, uint32_t *socket_id)
|
||||
static int nfn_linux_read_socket_id(int cpu_id, uint32_t *socket_id)
|
||||
{
|
||||
char path[160];
|
||||
char buffer[64];
|
||||
@@ -129,23 +129,23 @@ static int nif_linux_read_socket_id(int cpu_id, uint32_t *socket_id)
|
||||
long parsed_value;
|
||||
snprintf(path, sizeof(path),
|
||||
"/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu_id);
|
||||
if (!nif_linux_read_first_line(path, buffer, sizeof(buffer)))
|
||||
if (!nfn_linux_read_first_line(path, buffer, sizeof(buffer)))
|
||||
return 0;
|
||||
|
||||
errno = 0;
|
||||
parsed_value = strtol(buffer, &end, 10);
|
||||
if (end == buffer || errno == ERANGE || parsed_value < 0 || parsed_value > UINT32_MAX)
|
||||
if (end == buffer || errno == ERANGE || parsed_value < 0 || (uintmax_t)parsed_value > UINT32_MAX)
|
||||
return 0;
|
||||
|
||||
*socket_id = (uint32_t)parsed_value;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nif_linux_fill_cpu_id(int value, void *context)
|
||||
static int nfn_linux_fill_cpu_id(int value, void *context)
|
||||
{
|
||||
nif_linux_fill_cpu_context *fill_context = (nif_linux_fill_cpu_context *)context;
|
||||
nfn_linux_fill_cpu_context *fill_context = (nfn_linux_fill_cpu_context *)context;
|
||||
uint32_t socket_id = 0;
|
||||
if (!nif_linux_read_socket_id(value, &socket_id))
|
||||
if (!nfn_linux_read_socket_id(value, &socket_id))
|
||||
return 0;
|
||||
|
||||
fill_context->processor_group_ids[fill_context->index] = 0u;
|
||||
@@ -157,37 +157,37 @@ static int nif_linux_fill_cpu_id(int value, void *context)
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nif_linux_count_cpu_list(char const *path)
|
||||
static int nfn_linux_count_cpu_list(char const *path)
|
||||
{
|
||||
char buffer[256];
|
||||
if (!nif_linux_read_first_line(path, buffer, sizeof(buffer)))
|
||||
if (!nfn_linux_read_first_line(path, buffer, sizeof(buffer)))
|
||||
return 0;
|
||||
|
||||
nif_linux_counter_context counter = { 0 };
|
||||
if (!nif_linux_visit_id_list(buffer, nif_linux_count_id, &counter))
|
||||
nfn_linux_counter_context counter = { 0 };
|
||||
if (!nfn_linux_visit_id_list(buffer, nfn_linux_count_id, &counter))
|
||||
return 0;
|
||||
|
||||
return counter.count;
|
||||
}
|
||||
|
||||
static int nif_linux_fill_node_cpu_count(int value, void *context)
|
||||
static int nfn_linux_fill_node_cpu_count(int value, void *context)
|
||||
{
|
||||
char path[128];
|
||||
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", value);
|
||||
|
||||
((nif_linux_fill_node_count_context *)context)->count_of_each_numa[value] =
|
||||
nif_linux_count_cpu_list(path);
|
||||
((nfn_linux_fill_node_count_context *)context)->count_of_each_numa[value] =
|
||||
nfn_linux_count_cpu_list(path);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nif_linux_node_exists(int numa_id)
|
||||
static int nfn_linux_node_exists(int numa_id)
|
||||
{
|
||||
char path[128];
|
||||
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d", numa_id);
|
||||
return access(path, F_OK) == 0;
|
||||
}
|
||||
|
||||
static size_t nif_linux_round_up_to_page_size(size_t size)
|
||||
static size_t nfn_linux_round_up_to_page_size(size_t size)
|
||||
{
|
||||
int64_t page_size_value = (int64_t)sysconf(_SC_PAGESIZE);
|
||||
size_t effective_page_size = (page_size_value > 0) ? (size_t)page_size_value : 4096u;
|
||||
@@ -199,7 +199,7 @@ static size_t nif_linux_round_up_to_page_size(size_t size)
|
||||
return rounded - (rounded % effective_page_size);
|
||||
}
|
||||
|
||||
static int nif_linux_bind_memory_to_node(void *address, size_t size, int numa_id, uint32_t flags)
|
||||
static int nfn_linux_bind_memory_to_node(void *address, size_t size, int numa_id, uint32_t flags)
|
||||
{
|
||||
#if defined(SYS_mbind)
|
||||
size_t bits_per_word = sizeof(unsigned long) * 8u;
|
||||
@@ -223,7 +223,7 @@ static int nif_linux_bind_memory_to_node(void *address, size_t size, int numa_id
|
||||
#endif
|
||||
}
|
||||
|
||||
static int nif_linux_populate_writable_mapping(void *address, size_t size)
|
||||
static int nfn_linux_populate_writable_mapping(void *address, size_t size)
|
||||
{
|
||||
#if defined(MADV_POPULATE_WRITE)
|
||||
if (madvise(address, size, MADV_POPULATE_WRITE) == 0)
|
||||
@@ -238,7 +238,7 @@ static int nif_linux_populate_writable_mapping(void *address, size_t size)
|
||||
}
|
||||
|
||||
/* Linux has no Windows-style processor groups; processor_group_id is ignored. */
|
||||
NIF_API void NIF_DECL nif_set_thread_affinity(int cpu_index, int processor_group_id)
|
||||
NFN_API void NFN_DECL nfn_set_thread_affinity(int cpu_index, int processor_group_id)
|
||||
{
|
||||
(void)processor_group_id;
|
||||
|
||||
@@ -266,14 +266,14 @@ NIF_API void NIF_DECL nif_set_thread_affinity(int cpu_index, int processor_group
|
||||
CPU_FREE(set);
|
||||
}
|
||||
|
||||
NIF_API int NIF_DECL nif_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
|
||||
NFN_API int NFN_DECL nfn_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
|
||||
{
|
||||
char buffer[256];
|
||||
if (!nif_linux_read_first_line("/sys/devices/system/node/online", buffer, sizeof(buffer)))
|
||||
if (!nfn_linux_read_first_line("/sys/devices/system/node/online", buffer, sizeof(buffer)))
|
||||
return 0;
|
||||
|
||||
nif_linux_max_context max_context = { -1 };
|
||||
if (!nif_linux_visit_id_list(buffer, nif_linux_track_max_id, &max_context) ||
|
||||
nfn_linux_max_context max_context = { -1 };
|
||||
if (!nfn_linux_visit_id_list(buffer, nfn_linux_track_max_id, &max_context) ||
|
||||
max_context.max_value < 0)
|
||||
return 0;
|
||||
|
||||
@@ -284,19 +284,19 @@ NIF_API int NIF_DECL nif_get_numa_overall(int *out_numa_count, int *count_of_eac
|
||||
for (int index = 0; index <= max_context.max_value; index++)
|
||||
count_of_each_numa[index] = 0;
|
||||
|
||||
nif_linux_fill_node_count_context fill_context = { count_of_each_numa };
|
||||
if (!nif_linux_visit_id_list(buffer, nif_linux_fill_node_cpu_count, &fill_context))
|
||||
nfn_linux_fill_node_count_context fill_context = { count_of_each_numa };
|
||||
if (!nfn_linux_visit_id_list(buffer, nfn_linux_fill_node_cpu_count, &fill_context))
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
NIF_API int NIF_DECL nif_get_numa_each(int numa_id,
|
||||
uint32_t *processor_group_ids,
|
||||
uint32_t *socket_ids,
|
||||
uint32_t *cpu_indices,
|
||||
uint32_t *numa_node_ids)
|
||||
NFN_API int NFN_DECL nfn_get_numa_each(int numa_id,
|
||||
uint32_t *processor_group_ids,
|
||||
uint32_t *socket_ids,
|
||||
uint32_t *cpu_indices,
|
||||
uint32_t *numa_node_ids)
|
||||
{
|
||||
if (numa_id < 0 || processor_group_ids == NULL || socket_ids == NULL ||
|
||||
cpu_indices == NULL || numa_node_ids == NULL)
|
||||
@@ -305,10 +305,10 @@ NIF_API int NIF_DECL nif_get_numa_each(int numa_id,
|
||||
char path[128];
|
||||
char buffer[256];
|
||||
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", numa_id);
|
||||
if (!nif_linux_read_first_line(path, buffer, sizeof(buffer)))
|
||||
if (!nfn_linux_read_first_line(path, buffer, sizeof(buffer)))
|
||||
return 0;
|
||||
|
||||
nif_linux_fill_cpu_context fill_context = {
|
||||
nfn_linux_fill_cpu_context fill_context = {
|
||||
processor_group_ids,
|
||||
socket_ids,
|
||||
cpu_indices,
|
||||
@@ -316,12 +316,12 @@ NIF_API int NIF_DECL nif_get_numa_each(int numa_id,
|
||||
numa_id,
|
||||
0
|
||||
};
|
||||
return nif_linux_visit_id_list(buffer, nif_linux_fill_cpu_id, &fill_context);
|
||||
return nfn_linux_visit_id_list(buffer, nfn_linux_fill_cpu_id, &fill_context);
|
||||
}
|
||||
|
||||
NIF_API void *NIF_DECL nif_numa_alloc(size_t size, int numa_id)
|
||||
NFN_API void *NFN_DECL nfn_numa_alloc(size_t size, int numa_id)
|
||||
{
|
||||
size_t allocation_size = nif_linux_round_up_to_page_size(size);
|
||||
size_t allocation_size = nfn_linux_round_up_to_page_size(size);
|
||||
if (allocation_size == 0)
|
||||
return NULL;
|
||||
|
||||
@@ -343,9 +343,9 @@ NIF_API void *NIF_DECL nif_numa_alloc(size_t size, int numa_id)
|
||||
bind_flags = MPOL_MF_MOVE | MPOL_MF_STRICT;
|
||||
#endif
|
||||
|
||||
if (!nif_linux_node_exists(numa_id) ||
|
||||
!nif_linux_bind_memory_to_node(memory, allocation_size, numa_id, bind_flags) ||
|
||||
!nif_linux_populate_writable_mapping(memory, allocation_size)) {
|
||||
if (!nfn_linux_node_exists(numa_id) ||
|
||||
!nfn_linux_bind_memory_to_node(memory, allocation_size, numa_id, bind_flags) ||
|
||||
!nfn_linux_populate_writable_mapping(memory, allocation_size)) {
|
||||
munmap(memory, allocation_size);
|
||||
return NULL;
|
||||
}
|
||||
@@ -353,14 +353,14 @@ NIF_API void *NIF_DECL nif_numa_alloc(size_t size, int numa_id)
|
||||
return memory;
|
||||
}
|
||||
|
||||
NIF_API void NIF_DECL nif_numa_free(void *ptr, size_t size, int numa_id)
|
||||
NFN_API void NFN_DECL nfn_numa_free(void *ptr, size_t size, int numa_id)
|
||||
{
|
||||
(void)numa_id;
|
||||
|
||||
if (ptr == NULL || size == 0)
|
||||
return;
|
||||
|
||||
size_t allocation_size = nif_linux_round_up_to_page_size(size);
|
||||
size_t allocation_size = nfn_linux_round_up_to_page_size(size);
|
||||
if (allocation_size != 0)
|
||||
munmap(ptr, allocation_size);
|
||||
}
|
||||
+21
-17
@@ -1,9 +1,13 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
#include "NitloadFrameWork.h"
|
||||
|
||||
#if NIF_IS_OS(NIF_OS_WINDOWS)
|
||||
#if NFN_IS_OS(NFN_OS_WINDOWS)
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <windows.h>
|
||||
|
||||
static int nif_windows_get_socket_map(uint32_t **out_socket_map, DWORD *out_group_count)
|
||||
static int nfn_windows_get_socket_map(uint32_t **out_socket_map, DWORD *out_group_count)
|
||||
{
|
||||
DWORD group_count = GetActiveProcessorGroupCount();
|
||||
DWORD buffer_size = 0;
|
||||
@@ -64,7 +68,7 @@ static int nif_windows_get_socket_map(uint32_t **out_socket_map, DWORD *out_grou
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int nif_windows_get_processor_node(WORD group, BYTE cpu, USHORT *node_id)
|
||||
static int nfn_windows_get_processor_node(WORD group, BYTE cpu, USHORT *node_id)
|
||||
{
|
||||
PROCESSOR_NUMBER processor;
|
||||
memset(&processor, 0, sizeof(processor));
|
||||
@@ -74,7 +78,7 @@ static int nif_windows_get_processor_node(WORD group, BYTE cpu, USHORT *node_id)
|
||||
}
|
||||
|
||||
/* Pin the calling thread using its processor group and group-local CPU index. */
|
||||
NIF_API void NIF_DECL nif_set_thread_affinity(int cpu_index, int processor_group_id)
|
||||
NFN_API void NFN_DECL nfn_set_thread_affinity(int cpu_index, int processor_group_id)
|
||||
{
|
||||
if (cpu_index < 0 || processor_group_id < 0) {
|
||||
GROUP_AFFINITY current;
|
||||
@@ -91,7 +95,7 @@ NIF_API void NIF_DECL nif_set_thread_affinity(int cpu_index, int processor_group
|
||||
if ((DWORD)processor_group_id >= GetActiveProcessorGroupCount() || cpu_index >= 64)
|
||||
return;
|
||||
USHORT node_id = 0;
|
||||
if (!nif_windows_get_processor_node((WORD)processor_group_id, (BYTE)cpu_index, &node_id))
|
||||
if (!nfn_windows_get_processor_node((WORD)processor_group_id, (BYTE)cpu_index, &node_id))
|
||||
return;
|
||||
|
||||
GROUP_AFFINITY affinity;
|
||||
@@ -101,7 +105,7 @@ NIF_API void NIF_DECL nif_set_thread_affinity(int cpu_index, int processor_group
|
||||
SetThreadGroupAffinity(GetCurrentThread(), &affinity, NULL);
|
||||
}
|
||||
|
||||
NIF_API int NIF_DECL nif_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
|
||||
NFN_API int NFN_DECL nfn_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
|
||||
{
|
||||
ULONG highest;
|
||||
if (!GetNumaHighestNodeNumber(&highest))
|
||||
@@ -117,7 +121,7 @@ NIF_API int NIF_DECL nif_get_numa_overall(int *out_numa_count, int *count_of_eac
|
||||
for (DWORD group = 0; group < group_count; group++) {
|
||||
for (DWORD cpu = 0; cpu < 64; cpu++) {
|
||||
USHORT node_id = 0;
|
||||
if (nif_windows_get_processor_node((WORD)group, (BYTE)cpu, &node_id) &&
|
||||
if (nfn_windows_get_processor_node((WORD)group, (BYTE)cpu, &node_id) &&
|
||||
count_of_each_numa != NULL && node_id <= highest)
|
||||
count_of_each_numa[node_id]++;
|
||||
}
|
||||
@@ -125,11 +129,11 @@ NIF_API int NIF_DECL nif_get_numa_overall(int *out_numa_count, int *count_of_eac
|
||||
return 1;
|
||||
}
|
||||
|
||||
NIF_API int NIF_DECL nif_get_numa_each(int numa_id,
|
||||
uint32_t *processor_group_ids,
|
||||
uint32_t *socket_ids,
|
||||
uint32_t *cpu_indices,
|
||||
uint32_t *numa_node_ids)
|
||||
NFN_API int NFN_DECL nfn_get_numa_each(int numa_id,
|
||||
uint32_t *processor_group_ids,
|
||||
uint32_t *socket_ids,
|
||||
uint32_t *cpu_indices,
|
||||
uint32_t *numa_node_ids)
|
||||
{
|
||||
if (numa_id < 0 || processor_group_ids == NULL || socket_ids == NULL ||
|
||||
cpu_indices == NULL || numa_node_ids == NULL)
|
||||
@@ -137,14 +141,14 @@ NIF_API int NIF_DECL nif_get_numa_each(int numa_id,
|
||||
|
||||
uint32_t *socket_map = NULL;
|
||||
DWORD group_count = 0;
|
||||
if (!nif_windows_get_socket_map(&socket_map, &group_count))
|
||||
if (!nfn_windows_get_socket_map(&socket_map, &group_count))
|
||||
return 0;
|
||||
|
||||
int index = 0;
|
||||
for (DWORD group = 0; group < group_count; group++) {
|
||||
for (DWORD cpu = 0; cpu < 64; cpu++) {
|
||||
USHORT node_id = 0;
|
||||
if (!nif_windows_get_processor_node((WORD)group, (BYTE)cpu, &node_id) ||
|
||||
if (!nfn_windows_get_processor_node((WORD)group, (BYTE)cpu, &node_id) ||
|
||||
node_id != (USHORT)numa_id)
|
||||
continue;
|
||||
|
||||
@@ -166,7 +170,7 @@ NIF_API int NIF_DECL nif_get_numa_each(int numa_id,
|
||||
}
|
||||
|
||||
|
||||
NIF_API void *NIF_DECL nif_numa_alloc(size_t size, int numa_id)
|
||||
NFN_API void *NFN_DECL nfn_numa_alloc(size_t size, int numa_id)
|
||||
{
|
||||
if (size == 0)
|
||||
return NULL;
|
||||
@@ -182,7 +186,7 @@ NIF_API void *NIF_DECL nif_numa_alloc(size_t size, int numa_id)
|
||||
PAGE_READWRITE, (DWORD)numa_id);
|
||||
}
|
||||
|
||||
NIF_API void NIF_DECL nif_numa_free(void *ptr, size_t size, int numa_id)
|
||||
NFN_API void NFN_DECL nfn_numa_free(void *ptr, size_t size, int numa_id)
|
||||
{
|
||||
(void)size;
|
||||
(void)numa_id;
|
||||
@@ -1,7 +1,7 @@
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_internal.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_internal.h"
|
||||
|
||||
INTERNAL void NIF_DECL f32_normalize(VEC_SEGMENT(float) *restrict source, VEC_SEGMENT(float) *restrict target)
|
||||
INTERNAL void NFN_DECL f32_normalize(VEC_SEGMENT(float) *restrict source, VEC_SEGMENT(float) *restrict target)
|
||||
{
|
||||
register __m128 s = source->f32_x[0];
|
||||
register __m128 a_length_sq = _mm_mul_ps(s, s);
|
||||
@@ -1,9 +1,9 @@
|
||||
#include "nif_intrinsic.h"
|
||||
#include "nif_internal.h"
|
||||
#include "nfn_intrinsic.h"
|
||||
#include "nfn_internal.h"
|
||||
|
||||
static const int avxdot_indices[] = { 0, 4, 1, 5, 2, 3, 6, 7 };
|
||||
|
||||
INTERNAL void FORCEINLINE NIF_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
|
||||
INTERNAL void FORCEINLINE NFN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
|
||||
__m128 const *vector, __m128 *ans)
|
||||
{
|
||||
*ans = _mm_dp_ps(transpose->x[0], *vector, 0B11110001);
|
||||
@@ -15,7 +15,7 @@ INTERNAL void FORCEINLINE NIF_DECL dot_41_sse(MAT_KERNEL(float) const *transpose
|
||||
SSE_INSERT_SIMPLE(1, 4));
|
||||
}
|
||||
|
||||
INTERNAL void NIF_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
INTERNAL void NFN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans)
|
||||
{
|
||||
__m256 _double = _mm256_set_m128(vector[0], vector[0]);
|
||||
@@ -26,7 +26,7 @@ INTERNAL void NIF_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 con
|
||||
_mm256_permutevar8x32_ps(tmp, _mm256_loadu_epi32(avxdot_indices)));
|
||||
}
|
||||
|
||||
INTERNAL void NIF_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
INTERNAL void NFN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans)
|
||||
{
|
||||
ans[0] = _mm_dp_ps(transpose->x[0], vector[0], 0B11110001);
|
||||
@@ -46,7 +46,7 @@ INTERNAL void NIF_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 con
|
||||
SSE_INSERT_SIMPLE(1, 4));
|
||||
}
|
||||
|
||||
INTERNAL void NIF_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
INTERNAL void NFN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans)
|
||||
{
|
||||
__m256 temp = transpose->y[0];
|
||||
+9
-9
@@ -1,8 +1,8 @@
|
||||
#include "nif_target.h"
|
||||
#include "nfn_target.h"
|
||||
|
||||
#if NIF_IS_ARCH(NIF_ARCH_X86_64)
|
||||
#if NFN_IS_ARCH(NFN_ARCH_X86_64)
|
||||
|
||||
#define NIF_MK_FMA_F32_512_CYCLE(i) \
|
||||
#define NFN_MK_FMA_F32_512_CYCLE(i) \
|
||||
do { \
|
||||
c0 = _mm512_fmadd_ps(c0, a0, _mm512_permute_ps(b0, _FMA_SELECT(i))); \
|
||||
c1 = _mm512_fmadd_ps(c1, a1, _mm512_permute_ps(b1, _FMA_SELECT(i))); \
|
||||
@@ -10,7 +10,7 @@
|
||||
c3 = _mm512_fmadd_ps(c3, a3, _mm512_permute_ps(b3, _FMA_SELECT(i))); \
|
||||
} while (0)
|
||||
|
||||
#define NIF_MK_FMA_F32_AVX2_CYCLE(i) \
|
||||
#define NFN_MK_FMA_F32_AVX2_CYCLE(i) \
|
||||
do { \
|
||||
c0 = _mm256_fmadd_ps(c0, a0, _mm256_permute_ps(b0, _FMA_SELECT(i))); \
|
||||
c1 = _mm256_fmadd_ps(c1, a1, _mm256_permute_ps(b1, _FMA_SELECT(i))); \
|
||||
@@ -18,7 +18,7 @@
|
||||
c3 = _mm256_fmadd_ps(c3, a3, _mm256_permute_ps(b3, _FMA_SELECT(i))); \
|
||||
} while (0)
|
||||
|
||||
#define NIF_MK_FMA_F64_AVX2_CYCLE(i) \
|
||||
#define NFN_MK_FMA_F64_AVX2_CYCLE(i) \
|
||||
do { \
|
||||
c0 = _mm512_fmadd_ps(c0, a0, _mm512_permutex_pd(b0, _FMA_SELECT(i))); \
|
||||
c1 = _mm512_fmadd_ps(c1, a1, _mm512_permutex_pd(b1, _FMA_SELECT(i))); \
|
||||
@@ -26,7 +26,7 @@
|
||||
c3 = _mm512_fmadd_ps(c3, a3, _mm512_permutex_pd(b3, _FMA_SELECT(i))); \
|
||||
} while (0)
|
||||
|
||||
#define NIF_FMA_F32_512_PACK_COLUMN(col, target) \
|
||||
#define NFN_FMA_F32_512_PACK_COLUMN(col, target) \
|
||||
do { \
|
||||
register __m256 low = _mm512_castps512_ps256(col); \
|
||||
register __m256 high = _mm512_extractf32x8_ps(col, 1); \
|
||||
@@ -37,7 +37,7 @@
|
||||
_mm_store_ps(target, half128); \
|
||||
} while (0)
|
||||
|
||||
#define NIF_FMA_F32_AVX2_PACK_COLUMN(col, target) \
|
||||
#define NFN_FMA_F32_AVX2_PACK_COLUMN(col, target) \
|
||||
do { \
|
||||
register __m128 high128 = _mm256_extractf32x4_ps(col, 1); \
|
||||
register __m128 low128 = _mm256_castps256_ps128(col); \
|
||||
@@ -45,7 +45,7 @@
|
||||
_mm_store_ps(target, half128); \
|
||||
} while (0)
|
||||
|
||||
#define NIF_FMA_F64_512_PACK_COLUMN(col, target) \
|
||||
#define NFN_FMA_F64_512_PACK_COLUMN(col, target) \
|
||||
do { \
|
||||
register __m128 high128 = _mm256_extractf32x4_ps(col, 1); \
|
||||
register __m128 low128 = _mm256_castps256_ps128(col); \
|
||||
@@ -54,7 +54,7 @@
|
||||
} while (0)
|
||||
|
||||
// double
|
||||
#define NIF_MK_FMA_F64_SSE_PHASE(i) \
|
||||
#define NFN_MK_FMA_F64_SSE_PHASE(i) \
|
||||
do { \
|
||||
/* k = 0 */ \
|
||||
register __m128d b_lo = _mm_load_pd(right_values + (2 * i + 0)); \
|
||||
@@ -1,39 +0,0 @@
|
||||
#ifndef NIF_NATIVE_H
|
||||
#define NIF_NATIVE_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
#error "This header is not meant to be included in C++ code. Nitload.Native is a C library and should be used with C linkage."
|
||||
#endif
|
||||
|
||||
#include "nif_internal.h"
|
||||
#include "nif_target.h"
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <assert.h>
|
||||
#include <glfw_function.h>
|
||||
|
||||
#define byte char
|
||||
#define cs_char uint16_t
|
||||
#define nif_null_assert(ptr) assert(ptr != NULL)
|
||||
|
||||
#ifndef NIF_
|
||||
#endif
|
||||
|
||||
NIF_API void NIF_DECL nif_load_glfw_functions(GLFWFunctionTable *table);
|
||||
NIF_API void NIF_DECL nif_load_intrinsic_function(void *intrinsic_context, uint64_t id);
|
||||
NIF_API SIMDID NIF_DECL nif_get_simd_level_global();
|
||||
/* Per-core variant: always re-executes CPUID, never caches.
|
||||
* Must be called on the target core (after affinity is set). */
|
||||
NIF_API SIMDID NIF_DECL nif_get_simd_level_local();
|
||||
NIF_API SIMDID NIF_DECL nif_get_unite_simd(SIMDID left, SIMDID right);
|
||||
NIF_API int NIF_DECL nif_compare_simd(SIMDID left, SIMDID right);
|
||||
NIF_API void NIF_DECL nif_set_thread_affinity(int cpu_index, int processor_group_id);
|
||||
NIF_API void* NIF_DECL nif_numa_alloc(size_t size, int numa_id);
|
||||
NIF_API void NIF_DECL nif_numa_free(void *ptr, size_t size, int numa_id);
|
||||
NIF_API int NIF_DECL nif_get_numa_each(int numa_id,
|
||||
uint32_t *processor_group_ids,
|
||||
uint32_t *socket_ids,
|
||||
uint32_t *cpu_indices,
|
||||
uint32_t *numa_node_ids);
|
||||
NIF_API int NIF_DECL nif_get_numa_overall(int *out_numa_count, int *count_of_each_numa);
|
||||
#endif
|
||||
@@ -1,54 +0,0 @@
|
||||
#ifndef NIF_MATKERNEL
|
||||
#define NIF_MATKERNEL
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "nif_internal.h"
|
||||
|
||||
/* Kernel storage unions for a fixed 4x4 block. */
|
||||
#define ELEMENTSIZE(T) sizeof(T)
|
||||
#define XMMCOUNT(T) (sizeof(T))
|
||||
#define YMMCOUNT(T) (sizeof(T) / 2)
|
||||
#define ZMMCOUNT(T) (sizeof(T) / 4)
|
||||
|
||||
#define NIF_MAT_KERNEL_MEMBER(T) \
|
||||
{ \
|
||||
__m128 x[XMMCOUNT(T)]; \
|
||||
__m256 y[YMMCOUNT(T)]; \
|
||||
__m512 z[ZMMCOUNT(T)]; \
|
||||
T m[4][4]; \
|
||||
}
|
||||
|
||||
#define NIF_MAT_KERNEL_DEFINE(T) \
|
||||
typedef union NIF_MAT_KERNEL_TYPE_##T NIF_MAT_KERNEL_MEMBER(T) NIF_MAT_KERNEL_TYPE_##T
|
||||
|
||||
#define MAT_KERNEL(T) NIF_MAT_KERNEL_TYPE(T)
|
||||
#define NIF_MAT_KERNEL_TYPE(T) NIF_MAT_KERNEL_TYPE_##T
|
||||
|
||||
NIF_MAT_KERNEL_DEFINE(float);
|
||||
NIF_MAT_KERNEL_DEFINE(double);
|
||||
NIF_MAT_KERNEL_DEFINE(uint64_t);
|
||||
|
||||
|
||||
#define NIF_VEC_SEGMENT_MEMBER(T) \
|
||||
{ \
|
||||
__m128 f32_x[XMMCOUNT(T) / 4]; \
|
||||
__m256 f32_y[YMMCOUNT(T) / 4]; \
|
||||
__m512 f32_z[ZMMCOUNT(T) / 4]; \
|
||||
T v[4]; \
|
||||
}
|
||||
|
||||
#define NIF_VEC_SEGMENT_DEFINE(T) \
|
||||
typedef union NIF_VEC_SEGMENT_TYPE_##T NIF_VEC_SEGMENT_MEMBER(T) NIF_VEC_SEGMENT_TYPE_##T
|
||||
|
||||
#define VEC_SEGMENT(T) NIF_VEC_SEGMENT_TYPE(T)
|
||||
#define NIF_VEC_SEGMENT_TYPE(T) NIF_VEC_SEGMENT_TYPE_##T
|
||||
|
||||
NIF_VEC_SEGMENT_DEFINE(float);
|
||||
NIF_VEC_SEGMENT_DEFINE(double);
|
||||
|
||||
/* Global zero kernel declaration (defined in dllmain.c), aligned to 64 bytes */
|
||||
extern __declspec(align(64)) MAT_KERNEL(u64) NIF_ZERO_KERNEL;
|
||||
|
||||
#endif /* NIF_MATKERNEL */
|
||||
@@ -1,101 +0,0 @@
|
||||
#ifndef NIF_INTRINSIC_CALLCONV_STD
|
||||
#define NIF_INTRINSIC_CALLCONV_STD
|
||||
|
||||
#include "nif_internal.h"
|
||||
#include "nif_intrinsic.h"
|
||||
|
||||
#ifndef static_assert
|
||||
#define static_assert _Static_assert
|
||||
#endif
|
||||
|
||||
#define NIF_STD_TARGET_ATTR(T_arch, T_feature) NIF_STD_TARGET_ATTR_IMPL(T_arch, T_feature)
|
||||
|
||||
#pragma region mk_proc attributes
|
||||
#define MK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
|
||||
T_type##_multi_kernel_proc_##T_op##_##T_arch##_##T_feature
|
||||
#define GET_MK_PROC_STD(T_type, T_arch, T_feature, T_op) \
|
||||
((UNI_MK_PROC)MK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op))
|
||||
#define NIF_MK_PROC_DECL(T_type, P_name) INTERNAL void NIF_DECL P_name(MAT_TASK(T_type) const *task)
|
||||
#define NIF_MK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
|
||||
INTERNAL NOINLINE NIF_STD_TARGET_ATTR(T_arch, T_feature) void NIF_DECL MK_PROC_NAME_STD( \
|
||||
T_type, T_arch, T_feature, T_op)(MAT_TASK(T_type) const *task)
|
||||
#pragma endregion
|
||||
|
||||
#pragma region sk_proc attributes
|
||||
#define SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
|
||||
T_type##_single_kernel_proc_##T_op##_##T_arch##_##T_feature
|
||||
#define GET_SK_PROC_STD(T_type, T_arch, T_feature, T_op) \
|
||||
((UNI_SK_PROC)SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op))
|
||||
#define NIF_SK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
|
||||
INTERNAL NOINLINE NIF_STD_TARGET_ATTR(T_arch, T_feature) void NIF_DECL SK_PROC_NAME_STD( \
|
||||
T_type, T_arch, T_feature, \
|
||||
T_op)(MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
|
||||
MAT_KERNEL(T_type) *restrict ans, uint64_t scalar)
|
||||
#pragma endregion
|
||||
|
||||
#define _SN_STD__PROBE_0 0
|
||||
#define _SN_STD__PROBE_1 1
|
||||
#define NIF_STD__PROBE_PRINT(x) _SN_STD__PROBE_##x
|
||||
#define NIF_STD__PROBE(x) NIF_STD__PROBE_PRINT(x)
|
||||
|
||||
#pragma region param styles
|
||||
|
||||
#define _SN_MATPROC_LEFT_RIGHT_ANS 1
|
||||
#define _SN_MATPROC_RIGHT_ANS 1
|
||||
#define _SN_MATPROC_LEFT_RIGHT_ANS_SCALAR 1
|
||||
#define _SN_MATPROC_RIGHT_ANS_SCALAR 1
|
||||
#define _SN_MATPROC_ANS_SCALAR 1
|
||||
|
||||
#define _SN_MK_EXPAND_BUFFER_SIZE(i)\
|
||||
int offset_##i = task->offset_##i;\
|
||||
int count_##i = task->count_##i;\
|
||||
|
||||
#define _SN_MK_EXPAND_RIGHT_ANS \
|
||||
NIF_MAT_KERNEL_LOCAL const *right = (NIF_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans = (NIF_MAT_KERNEL_LOCAL *)task->buffer_2.ptr;\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(1); \
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS \
|
||||
NIF_MAT_KERNEL_LOCAL const *left = (NIF_MAT_KERNEL_LOCAL const *)task->buffer_0.ptr; \
|
||||
NIF_MAT_KERNEL_LOCAL const *right = (NIF_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans = (NIF_MAT_KERNEL_LOCAL *)task->buffer_2.ptr;\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(0); \
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(1);\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _SN_MK_EXPAND_RIGHT_ANS_SCALAR \
|
||||
NIF_MAT_KERNEL_LOCAL const *right = (NIF_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans = (NIF_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
|
||||
M128 scalar = task->scalar_pack;\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(1);\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS_SCALAR \
|
||||
NIF_MAT_KERNEL_LOCAL const *left = (NIF_MAT_KERNEL_LOCAL const *)task->buffer_0.ptr; \
|
||||
NIF_MAT_KERNEL_LOCAL const *right = (NIF_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans = (NIF_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
|
||||
M128 scalar = task->scalar_pack;\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(0);\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(1);\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _SN_MK_EXPAND_ANS_SCALAR \
|
||||
NIF_MAT_KERNEL_LOCAL *restrict ans = (NIF_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
|
||||
M128 scalar = task->scalar_pack;\
|
||||
_SN_MK_EXPAND_BUFFER_SIZE(2);
|
||||
|
||||
#define _SN_PARAM_STYLE(T) _SN_MATPROC_##T
|
||||
#define _SN_MK_EXPAND_PARAM(style) _SN_MK_EXPAND_##style
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#define __mk_param_std(param, T) \
|
||||
typedef MAT_KERNEL(T) NIF_MAT_KERNEL_LOCAL; \
|
||||
matrix_task const *_ = (_SN_PARAM_STYLE(param), task); \
|
||||
_SN_MK_EXPAND_PARAM(param)
|
||||
|
||||
#define __sk_param_std(T) \
|
||||
typedef MAT_KERNEL(T) NIF_MAT_KERNEL_LOCAL; \
|
||||
(void)scalar
|
||||
#endif // !NIF_INTRINSIC_PARAM_EXPAND
|
||||
@@ -1,94 +0,0 @@
|
||||
#ifndef NIF_TARGET
|
||||
#define NIF_TARGET
|
||||
|
||||
#include "nif_internal.h"
|
||||
#include <stdint.h>
|
||||
|
||||
#define NIF_OS_WINDOWS 0
|
||||
#define NIF_OS_LINUX 1
|
||||
#define NIF_OS_MACOS 2
|
||||
#define NIF_OS_IOS 3
|
||||
#define NIF_OS_ANDROID 4
|
||||
#define NIF_OS_UNKNOWN 5
|
||||
|
||||
#define NIF_ARCH_X86_64 0
|
||||
#define NIF_ARCH_ARM64 1
|
||||
#define NIF_ARCH_RISCV64 2
|
||||
#define NIF_ARCH_PPC64le 3
|
||||
|
||||
#if defined(_M_X64) || defined(_M_AMD64) || defined(__x86_64__) || defined(__amd64__)
|
||||
#define NIF_CURRENT_ARCH NIF_ARCH_X86_64
|
||||
|
||||
#elif defined(_M_ARM64) || defined(__aarch64__)
|
||||
#define NIF_CURRENT_ARCH NIF_ARCH_ARM64
|
||||
|
||||
#elif defined(_M_ARM) || defined(__arm__)
|
||||
#define NIF_CURRENT_ARCH NIF_ARCH_ARM32
|
||||
|
||||
#elif defined(_M_IX86) || defined(__i386__)
|
||||
#define NIF_CURRENT_ARCH NIF_ARCH_X86_32
|
||||
|
||||
#elif defined(__powerpc64__) && defined(__LITTLE_ENDIAN__)
|
||||
#define NIF_CURRENT_ARCH NIF_ARCH_PPC64LE
|
||||
|
||||
#elif defined(__riscv) && (__riscv_xlen == 64)
|
||||
#define NIF_CURRENT_ARCH NIF_ARCH_RISCV64
|
||||
|
||||
#else
|
||||
#error "SN: Unsupported target architecture"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Target selection policy:
|
||||
* - Default (no NIF_EXTERN_TARGET): fixed Windows + x86_64.
|
||||
* - External (NIF_EXTERN_TARGET): build system must provide NIF_OS and NIF_ARCH.
|
||||
*/
|
||||
#ifndef NIF_EXTERN_TARGET
|
||||
|
||||
#define NIF_OS_TARGET NIF_OS_WINDOWS
|
||||
#define NIF_ARCH_TARGET NIF_ARCH_X86_64
|
||||
|
||||
#else
|
||||
|
||||
#ifndef NIF_OS_TARGET
|
||||
#error "NIF_EXTERN_TARGET is defined but NIF_OS is not provided."
|
||||
#endif
|
||||
|
||||
#ifndef NIF_ARCH_TARGET
|
||||
#error "NIF_EXTERN_TARGET is defined but NIF_ARCH is not provided."
|
||||
#endif
|
||||
|
||||
/* Validate externally supplied values are within known enum constants. */
|
||||
#if (NIF_OS_TARGET != NIF_OS_WINDOWS) && (NIF_OS_TARGET != NIF_OS_LINUX) && \
|
||||
(NIF_OS_TARGET != NIF_OS_MACOS) && (NIF_OS_TARGET != NIF_OS_IOS) && \
|
||||
(NIF_OS_TARGET != NIF_OS_ANDROID) && (NIF_OS_TARGET != NIF_OS_UNKNOWN)
|
||||
#error "NIF_OS has an invalid value. Use NIF_OS_<...> enum constants."
|
||||
#endif
|
||||
|
||||
#if (NIF_ARCH_TARGET != NIF_ARCH_X86_64) && (NIF_ARCH_TARGET != NIF_ARCH_ARM64) && \
|
||||
(NIF_ARCH_TARGET != NIF_ARCH_RISCV64) && (NIF_ARCH_TARGET != NIF_ARCH_PPC64le)
|
||||
#error "NIF_ARCH has an invalid value. Use NIF_ARCH_<...> enum constants."
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#define NIF_IS_OS(os) ((NIF_OS_TARGET) == (os))
|
||||
|
||||
#define NIF_IS_ARCH(arch) ((NIF_ARCH_TARGET) == (arch))
|
||||
|
||||
/*
|
||||
* Per-function ISA selection is required for runtime-dispatched kernels that
|
||||
* mix multiple x86 feature levels in one build target.
|
||||
*/
|
||||
#if defined(__clang__) && ((NIF_ARCH_TARGET) == NIF_ARCH_X86_64)
|
||||
#define NIF_CLANG_TARGET_ATTR(options) __attribute__((target(options)))
|
||||
#else
|
||||
#define NIF_CLANG_TARGET_ATTR(options)
|
||||
#endif
|
||||
|
||||
typedef union SIMDID {
|
||||
uint64_t mask;
|
||||
char make_byte[8];
|
||||
} SIMDID;
|
||||
|
||||
#endif // NIF_TARGET
|
||||
@@ -32,7 +32,6 @@ using StgSharp;
|
||||
using StgSharp.Collections;
|
||||
using StgSharp.Script;
|
||||
using StgSharp.Script.Express;
|
||||
using Nitload.Common.Collections;
|
||||
|
||||
|
||||
//using StgSharp.Model;
|
||||
@@ -47,22 +46,6 @@ namespace STgSharpTest
|
||||
{
|
||||
string testExpPath = @"D:\Programmer\stepcode\data\ap203\ap203.exp";
|
||||
|
||||
[TestMethod]
|
||||
public void SwissTableOperationsAreNotImplemented()
|
||||
{
|
||||
using SwissTable table = SwissTable.Create();
|
||||
nuint key = 17;
|
||||
|
||||
Assert.ThrowsException<NotImplementedException>(
|
||||
() => SwissTable.Reader.TryGet(table, key, out _));
|
||||
Assert.ThrowsException<NotImplementedException>(
|
||||
() => SwissTable.Writer.TryAdd(table, key, 100));
|
||||
Assert.ThrowsException<NotImplementedException>(
|
||||
() => SwissTable.Writer.TrySet(table, key, 100));
|
||||
Assert.ThrowsException<NotImplementedException>(
|
||||
() => SwissTable.Writer.TryRemove(table, key, out _));
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public void RegexTest()
|
||||
{
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="TestSwiss"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Nitload.Common.Collections;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Nitload
|
||||
{
|
||||
internal static partial class Program
|
||||
{
|
||||
|
||||
public static void TestSwiss()
|
||||
{
|
||||
SwissTable swiss = SwissTable.Create();
|
||||
for (nuint i = 0; i < 5000; i++)
|
||||
{
|
||||
Console.Write($"Set {i}: ");
|
||||
_ = SwissTable.Writer.TrySet(swiss, i, i * 2);
|
||||
Console.WriteLine($"Set a kvp of ({i},{i * 2})");
|
||||
}
|
||||
Random rand = new Random();
|
||||
for (int i = 0; i < 100; i++)
|
||||
{
|
||||
Console.Write($"Get {i}: ");
|
||||
nuint key = (nuint)rand.Next(4000, 5000);
|
||||
if (SwissTable.Reader.TryGet(swiss, key, out nuint value))
|
||||
{
|
||||
Console.WriteLine($"Success in get a kvp of ({key},{value})");
|
||||
} else
|
||||
{
|
||||
Console.WriteLine($"Failed to get a kvp of ({key},?)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,215 +0,0 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[switch] $Apply
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$taskWorkspaceRoot = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..'))
|
||||
$nativeProjectRoot = Join-Path $taskWorkspaceRoot 'src\Nitload.Native'
|
||||
$textRoots = @(
|
||||
$nativeProjectRoot
|
||||
(Join-Path $taskWorkspaceRoot 'src\Nitload.Common')
|
||||
(Join-Path $taskWorkspaceRoot 'src\Nitload.Mathematics')
|
||||
(Join-Path $taskWorkspaceRoot 'src\Nitload.Graphics')
|
||||
)
|
||||
|
||||
$textExtensions = [Collections.Generic.HashSet[string]]::new(
|
||||
[StringComparer]::OrdinalIgnoreCase
|
||||
)
|
||||
@(
|
||||
'.c', '.h', '.inl', '.inc', '.ipp',
|
||||
'.cs', '.cmake', '.txt', '.md',
|
||||
'.csproj', '.vcxproj', '.filters',
|
||||
'.json', '.props', '.targets'
|
||||
) | ForEach-Object { [void] $textExtensions.Add($_) }
|
||||
|
||||
$replacementRules = @(
|
||||
@{ Pattern = '\bSTGSHARP_'; Replacement = 'NIF_' }
|
||||
@{ Pattern = '\bSN_'; Replacement = 'NIF_' }
|
||||
@{ Pattern = '(?<![A-Za-z0-9])sn_'; Replacement = 'nif_' }
|
||||
@{ Pattern = '\bload_glfw_functions\b'; Replacement = 'nif_load_glfw_functions' }
|
||||
@{ Pattern = '\bload_intrinsic_function\b'; Replacement = 'nif_load_intrinsic_function' }
|
||||
@{ Pattern = '\bStgSharpNative\b'; Replacement = 'NitloadFrameWork' }
|
||||
@{ Pattern = '\bNitloadInnovationFramework\b'; Replacement = 'NitloadFrameWork' }
|
||||
@{ Pattern = '\bNitloadFramework\b'; Replacement = 'NitloadFrameWork' }
|
||||
@{ Pattern = '\bSNative\b'; Replacement = 'NIF_NATIVE_H' }
|
||||
@{ Pattern = 'sn-head'; Replacement = 'nif-head' }
|
||||
@{ Pattern = 'sn-src'; Replacement = 'nif-src' }
|
||||
@{ Pattern = 'sn-template'; Replacement = 'nif-template' }
|
||||
)
|
||||
|
||||
function Test-TaskPath {
|
||||
param([Parameter(Mandatory)][string] $Path)
|
||||
|
||||
$resolvedPath = [IO.Path]::GetFullPath($Path)
|
||||
if (-not $resolvedPath.StartsWith(
|
||||
$taskWorkspaceRoot + [IO.Path]::DirectorySeparatorChar,
|
||||
[StringComparison]::OrdinalIgnoreCase
|
||||
)) {
|
||||
throw "Path escapes workspace: $resolvedPath"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-TaskTextFile {
|
||||
foreach ($textRoot in $textRoots) {
|
||||
Get-ChildItem -LiteralPath $textRoot -Recurse -File | Where-Object {
|
||||
$relativePath = [IO.Path]::GetRelativePath($taskWorkspaceRoot, $_.FullName)
|
||||
if ($relativePath -match '(^|[\\/])(bin|obj|\.discard)([\\/]|$)') {
|
||||
return $false
|
||||
}
|
||||
|
||||
$nativeLibRoot = Join-Path $nativeProjectRoot 'lib'
|
||||
if ($_.FullName.StartsWith(
|
||||
$nativeLibRoot + [IO.Path]::DirectorySeparatorChar,
|
||||
[StringComparison]::OrdinalIgnoreCase
|
||||
) -and $_.Name -ne 'CMakeLists.txt') {
|
||||
return $false
|
||||
}
|
||||
|
||||
return $textExtensions.Contains($_.Extension)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Read-TaskText {
|
||||
param([Parameter(Mandatory)][string] $Path)
|
||||
|
||||
$bytes = [IO.File]::ReadAllBytes($Path)
|
||||
$encoding = [Text.UTF8Encoding]::new($false, $true)
|
||||
$preambleLength = 0
|
||||
$preambleBytes = [byte[]]::new(0)
|
||||
|
||||
if ($bytes.Length -ge 3 -and
|
||||
$bytes[0] -eq 0xEF -and $bytes[1] -eq 0xBB -and $bytes[2] -eq 0xBF) {
|
||||
$preambleLength = 3
|
||||
}
|
||||
elseif ($bytes.Length -ge 2 -and $bytes[0] -eq 0xFF -and $bytes[1] -eq 0xFE) {
|
||||
$encoding = [Text.UnicodeEncoding]::new($false, $false, $true)
|
||||
$preambleLength = 2
|
||||
}
|
||||
elseif ($bytes.Length -ge 2 -and $bytes[0] -eq 0xFE -and $bytes[1] -eq 0xFF) {
|
||||
$encoding = [Text.UnicodeEncoding]::new($true, $false, $true)
|
||||
$preambleLength = 2
|
||||
}
|
||||
|
||||
if ($preambleLength -ne 0) {
|
||||
$preambleBytes = [byte[]] $bytes[0..($preambleLength - 1)]
|
||||
}
|
||||
|
||||
[PSCustomObject]@{
|
||||
Text = $encoding.GetString($bytes, $preambleLength, $bytes.Length - $preambleLength)
|
||||
Encoding = $encoding
|
||||
Preamble = $preambleBytes
|
||||
}
|
||||
}
|
||||
|
||||
function Write-TaskText {
|
||||
param(
|
||||
[Parameter(Mandatory)][string] $Path,
|
||||
[Parameter(Mandatory)][string] $Text,
|
||||
[Parameter(Mandatory)] $Encoding,
|
||||
[Parameter(Mandatory)][AllowEmptyCollection()][byte[]] $Preamble
|
||||
)
|
||||
|
||||
$body = $Encoding.GetBytes($Text)
|
||||
$output = [byte[]]::new($Preamble.Length + $body.Length)
|
||||
if ($Preamble.Length -ne 0) {
|
||||
[Buffer]::BlockCopy($Preamble, 0, $output, 0, $Preamble.Length)
|
||||
}
|
||||
[Buffer]::BlockCopy($body, 0, $output, $Preamble.Length, $body.Length)
|
||||
[IO.File]::WriteAllBytes($Path, $output)
|
||||
}
|
||||
|
||||
$changedTextFiles = [Collections.Generic.List[string]]::new()
|
||||
foreach ($textFile in Get-TaskTextFile) {
|
||||
$fileContent = Read-TaskText -Path $textFile.FullName
|
||||
$updatedText = $fileContent.Text
|
||||
foreach ($rule in $replacementRules) {
|
||||
$updatedText = [Text.RegularExpressions.Regex]::Replace(
|
||||
$updatedText,
|
||||
$rule.Pattern,
|
||||
$rule.Replacement,
|
||||
[Text.RegularExpressions.RegexOptions]::CultureInvariant
|
||||
)
|
||||
}
|
||||
|
||||
if ($updatedText -ceq $fileContent.Text) {
|
||||
continue
|
||||
}
|
||||
|
||||
$relativePath = [IO.Path]::GetRelativePath($taskWorkspaceRoot, $textFile.FullName)
|
||||
$changedTextFiles.Add($relativePath)
|
||||
if ($Apply) {
|
||||
Write-TaskText -Path $textFile.FullName `
|
||||
-Text $updatedText `
|
||||
-Encoding $fileContent.Encoding `
|
||||
-Preamble $fileContent.Preamble
|
||||
}
|
||||
}
|
||||
|
||||
$pathMoves = [Collections.Generic.List[object]]::new()
|
||||
Get-ChildItem -LiteralPath $nativeProjectRoot -Recurse -File |
|
||||
Where-Object {
|
||||
$_.Name -match '^sn_' -or
|
||||
$_.Name -cin @(
|
||||
'StgSharpNative.h',
|
||||
'NitloadInnovationFramework.h',
|
||||
'NitloadFramework.h'
|
||||
)
|
||||
} |
|
||||
ForEach-Object {
|
||||
$newName = if ($_.Name -cin @(
|
||||
'StgSharpNative.h',
|
||||
'NitloadInnovationFramework.h',
|
||||
'NitloadFramework.h'
|
||||
)) {
|
||||
'NitloadFrameWork.h'
|
||||
}
|
||||
else {
|
||||
$_.Name -replace '^sn_', 'nif_'
|
||||
}
|
||||
$pathMoves.Add([PSCustomObject]@{
|
||||
Source = $_.FullName
|
||||
Target = Join-Path $_.DirectoryName $newName
|
||||
})
|
||||
}
|
||||
|
||||
Get-ChildItem -LiteralPath $nativeProjectRoot -Recurse -Directory |
|
||||
Where-Object { $_.Name -in @('sn-head', 'sn-src', 'sn-template') } |
|
||||
Sort-Object { $_.FullName.Length } -Descending |
|
||||
ForEach-Object {
|
||||
$pathMoves.Add([PSCustomObject]@{
|
||||
Source = $_.FullName
|
||||
Target = Join-Path $_.Parent.FullName ($_.Name -replace '^sn-', 'nif-')
|
||||
})
|
||||
}
|
||||
|
||||
foreach ($pathMove in $pathMoves) {
|
||||
Test-TaskPath -Path $pathMove.Source
|
||||
Test-TaskPath -Path $pathMove.Target
|
||||
if (Test-Path -LiteralPath $pathMove.Target) {
|
||||
throw "Rename target already exists: $($pathMove.Target)"
|
||||
}
|
||||
}
|
||||
|
||||
"Text files to update: $($changedTextFiles.Count)"
|
||||
$changedTextFiles | ForEach-Object { " TEXT $_" }
|
||||
"Paths to rename: $($pathMoves.Count)"
|
||||
$pathMoves | ForEach-Object {
|
||||
$source = [IO.Path]::GetRelativePath($taskWorkspaceRoot, $_.Source)
|
||||
$target = [IO.Path]::GetRelativePath($taskWorkspaceRoot, $_.Target)
|
||||
" MOVE $source -> $target"
|
||||
}
|
||||
|
||||
if (-not $Apply) {
|
||||
'Dry run only. Re-run with -Apply to perform the migration.'
|
||||
exit 0
|
||||
}
|
||||
|
||||
foreach ($pathMove in $pathMoves) {
|
||||
Move-Item -LiteralPath $pathMove.Source -Destination $pathMove.Target
|
||||
}
|
||||
|
||||
'NIF prefix migration completed.'
|
||||
@@ -0,0 +1,77 @@
|
||||
[CmdletBinding()]
|
||||
param([switch] $Apply)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$workspaceRoot = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..'))
|
||||
$nativeRoot = Join-Path $workspaceRoot 'src/Nitload.Native'
|
||||
$textRoots = @(
|
||||
(Join-Path $workspaceRoot 'Directory.Build.targets'),
|
||||
$nativeRoot,
|
||||
(Join-Path $workspaceRoot 'src/Nitload.Common'),
|
||||
(Join-Path $workspaceRoot 'src/Nitload.Mathematics'),
|
||||
(Join-Path $workspaceRoot 'src/Nitload.Graphics')
|
||||
)
|
||||
$textExtensions = @('.c', '.h', '.cs', '.cmake', '.txt', '.md', '.vcxproj', '.filters', '.targets')
|
||||
$utf8 = [Text.UTF8Encoding]::new($false, $true)
|
||||
|
||||
$files = foreach ($root in $textRoots) {
|
||||
if (Test-Path -LiteralPath $root -PathType Leaf) {
|
||||
Get-Item -LiteralPath $root
|
||||
} else {
|
||||
Get-ChildItem -LiteralPath $root -Recurse -File | Where-Object {
|
||||
$_.Extension -in $textExtensions -and
|
||||
$_.FullName -notmatch '[\\/](bin|obj|cmake_build)[\\/]'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$changes = [Collections.Generic.List[string]]::new()
|
||||
foreach ($file in $files) {
|
||||
$bytes = [IO.File]::ReadAllBytes($file.FullName)
|
||||
try { $content = $utf8.GetString($bytes) }
|
||||
catch [Text.DecoderFallbackException] { continue }
|
||||
if ($file.FullName -eq (Join-Path $nativeRoot 'README.md') -and
|
||||
$content.Contains('The C implementation uses `nfn`')) { continue }
|
||||
|
||||
$updated = $content.Replace('NIF_', 'NFN_').Replace('nif_', 'nfn_').Replace('nif-', 'nfn-')
|
||||
$updated = $updated.Replace('NF_', 'NFN_').Replace('nf_', 'nfn_').Replace('nf-', 'nfn-')
|
||||
# The previous migration left private SN macros in this header.
|
||||
if ($file.Name -in @('nif_intrinsic.std.h', 'nf_intrinsic.std.h')) {
|
||||
$updated = $updated.Replace('_SN_', '_NFN_')
|
||||
}
|
||||
if ($updated -ceq $content) { continue }
|
||||
|
||||
$changes.Add([IO.Path]::GetRelativePath($workspaceRoot, $file.FullName))
|
||||
if ($Apply) { [IO.File]::WriteAllText($file.FullName, $updated, $utf8) }
|
||||
}
|
||||
|
||||
$moves = [Collections.Generic.List[object]]::new()
|
||||
foreach ($directory in @('nif-head', 'nif-src', 'nif-template', 'nf-head', 'nf-src', 'nf-template')) {
|
||||
$sourceDir = Join-Path $nativeRoot $directory
|
||||
if (-not (Test-Path -LiteralPath $sourceDir)) { continue }
|
||||
foreach ($file in Get-ChildItem -LiteralPath $sourceDir -File | Where-Object { $_.Name -cmatch '^(nif|nf)_' }) {
|
||||
$moves.Add(@($file.FullName, (Join-Path $sourceDir ($file.Name -creplace '^(nif|nf)_', 'nfn_'))))
|
||||
}
|
||||
$moves.Add(@($sourceDir, (Join-Path $nativeRoot ($directory -creplace '^(nif|nf)-', 'nfn-'))))
|
||||
}
|
||||
|
||||
foreach ($move in $moves) {
|
||||
if (Test-Path -LiteralPath $move[1]) { throw "Rename target already exists: $($move[1])" }
|
||||
}
|
||||
|
||||
"Text files to update: $($changes.Count)"
|
||||
$changes | ForEach-Object { " TEXT $_" }
|
||||
"Paths to rename: $($moves.Count)"
|
||||
$moves | ForEach-Object {
|
||||
" MOVE $([IO.Path]::GetRelativePath($workspaceRoot, $_[0])) -> $([IO.Path]::GetRelativePath($workspaceRoot, $_[1]))"
|
||||
}
|
||||
|
||||
if (-not $Apply) {
|
||||
'Dry run only. Re-run with -Apply to perform the migration.'
|
||||
exit 0
|
||||
}
|
||||
|
||||
foreach ($move in $moves) { Move-Item -LiteralPath $move[0] -Destination $move[1] }
|
||||
'NFN prefix migration completed.'
|
||||
Binary file not shown.
@@ -1,32 +0,0 @@
|
||||
#import "template.typ": cover, conf
|
||||
|
||||
// ===== 封面 =====
|
||||
// 字段均可选,自动排版:留空的字段会自动跳过。
|
||||
#cover(
|
||||
title: "Very长的文档标题示",
|
||||
title-en: "Very Long Document Title Example",
|
||||
subtitle: "副标题 Subtitle",
|
||||
date: "2026 年 6 月",
|
||||
)
|
||||
|
||||
// ===== 正文 =====
|
||||
#show: conf.with(number-pages: true)
|
||||
|
||||
= 第一章 引言
|
||||
|
||||
这是正文示例。正文中文使用宋体,English text uses Times New Roman by default。
|
||||
段落首行自动缩进两字符,两端对齐。
|
||||
|
||||
== 1.1 小节
|
||||
|
||||
行内代码示例:`let x = 1`,以及代码块:
|
||||
|
||||
```rust
|
||||
fn main() {
|
||||
println!("Hello, Typst!");
|
||||
}
|
||||
```
|
||||
|
||||
= 第二章 结论
|
||||
|
||||
这里是结论部分的内容。
|
||||
@@ -1,172 +0,0 @@
|
||||
// ============================================================
|
||||
// A4 文档模板
|
||||
// 封面:页边距 0,中文黑体 / 英文 Source Code Pro
|
||||
// 正文:上下 3.5cm,左右 2cm,中文宋体 / 英文 Times New Roman
|
||||
// 代码块:Source Code Pro
|
||||
// ============================================================
|
||||
|
||||
// ---- 字体配置(可按需修改)----
|
||||
#let cover-cjk = "SimHei" // 封面中文:黑体
|
||||
#let cover-latin = "Source Code Pro" // 封面英文
|
||||
#let body-cjk = "SimSun" // 正文中文:宋体
|
||||
#let body-latin = "Times New Roman" // 正文英文:新罗马
|
||||
#let mono-font = "Source Code Pro" // 代码块
|
||||
|
||||
// ---- 封面网格系统 ----
|
||||
// 坐标系:左上角为原点,x 向右、y 向下;坐标记为 (x, y) 即 (列, 行)。
|
||||
// 1 单位 = 1.2cm,一个单位方块为边长 1.5cm 的正方形。
|
||||
#let unit = 1.2cm
|
||||
#let sky = rgb("#4FC3F7") // 天蓝色
|
||||
#let ochre = rgb("#C9A24B") // 土黄色
|
||||
|
||||
// 在网格 (x, y) 处放置内容(以页面左上角为基准)
|
||||
#let cell(x, y, body) = place(top + left, dx: x * unit, dy: y * unit, body)
|
||||
|
||||
// 实心单位正方形
|
||||
#let sq(x, y, color) = cell(x, y, rect(width: unit, height: unit, fill: color, stroke: none))
|
||||
|
||||
// 单位等腰直角三角形,corner 指定直角所在角:
|
||||
// "tl" 左上 / "tr" 右上 / "bl" 左下 / "br" 右下
|
||||
#let tri(x, y, corner, color) = cell(x, y, {
|
||||
let pts = if corner == "tr" { ((0pt, 0pt), (unit, 0pt), (unit, unit)) }
|
||||
else if corner == "tl" { ((0pt, 0pt), (unit, 0pt), (0pt, unit)) }
|
||||
else if corner == "bl" { ((0pt, 0pt), (0pt, unit), (unit, unit)) }
|
||||
else { ((unit, 0pt), (unit, unit), (0pt, unit)) }
|
||||
polygon(fill: color, stroke: none, ..pts)
|
||||
})
|
||||
|
||||
// 单元格字母字号(NITLOAD 与主标题共用,保证视觉一致)
|
||||
#let glyph-size = 30pt
|
||||
|
||||
// 在单位格内居中放置一个字母
|
||||
#let glyph(x, y, ch, color) = cell(x, y, box(
|
||||
width: unit, height: unit, inset: 0pt,
|
||||
align(center + horizon, text(font: cover-cjk, size: glyph-size, weight: "medium", fill: color)[#ch]),
|
||||
))
|
||||
|
||||
// ---- 封面 ----
|
||||
// 所有字段均可选,留空则自动跳过,实现自动排版。
|
||||
#let cover(
|
||||
title: none, // 主标题(中文)
|
||||
title-en: none, // 主标题(英文,Source Code Pro 自行排版)
|
||||
subtitle: none, // 副标题
|
||||
logo: none, // 图片路径,如 image("logo.png")
|
||||
author: none, // 作者(可为单个字符串或数组)
|
||||
info: (:), // 附加信息字典,如 (学号: "123", 指导教师: "张三")
|
||||
date: none, // 日期
|
||||
fill: white, // 封面背景色
|
||||
decorate: true, // 是否绘制天蓝/土黄网格装饰色块
|
||||
) = {
|
||||
// 封面页:页边距为 0
|
||||
set page(
|
||||
paper: "a4",
|
||||
margin: 0pt,
|
||||
fill: fill,
|
||||
header: none,
|
||||
footer: none,
|
||||
)
|
||||
set text(font: (cover-latin, cover-cjk), lang: "zh")
|
||||
|
||||
// 网格装饰:NITLOAD 标识(坐标 (x, y) = (列, 行))。
|
||||
if decorate {
|
||||
// 天蓝色实心单位正方形
|
||||
// 土黄色实心单位正方形
|
||||
sq(0, 0, sky); sq(1, 0, sky); sq(2, 0, ochre)
|
||||
sq(0, 1, sky); sq(2, 1, sky)
|
||||
sq(0, 2, sky); sq(2, 2, sky)
|
||||
sq(0, 3, sky); sq(2, 3, sky)
|
||||
sq(0, 4, sky); sq(2, 4, sky)
|
||||
sq(0, 5, sky); sq(2, 5, sky)
|
||||
|
||||
// NITLOAD 竖排字样(列 x=1, 行 y=0..6),N 为土黄,其余天蓝
|
||||
let letters = ("N", "I", "T", "L", "O", "A", "D")
|
||||
for (i, ch) in letters.enumerate() {
|
||||
glyph(1, i, ch, if i == 0 { ochre } else { sky })
|
||||
}
|
||||
|
||||
// 主标题:中文逐字对齐网格,英文居中于单元格行。
|
||||
// 第 1 列到右缘的整格数 = 中文每行字数。
|
||||
let title-cols = calc.floor((21cm - 1 * unit) / unit)
|
||||
let title-width = 21cm - 1 * unit
|
||||
let cs = if type(title) == str { title.clusters() } else { () }
|
||||
|
||||
// 中文逐字对齐网格;混排西文字母按 2/3 格宽打包,预留整数格。
|
||||
// 中文视觉重心偏大,整体右移 0.1 格、下移 0.2 格
|
||||
let cn-dx = 0.1 * unit
|
||||
let cn-dy = 0.2 * unit
|
||||
let is-narrow = c => str.to-unicode(c.codepoints().first()) < 0x2E80
|
||||
// 分段:连续西文合为一段,其余每字一段
|
||||
let segs = ()
|
||||
let run = ""
|
||||
for ch in cs {
|
||||
if is-narrow(ch) { run += ch }
|
||||
else {
|
||||
if run != "" { segs.push(run); run = "" }
|
||||
segs.push(ch)
|
||||
}
|
||||
}
|
||||
if run != "" { segs.push(run) }
|
||||
|
||||
// 逐段放置:CJK 占 1 格,西文段占 ceil(字数 * 1/2) 格,溢出换行
|
||||
let cur = 0
|
||||
let row = 0
|
||||
for seg in segs {
|
||||
let w = if is-narrow(seg.first()) { calc.ceil(seg.len() * 1 / 2) } else { 1 }
|
||||
if cur + w > title-cols { cur = 0; row += 1 }
|
||||
let col = 1 + cur
|
||||
place(top + left, dx: col * unit + cn-dx, dy: (7 + row) * unit + cn-dy,
|
||||
box(width: w * unit, height: unit, inset: 0pt, align(center + horizon,
|
||||
text(font: (cover-cjk, cover-latin), size: glyph-size, fill: black)[#seg])))
|
||||
cur += w
|
||||
}
|
||||
let cn-rows = if segs.len() == 0 { 0 } else { row + 1 }
|
||||
|
||||
// 英文:左对齐于中文下方;多行时行距对齐网格(行距 = unit),左/上边距相等。
|
||||
// 英文行数需测量,故把依赖行数的笔画/三角形一并放入 context。
|
||||
context {
|
||||
let en-rows = 0
|
||||
if title-en != none {
|
||||
let en1 = text(font: (cover-latin, cover-cjk), size: glyph-size, fill: black)[#title-en]
|
||||
let lh = measure(en1).height
|
||||
let pad = (unit - lh) / 2
|
||||
let inner = {
|
||||
set par(leading: unit - lh)
|
||||
en1
|
||||
}
|
||||
let h = measure(box(width: title-width - pad, inner)).height
|
||||
en-rows = calc.max(1, int(calc.round((h - lh) / unit)) + 1)
|
||||
cell(1, 7 + cn-rows, box(width: title-width, inset: (left: pad, top: pad), inner))
|
||||
}
|
||||
let title-rows = cn-rows + en-rows
|
||||
// 左侧一笔向下延伸至主标题下端,三角形收尾;右侧保持原样
|
||||
for r in range(6, 6 + title-rows) { sq(0, r, sky) }
|
||||
tri(0, 6 + title-rows, "tr", sky)
|
||||
}
|
||||
tri(2, 6, "tl", sky)
|
||||
}
|
||||
|
||||
pagebreak(weak: true)
|
||||
}
|
||||
|
||||
// ---- 正文配置 ----
|
||||
#let conf(
|
||||
number-pages: true, // 是否显示页码
|
||||
doc,
|
||||
) = {
|
||||
set page(
|
||||
paper: "a4",
|
||||
margin: (top: 3.5cm, bottom: 3.5cm, left: 2cm, right: 2cm),
|
||||
numbering: if number-pages { "1" } else { none },
|
||||
number-align: center,
|
||||
)
|
||||
set text(font: (body-latin, body-cjk), lang: "zh", size: 12pt)
|
||||
set par(justify: true, leading: 1em, first-line-indent: (amount: 2em, all: true))
|
||||
|
||||
// 标题字体:中文黑体,英文 Source Code Pro
|
||||
show heading: set text(font: (cover-latin, cover-cjk))
|
||||
|
||||
// 代码块与行内代码:Source Code Pro
|
||||
show raw: set text(font: mono-font)
|
||||
|
||||
doc
|
||||
}
|
||||
Reference in New Issue
Block a user