diff --git a/StgSharp.Common/Environment/Dialogue.cs b/StgSharp.Common/Environment/Dialogue.cs index 9f1b507c..3127bb1d 100644 --- a/StgSharp.Common/Environment/Dialogue.cs +++ b/StgSharp.Common/Environment/Dialogue.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="Dialogue" // Project: StgSharp @@ -43,7 +43,6 @@ namespace StgSharp private static Process? _dialogueProcess; private static AnonymousPipeServerStream _dialogueClientPipe,_dialogueServerPipe; - private static ConcurrentBag msgCache = new ConcurrentBag(); private static int _readAndWriteThreadId; private static ProcessStartInfo _dialogueProcessInfo; private static StreamReader _dr; @@ -51,7 +50,9 @@ namespace StgSharp public static bool NeedShowWhenStartup { get; set; } - public static void PostError(Exception ex) + public static void PostError( + Exception ex + ) { CreateDialogueProcessIfNotExist(); @@ -60,7 +61,9 @@ namespace StgSharp _dialogueProcess!.Exited -= DialogueProcessExitCallback; } - public static void SetReadAndWriteThread(Thread thread) + public static void SetReadAndWriteThread( + Thread thread + ) { _readAndWriteThreadId = thread.ManagedThreadId; } @@ -70,7 +73,9 @@ namespace StgSharp if (_dialogueProcess != null) { return; } - if (!File.Exists(_dialogueProcessInfo.FileName)) { } + if (!File.Exists(_dialogueProcessInfo.FileName)) { + return; + } _dialogueProcess = Process.Start(_dialogueProcessInfo); _dr = new StreamReader(_dialogueClientPipe); @@ -89,8 +94,8 @@ namespace StgSharp _dialogueClientPipe = new AnonymousPipeServerStream( PipeDirection.In, HandleInheritability.Inheritable); - string route = Assembly.GetAssembly(typeof(World)).Location; - string path = Path.GetDirectoryName(route); + string route = Assembly.GetAssembly(typeof(World))?.Location ?? string.Empty; + string path = Path.GetDirectoryName(route) ?? string.Empty; string exeName; @@ -107,7 +112,7 @@ namespace StgSharp } string consoleRoute = Path.Combine( - path, @"StgSharpTerminalDialogue", $@"StgSharpTerminalDialogue{exeName}"); + path, @"StgSharp.TerminalDialogue", $@"StgSharp.TerminalDialogue{exeName}"); _dialogueProcessInfo = new ProcessStartInfo { @@ -119,7 +124,10 @@ namespace StgSharp }; } - internal static void PostMessage(string instruction, params string[] operands) + internal static void PostMessage( + string instruction, + params string[] operands + ) { CreateDialogueProcessIfNotExist(); @@ -137,7 +145,9 @@ namespace StgSharp _dialogueClientPipe.Dispose(); } - internal static string WaitCertainReturn(string target) + internal static string WaitCertainReturn( + string target + ) { if (_dialogueProcess == null) { return string.Empty; @@ -166,7 +176,10 @@ namespace StgSharp } } - private static void DialogueProcessExitCallback(object? sender, EventArgs e) + private static void DialogueProcessExitCallback( + object? sender, + EventArgs e + ) { _dialogueProcess = null; _dr.Close(); diff --git a/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs b/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs index a41cdd31..93f5020c 100644 --- a/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs +++ b/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs @@ -45,7 +45,7 @@ namespace StgSharp.Internal.Intrinsic public delegate* unmanaged[Cdecl] buffer_transpose; //f32_kernel_transpose public delegate* unmanaged[Cdecl] build_panel; //f32_build_panel public delegate* unmanaged[Cdecl] clear_panel; //f32_clear_panel - public delegate* unmanaged[Cdecl] panel_fma; //f32_panel_fma + public delegate* unmanaged[Cdecl] kernel_fma; //f32_panel_fma public delegate* unmanaged[Cdecl] pivot; //f32_panel_mul public delegate* unmanaged[Cdecl] store_panel; //f32_store_panel diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.Float.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.Float.cs index 71a330ea..9e6f6731 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.Float.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.Float.cs @@ -143,72 +143,32 @@ namespace StgSharp.Mathematics.Numeric throw new IndexOutOfRangeException("Matrix dimensions must agree."); } long count = (long)left.KernelColumnLength * left.KernelRowLength; - int size = MatrixPanel.KernelSideCount(); + + // int size = MatrixPanel.KernelSideCount(); if (count <= 1024) { MatrixParallelThread[] pool = null; if (count > 256) { pool = MatrixParallel.LeadParallel(1); } - if (NativeIntrinsic.Context.mat[f32].build_panel == null) + for (int i = 0; i < ans.KernelRowLength; i++) { - for (int i = 0; i < ans.KernelRowLength; i += size) + for (int j = 0; j < ans.KernelColumnLength; j++) { - for (int j = 0; j < ans.KernelColumnLength; j += size) - { - MatrixKernel* ansPanel = ans.GetKernelAddressUnsafe(i, j); - NativeIntrinsic.Context.mat[f32].clear_panel((MatrixPanel*)ansPanel); + MatrixKernel* ansPanel = ans.GetKernelAddressUnsafe(i, j); + NativeIntrinsic.Context.mat[f32].clear_panel((MatrixPanel*)ansPanel); - // Console.WriteLine($"{i}{j}"); - for (int k = 0; k < left.KernelColumnLength; k += size) - { - NativeIntrinsic.Context - .mat[f32].panel_fma((MatrixPanel*)left.GetKernelAddressUnsafe(i, k), + // Console.WriteLine($"{i}{j}"); + for (int k = 0; k < left.KernelColumnLength; k++) + { + NativeIntrinsic.Context + .mat[f32].kernel_fma((MatrixPanel*)left.GetKernelAddressUnsafe(i, k), (MatrixPanel*)right.GetKernelAddressUnsafe(k, j), (MatrixPanel*)ansPanel); - } } } - } else - { - MatrixPanel* leftPanel = MatrixPanel.Create(); - MatrixPanel* rightPanel = MatrixPanel.Create(); - MatrixPanel* ansPanel = MatrixPanel.Create(); - for (int i = 0; i < ans.KernelRowLength; i += size) - { - for (int j = 0; j < ans.KernelColumnLength; j += size) - { - NativeIntrinsic.Context.mat[f32].clear_panel((MatrixPanel*)ansPanel); - - // Console.WriteLine($"{i}{j}"); - for (int k = 0; k < left.KernelColumnLength; k += size) - { - NativeIntrinsic.Context - .mat[f32].build_panel((MatrixPanel*)leftPanel, - (MatrixKernel*)left.Buffer, - left.KernelColumnLength, - left.KernelRowLength, i, k); - NativeIntrinsic.Context - .mat[f32].build_panel((MatrixPanel*)rightPanel, - (MatrixKernel*)right.Buffer, - right.KernelColumnLength, - right.KernelRowLength, k, j); - NativeIntrinsic.Context - .mat[f32].panel_fma((MatrixPanel*)leftPanel, - (MatrixPanel*)rightPanel, - (MatrixPanel*)ansPanel); - } - NativeIntrinsic.Context - .mat[f32].store_panel((MatrixPanel*)ansPanel, - (MatrixKernel*)ans.Buffer, - ans.KernelColumnLength, - ans.KernelRowLength, i, j); - } - } - MatrixPanel.Destroy(leftPanel); - MatrixPanel.Destroy(rightPanel); - MatrixPanel.Destroy(ansPanel); } + if (count > 256) { MatrixParallel.ReturnParallelResources(ref pool!); } @@ -221,8 +181,8 @@ namespace StgSharp.Mathematics.Numeric package->AnsPrimOffset = ans.KernelRowLength; package->AnsSecOffset = ans.KernelColumnLength; package->LeftPrimOffset = left.KernelColumnLength; - package->PrimCount = ans.KernelColumnLength / size; - package->SecCount = ans.KernelRowLength / size; + package->PrimCount = ans.KernelColumnLength; + package->SecCount = ans.KernelRowLength; package->ElementSize = (int)MatrixElementType.F32; package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationSpecial.SEQ_FMA); MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.cs index 768d4172..27fff0db 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.cs @@ -54,55 +54,26 @@ namespace StgSharp.Mathematics.Numeric int ansWidth = p->AnsPrimOffset; int ansHeight = p->AnsSecOffset; int commonK = p->LeftPrimOffset; - int blocksPerRow = ansWidth / size; - if (NativeIntrinsic.Context.mat[f32].build_panel == null) + + // int blocksPerRow = ansWidth / size; + + // in this occasion size must be 1 + for (long c = offsetRef; c < offsetRef + lengthRef; c++) { - // in this occasion size must be 1 - for (long c = offsetRef; c < offsetRef + lengthRef; c++) - { - int i = (int)(c / blocksPerRow); - int j = (int)(c % blocksPerRow); - MatrixKernel* ansPanel = GetKernelAddressUnsafe((MatrixKernel*)ansBuffer, - ansHeight, i, j); - NativeIntrinsic.Context.mat[f32].clear_panel((MatrixPanel*)ansPanel); - for (int k = 0; k < commonK; k += size) { - NativeIntrinsic.Context - .mat[f32].panel_fma( + int i = (int)(c / ansWidth); + int j = (int)(c % ansWidth); + MatrixKernel* ansPanel = GetKernelAddressUnsafe((MatrixKernel*)ansBuffer, + ansHeight, i, j); + NativeIntrinsic.Context.mat[f32].clear_panel((MatrixPanel*)ansPanel); + for (int k = 0; k < commonK; k += size) { + NativeIntrinsic.Context + .mat[f32].kernel_fma( (MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel*)leftBuffer, ansHeight, k, j), (MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel*)rightBuffer, commonK, j, k), (MatrixPanel*)ansPanel); - } } - } else - { - MatrixPanel* leftPanel = MatrixPanel.Create(); - MatrixPanel* rightPanel = MatrixPanel.Create(); - MatrixPanel* ansPanel = MatrixPanel.Create(); - for (long c = offsetRef; c < offsetRef + lengthRef; c++) - { - int i = (int)(c / blocksPerRow) * size; - int j = (int)(c % blocksPerRow) * size; - NativeIntrinsic.Context.mat[f32].clear_panel((MatrixPanel*)ansPanel); - for (int k = 0; k < commonK; k += size) - { - NativeIntrinsic.Context - .mat[f32].build_panel((MatrixPanel*)leftPanel, leftBuffer, - commonK, ansHeight, k, i); - NativeIntrinsic.Context - .mat[f32].build_panel((MatrixPanel*)rightPanel, rightBuffer, - ansWidth, commonK, j, k); - NativeIntrinsic.Context - .mat[f32].panel_fma((MatrixPanel*)leftPanel, (MatrixPanel*)rightPanel, (MatrixPanel*)ansPanel); - } - NativeIntrinsic.Context - .mat[f32].store_panel((MatrixPanel*)ansPanel, ansBuffer, - ansWidth, ansHeight, j, i); - } - MatrixPanel.Destroy(leftPanel); - MatrixPanel.Destroy(rightPanel); - MatrixPanel.Destroy(ansPanel); } } @@ -111,7 +82,8 @@ namespace StgSharp.Mathematics.Numeric ) { int f64 = (int)MatrixElementType.F64; - int size = MatrixPanel.KernelSideCount(); + + // int size = 1; MatrixKernel* leftBuffer = (MatrixKernel*)p->Left; MatrixKernel* rightBuffer = (MatrixKernel*)p->Right; MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer; @@ -120,55 +92,24 @@ namespace StgSharp.Mathematics.Numeric int ansWidth = p->AnsPrimOffset; int ansHeight = p->AnsSecOffset; int commonK = p->LeftPrimOffset; - int blocksPerRow = ansWidth / size; - if (NativeIntrinsic.Context.mat[f64].build_panel == null) + + // in this occasion size must be 1 + for (long c = offsetRef; c < offsetRef + lengthRef; c++) { - // in this occasion size must be 1 - for (long c = offsetRef; c < offsetRef + lengthRef; c++) - { - int i = (int)(c / blocksPerRow); - int j = (int)(c % blocksPerRow); - MatrixKernel* ansPanel = GetKernelAddressUnsafe((MatrixKernel*)ansBuffer, - ansHeight, i, j); - NativeIntrinsic.Context.mat[f64].clear_panel((MatrixPanel*)ansPanel); - for (int k = 0; k < commonK; k += size) { - NativeIntrinsic.Context - .mat[f64].panel_fma( + int i = (int)(c / ansWidth); + int j = (int)(c % ansWidth); + MatrixKernel* ansPanel = GetKernelAddressUnsafe((MatrixKernel*)ansBuffer, + ansHeight, i, j); + NativeIntrinsic.Context.mat[f64].clear_panel((MatrixPanel*)ansPanel); + for (int k = 0; k < commonK; k += 1) { + NativeIntrinsic.Context + .mat[f64].kernel_fma( (MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel*)leftBuffer, ansHeight, k, j), (MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel*)rightBuffer, commonK, j, k), (MatrixPanel*)ansPanel); - } } - } else - { - MatrixPanel* leftPanel = MatrixPanel.Create(); - MatrixPanel* rightPanel = MatrixPanel.Create(); - MatrixPanel* ansPanel = MatrixPanel.Create(); - for (long c = offsetRef; c < offsetRef + lengthRef; c++) - { - int i = (int)(c / blocksPerRow) * size; - int j = (int)(c % blocksPerRow) * size; - NativeIntrinsic.Context.mat[f64].clear_panel((MatrixPanel*)ansPanel); - for (int k = 0; k < commonK; k += size) - { - NativeIntrinsic.Context - .mat[f64].build_panel((MatrixPanel*)leftPanel, leftBuffer, - commonK, ansHeight, k, i); - NativeIntrinsic.Context - .mat[f64].build_panel((MatrixPanel*)rightPanel, rightBuffer, - ansWidth, commonK, j, k); - NativeIntrinsic.Context - .mat[f64].panel_fma((MatrixPanel*)leftPanel, (MatrixPanel*)rightPanel, (MatrixPanel*)ansPanel); - } - NativeIntrinsic.Context - .mat[f64].store_panel((MatrixPanel*)ansPanel, ansBuffer, - ansWidth, ansHeight, j, i); - } - MatrixPanel.Destroy(leftPanel); - MatrixPanel.Destroy(rightPanel); - MatrixPanel.Destroy(ansPanel); } } diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.double.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.double.cs index 6d9eccf3..b782278d 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.double.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.double.cs @@ -159,7 +159,7 @@ namespace StgSharp.Mathematics.Numeric for (int k = 0; k < left.KernelColumnLength; k += size) { NativeIntrinsic.Context - .mat[f64].panel_fma((MatrixPanel*)left.GetKernelAddressUnsafe(i, k), + .mat[f64].kernel_fma((MatrixPanel*)left.GetKernelAddressUnsafe(i, k), (MatrixPanel*)right.GetKernelAddressUnsafe(k, j), (MatrixPanel*)ansPanel); } @@ -190,7 +190,7 @@ namespace StgSharp.Mathematics.Numeric right.KernelColumnLength, right.KernelRowLength, k, j); NativeIntrinsic.Context - .mat[f64].panel_fma((MatrixPanel*)leftPanel, + .mat[f64].kernel_fma((MatrixPanel*)leftPanel, (MatrixPanel*)rightPanel, (MatrixPanel*)ansPanel); } diff --git a/StgSharp.Native/SIMDID.x86.md b/StgSharp.Native/SIMDID.x86.md index 31b2f213..c2e939d2 100644 --- a/StgSharp.Native/SIMDID.x86.md +++ b/StgSharp.Native/SIMDID.x86.md @@ -1,54 +1,55 @@ -# SIMDID（x86-64）64-bit 规范（按你给定的固定布局补全） +# SIMDID (x86-64) 64-bit Specification (Completed per fixed layout) -本文按你指定的 **64-bit SIMDID** 位布局进行补全，重点给出 **AVX512 Feature 区段（[31..16]）** 的设计，使其既能覆盖 AVX-512 混乱的子集，又能保持分支逻辑尽量精简。 +This document completes the **64-bit SIMDID** bit layout as specified, with emphasis on the design of the **AVX512 Feature section ([31..16])**, so it can cover the messy AVX-512 subsets while keeping branching logic compact. It also allocates an **AVX Feature byte** adjacent to the AVX512/AMX regions for AVX-family capabilities. -> 约定：bit0 为最低位（LSB）。本文只定义 x86-64 解释器下的语义；Header 的 Root ISA 允许扩展到其他架构。 +> Convention: bit0 is the least significant bit (LSB). Only x86-64 interpreter semantics are defined here; the header’s Root ISA allows extension to other architectures. --- -## 1) ASCII 位布局（最终） +## 1) ASCII Bit Layout (Final) ``` SIMDID (64 bits) [63 ........................................................ 0] -+--------------------------+-----------+----------------+----------------+----------------+----------------+ -| [63..40] uArch/Policy | [39..32] | [31..16] | [15..8] | [7..4] | [3..0] | -| (24 bits, optional) | AMX Future| AVX512 Feature | MainLevel | Manufacture | Root ISA | -+--------------------------+-----------+----------------+----------------+----------------+----------------+ ++----------+-----------+----------------+----------------+----------------+----------------+----------------+ +| [63..56] | [47..40] | [39..32] | [31..16] | [15..8] | [7..4] | [3..0] | +| uArchHi | AVX Feat | AMX Future | AVX512 Feature | MainLevel | Manuf | Root | ++----------+-----------+----------------+----------------+----------------+----------------+----------------+ ``` -- **[3..0] Root ISA**：根指令集类型（用于选择解释器） -- **[7..4] Manufacture**：制造商信息（对 x86 来说 Intel/AMD 等） -- **[15..8] MainLevel**：主 ISA 等级（递增、共享比特语义） -- **[31..16] AVX512 Feature**：AVX-512 特征/子集/实现形态（重点） -- **[39..32] AMX Future**：AMX 特征（目前可只放少量位） -- **[63..40] uArch/Policy**：可选策略/微架构提示（默认 0，不影响功能） +- **[3..0] Root ISA**: Root ISA type (selects interpreter) +- **[7..4] Manufacture**: Vendor info (Intel/AMD for x86) +- **[15..8] MainLevel**: Main ISA level (ascending, shared bit semantics) +- **[31..16] AVX512 Feature**: AVX-512 features/subsets/implementation form (key part) +- **[39..32] AMX Future**: AMX features (few bits for now) +- **[47..40] AVX Feature**: AVX-family capabilities (base requires AVX+AVX2; key extensions include FMA/F16C) +- **[63..56] uArchHi**: Optional microarchitecture/policy hints (default 0, no functional impact). --- -## 2) Root ISA（[3..0]） +## 2) Root ISA ([3..0]) -建议编码（你列出的架构较多，这里给一个可用的最小集合；不够可继续扩）： +Suggested encoding (minimal usable set, expandable if needed): -| 值 | Root ISA | +| Value | Root ISA | |---:|----------| | 0 | Unknown | | 1 | x86-64 | | 2 | AArch64 | | 3 | PPC64LE | -| 4 | Apple Silicon（如果你想把它当“制造商+ARM”以外的 note，也可放在 Manufacture；但你要求 Root ISA 里有它，那就给它一个值） | +| 4 | Apple Silicon (distinct value as required) | | 5 | RISC-V | | 6 | LoongArch | | 7..15 | Reserved | -> 注：从体系结构角度 Apple Silicon 属于 AArch64；但你既然希望在 Root ISA 层面区分，那就这样定义即可（解释器由你决定）。 +> Note: Architecturally Apple Silicon is AArch64; a distinct value is provided here because the Root ISA layer may distinguish it. Interpreter choice is up to implementation. --- -## 3) Manufacture（[7..4]） +## 3) Manufacture ([7..4]) -在 **x86-64** 下建议： +For **x86-64**: -| 值 | Manufacture | +| Value | Manufacture | |---:|-------------| | 0 | Unknown/Other | | 1 | Intel | @@ -56,42 +57,39 @@ SIMDID (64 bits) [63 ........................................................ 0 | 3 | VIA/Zhaoxin | | 4..15 | Reserved | -在其它 Root ISA 下，该 nibble 由各自解释器自行定义（例如 ARM 下可放 Apple/Qualcomm/MediaTek 等）。 +For other Root ISAs, this nibble is defined by their interpreters (e.g., Apple/Qualcomm/MediaTek for ARM). --- -## 4) MainLevel（[15..8]，递增） +## 4) MainLevel ([15..8], ascending) -你要求“共用比特且递增”，也就是说数值越大能力越强（或至少在你的库调度中越靠后）。建议定为： +"Shared bits and ascending" means higher values indicate higher capability (or later scheduling order). Suggested values: -| 值 | MainLevel | +| Value | MainLevel | |---:|-----------| | 0 | NONE | -| 1 | SSE（完整 SSE 家族：SSE1..SSE4.2） | +| 1 | SSE (full SSE family: SSE1..SSE4.2) | | 2 | AVX2 | | 3 | AVX2_FMA | -| 4 | AVX2_FMA_F16C（从该级开始可处理 16-bit 类型） | -| 5 | AVX512（需要查询 [31..16]） | +| 4 | AVX2_FMA_F16C (from this level onward, 16-bit types are supported) | +| 5 | AVX512 (check [31..16]) | | 6 | AMX | | 7 | AVX10 | | 8..255 | Reserved | -> 建议调度：`MainLevel` 是主 switch；只有当 `MainLevel>=AVX512` 才解析 `AVX512 Feature`；当 `MainLevel>=AMX` 才解析 `AMX Future`。 +> Scheduling suggestion: `MainLevel` is the primary switch; only when `MainLevel >= AVX512` should `AVX512 Feature` be parsed; only when `MainLevel >= AMX` should `AMX Future` be parsed. AVX Feature sits beside AMX/AVX512 and is meaningful when AVX or higher is present. --- -## 5) AVX512 Feature（[31..16]）设计目标（精简版） +## 5) AVX512 Feature ([31..16]) Design Goals (Compact) -目标改为： - -1) **基础 AVX512 一次性判定**：用 1 bit 表示“F+CD+VL+DQ+BW”这一捆绑基础包，避免多次子集判断。 -2) **胶水 vs 原生**：用实现字段直接区分 `AVX512-DUAL`（胶水/降频风险）与 `AVX512-NATIVE`（完整 512 位 ALU）。 -3) **小众/AI 扩展单独留位**：VNNI、BF16/FP16 等稀疏出现的扩展单独占位，基础路径不受干扰。 +1) **Base AVX512 one-shot check**: Use 1 bit for the bundle `F + CD + VL + DQ + BW`, avoiding multiple subset checks. +2) **Glue vs native**: Use an implementation field to distinguish `AVX512-DUAL` (glue/downclock risk) from `AVX512-NATIVE` (full 512-bit ALU). +3) **Niche/AI extensions separated**: VNNI, BF16/FP16, and similar extensions each get a dedicated bit so the base path stays simple. --- -## 6) AVX512 Feature 位布局（[31..16] = 16 bits，精简） - +## 6) AVX512 Feature Bit Layout ([31..16] = 16 bits, compact) AVX512 Feature (bits 31..16) @@ -99,84 +97,102 @@ AVX512 Feature (bits 31..16) |-----------|-----|-----|------|------|------|-------|------|------|-------|-------|----|------|----|----|----|----| | **Usage** | R | R | FP16 | BF16 | VNNI | VBMI2 | VBMI | R/AI | Impl1 | Impl0 | R | BASE | R | R | R | R | +Notes: -说明： +- **BASE (bit 20)**: Base AVX512 bundle, indicating `F + CD + VL + DQ + BW` are all present. If set to 1, the detecting side must guarantee these subsets are all available. +- **AVX512_IMPL ([23..22], 2-bit)**: Implementation form (width policy). -- **BASE（bit 20）**：基础 AVX512 一揽子，表示同时具备 `F + CD + VL + DQ + BW`。这是 AVX512 最低可用门槛，探测端若设为 1，应保证上述子集都成立。 -- **AVX512_IMPL（[23..22]，2-bit）**：实现形态（宽度策略）。 - -| 值 | 含义 | +| Value | Meaning | |---:|------| -| 0 | Unknown（需结合 Manufacture/uArch 决定策略） | -| 1 | AVX512-DUAL（胶水/降频风险，建议默认 256 宽路径） | -| 2 | AVX512-NATIVE（完整 512 位 ALU，可走 512 宽路径） | +| 0 | Unknown (strategy decided by Manufacture/uArch) | +| 1 | AVX512-DUAL (glue/downclock risk; prefer 256-bit path) | +| 2 | AVX512-NATIVE (full 512-bit ALU; 512-bit path is usable) | | 3 | Reserved | -- **VBMI/VBMI2/VNNI/BF16/FP16（[29..26]）**：稀疏但常用的小类型/AI 扩展，按需设置。 -- **R/AI（bit 24）**：预留给未来 AI 或其他非密码学的自选扩展。 -- **[31..30]、[21]、[19..16]**：Reserved，按需再分配。 +- **VBMI/VBMI2/VNNI/BF16/FP16 ([29..26])**: Commonly needed niche or AI-related small-type extensions; set as needed. +- **R/AI (bit 24)**: Reserved for future AI or other non-crypto extensions. +- **[31..30], [21], [19..16]**: Reserved for future allocation. --- -## 7) “AVX512BASE” 判定（一次性检查） +## 7) AVX Feature Byte ([47..40], 8 bits) -- **规则**：`BASE=1` 即视为具备 `F+CD+VL+DQ+BW`，无需逐项再查。探测端必须保持一致性，若无法保证全部子集齐备则应置 0。 -- **运行时分支**： - - 若 `MainLevel < AVX512`：直接降级到 AVX2/FMA 路径。 - - 否则： - - 检查 `BASE`；若为 0 则降级到 AVX2/FMA（或视为探测错误）。 - - 查看 `AVX512_IMPL`：`DUAL` 走 256 宽/降频安全路径，`NATIVE` 走 512 宽路径。 - - 需要小类型/AI 能力时，按位查看 `VNNI/BF16/FP16/VBMI/VBMI2`。 +Purpose: compact AVX-family capability encoding adjacent to AVX512/AMX. Base requirement: AVX **and** AVX2 present. Key extensions: FMA and F16C. Additional bits reserved for other extensions. + +AVX Feature (bits 47..40) + +| bit | 47 | 46 | 45 | 44 | 43 | 42 | 41 | 40 | +|-----|----|----|----|----|----|----|----|----| +| use | R | R | R | R | F16C | FMA | AVX2 | AVX | + +Rules: +- **AVX (bit 40)**: AVX present. +- **AVX2 (bit 41)**: AVX2 present. For the AVX feature byte to be considered valid, both AVX and AVX2 must be set. +- **FMA (bit 42)**: FMA present. +- **F16C (bit 43)**: F16C present. +- **[44..47]**: Reserved for other AVX-family extensions (e.g., BMI/BMI2, IFMA, XOP) as needed. --- -## 8) AMX Future（[39..32]，8 bits） +## 8) "AVX512BASE" Determination (Single Check) -你已经留了 1 个字节，建议先只定义 3 个最常见位，其余保留： +- **Rule**: `BASE=1` implies `F+CD+VL+DQ+BW` are all present; no need for per-subset checks. Detection must keep this consistent; if not all subsets are present, set to 0. +- **Runtime branching**: + - If `MainLevel < AVX512`: fall back to AVX2/FMA path. + - Otherwise: + - Check `BASE`; if 0, fall back to AVX2/FMA (or treat as detection failure). + - Check `AVX512_IMPL`: `DUAL` uses 256-bit/downclock-safe path, `NATIVE` uses 512-bit path. + - When small-type/AI capabilities are required, consult `VNNI/BF16/FP16/VBMI/VBMI2` bits. -| bit（相对 [39..32]） | 名称 | 说明 | +--- + +## 9) AMX Future ([39..32], 8 bits) + +A single byte is reserved; define three common bits for now, leave the rest reserved: + +| bit (relative to [39..32]) | Name | Note | |---:|------|------| -| 0 | AMX_TILE | tile 基础 | +| 0 | AMX_TILE | Tile base | | 1 | AMX_INT8 | INT8 tile | | 2 | AMX_BF16 | BF16 tile | | 3..7 | Reserved | | -当 `MainLevel < AMX` 时，这个字节可全为 0。 +When `MainLevel < AMX`, this byte can be all zeros. --- -## 9) uArch/Policy（[63..40]，24 bits） +## 10) uArchHi ([63..56], 8 bits) -用途：**你不想浪费比特，但又需要解决“Intel 早期/AMD 胶水”这类非 CPUID 纯子集能解决的问题**。 -把它留作可选策略字段，默认 0；你需要时再启用。 +Purpose: optional high-byte strategy field to address issues like early Intel/AMD "glue" AVX512 that are not expressible purely via CPUID subsets. Defaults to 0. -建议最小化用法： -- `0`：未使用 -- `1`：ForceDual（即使 impl unknown 也当 AVX512-DUAL） -- `2`：ForceNative（即使 impl unknown 也当 AVX512-NATIVE） -- `3..`：你自己维护的 uArch 枚举（例如 SKX/ICL/SPR/ZEN4/ZEN5…） +Suggested minimal use: +- `0`: unused +- `1`: ForceDual (treat as AVX512-DUAL even if impl is unknown) +- `2`: ForceNative (treat as AVX512-NATIVE even if impl is unknown) +- `3..`: Implementation-maintained uArch enumeration (e.g., SKX/ICL/SPR/ZEN4/ZEN5…) -> 由于你是单人维护，最现实的策略是： -> - AMD：探测端直接把 `AVX512_IMPL=DUAL`（无需 uArch） -> - Intel：仅对你关心的少数型号做 uArch 枚举，否则 unknown 走默认策略 +Practical strategy for a single maintainer: +- AMD: detection side sets `AVX512_IMPL = DUAL` directly (no uArch needed) +- Intel: enumerate only selected models of interest; otherwise Unknown follows default policy --- -## 10) 建议的最简探测端填充规则（x86-64） +## 11) Minimal Detection-Side Fill Rules (x86-64) -1) 先填 `RootISA=x86-64`、`Manufacture=Intel/AMD` -2) 计算 `MainLevel`（按你递增规则） -3) 若 `MainLevel >= AVX512`： - - 若探测到 `F+CD+VL+DQ+BW` 全部成立，则置 `BASE=1`，否则 `BASE=0` - - 填 `AVX512_IMPL`： - - AMD：DUAL（按你的定义） - - Intel：可按 family/model 或 policy 表决定 DUAL/NATIVE；不确定就 Unknown - - 需要 VNNI/BF16/FP16/VBMI/VBMI2 时，按位填写对应扩展 -4) 若 `MainLevel >= AMX`：填 AMX byte +1) Set `RootISA = x86-64`, `Manufacture = Intel/AMD`. +2) Compute `MainLevel` (per ascending rule). +3) If `MainLevel >= AVX512`: + - If `F+CD+VL+DQ+BW` are all present, set `BASE = 1`; otherwise `BASE = 0`. + - Set `AVX512_IMPL`: + - AMD: DUAL (per definition) + - Intel: Decide DUAL/NATIVE based on family/model or policy table; if uncertain, use Unknown + - When VNNI/BF16/FP16/VBMI/VBMI2 are detected, set the corresponding bits. +4) If `MainLevel >= AMX`: fill the AMX byte. +5) When AVX/AVX2 are present, populate the AVX Feature byte: set AVX and AVX2; set FMA and F16C when detected; leave reserved bits for future extensions. +6) Use uArchHi only when additional policy is needed; otherwise leave as 0. --- -## 11) 你需要做的一个选择（高位保留位如何分配） +## 12) Choice for High Bits Reservation -目前 `[31..24]` 留作扩展/AI：示例定义为 FP16/BF16/VNNI/VBMI2/VBMI + 1 个预留位。 -如果你更需要 IFMA 或其他非密码学扩展，请替换 bit26~29 的定义；`BASE` 与 `Impl` 保持不变即可。 \ No newline at end of file +Currently `[31..24]` is reserved for AVX512 extensions/AI. `[47..44]` is reserved for AVX-family extensions beside the AVX byte. uArchHi `[63..56]` remains available for minimal policy hints. If IFMA or other extensions are more important, repurpose the reserved bits while keeping `BASE` and `Impl` unchanged. \ No newline at end of file diff --git a/StgSharp.Native/StgSharp.Native.vcxproj b/StgSharp.Native/StgSharp.Native.vcxproj index d17c7d2a..a267628a 100644 --- a/StgSharp.Native/StgSharp.Native.vcxproj +++ b/StgSharp.Native/StgSharp.Native.vcxproj @@ -116,6 +116,7 @@ CompileAsC true $(ProjectDir)lib;$(ProjectDir)sn-head;%(AdditionalIncludeDirectories) + AnySuitable Console @@ -192,7 +193,7 @@ - + diff --git a/StgSharp.Native/StgSharp.Native.vcxproj.filters b/StgSharp.Native/StgSharp.Native.vcxproj.filters index a5578a5e..f5e7ef25 100644 --- a/StgSharp.Native/StgSharp.Native.vcxproj.filters +++ b/StgSharp.Native/StgSharp.Native.vcxproj.filters @@ -114,7 +114,7 @@ - + Document diff --git a/StgSharp.Native/sn-head/sn_intrincontext.x86.h b/StgSharp.Native/sn-head/sn_intrincontext.x86.h index 1956b3e3..8bbd39a4 100644 --- a/StgSharp.Native/sn-head/sn_intrincontext.x86.h +++ b/StgSharp.Native/sn-head/sn_intrincontext.x86.h @@ -5,27 +5,30 @@ #include "sn_intrinsic.h" // ----------------------------------------------------------------------------- -// SIMDID layout helpers (see SIMDID.simd.md) +// SIMDID layout helpers (see SIMDID.x86.md) // bits [3..0] : Root ISA // bits [7..4] : Manufacture (ISA-specific) // bits [15..8] : MainLevel (monotonic capability) // bits [31..16]: AVX512 Feature // bits [39..32]: AMX Feature -// bits [63..40]: uArch / Policy (optional) +// bits [47..40]: AVX Feature +// bits [63..56]: uArchHi / Policy (optional) // ----------------------------------------------------------------------------- #define SIMDID_SHIFT_ROOT 0 #define SIMDID_SHIFT_MANU 4 #define SIMDID_SHIFT_MAIN 8 #define SIMDID_SHIFT_AVX512 16 #define SIMDID_SHIFT_AMX 32 -#define SIMDID_SHIFT_UARCH 40 +#define SIMDID_SHIFT_AVX 40 +#define SIMDID_SHIFT_UARCH 56 #define SIMDID_MASK_ROOT 0xFULL #define SIMDID_MASK_MANU (0xFULL << SIMDID_SHIFT_MANU) #define SIMDID_MASK_MAIN (0xFFULL << SIMDID_SHIFT_MAIN) #define SIMDID_MASK_AVX512 (0xFFFFULL << SIMDID_SHIFT_AVX512) #define SIMDID_MASK_AMX (0xFFULL << SIMDID_SHIFT_AMX) -#define SIMDID_MASK_UARCH (0xFFFFFFULL << SIMDID_SHIFT_UARCH) +#define SIMDID_MASK_AVX (0xFFULL << SIMDID_SHIFT_AVX) +#define SIMDID_MASK_UARCH (0xFFULL << SIMDID_SHIFT_UARCH) // Root ISA (low nibble) #define SIMDID_ROOT_UNKNOWN 0x0 @@ -46,11 +49,9 @@ #define SIMDID_MAIN_LVL_NONE 0x00u #define SIMDID_MAIN_LVL_SSE 0x01u /* SSE1..SSE4.2 */ #define SIMDID_MAIN_LVL_AVX2 0x02u -#define SIMDID_MAIN_LVL_AVX2_FMA 0x03u -#define SIMDID_MAIN_LVL_AVX2_FMA_F16C 0x04u /* 16-bit scalar/vector ready */ -#define SIMDID_MAIN_LVL_AVX512 0x05u -#define SIMDID_MAIN_LVL_AMX 0x06u -#define SIMDID_MAIN_LVL_AVX10 0x07u +#define SIMDID_MAIN_LVL_AVX512 0x03u +#define SIMDID_MAIN_LVL_AMX 0x04u +#define SIMDID_MAIN_LVL_AVX10 0x05u #ifndef SIMDID_AVX512_BASE #define SIMDID_AVX512_FP16 (1U << 29) @@ -67,6 +68,13 @@ #define SIMDID_AVX512_IMPL_NATIVE (0x2U << SIMDID_AVX512_IMPL_SHIFT) #endif +#ifndef SIMDID_AVX_BASE +#define SIMDID_AVX (1ULL << 40) +#define SIMDID_AVX2 (1ULL << 41) +#define SIMDID_AVX_FMA (1ULL << 42) +#define SIMDID_AVX_F16C (1ULL << 43) +#endif + #ifndef SIMDID_AMX_TILE #define SIMDID_AMX_TILE (1ULL << 32) #define SIMDID_AMX_INT8 (1ULL << 33) @@ -81,14 +89,14 @@ #define SIMDID_PACK_AVX512(avx512_bits) \ (((uint64_t)(avx512_bits) & 0xFFFFULL) << SIMDID_SHIFT_AVX512) #define SIMDID_PACK_AMX(amx_bits) (((uint64_t)(amx_bits) & 0xFFULL) << SIMDID_SHIFT_AMX) +#define SIMDID_PACK_AVX(avx_bits) (((uint64_t)(avx_bits) & 0xFFULL) << SIMDID_SHIFT_AVX) +#define SIMDID_PACK_UARCH(uarch_hi) (((uint64_t)(uarch_hi) & 0xFFULL) << SIMDID_SHIFT_UARCH) #endif /* Pre-shifted helpers for convenience when constructing SIMDID */ #define SIMDID_MAIN_ENC_NONE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE) #define SIMDID_MAIN_ENC_SSE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE) #define SIMDID_MAIN_ENC_AVX2 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2) -#define SIMDID_MAIN_ENC_AVX2_FMA SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2_FMA) -#define SIMDID_MAIN_ENC_AVX2_FMA_F16C SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2_FMA_F16C) #define SIMDID_MAIN_ENC_AVX512 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512) #define SIMDID_MAIN_ENC_AMX SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX) #define SIMDID_MAIN_ENC_AVX10 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10) @@ -113,9 +121,9 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub); DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub); DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub); -DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose); -DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose); -DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose); +DECLARE_KER_PROC_RIGHT_ANS(float, sse, , transpose); +DECLARE_KER_PROC_RIGHT_ANS(float, avx, , transpose); +DECLARE_KER_PROC_RIGHT_ANS(float, 512, , transpose); DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul); DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul); @@ -133,9 +141,6 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill); DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill); DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill); -#pragma endregion - -#pragma region fma DECLARE_PNL_PROC_ANS(float, sse, , clear); DECLARE_PNL_PROC_ANS(float, avx, , clear); DECLARE_PNL_PROC_ANS(float, 512, , clear); @@ -144,10 +149,10 @@ DECLARE_PNL_PROC_ANS(double, sse, , clear); DECLARE_PNL_PROC_ANS(double, avx, , clear); DECLARE_PNL_PROC_ANS(double, 512, , clear); -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, , fma); -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, , fma); -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma); -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, , fma); +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma); +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma); +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma); +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma); DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma); DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, , fma); diff --git a/StgSharp.Native/sn-head/sn_intrinsic.h b/StgSharp.Native/sn-head/sn_intrinsic.h index 2809fed9..36a6b2f7 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.h @@ -167,22 +167,23 @@ typedef void(SN_DECL *KER_PROC_RA)(mat_kernel *left, mat_kernel *ans); typedef void(SN_DECL *KER_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans); typedef void(SN_DECL *KER_PROC_LAS)(mat_kernel *matrix, mat_kernel *ans, scalar_pack *scalar); -#define KER_PROC(T_type, T_arch, T_op_name) T_type##_kernel_##T_op_name##_##T_arch +#define KER_PROC(T_type, T_arch, T_feature, T_op_name) \ + T_type##_kernel_##T_op_name##_##T_arch##T_feature -#define DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ - INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \ - MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \ +#define DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \ + INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_feature, T_op_name)( \ + MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \ MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar) -#define DECLARE_KER_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ - INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \ +#define DECLARE_KER_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \ + INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_feature, T_op_name)( \ MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \ scalar_pack const *scalar) -#define DECLARE_KER_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \ - INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \ +#define DECLARE_KER_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_feature, T_op_name) \ + INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_feature, T_op_name)( \ MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \ MAT_KERNEL(T_type) *restrict ans) -#define DECLARE_KER_PROC_RIGHT_ANS(T_type, T_arch, T_op_name) \ - INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \ +#define DECLARE_KER_PROC_RIGHT_ANS(T_type, T_arch, T_feature, T_op_name) \ + INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_feature, T_op_name)( \ MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans) #pragma endregion @@ -248,159 +249,4 @@ typedef enum most_advanced_instruction { #include "sn_target.h" -#if SN_IS_ARCH(SN_ARCH_X86_64) - -// ----------------------------------------------------------------------------- -// SIMDID layout helpers (see SIMDID.simd.md) -// bits [3..0] : Root ISA -// bits [7..4] : Manufacture (ISA-specific) -// bits [15..8] : MainLevel (monotonic capability) -// bits [31..16]: AVX512 Feature -// bits [39..32]: AMX Feature -// bits [63..40]: uArch / Policy (optional) -// ----------------------------------------------------------------------------- -#define SIMDID_SHIFT_ROOT 0 -#define SIMDID_SHIFT_MANU 4 -#define SIMDID_SHIFT_MAIN 8 -#define SIMDID_SHIFT_AVX512 16 -#define SIMDID_SHIFT_AMX 32 -#define SIMDID_SHIFT_UARCH 40 - -#define SIMDID_MASK_ROOT 0xFULL -#define SIMDID_MASK_MANU (0xFULL << SIMDID_SHIFT_MANU) -#define SIMDID_MASK_MAIN (0xFFULL << SIMDID_SHIFT_MAIN) -#define SIMDID_MASK_AVX512 (0xFFFFULL << SIMDID_SHIFT_AVX512) -#define SIMDID_MASK_AMX (0xFFULL << SIMDID_SHIFT_AMX) -#define SIMDID_MASK_UARCH (0xFFFFFFULL << SIMDID_SHIFT_UARCH) - -// Root ISA (low nibble) -#define SIMDID_ROOT_UNKNOWN 0x0 -#define SIMDID_ROOT_X86_64 0x1 -#define SIMDID_ROOT_AARCH64 0x2 -#define SIMDID_ROOT_PPC64LE 0x3 -#define SIMDID_ROOT_APPLESILICON 0x4 -#define SIMDID_ROOT_RISCV64 0x5 -#define SIMDID_ROOT_LOONGARCH 0x6 - -// Manufacture (x86-64 mapping; other ISAs may reinterpret) -#define SIMDID_MANU_UNKNOWN 0x0 -#define SIMDID_MANU_INTEL 0x1 -#define SIMDID_MANU_AMD 0x2 -#define SIMDID_MANU_VIA 0x3 - -// MainLevel (ordinal, shared bits 8..15; not bit-flags) -#define SIMDID_MAIN_LVL_NONE 0x00u -#define SIMDID_MAIN_LVL_SSE 0x01u /* SSE1..SSE4.2 */ -#define SIMDID_MAIN_LVL_AVX2 0x02u -#define SIMDID_MAIN_LVL_AVX2_FMA 0x03u -#define SIMDID_MAIN_LVL_AVX2_FMA_F16C 0x04u /* 16-bit scalar/vector ready */ -#define SIMDID_MAIN_LVL_AVX512 0x05u -#define SIMDID_MAIN_LVL_AMX 0x06u -#define SIMDID_MAIN_LVL_AVX10 0x07u - -/* Pre-shifted helpers for convenience when constructing SIMDID */ -#define SIMDID_MAIN_ENC_NONE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE) -#define SIMDID_MAIN_ENC_SSE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE) -#define SIMDID_MAIN_ENC_AVX2 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2) -#define SIMDID_MAIN_ENC_AVX2_FMA SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2_FMA) -#define SIMDID_MAIN_ENC_AVX2_FMA_F16C SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2_FMA_F16C) -#define SIMDID_MAIN_ENC_AVX512 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512) -#define SIMDID_MAIN_ENC_AMX SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX) -#define SIMDID_MAIN_ENC_AVX10 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10) - -#pragma region matix function - -INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target); - -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , add); -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , add); -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , add); - -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , add); -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , add); -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , add); - -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub); -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub); -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub); - -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub); -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub); -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub); - -DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose); -DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose); -DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose); - -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul); -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul); -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul); - -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul); -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul); -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, , scalar_mul); - -DECLARE_BUF_PROC_ANS_SCALAR(float, sse, , fill); -DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill); -DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill); - -DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill); -DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill); -DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill); - -#pragma endregion - -#pragma region fma -DECLARE_PNL_PROC_ANS(float, sse, , clear); -DECLARE_PNL_PROC_ANS(float, avx, , clear); -DECLARE_PNL_PROC_ANS(float, 512, , clear); - -DECLARE_PNL_PROC_ANS(double, sse, , clear); -DECLARE_PNL_PROC_ANS(double, avx, , clear); -DECLARE_PNL_PROC_ANS(double, 512, , clear); - -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, , fma); -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, , fma); -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma); -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, , fma); - -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma); -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, , fma); -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma); -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, 512, , fma); - -#pragma endregion - -#pragma region scaler_simd - -INTERNAL uint64_t SN_DECL factorial_simd_sse(int n); - -#pragma endregion - -#pragma region dot_product - -INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose, - __m128 const *vector, __m128 *ans); -INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector, - __m128 *ans); -INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector, - __m128 *ans); -INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector, - __m128 *ans); - -#pragma endregion - -#pragma region string - -INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int length); -INTERNAL int SN_DECL index_pair_sse(short const *stram, uint32_t target, int length); - -#pragma endregion - -#else - -typedef enum intrinsic_level { SIMD0 = 0 } intrinsic_level; - -#endif - #endif diff --git a/StgSharp.Native/sn-head/sn_target.h b/StgSharp.Native/sn-head/sn_target.h index e5d6ea5c..7e4ff9ad 100644 --- a/StgSharp.Native/sn-head/sn_target.h +++ b/StgSharp.Native/sn-head/sn_target.h @@ -56,7 +56,7 @@ typedef union SIMDID { uint64_t mask; - char make_byte[]; + char make_byte[8]; } SIMDID; SN_API SIMDID SN_DECL sn_get_simd_level(); diff --git a/StgSharp.Native/sn-src/cpuid.x86.c b/StgSharp.Native/sn-src/cpuid.x86.c index 3a29009c..85156cc4 100644 --- a/StgSharp.Native/sn-src/cpuid.x86.c +++ b/StgSharp.Native/sn-src/cpuid.x86.c @@ -56,7 +56,7 @@ static int xgetbv_has_avx512_state(void) return ((xcr0 & mask) == mask); } -static int is_intel_glue_avx512(int family, int model) +static int is_intel_dual_avx512(int family, int model) { return (family == 6) && (model == 0x55); /* SKX/CLX/CPX */ } @@ -104,6 +104,7 @@ SN_API SIMDID SN_DECL sn_get_simd_level() int has_avx = (ecx & (1 << 28)) != 0; int has_osxsave = (ecx & (1 << 27)) != 0; int has_fma = (ecx & (1 << 12)) != 0; + int has_f16c = (ecx & (1 << 29)) != 0; uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64); uint64_t manu = SIMDID_PACK_MANU(is_intel ? SIMDID_MANU_INTEL : @@ -112,12 +113,15 @@ SN_API SIMDID SN_DECL sn_get_simd_level() uint64_t main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE); uint64_t avx512_bits = 0; uint64_t amx_bits = 0; + uint64_t avx_bits = 0; /* Baseline main level */ if (has_sse42) main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE); if (has_avx && has_osxsave && xgetbv_has_xmm_ymm()) { + avx_bits |= SIMDID_AVX; + /* CPUID.7.0 */ get_cpuid(7, 0, &eax, &ebx, &ecx, &edx); const int has_bmi1 = (ebx & (1 << 3)) != 0; @@ -142,13 +146,13 @@ SN_API SIMDID SN_DECL sn_get_simd_level() const int has_amx_bf16 = (edx & (1 << 26)) != 0; if (has_avx2) { - if (has_fma) { - main_lvl = SIMDID_PACK_MAIN((has_bmi1 && has_bmi2) ? - SIMDID_MAIN_LVL_AVX2_FMA_F16C : - SIMDID_MAIN_LVL_AVX2_FMA); - } else { - main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2); - } + avx_bits |= SIMDID_AVX2; + if (has_fma) + avx_bits |= SIMDID_AVX_FMA; + if (has_f16c) + avx_bits |= SIMDID_AVX_F16C; + + main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2); } if (has_avx512f && xgetbv_has_avx512_state()) { @@ -168,7 +172,7 @@ SN_API SIMDID SN_DECL sn_get_simd_level() avx512_bits |= SIMDID_AVX512_VBMI2; /* impl classification */ - const int glue = is_amd || is_intel_glue_avx512(family, model); + const int glue = is_amd || is_intel_dual_avx512(family, model); const int native512 = is_intel_native_avx512(family, model); if (glue) { avx512_bits |= SIMDID_AVX512_IMPL_DUAL; @@ -194,7 +198,7 @@ SN_API SIMDID SN_DECL sn_get_simd_level() } global_simdid.mask = root | manu | main_lvl | SIMDID_PACK_AVX512(avx512_bits) | - SIMDID_PACK_AMX(amx_bits); + SIMDID_PACK_AMX(amx_bits) | SIMDID_PACK_AVX(avx_bits); return global_simdid; } diff --git a/StgSharp.Native/sn-src/load_intrinsic.x86.c b/StgSharp.Native/sn-src/load_intrinsic.x86.c index d4e37caa..ee4edb7a 100644 --- a/StgSharp.Native/sn-src/load_intrinsic.x86.c +++ b/StgSharp.Native/sn-src/load_intrinsic.x86.c @@ -13,10 +13,9 @@ #include #endif // _MSC_VER -PRIVATE void load_intrinsic_sse(sn_intrinsic *context); -PRIVATE void load_intrinsic_avx(sn_intrinsic *context); -PRIVATE void load_intrinsic_avx_fma(sn_intrinsic *context); -PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID simdid); +PRIVATE void load_intrinsic_sse(sn_intrinsic *context, SIMDID id); +PRIVATE void load_intrinsic_avx(sn_intrinsic *context, SIMDID id); +PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID id); SN_API SIMDID load_intrinsic_function(void *intrinsic_context) { @@ -26,18 +25,21 @@ SN_API SIMDID load_intrinsic_function(void *intrinsic_context) sn_intrinsic *context = (sn_intrinsic *)intrinsic_context; - if (check_isa(AVX512)) { + switch (simdid.make_byte[1]) { + case SIMDID_MAIN_LVL_SSE: + load_intrinsic_sse(context, simdid); + break; + case SIMDID_MAIN_LVL_AVX2: + load_intrinsic_avx(context, simdid); + break; + case SIMDID_MAIN_LVL_AVX512: load_intrinsic_512(context, simdid); - } else if (check_isa(AVX256)) { - load_intrinsic_avx(context); - } else if (check_isa(SSE)) { - load_intrinsic_sse(context); + break; } - return simdid; } -void load_intrinsic_sse(sn_intrinsic *context) +void load_intrinsic_sse(sn_intrinsic *context, SIMDID id) { context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; @@ -47,10 +49,10 @@ void load_intrinsic_sse(sn_intrinsic *context) context->mat[F32].buffer_fill = BUF_PROC(float, sse, , fill); context->mat[F32].buffer_scalar_mul = BUF_PROC(float, sse, , scalar_mul); context->mat[F32].buffer_sub = BUF_PROC(float, sse, , sub); - context->mat[F32].buffer_transpose = KER_PROC(float, sse, transpose); + context->mat[F32].buffer_transpose = KER_PROC(float, sse, , transpose); context->mat[F32].build_panel = NULL; context->mat[F32].clear_panel = PNL_PROC(float, sse, , clear); - context->mat[F32].panel_fma = PNL_PROC(float, sse, , fma); + context->mat[F32].panel_fma = KER_PROC(float, sse, , fma); context->mat[F32].pivot = pivot_float; context->mat[F32].store_panel = NULL; @@ -68,8 +70,12 @@ void load_intrinsic_sse(sn_intrinsic *context) context->vec_f32_normalize_3 = f32_normalize; } -void load_intrinsic_avx(sn_intrinsic *context) +void load_intrinsic_avx(sn_intrinsic *context, SIMDID id) { + char avx_seg = id.make_byte[5]; + char is_fma = (avx_seg & 1 << 3) != 0; + char is_f16c = (avx_seg & 1 << 4) != 0; + context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; @@ -78,10 +84,11 @@ void load_intrinsic_avx(sn_intrinsic *context) context->mat[F32].buffer_fill = BUF_PROC(float, avx, , fill); context->mat[F32].buffer_scalar_mul = BUF_PROC(float, avx, , scalar_mul); context->mat[F32].buffer_sub = BUF_PROC(float, avx, , sub); - context->mat[F32].buffer_transpose = KER_PROC(float, avx, transpose); + context->mat[F32].buffer_transpose = KER_PROC(float, avx, , transpose); context->mat[F32].build_panel = BUILD_PANEL(avx, float); context->mat[F32].clear_panel = PNL_PROC(float, avx, , clear); - context->mat[F32].panel_fma = PNL_PROC(float, avx, , fma); + context->mat[F32].panel_fma = is_fma ? KER_PROC(float, avx, , fma) : + KER_PROC(float, avx, _fma, fma); context->mat[F32].pivot = pivot_float; context->mat[F32].store_panel = STORE_PANEL(avx, float); @@ -92,7 +99,8 @@ void load_intrinsic_avx(sn_intrinsic *context) context->mat[F64].buffer_transpose = NULL; context->mat[F64].build_panel = NULL; context->mat[F64].clear_panel = PNL_PROC(double, avx, , clear); - context->mat[F64].panel_fma = PNL_PROC(double, avx, , fma); + context->mat[F64].panel_fma = is_fma ? PNL_PROC(double, avx, , fma) : + PNL_PROC(double, avx, _fma, fma); context->mat[F64].pivot = NULL; context->mat[F64].store_panel = NULL; @@ -109,10 +117,10 @@ void load_intrinsic_512(sn_intrinsic *context, SIMDID id) context->mat[F32].buffer_fill = BUF_PROC(float, 512, , fill); context->mat[F32].buffer_scalar_mul = BUF_PROC(float, 512, , scalar_mul); context->mat[F32].buffer_sub = BUF_PROC(float, 512, , sub); - context->mat[F32].buffer_transpose = KER_PROC(float, 512, transpose); + context->mat[F32].buffer_transpose = KER_PROC(float, 512, , transpose); context->mat[F32].build_panel = BUILD_PANEL(512, float); context->mat[F32].clear_panel = PNL_PROC(float, 512, , clear); - context->mat[F32].panel_fma = PNL_PROC(float, 512, , fma); + context->mat[F32].panel_fma = KER_PROC(float, 512, , fma); context->mat[F32].pivot = pivot_float; context->mat[F32].store_panel = STORE_PANEL(512, float); diff --git a/StgSharp.Native/sn-src/m.basic.x86.c b/StgSharp.Native/sn-src/m.basic.x86.c index 7d44e6e7..ca461203 100644 --- a/StgSharp.Native/sn-src/m.basic.x86.c +++ b/StgSharp.Native/sn-src/m.basic.x86.c @@ -55,7 +55,7 @@ DECLARE_PNL_PROC_ANS(double, 512, , clear) } // Matrix transpose implementations -DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose) +DECLARE_KER_PROC_RIGHT_ANS(float, sse, ,transpose) { register __m128 col0 = right->f32_x[0]; register __m128 col1 = right->f32_x[1]; @@ -72,7 +72,7 @@ DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose) ans->f32_x[3] = _mm_movehl_ps(tmp3, tmp1); } -DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose) +DECLARE_KER_PROC_RIGHT_ANS(float, avx, ,transpose) { register __m256 col_low = right->f32_y[0]; register __m256 col_high = right->f32_y[1]; @@ -87,7 +87,7 @@ DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose) ans->f32_y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1)); } -DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose) +DECLARE_KER_PROC_RIGHT_ANS(float, 512, ,transpose) { register __m512 matrix = right->f32_z[0]; diff --git a/StgSharp.Native/sn-src/m.fma.x86.c b/StgSharp.Native/sn-src/m.fma.x86.c index 1fe8db8b..a620bb52 100644 --- a/StgSharp.Native/sn-src/m.fma.x86.c +++ b/StgSharp.Native/sn-src/m.fma.x86.c @@ -9,204 +9,163 @@ // ans = left * right + ans // right is not transposed -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, , fma) +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma) { - const __m128 a0 = PANEL_COL_VEC((*left), 0, 0); - const __m128 a1 = PANEL_COL_VEC((*left), 1, 0); - const __m128 a2 = PANEL_COL_VEC((*left), 2, 0); - const __m128 a3 = PANEL_COL_VEC((*left), 3, 0); + const __m128 a0 = _mm_load_ps((float *)&left->f32_x[0]); + const __m128 a1 = _mm_load_ps((float *)&left->f32_x[1]); + const __m128 a2 = _mm_load_ps((float *)&left->f32_x[2]); + const __m128 a3 = _mm_load_ps((float *)&left->f32_x[3]); - __m128 c0 = PANEL_COL_VEC((*ans), 0, 0); - __m128 c1 = PANEL_COL_VEC((*ans), 1, 0); - __m128 c2 = PANEL_COL_VEC((*ans), 2, 0); - __m128 c3 = PANEL_COL_VEC((*ans), 3, 0); + __m128 c0 = _mm_load_ps((float *)&ans->f32_x[0]); + __m128 c1 = _mm_load_ps((float *)&ans->f32_x[1]); + __m128 c2 = _mm_load_ps((float *)&ans->f32_x[2]); + __m128 c3 = _mm_load_ps((float *)&ans->f32_x[3]); __m128 bcol; /* Column 0 */ - bcol = PANEL_COL_VEC((*right), 0, 0); + bcol = _mm_load_ps((float *)&right->f32_x[0]); c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); /* Column 1 */ - bcol = PANEL_COL_VEC((*right), 1, 0); + bcol = _mm_load_ps((float *)&right->f32_x[1]); c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); /* Column 2 */ - bcol = PANEL_COL_VEC((*right), 2, 0); + bcol = _mm_load_ps((float *)&right->f32_x[2]); c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); /* Column 3 */ - bcol = PANEL_COL_VEC((*right), 3, 0); + bcol = _mm_load_ps((float *)&right->f32_x[3]); c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); - PANEL_COL_VEC((*ans), 0, 0) = c0; - PANEL_COL_VEC((*ans), 1, 0) = c1; - PANEL_COL_VEC((*ans), 2, 0) = c2; - PANEL_COL_VEC((*ans), 3, 0) = c3; + ans->f32_x[0] = c0; + ans->f32_x[1] = c1; + ans->f32_x[2] = c2; + ans->f32_x[3] = c3; } -// ans = left * right + ans -// right is not transposed -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma) +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma) { - static const __m256i idx0 = { 1, 1, 1, 1, 1, 1, 1, 1 }; + register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]); + register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]); - for (int col = 0; col < 8; ++col) { - register __m256 bcol = _mm256_load_ps((float *)&right->vec[col][0]); - register __m256 ccol = _mm256_load_ps((float *)&ans->vec[col][0]); - register __m256i idx = _mm256_setzero_si256(); - register __m256i increament = _mm256_load_si256(&idx0); + register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]); + register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]); - register __m256 a0 = _mm256_load_ps((float *)&left->vec[0][0]); - register __m256 a1 = _mm256_load_ps((float *)&left->vec[1][0]); - register __m256 a2 = _mm256_load_ps((float *)&left->vec[2][0]); - register __m256 a3 = _mm256_load_ps((float *)&left->vec[3][0]); + register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]); + register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]); - ccol = _mm256_fmadd_ps(a0, _mm256_permutevar8x32_ps(bcol, idx), ccol); - idx = _mm256_add_epi32(idx, increament); - ccol = _mm256_fmadd_ps(a1, _mm256_permutevar8x32_ps(bcol, idx), ccol); - idx = _mm256_add_epi32(idx, increament); - ccol = _mm256_fmadd_ps(a2, _mm256_permutevar8x32_ps(bcol, idx), ccol); - idx = _mm256_add_epi32(idx, increament); - ccol = _mm256_fmadd_ps(a3, _mm256_permutevar8x32_ps(bcol, idx), ccol); - idx = _mm256_add_epi32(idx, increament); + //group 0 + register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0)); + register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0)); + c01 = _mm256_fmadd_ps(a01, b01_perm, c01); + c23 = _mm256_fmadd_ps(a23, b23_perm, c23); - a0 = _mm256_load_ps((float *)&left->vec[4][0]); - a1 = _mm256_load_ps((float *)&left->vec[5][0]); - a2 = _mm256_load_ps((float *)&left->vec[6][0]); - a3 = _mm256_load_ps((float *)&left->vec[7][0]); + //group 2 + b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2)); + b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2)); + c01 = _mm256_fmadd_ps(a23, b01_perm, c01); + c23 = _mm256_fmadd_ps(a01, b23_perm, c23); - ccol = _mm256_fmadd_ps(a0, _mm256_permutevar8x32_ps(bcol, idx), ccol); - idx = _mm256_add_epi32(idx, increament); - ccol = _mm256_fmadd_ps(a1, _mm256_permutevar8x32_ps(bcol, idx), ccol); - idx = _mm256_add_epi32(idx, increament); - ccol = _mm256_fmadd_ps(a2, _mm256_permutevar8x32_ps(bcol, idx), ccol); - idx = _mm256_add_epi32(idx, increament); - ccol = _mm256_fmadd_ps(a3, _mm256_permutevar8x32_ps(bcol, idx), ccol); + //group 1 + register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21); + register __m256 a03 = _mm256_permute2f128_ps(a01, a23, 0x30); + b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1)); + b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1)); + c01 = _mm256_fmadd_ps(a12, b01_perm, c01); + c23 = _mm256_fmadd_ps(a03, b23_perm, c23); - _mm256_store_ps((float *)&ans->vec[col][0], ccol); - } + //group 3 + a12 = _mm256_permute2f128_ps(a01, a23, 0x21); + a03 = _mm256_permute2f128_ps(a01, a23, 0x30); + b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3)); + b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3)); + c01 = _mm256_fmadd_ps(a03, b01_perm, c01); + c23 = _mm256_fmadd_ps(a12, b23_perm, c23); + + ans->f32_y[0] = c01; + ans->f32_y[1] = c23; } -// ans = left * right + ans -// right is not transposed -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, , fma) +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma) { - static const __m256i idx0 = { 1, 1, 1, 1, 1, 1, 1, 1 }; + register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]); + register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]); - for (int col = 0; col < 8; ++col) { - register __m256 bcol = _mm256_load_ps((float *)&right->vec[col][0]); - register __m256 ccol = _mm256_load_ps((float *)&ans->vec[col][0]); - register __m256i idx = _mm256_setzero_si256(); - register __m256i increament = _mm256_load_si256(&idx0); + register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]); + register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]); - register __m256 a0 = _mm256_load_ps((float *)&left->vec[0][0]); - register __m256 a1 = _mm256_load_ps((float *)&left->vec[1][0]); - register __m256 a2 = _mm256_load_ps((float *)&left->vec[2][0]); - register __m256 a3 = _mm256_load_ps((float *)&left->vec[3][0]); + register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]); + register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]); - ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a0, _mm256_permutevar8x32_ps(bcol, idx))); - idx = _mm256_add_epi32(idx, increament); - ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a1, _mm256_permutevar8x32_ps(bcol, idx))); - idx = _mm256_add_epi32(idx, increament); - ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a2, _mm256_permutevar8x32_ps(bcol, idx))); - idx = _mm256_add_epi32(idx, increament); - ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a3, _mm256_permutevar8x32_ps(bcol, idx))); - idx = _mm256_add_epi32(idx, increament); + //group 0 + register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0)); + register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0)); + c01 = _mm256_add_ps(_mm256_mul_ps(a01, b01_perm), c01); + c23 = _mm256_add_ps(_mm256_mul_ps(a23, b23_perm), c23); - a0 = _mm256_load_ps((float *)&left->vec[4][0]); - a1 = _mm256_load_ps((float *)&left->vec[5][0]); - a2 = _mm256_load_ps((float *)&left->vec[6][0]); - a3 = _mm256_load_ps((float *)&left->vec[7][0]); + //group 2 + b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2)); + b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2)); + c01 = _mm256_add_ps(_mm256_mul_ps(a23, b01_perm), c01); + c23 = _mm256_add_ps(_mm256_mul_ps(a01, b23_perm), c23); - ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a0, _mm256_permutevar8x32_ps(bcol, idx))); - idx = _mm256_add_epi32(idx, increament); - ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a1, _mm256_permutevar8x32_ps(bcol, idx))); - idx = _mm256_add_epi32(idx, increament); - ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a2, _mm256_permutevar8x32_ps(bcol, idx))); - idx = _mm256_add_epi32(idx, increament); - ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a3, _mm256_permutevar8x32_ps(bcol, idx))); + //group 1 + register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21); + register __m256 a03 = _mm256_permute2f128_ps(a01, a23, 0x30); + b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1)); + b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1)); + c01 = _mm256_add_ps(_mm256_mul_ps(a12, b01_perm), c01); + c23 = _mm256_add_ps(_mm256_mul_ps(a03, b23_perm), c23); - _mm256_store_ps((float *)&ans->vec[col][0], ccol); - } + //group 3 + a12 = _mm256_permute2f128_ps(a01, a23, 0x21); + a03 = _mm256_permute2f128_ps(a01, a23, 0x30); + b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3)); + b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3)); + c01 = _mm256_add_ps(_mm256_mul_ps(a03, b01_perm), c01); + c23 = _mm256_add_ps(_mm256_mul_ps(a12, b23_perm), c23); + + ans->f32_y[0] = c01; + ans->f32_y[1] = c23; } -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, , fma) +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma) { - const __m512i ones = _mm512_set1_epi32(1); + register __m512 a = _mm512_load_ps((float *)left); + register __m512 b = _mm512_load_ps((float *)right); + register __m512 c = _mm512_load_ps((float *)ans); - for (int col = 0; col < 16; ++col) { - register __m512 bcol = _mm512_load_ps((float *)&right->vec[col][0]); - register __m512 ccol = _mm512_load_ps((float *)&ans->vec[col][0]); - register __m512i idx = _mm512_setzero_si512(); - register __m512i increament = _mm512_load_si512(&ones); + register __m512 a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0)); + register __m512 b_perm = _mm512_permute_ps(b, _FMA_SELECT(0)); + c = _mm512_fmadd_ps(a, b_perm, c); - register __m512 a0 = _mm512_load_ps((float *)&left->vec[0][0]); - register __m512 a1 = _mm512_load_ps((float *)&left->vec[1][0]); - register __m512 a2 = _mm512_load_ps((float *)&left->vec[2][0]); - register __m512 a3 = _mm512_load_ps((float *)&left->vec[3][0]); - register __m512 a4 = _mm512_load_ps((float *)&left->vec[4][0]); - register __m512 a5 = _mm512_load_ps((float *)&left->vec[5][0]); - register __m512 a6 = _mm512_load_ps((float *)&left->vec[6][0]); - register __m512 a7 = _mm512_load_ps((float *)&left->vec[7][0]); + a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3)); + b_perm = _mm512_permute_ps(b, _FMA_SELECT(1)); + c = _mm512_fmadd_ps(a, b_perm, c); - ccol = _mm512_fmadd_ps(a0, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); - ccol = _mm512_fmadd_ps(a1, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); - ccol = _mm512_fmadd_ps(a2, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); - ccol = _mm512_fmadd_ps(a3, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); - ccol = _mm512_fmadd_ps(a4, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); - ccol = _mm512_fmadd_ps(a5, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); - ccol = _mm512_fmadd_ps(a6, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); - ccol = _mm512_fmadd_ps(a7, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); + a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2)); + b_perm = _mm512_permute_ps(b, _FMA_SELECT(2)); + c = _mm512_fmadd_ps(a, b_perm, c); - a0 = _mm512_load_ps((float *)&left->vec[8][0]); - a1 = _mm512_load_ps((float *)&left->vec[9][0]); - a2 = _mm512_load_ps((float *)&left->vec[10][0]); - a3 = _mm512_load_ps((float *)&left->vec[11][0]); - a4 = _mm512_load_ps((float *)&left->vec[12][0]); - a5 = _mm512_load_ps((float *)&left->vec[13][0]); - a6 = _mm512_load_ps((float *)&left->vec[14][0]); - a7 = _mm512_load_ps((float *)&left->vec[15][0]); + a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1)); + b_perm = _mm512_permute_ps(b, _FMA_SELECT(3)); + c = _mm512_fmadd_ps(a, b_perm, c); - ccol = _mm512_fmadd_ps(a0, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); - ccol = _mm512_fmadd_ps(a1, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); - ccol = _mm512_fmadd_ps(a2, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); - ccol = _mm512_fmadd_ps(a3, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); - ccol = _mm512_fmadd_ps(a4, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); - ccol = _mm512_fmadd_ps(a5, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); - ccol = _mm512_fmadd_ps(a6, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); - ccol = _mm512_fmadd_ps(a7, _mm512_permutexvar_ps(idx, bcol), ccol); - idx = _mm512_add_epi32(idx, increament); - - _mm512_store_ps((float *)&ans->vec[col][0], ccol); - } + _mm512_store_ps((float *)ans, c); } DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma) diff --git a/StgSharp.TerminalDialogue/StgSharp.TerminalDialogue.csproj b/StgSharp.TerminalDialogue/StgSharp.TerminalDialogue.csproj index a1ecd2f3..166043db 100644 --- a/StgSharp.TerminalDialogue/StgSharp.TerminalDialogue.csproj +++ b/StgSharp.TerminalDialogue/StgSharp.TerminalDialogue.csproj @@ -41,17 +41,13 @@ True - - - - - + diff --git a/StgSharpDebug/Program.cs b/StgSharpDebug/Program.cs index 9026d7ba..9f67c56f 100644 --- a/StgSharpDebug/Program.cs +++ b/StgSharpDebug/Program.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="Program" // Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -39,11 +39,16 @@ namespace StgSharp private static string testExpPath = @"D:\Programmer\stepcode\data\ap203\ap203.exp"; - internal static unsafe void Main(string[] args) + internal static unsafe void Main( + string[] args + ) { try { - World.Initialize().UseModule().UseModule().InitializeModule(); + World.Initialize() + .UseModule() + .UseModule() + .InitializeModule(); // TestRender(); /**/ @@ -51,11 +56,14 @@ namespace StgSharp unchecked { hlsf = new((nuint)(64 * (ulong)10000 * 10000)); } - Matrix m = Matrix.Create(4000, 4000, MatrixLayout.DenseRectangle, hlsf); + Matrix m = Matrix.Create(4000, 4000, MatrixLayout.DenseRectangle, + hlsf); MatrixCompute.Fill(m, 1.0f); - Matrix n = Matrix.Create(4000, 4000, MatrixLayout.DenseRectangle, hlsf); + Matrix n = Matrix.Create(4000, 4000, MatrixLayout.DenseRectangle, + hlsf); MatrixCompute.Fill(n, 1.0f); - Matrix p = Matrix.Create(4000, 4000, MatrixLayout.DenseRectangle, hlsf); + Matrix p = Matrix.Create(4000, 4000, MatrixLayout.DenseRectangle, + hlsf); MatrixCompute.Fill(p, 0.0f); /*