diff --git a/StgSharp.Common/Controlling/KeyEnum.cs b/StgSharp.Common/Controlling/KeyEnum.cs index 4168a2b1..c2ac0553 100644 --- a/StgSharp.Common/Controlling/KeyEnum.cs +++ b/StgSharp.Common/Controlling/KeyEnum.cs @@ -27,7 +27,6 @@ // ----------------------------------------------------------------------- using StgSharp.Graphics; using StgSharp.Graphics.OpenGL; -using StgSharp.Internal.OpenGL; using System; diff --git a/StgSharp.Common/Global.cs b/StgSharp.Common/Global.cs index af546338..b4d9803e 100644 --- a/StgSharp.Common/Global.cs +++ b/StgSharp.Common/Global.cs @@ -27,7 +27,7 @@ // ----------------------------------------------------------------------- global using StgSharp.Internal; global using StgSharp.Internal.Intrinsic; -global using StgSharp.Internal.OpenGL; +global using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; diff --git a/StgSharp.Common/Graphics/OpenGL/Buffer/EBO.cs b/StgSharp.Common/Graphics/OpenGL/Buffer/EBO.cs index 86aa5f9b..b137ca90 100644 --- a/StgSharp.Common/Graphics/OpenGL/Buffer/EBO.cs +++ b/StgSharp.Common/Graphics/OpenGL/Buffer/EBO.cs @@ -27,7 +27,7 @@ // ----------------------------------------------------------------------- using StgSharp.Graphics; using StgSharp.Graphics.OpenGL; -using StgSharp.Internal.OpenGL; + using StgSharp.Mathematics; using System; diff --git a/StgSharp.Common/Graphics/OpenGL/glType.cs b/StgSharp.Common/Graphics/OpenGL/glType.cs index f4d5dff4..2f180a06 100644 --- a/StgSharp.Common/Graphics/OpenGL/glType.cs +++ b/StgSharp.Common/Graphics/OpenGL/glType.cs @@ -31,7 +31,7 @@ using System.Collections.ObjectModel; using System.Runtime.InteropServices; using System.Text; -namespace StgSharp.Internal.OpenGL +namespace StgSharp.Graphics.OpenGL { internal delegate void GLVULKANPROCNV(); diff --git a/StgSharp.Common/HighPerformance/M512.cs b/StgSharp.Common/HighPerformance/M512.cs index 6db9a3e7..18335a41 100644 --- a/StgSharp.Common/HighPerformance/M512.cs +++ b/StgSharp.Common/HighPerformance/M512.cs @@ -41,20 +41,20 @@ namespace StgSharp.HighPerformance public unsafe struct M512 : IRegisterPresentation { - [FieldOffset(0)]private fixed byte _buffer[64]; - [FieldOffset(0)]private Vector512 Vec; + [FieldOffset(0)] private fixed byte _buffer[64]; + [FieldOffset(0)] private Vector512 Vec; - public ref T AsRef() where T: unmanaged, INumber + public ref T AsRef() where T : unmanaged, INumber { return ref Unsafe.As(ref _buffer[0]); } - public void BroadCastFrom(T value) where T: unmanaged, INumber + public void BroadCastFrom(T value) where T : unmanaged, INumber { Vec = Vector512.Create(value).AsSingle(); } - public ref T Member(int index) where T: unmanaged, INumber + public ref T Member(int index) where T : unmanaged, INumber { return ref Unsafe.As(ref _buffer[0]); } diff --git a/StgSharp.Common/HighPerformance/IMatrixPanel.cs b/StgSharp.Common/HighPerformance/MatrixPanel.cs similarity index 76% rename from StgSharp.Common/HighPerformance/IMatrixPanel.cs rename to StgSharp.Common/HighPerformance/MatrixPanel.cs index 64887221..b2543a55 100644 --- a/StgSharp.Common/HighPerformance/IMatrixPanel.cs +++ b/StgSharp.Common/HighPerformance/MatrixPanel.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="IMatrixPanel" +// file="MatrixPanel" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -35,11 +35,7 @@ namespace StgSharp.HighPerformance public interface IMatrixPanel : IDisposable where T: unmanaged,INumber { - static abstract int KernelColumnCount { get; } - - static abstract int KernelRowCount { get; } - - static abstract int ElementCount { get; } + static abstract int MaxElementCount { get; } static abstract nuint Size { get; } @@ -54,52 +50,26 @@ namespace StgSharp.HighPerformance /// /// Type of matrix element. /// + [InlineArray(16)] public unsafe struct MatrixPanelSSE : IMatrixPanel where T: unmanaged, INumber { - private MatrixKernel _buffer; + private M128 vector; [Obsolete("Compute kernel should not be created from new(), use Create instead", true)] public MatrixPanelSSE() { } - public static int KernelColumnCount + public static int MaxElementCount { [MethodImpl(MethodImplOptions.AggressiveInlining)] get { int size = sizeof(T); - return size < 4 ? 2 : 1; + return 16 * 16 / size; } } - public static int KernelRowCount - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get - { - int size = sizeof(T); - return size < 4 ? 2 : 1; - } - } - - public static int ElementCount - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get - { - int size = sizeof(T); - return size < 4 ? 64 / size : 16; - } - } - - public static unsafe nuint Size - { - get - { - nuint size = (nuint)sizeof(T); - return size < 4 ? 64 : 16 * size; - } - } + public static unsafe nuint Size { get; } = 16 * 16; [MethodImpl(MethodImplOptions.AggressiveInlining)] public static MatrixPanelSSE* Create() @@ -130,15 +100,16 @@ namespace StgSharp.HighPerformance /// /// Type of matrix element. /// + [InlineArray(16)] public unsafe struct MatrixPanelAVX : IMatrixPanel where T: unmanaged, INumber { - private M512 _buffer; + private M256 _buffer; [Obsolete("Compute kernel should not be created from new(), use Create instead", true)] public MatrixPanelAVX() { } - public static unsafe int ElementCount + public static unsafe int MaxElementCount { [MethodImpl(MethodImplOptions.AggressiveInlining)] get @@ -148,18 +119,7 @@ namespace StgSharp.HighPerformance } } - public static int KernelColumnCount => 2; - - public static int KernelRowCount => 2; - - public static unsafe nuint Size - { - get - { - nuint size = (nuint)sizeof(T); - return size < 4 ? 128 : 16 * size; - } - } + public static unsafe nuint Size { get; } = 32 * 16; [MethodImpl(MethodImplOptions.AggressiveInlining)] public static MatrixPanelAVX* Create() @@ -190,6 +150,7 @@ namespace StgSharp.HighPerformance /// /// Type of matrix element. /// + [InlineArray(16)] public unsafe struct MatrixComputeKernelAVX512 : IMatrixPanel where T: unmanaged, INumber { @@ -198,7 +159,7 @@ namespace StgSharp.HighPerformance [Obsolete("Compute kernel should not be created from new(), use Create instead", true)] public MatrixComputeKernelAVX512() { } - public readonly unsafe int ElementCount + public static unsafe int MaxElementCount { [MethodImpl(MethodImplOptions.AggressiveInlining)] get @@ -208,18 +169,7 @@ namespace StgSharp.HighPerformance } } - public static int KernelColumnCount { get; } - - public static int KernelRowCount { get; } - - public static unsafe nuint Size - { - get - { - nuint size = (nuint)sizeof(T); - return size < 4 ? 128 : 16 * size; - } - } + public static unsafe nuint Size { get; } = 32 * 16; [MethodImpl(MethodImplOptions.AggressiveInlining)] public static MatrixComputeKernelAVX512* Create() diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs b/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs index 2de055e7..c4d14874 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="MatrixParallelHandle" +// file="MatrixParallelTaskPackage" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -117,8 +117,8 @@ namespace StgSharp.Mathematics.Numeric #endregion } - [StructLayout(LayoutKind.Sequential)] - internal unsafe struct MatrixParallelTaskPackage where T: unmanaged, INumber + [StructLayout(LayoutKind.Explicit, Size =128)] + internal unsafe struct MatrixParallelTaskPackage where T : unmanaged, INumber { public MatrixParallelTaskPackage() @@ -136,69 +136,69 @@ namespace StgSharp.Mathematics.Numeric #region source // Group 1: Left/Right matrix kernel base address - public MatrixKernel* Left; // 8 - public MatrixKernel* Right; // 8 + [FieldOffset(0)]public MatrixKernel* Left; // 8 + [FieldOffset(8)] public MatrixKernel* Right; // 8 #endregion #region ans // Group 2: Result kernel base address + reserved - public MatrixKernel* Result; // 8 - public int ElementSize = sizeof(T); - public int ReservedPtr0; // 4 (extension/reserved) + [FieldOffset(16)] public MatrixKernel* Result; // 8 + [FieldOffset(24)] public int ElementSize = sizeof(T); + [FieldOffset(28)] public int ReservedPtr0; // 4 (extension/reserved) #endregion #region left enum - public int LeftPrimOffset; // 4 - public int LeftPrimStride; // 4 - public int LeftSecOffset; // 4 - public int LeftSecStride; // 4 + [FieldOffset(32)]public int LeftPrimOffset; // 4 + [FieldOffset(36)]public int LeftPrimStride; // 4 + [FieldOffset(40)]public int LeftSecOffset; // 4 + [FieldOffset(44)] public int LeftSecStride; // 4 #endregion #region right enum - public int RightPrimOffset; // 4 - public int RightPrimStride; // 4 - public int RightSecOffset; // 4 - public int RightSecStride; // 4 + [FieldOffset(48)]public int RightPrimOffset; // 4 + [FieldOffset(52)]public int RightPrimStride; // 4 + [FieldOffset(56)]public int RightSecOffset; // 4 + [FieldOffset(60)]public int RightSecStride; // 4 #endregion #region ans enum - public int ResultPrimOffset; // 4 - public int ResultPrimStride; // 4 - public int ResultSecOffset; // 4 - public int ResultSecStride; // 4 + [FieldOffset(64)]public int ResultPrimOffset; // 4 + [FieldOffset(68)]public int ResultPrimStride; // 4 + [FieldOffset(72)]public int ResultSecOffset; // 4 + [FieldOffset(76)] public int ResultSecStride; // 4 #endregion #region scalar/compute - public ScalarPacket* Scalar; // 8 - public IntPtr ComputeHandle; // 8 + [FieldOffset(80)] public ScalarPacket* Scalar; // 8 + [FieldOffset(88)]public IntPtr ComputeHandle; // 8 #endregion #region global profile - public int PrimCount; // 4 - public int ComputeMode; // 4 - public int SecCount; // 4 - private readonly int ReservedInt0; // 4 + [FieldOffset(96)]public int PrimCount; // 4 + [FieldOffset(100)]public int ComputeMode; // 4 + [FieldOffset(104)]public int SecCount; // 4 + [FieldOffset(108)] private readonly int ReservedInt0; // 4 #endregion #region count offset - public int PrimTileOffset; // 4 - public int SecTileOffset; // 4 - public int PrimColumnCountInTile; // 4 - public int SecColumnCountInTile; // 4 + [FieldOffset(112)]public int PrimTileOffset; // 4 + [FieldOffset(116)]public int SecTileOffset; // 4 + [FieldOffset(120)]public int PrimColumnCountInTile; // 4 + [FieldOffset(124)]public int SecColumnCountInTile; // 4 #endregion } diff --git a/StgSharp.Native/StgSharp.Native.vcxproj b/StgSharp.Native/StgSharp.Native.vcxproj index 63c9665c..cdaa152a 100644 --- a/StgSharp.Native/StgSharp.Native.vcxproj +++ b/StgSharp.Native/StgSharp.Native.vcxproj @@ -104,6 +104,7 @@ + diff --git a/StgSharp.Native/StgSharp.Native.vcxproj.filters b/StgSharp.Native/StgSharp.Native.vcxproj.filters index 79996cf6..e98500dc 100644 --- a/StgSharp.Native/StgSharp.Native.vcxproj.filters +++ b/StgSharp.Native/StgSharp.Native.vcxproj.filters @@ -51,6 +51,9 @@ 源文件 + + 源文件 + diff --git a/StgSharp.Native/k_bufop.c b/StgSharp.Native/k_bufop.c new file mode 100644 index 00000000..e6976ec5 --- /dev/null +++ b/StgSharp.Native/k_bufop.c @@ -0,0 +1,29 @@ +#include "sn_intrinsic.h" + +DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill) +{ + register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]); + for (size_t i = 0; i < count; i++) { + ans[i].xmm[0] = vec; + ans[i].xmm[1] = vec; + ans[i].xmm[2] = vec; + ans[i].xmm[3] = vec; + } +} + +DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill) +{ + register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data))); + for (size_t i = 0; i < count; i++) { + ans[i].ymm[0] = vec; + ans[i].ymm[1] = vec; + } +} + +DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill) +{ + register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]); + for (size_t i = 0; i < count; i++) { + ans[i].zmm[0] = vec; + } +} diff --git a/StgSharp.Native/sn-head/sn_internal.h b/StgSharp.Native/sn-head/sn_internal.h index 1d0077fb..c2d751f0 100644 --- a/StgSharp.Native/sn-head/sn_internal.h +++ b/StgSharp.Native/sn-head/sn_internal.h @@ -1,6 +1,3 @@ -#ifdef _MSC_VER -#pragma once -#endif #ifndef SN_INTERNAL #define SN_INTERNAL @@ -24,6 +21,12 @@ #define FORCEINLINE #endif +#if defined(_MSC_VER) +#define SN_ALIGNAS(n) __declspec(align(n)) +#else +#define SN_ALIGNAS(n) __attribute__((aligned(n))) +#endif + #define SN_APIDEF #define PRIVATE static diff --git a/StgSharp.Native/sn-head/sn_intrinsic.h b/StgSharp.Native/sn-head/sn_intrinsic.h index 3c922d18..184ee8a8 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.h @@ -33,71 +33,179 @@ typedef union uint64_u { typedef struct mat_kernel mat_kernel; +typedef struct mat_panel mat_panel; + typedef union scalar_pack { uint64_t data[8]; } scalar_pack; +/* Mirrors StgSharp.Mathematics.Numeric.MatrixParallelTaskPackage (128 bytes). */ +#define MAT_TASK(T) \ + struct { \ + MAT_KERNEL(T) * left; /* Offset 0: left matrix kernel base */ \ + MAT_KERNEL(T) * right; /* Offset 8: right matrix kernel base */ \ + MAT_KERNEL(T) * result; /* Offset 16: result matrix kernel base */ \ + int32_t element_size; /* Offset 24: sizeof(T) */ \ + int32_t reserved_ptr0; /* Offset 28: reserved/extension slot */ \ + int32_t left_prim_offset; /* Offset 32 */ \ + int32_t left_prim_stride; /* Offset 36 */ \ + int32_t left_sec_offset; /* Offset 40 */ \ + int32_t left_sec_stride; /* Offset 44 */ \ + int32_t right_prim_offset; /* Offset 48 */ \ + int32_t right_prim_stride; /* Offset 52 */ \ + int32_t right_sec_offset; /* Offset 56 */ \ + int32_t right_sec_stride; /* Offset 60 */ \ + int32_t result_prim_offset; /* Offset 64 */ \ + int32_t result_prim_stride; /* Offset 68 */ \ + int32_t result_sec_offset; /* Offset 72 */ \ + int32_t result_sec_stride; /* Offset 76 */ \ + scalar_pack *scalar; /* Offset 80 */ \ + void *compute_handle; /* Offset 88 */ \ + int32_t prim_count; /* Offset 96 */ \ + int32_t compute_mode; /* Offset 100 */ \ + int32_t sec_count; /* Offset 104 */ \ + int32_t reserved_int0; /* Offset 108 */ \ + int32_t prim_tile_offset; /* Offset 112 */ \ + int32_t sec_tile_offset; /* Offset 116 */ \ + int32_t prim_cols_in_tile; /* Offset 120 */ \ + int32_t sec_cols_in_tile; /* Offset 124 */ \ + } +typedef struct matrix_task matrix_task; + +#if defined(__cplusplus) +static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mismatch"); +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +_Static_assert(sizeof(MAT_TASK(float)) == 128, "matrix_parallel_task_package size mismatch"); +#endif + +#pragma region matrix buffer operation function type + +typedef void(SN_DECL *BUF_PROC_AS)(mat_kernel *ans, scalar_pack *scalar, int count); +typedef void(SN_DECL *BUF_PROC_RAS)(mat_kernel *right, mat_kernel *ans, scalar_pack *scalar, + int count); +typedef void(SN_DECL *BUF_PROC_LRAS)(mat_kernel *left, mat_kernel *right, mat_kernel *ans, + scalar_pack *scalar, int count); +typedef void(SN_DECL *BUF_PROC_RA)(mat_kernel *right, mat_kernel *ans, int count); +typedef void(SN_DECL *BUF_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans, + int count); + +#define BUF_PROC(T_type, T_arch, T_op_name) T_type##_buf_##T_op_name##_##T_arch + +#define DECLARE_BUF_PROC_ANS_SCALAR(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \ + MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, int count) +#define DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \ + MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \ + scalar_pack const *scalar, int count) +#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \ + MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \ + MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, int count) +#define DECLARE_BUF_PROC_RIGHT_ANS(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \ + MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, int count) +#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \ + MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \ + MAT_KERNEL(T_type) *restrict ans, int count) + +#pragma endregion + +#pragma region matrix panel function type + +typedef void(SN_DECL *PNL_PROC_LRA)(mat_panel *left, mat_panel *right, mat_panel *ans); +typedef void(SN_DECL *PNL_PROC_RA)(mat_panel *right, mat_panel *ans); +typedef void(SN_DECL *PNL_PROC_LRAS)(mat_panel *left, mat_panel *right, mat_panel *ans, + scalar_pack *scalar); +typedef void(SN_DECL *PNL_PROC_RAS)(mat_panel *right, mat_panel *ans, scalar_pack *scalar); + +#define PNL_PROC(T_type, T_arch, T_op_name) T_type##_pnl_##T_op_name##_##T_arch + +#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \ + MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \ + MAT_PANEL##T_arch##(T_type) *restrict ans) +#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \ + MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \ + MAT_PANEL##T_arch##(T_type) *restrict ans) +#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \ + MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \ + MAT_PANEL##T_arch##(T_type) *restrict ans, scalar_pack const *scalar) +#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \ + MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \ + MAT_PANEL##T_arch##(T_type) *restrict ans, scalar_pack const *scalar) + +#pragma endregion + +#pragma region matrix kernel function type + +typedef void(SN_DECL *KER_PROC_AS)(mat_kernel *ans, scalar_pack *scalar); +typedef void(SN_DECL *KER_PROC_RA)(mat_kernel *left, mat_kernel *ans); +typedef void(SN_DECL *KER_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans); +typedef void(SN_DECL *KER_PROC_LAS)(mat_kernel *matrix, mat_kernel *ans, scalar_pack *scalar); + +#define KER_PROC(T_type, T_arch, T_op_name) T_type##_kernel_##T_op_name##_##T_arch + +#define DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \ + MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \ + MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar) +#define DECLARE_KER_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \ + MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \ + scalar_pack const *scalar) +#define DECLARE_KER_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \ + MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \ + MAT_KERNEL(T_type) *restrict ans) +#define DECLARE_KER_PROC_RIGHT_ANS(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \ + MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans) + +#pragma endregion + typedef void(SN_DECL *VECTORNORMALIZEPROC)(__m128 *source, __m128 *target); typedef void(SN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans); -typedef void(SN_DECL *PANEL_PROC_AS)(MAT_PANEL_SSE(int)); -typedef void(SN_DECL *MAT_PROC_AS)(mat_kernel *ans, scalar_pack *scalar); -typedef void(SN_DECL *MAT_PROC_LA)(mat_kernel *left, mat_kernel *ans); -typedef void(SN_DECL *MAT_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans); -typedef void(SN_DECL *MAT_PROC_LSA)(mat_kernel *matrix, scalar_pack *scalar, mat_kernel *ans); typedef uint64_t(SN_DECL *FACTORIALROC)(int n); #pragma region matix function INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target); -INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) *restrict ans); -INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) *restrict ans); -INTERNAL void SN_DECL f32_add_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) *restrict ans); -INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) *restrict ans); -INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) *restrict ans); -INTERNAL void SN_DECL f32_sub_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) *restrict ans); -INTERNAL void SN_DECL f32_transpose_sse(MAT_KERNEL(float) const *source, - MAT_KERNEL(float) *restrict target); -INTERNAL void SN_DECL f32_transpose_avx(MAT_KERNEL(float) const *source, - MAT_KERNEL(float) *restrict target); -INTERNAL void SN_DECL f32_transpose_512(MAT_KERNEL(float) const *source, - MAT_KERNEL(float) *restrict target); -INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix, - MAT_KERNEL(float) *restrict ans, - scalar_pack const *scalar); -INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix, - MAT_KERNEL(float) *restrict ans, - scalar_pack const *scalar); -INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix, - MAT_KERNEL(float) *restrict ans, - scalar_pack const *scalar); -INTERNAL void SN_DECL f32_fill_sse(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler); -INTERNAL void SN_DECL f32_fill_avx(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler); -INTERNAL void SN_DECL f32_fill_512(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler); +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, add); +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, add); +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, add); + +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, sub); +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, sub); +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, sub); + +DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose); +DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose); +DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose); + +DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul); +DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul); +DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul); + +DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill); + #pragma endregion #pragma region fma -INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left, - MAT_KERNEL(float) const *restrict right, - MAT_KERNEL(float) *restrict ans); -INTERNAL void SN_DECL f32_fma_avx(MAT_KERNEL(float) const *restrict left, - MAT_KERNEL(float) const *restrict right, - MAT_KERNEL(float) *restrict ans); -INTERNAL void SN_DECL f32_fma_512(MAT_KERNEL(float) const *restrict left, - MAT_KERNEL(float) const *restrict right, - MAT_KERNEL(float) *restrict ans); +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, fma); #pragma endregion #pragma region scaler_simd + INTERNAL uint64_t SN_DECL factorial_simd_sse(int n); + #pragma endregion #pragma region dot_product @@ -115,21 +223,24 @@ INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 cons #pragma region string +typedef int(SN_DECL *HASH)(byte const *str, int length); +typedef int(SN_DECL *INDEX_PAIR)(short const *stram, uint32_t target, int length); + INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int length); INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int length); #pragma endregion typedef struct sn_intrinsic { - int (*city_hash_simplify)(byte *str, int length); + HASH city_hash_simplify; FACTORIALROC factorial_simd; - int (*index_pair)(short const *str, uint32_t target, int length); - MAT_PROC_LRA f32_add; - MAT_PROC_AS f32_fill; - MAT_PROC_LRA f32_fma; - MAT_PROC_LRA f32_scalar_mul; - MAT_PROC_LRA f32_sub; - MAT_PROC_LA f32_transpose; + INDEX_PAIR index_pair; + KER_PROC_LRA f32_kernel_add; + KER_PROC_AS f32_kernel_fill; + PNL_PROC_LRA f32_panel_fma; + KER_PROC_LRA f32_kernel_scalar_mul; + KER_PROC_LRA f32_kernel_sub; + KER_PROC_RA f32_kernel_transpose; VECTORNORMALIZEPROC f32_normalize_3; } sn_intrinsic; diff --git a/StgSharp.Native/sn-head/sn_matkernel.h b/StgSharp.Native/sn-head/sn_matkernel.h index 75df2c95..168c740f 100644 --- a/StgSharp.Native/sn-head/sn_matkernel.h +++ b/StgSharp.Native/sn-head/sn_matkernel.h @@ -26,98 +26,58 @@ #endif -#include -#include +/* Helpers (all compile-time): */ +#define SN_LANES(T_VEC, T) (sizeof(T_VEC) / sizeof(T)) +#define SN_MAX(a, b) (((a)) > ((b)) ? (a) : (b)) +#define SN_CEIL_DIV(a, b) (((a) + (b) - 1) / (b)) -/* align decoration */ +/* Panel geometry helpers (all compile-time): */ +#define PANEL_Q(T, T_VEC, MINQ) SN_MAX(SN_LANES(T_VEC, T), (MINQ)) +#define PANEL_CHUNKS(T, T_VEC, MINQ) SN_CEIL_DIV(PANEL_Q(T, T_VEC, MINQ), SN_LANES(T_VEC, T)) -#if __cplusplus && defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) -#define ML_ALIGNAS(n) _Alignas(n) -#elif defined(_MSC_VER) -#define ML_ALIGNAS(n) __declspec(align(n)) -#else -#define ML_ALIGNAS(n) __attribute__((aligned(n))) -#endif - -/* Round up to the alignment boundary in bytes */ -#ifndef ML_ALIGN_UP -#define ML_ALIGN_UP(x, a) (((size_t)(x) + ((size_t)(a) - 1)) & ~((size_t)(a) - 1)) -#endif - -/* Vector register width in bytes */ -#define ML_BYTES_XMM 16u -#define ML_BYTES_YMM 32u -#define ML_BYTES_ZMM 64u - -/* Compute the aligned leading-direction element stride (LEp) */ -#define ML_PANEL_STRIDE_ELEMS(T, LE, ALIGN) \ - (ML_ALIGN_UP((size_t)(LE) * sizeof(T), (size_t)(ALIGN)) / sizeof(T)) - -/* Total element count and total byte count */ -#define ML_PANEL_TOT_ELEMS(T, LE, MINOR, ALIGN) \ - ((size_t)ML_PANEL_STRIDE_ELEMS(T, (LE), (ALIGN)) * (size_t)(MINOR)) - -#define ML_TOT_BYTES(T, ELEMS) ((size_t)(ELEMS) * sizeof(T)) - -/* Number of SIMD view slots (ceil by total bytes) */ -#define ML_XMM_COUNT(TOT_BYTES) (((size_t)(TOT_BYTES) + ML_BYTES_XMM - 1) / ML_BYTES_XMM) -#define ML_YMM_COUNT(TOT_BYTES) (((size_t)(TOT_BYTES) + ML_BYTES_YMM - 1) / ML_BYTES_YMM) -#define ML_ZMM_COUNT(TOT_BYTES) (((size_t)(TOT_BYTES) + ML_BYTES_ZMM - 1) / ML_BYTES_ZMM) - -/* ========== Unified "panel" union (no enum/descriptor fields) ========== - Usage: - MAT_PANEL(float, 8, 32, 32) P; // LE=8, MINOR=32, aligned to 32 bytes - Views: - P.xmm[] / P.ymm[] / P.zmm[] // raw SIMD blocks - P.blk[MINOR][LEp] // 2D scalar view (includes padding for alignment) - P.flat[MINOR*LEp] // flat array - Notes: - - LE/MINOR/ALIGN must be compile-time constants - - LEp = ML_PANEL_STRIDE_ELEMS(T, LE, ALIGN) - - Layout/semantics defined by the calling convention (e.g. blk[k][r] is element r of block k) +/* + Column-major square panel with minimum side Q: + - Let R = lanes per vector = sizeof(T_VEC)/sizeof(T). + - Side length Q = max(R, MINQ) to enforce the requested minimum extent. + - Each column is stored contiguously via CH = ceil(Q/R) vector chunks. + - Memory layout: + vec[col][chunk] is a T_VEC holding up to R scalars of column 'col' + spanning rows chunk*R .. chunk*R + (R - 1). + flat[] mirrors the same column-major order (column stride = Q). + - SIMD view remains integer-typed; cast inside micro-kernels to the + appropriate floating-point/integer vector type on demand. */ -#define MAT_PANEL(T, LE, MINOR, ALIGN) \ - ML_ALIGNAS(ALIGN) union { \ - /* Raw SIMD view: cast to __m256/__m512d inside the micro-kernel on demand */ \ - __m128i xmm[ML_XMM_COUNT( \ - ML_TOT_BYTES(T, ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))))]; \ - __m256i ymm[ML_YMM_COUNT( \ - ML_TOT_BYTES(T, ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))))]; \ - __m512i zmm[ML_ZMM_COUNT( \ - ML_TOT_BYTES(T, ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))))]; \ - /* Scalar 2D view: minor �� LEp (LE dimension contains alignment padding) */ \ - T blk[(MINOR)][ML_PANEL_STRIDE_ELEMS(T, (LE), (ALIGN))]; \ - /* Flat view */ \ - T flat[ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))]; \ +#define MAT_PANEL_SQ_MIN(T, T_VEC, MINQ) \ + SN_ALIGNAS(sizeof(T_VEC)) union { \ + T_VEC vec[PANEL_Q(T, T_VEC, MINQ)][PANEL_CHUNKS(T, T_VEC, MINQ)]; \ + T flat[PANEL_Q(T, T_VEC, MINQ) * PANEL_Q(T, T_VEC, MINQ)]; \ } -/* ---------- SSE preset ---------- */ -#undef PANEL_LE -#undef PANEL_MINOR -#undef PANEL_ALIGN -#define PANEL_LE 4 -#define PANEL_MINOR 32 -#define PANEL_ALIGN 16 -/* ======== SSE preset ======== - 4 x 32, 16B aligned +/* ISA presets with minimum square side: + - SSE : MINQ=4 (avoid 2x2 for double) + - AVX2 : MINQ=8 + - AVX-512: MINQ=8 (16 for float/i32 naturally, so this is a no-op there) */ -#define MAT_PANEL_SSE(T) MAT_PANEL(T, PANEL_LE, PANEL_MINOR, PANEL_ALIGN) -/* ---------- AVX2 preset ---------- */ -#undef PANEL_LE -#undef PANEL_MINOR -#undef PANEL_ALIGN -#define PANEL_LE 8 -#define PANEL_MINOR 64 -#define PANEL_ALIGN 32 -#define MAT_PANEL_AVX(T) MAT_PANEL(T, PANEL_LE, PANEL_MINOR, PANEL_ALIGN) /* 8 x 64, 32B aligned */ +/* SSE panel layout: column-major buffer holding 4 logical columns with MINQ=4. + Columns: each column packs up to four scalars so ALIGN equals sizeof(__m128). + Alignment: every column boundary lands on a 16-byte multiple for XMM loads. */ +#define MAT_PANEL_SSE(T) MAT_PANEL_SQ_MIN(T, __m128, 4) +/* AVX2 panel layout: column-major buffer holding 8 logical columns with MINQ=8. + Columns: each column stores eight elements via two 256-bit chunks when T is 32-bit. + Alignment: column starts are 32-byte aligned so AVX2 loads/stores remain contiguous. */ +#define MAT_PANEL_AVX(T) MAT_PANEL_SQ_MIN(T, __m256, 8) +/* AVX-512 panel layout: column-major buffer holding 8 or more columns depending on T. + Columns: each column expands to ceil(Q/lanes) ZMM chunks; default MINQ=8 keeps double-wide tiles. + Alignment: column starts sit on 64-byte boundaries to match AVX-512 streaming access. */ +#define MAT_PANEL_512(T) MAT_PANEL_SQ_MIN(T, __m512, 8) -/* ---------- AVX-512 preset ---------- */ -#undef PANEL_LE -#undef PANEL_MINOR -#undef PANEL_ALIGN -#define PANEL_LE 16 -#define PANEL_MINOR 64 -#define PANEL_ALIGN 64 -#define MAT_PANEL_512(T) \ - MAT_PANEL(T, PANEL_LE, PANEL_MINOR, PANEL_ALIGN); /* 16 x 64, 64B aligned */ +/* Access helpers (all compile-time friendly): + Given Q = PANEL_Q(...), CH = PANEL_CHUNKS(...), R = lanes per vector: + - Column c occupies vec[c][0..CH-1]. + - Row r inside column c is located at vec[c][r / R] with lane (r % R). +*/ +#define PANEL_COL_VEC(panel_obj, c, chunk) ((panel_obj).vec[(int)(c)][(int)(chunk)]) +/* Scalar flat index helper for column-major addressing */ +#define PANEL_ELEM(panel_obj, T, T_VEC, MINQ, r, c) \ + ((panel_obj).flat[(int)(c) * PANEL_Q(T, T_VEC, MINQ) + (int)(r)]) diff --git a/StgSharp.Native/sn-src/dllmain.c b/StgSharp.Native/sn-src/dllmain.c index 487db88b..13a88745 100644 --- a/StgSharp.Native/sn-src/dllmain.c +++ b/StgSharp.Native/sn-src/dllmain.c @@ -3,16 +3,9 @@ #ifdef _WIN32 -// clang-format off - -BOOL APIENTRY DllMain( - HMODULE hModule, - DWORD ul_reason_for_call, - LPVOID lpReserved -) +BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) { - switch (ul_reason_for_call) - { + switch (ul_reason_for_call) { case DLL_PROCESS_ATTACH: return 1; case DLL_THREAD_ATTACH: @@ -21,9 +14,9 @@ BOOL APIENTRY DllMain( return 1; case DLL_PROCESS_DETACH: return 1; - break; + break; } - return TRUE; + return TRUE; } #else attribute((constructor)) void DllMain() @@ -31,5 +24,3 @@ attribute((constructor)) void DllMain() } #endif // _WIN32 - -// clang-format on diff --git a/StgSharp.Native/sn-src/k_basicop.c b/StgSharp.Native/sn-src/k_basicop.c index 31a7cda8..d0e4f44b 100644 --- a/StgSharp.Native/sn-src/k_basicop.c +++ b/StgSharp.Native/sn-src/k_basicop.c @@ -1,29 +1,34 @@ #include "sn_intrinsic.h" -INTERNAL void SN_DECL f32_fill_sse(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler) +DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill) { - register __m128 vec = _mm_set_ps1(((float *)(scaler->data))[0]); - dest->xmm[0] = vec; - dest->xmm[1] = vec; - dest->xmm[2] = vec; - dest->xmm[3] = vec; + register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]); + for (size_t i = 0; i < count; i++) { + ans[i].xmm[0] = vec; + ans[i].xmm[1] = vec; + ans[i].xmm[2] = vec; + ans[i].xmm[3] = vec; + } } -INTERNAL void SN_DECL f32_fill_avx(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler) +DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill) { - register __m256 vec = _mm256_broadcast_ss(((float *)(scaler->data))); - dest->ymm[0] = vec; - dest->ymm[1] = vec; + register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data))); + for (size_t i = 0; i < count; i++) { + ans[i].ymm[0] = vec; + ans[i].ymm[1] = vec; + } } -INTERNAL void SN_DECL f32_fill_512(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler) +DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill) { - register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scaler->data))[0]); - dest->zmm[0] = vec; + register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]); + for (size_t i = 0; i < count; i++) { + ans[i].zmm[0] = vec; + } } -INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) *restrict ans) +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, add) { register __m128 c0 = _mm_add_ps(left->xmm[0], right->xmm[0]); register __m128 c1 = _mm_add_ps(left->xmm[1], right->xmm[1]); @@ -36,8 +41,7 @@ INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(floa ans->xmm[3] = c3; } -INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) *restrict ans) +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, add) { register __m256 c0 = _mm256_add_ps(left->ymm[0], right->ymm[0]); register __m256 c1 = _mm256_add_ps(left->ymm[1], right->ymm[1]); @@ -45,15 +49,13 @@ INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(floa ans->ymm[1] = c1; } -INTERNAL void SN_DECL f32_add_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) *restrict ans) +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, add) { register __m512 c0 = _mm512_add_ps(left->zmm[0], right->zmm[0]); ans->zmm[0] = c0; } -INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) *restrict ans) +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, sub) { register __m128 c0 = _mm_sub_ps(left->xmm[0], right->xmm[0]); register __m128 c1 = _mm_sub_ps(left->xmm[1], right->xmm[1]); @@ -66,8 +68,7 @@ INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(floa ans->xmm[3] = c3; } -INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) *restrict ans) +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, sub) { register __m256 c0 = _mm256_sub_ps(left->ymm[0], right->ymm[0]); register __m256 c1 = _mm256_sub_ps(left->ymm[1], right->ymm[1]); @@ -75,37 +76,34 @@ INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(floa ans->ymm[1] = c1; } -INTERNAL void SN_DECL f32_sub_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) *restrict ans) +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, sub) { register __m512 c0 = _mm512_sub_ps(left->zmm[0], right->zmm[0]); ans->zmm[0] = c0; } // Matrix transpose implementations -INTERNAL void SN_DECL f32_transpose_sse(MAT_KERNEL(float) const *source, - MAT_KERNEL(float) *restrict target) +DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose) { - register __m128 col0 = source->xmm[0]; - register __m128 col1 = source->xmm[1]; - register __m128 col2 = source->xmm[2]; - register __m128 col3 = source->xmm[3]; + register __m128 col0 = right->xmm[0]; + register __m128 col1 = right->xmm[1]; + register __m128 col2 = right->xmm[2]; + register __m128 col3 = right->xmm[3]; register __m128 tmp0 = _mm_unpacklo_ps(col0, col1); register __m128 tmp1 = _mm_unpackhi_ps(col0, col1); register __m128 tmp2 = _mm_unpacklo_ps(col2, col3); register __m128 tmp3 = _mm_unpackhi_ps(col2, col3); - target->xmm[0] = _mm_movelh_ps(tmp0, tmp2); - target->xmm[1] = _mm_movehl_ps(tmp2, tmp0); - target->xmm[2] = _mm_movelh_ps(tmp1, tmp3); - target->xmm[3] = _mm_movehl_ps(tmp3, tmp1); + ans->xmm[0] = _mm_movelh_ps(tmp0, tmp2); + ans->xmm[1] = _mm_movehl_ps(tmp2, tmp0); + ans->xmm[2] = _mm_movelh_ps(tmp1, tmp3); + ans->xmm[3] = _mm_movehl_ps(tmp3, tmp1); } -INTERNAL void SN_DECL f32_transpose_avx(MAT_KERNEL(float) const *source, - MAT_KERNEL(float) *restrict target) +DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose) { - register __m256 col_low = source->ymm[0]; - register __m256 col_high = source->ymm[1]; + register __m256 col_low = right->ymm[0]; + register __m256 col_high = right->ymm[1]; register __m256 unpack_low = _mm256_unpacklo_ps(col_low, col_high); register __m256 unpack_high = _mm256_unpackhi_ps(col_low, col_high); @@ -113,30 +111,28 @@ INTERNAL void SN_DECL f32_transpose_avx(MAT_KERNEL(float) const *source, col_low = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x20); col_high = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x31); - target->ymm[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0)); - target->ymm[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1)); + ans->ymm[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0)); + ans->ymm[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1)); } -INTERNAL void SN_DECL f32_transpose_512(MAT_KERNEL(float) const *source, - MAT_KERNEL(float) *restrict target) +DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose) { - register __m512 matrix = source->zmm[0]; + register __m512 matrix = right->zmm[0]; const __m512i transpose_indices = _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - target->zmm[0] = _mm512_permutexvar_ps(transpose_indices, matrix); + ans->zmm[0] = _mm512_permutexvar_ps(transpose_indices, matrix); } -INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix, - MAT_KERNEL(float) *restrict ans, scalar_pack const *scalar) +DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul) { register __m128 scalar_vec = ((__m128 *)(scalar->data))[0]; - register __m128 c0 = _mm_mul_ps(matrix->xmm[0], scalar_vec); - register __m128 c1 = _mm_mul_ps(matrix->xmm[1], scalar_vec); - register __m128 c2 = _mm_mul_ps(matrix->xmm[2], scalar_vec); - register __m128 c3 = _mm_mul_ps(matrix->xmm[3], scalar_vec); + register __m128 c0 = _mm_mul_ps(right->xmm[0], scalar_vec); + register __m128 c1 = _mm_mul_ps(right->xmm[1], scalar_vec); + register __m128 c2 = _mm_mul_ps(right->xmm[2], scalar_vec); + register __m128 c3 = _mm_mul_ps(right->xmm[3], scalar_vec); ans->xmm[0] = c0; ans->xmm[1] = c1; @@ -144,24 +140,22 @@ INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix, ans->xmm[3] = c3; } -INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix, - MAT_KERNEL(float) *restrict ans, scalar_pack const *scalar) +DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul) { register __m256 scalar_vec = ((__m256 *)(scalar->data))[0]; - register __m256 c0 = _mm256_mul_ps(matrix->ymm[0], scalar_vec); - register __m256 c1 = _mm256_mul_ps(matrix->ymm[1], scalar_vec); + register __m256 c0 = _mm256_mul_ps(right->ymm[0], scalar_vec); + register __m256 c1 = _mm256_mul_ps(right->ymm[1], scalar_vec); ans->ymm[0] = c0; ans->ymm[1] = c1; } -INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix, - MAT_KERNEL(float) *restrict ans, scalar_pack const *scalar) +DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul) { register __m512 scalar_vec = ((__m512 *)(scalar->data))[0]; - register __m512 c0 = _mm512_mul_ps(matrix->zmm[0], scalar_vec); + register __m512 c0 = _mm512_mul_ps(right->zmm[0], scalar_vec); ans->zmm[0] = c0; } diff --git a/StgSharp.Native/sn-src/k_fma.c b/StgSharp.Native/sn-src/k_fma.c index 528f4e1a..a600e7cf 100644 --- a/StgSharp.Native/sn-src/k_fma.c +++ b/StgSharp.Native/sn-src/k_fma.c @@ -1,14 +1,8 @@ #include "sn_intrinsic.h" -INTERNAL void SN_DECL f32_fma_sse(MAT_PANEL_SSE(int) * left) -{ -} - // ans = left * right + ans // right is not transposed -INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left, - MAT_KERNEL(float) const *restrict right, - MAT_KERNEL(float) *restrict ans) +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, fma) { register __m128 source0 = _mm_load_ps(&left->xmm[0]); register __m128 source1 = _mm_load_ps(&left->xmm[1]); @@ -39,46 +33,3 @@ INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left, _mm_store_ps(&ans->xmm[2], ans2); _mm_store_ps(&ans->xmm[3], ans3); } - -// ans = left * right + ans -INTERNAL void SN_DECL f32_fma_avx(MAT_KERNEL(float) const *restrict left, - MAT_KERNEL(float) const *restrict right, - MAT_KERNEL(float) *restrict ans) -{ - register __m256 source0 = _mm256_load_ps(&left->ymm[0]); - register __m256 source1 = _mm256_load_ps(&left->ymm[1]); - - register __m256 ans0 = _mm256_load_ps(&ans->ymm[0]); - register __m256 ans1 = _mm256_load_ps(&ans->ymm[1]); - - register __m128 bs0, bs1; - register __m256 broadcast; - for (size_t i = 0; i < 4; i++) { - bs0 = _mm_load_ps(&right->xmm[i]); - - __m128 lo = _mm_shuffle_ps(bs0, bs0, 0x00); - __m128 hi = _mm_shuffle_ps(bs0, bs0, 0x55); - broadcast = _mm256_castps128_ps256(lo); - broadcast = _mm256_insertf128_ps(broadcast, bs0, 1); - ans0 = _mm256_add_ps(ans0, _mm256_mul_ps(source0, broadcast)); - - lo = _mm_shuffle_ps(bs0, bs0, 0xAA); - hi = _mm_shuffle_ps(bs0, bs0, 0xFF); - broadcast = _mm256_castps128_ps256(lo); - broadcast = _mm256_insertf128_ps(broadcast, bs0, 1); - ans1 = _mm256_add_ps(ans1, _mm256_mul_ps(source1, broadcast)); - } - - _mm256_store_ps(&ans->ymm[0], ans0); - _mm256_store_ps(&ans->ymm[1], ans1); -} - -// ans = left * right + ans -INTERNAL void SN_DECL f32_fma_512(MAT_KERNEL(float) const *restrict left, - MAT_KERNEL(float) const *restrict right, - MAT_KERNEL(float) *restrict ans) -{ - /* - * to be implemented - */ -} diff --git a/StgSharp.Native/sn-src/load_intrinsic.c b/StgSharp.Native/sn-src/load_intrinsic.c index ffb88195..af5b8062 100644 --- a/StgSharp.Native/sn-src/load_intrinsic.c +++ b/StgSharp.Native/sn-src/load_intrinsic.c @@ -65,11 +65,11 @@ void load_intrinsic_sse(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; - context->f32_add = (MAT_PROC_LRA)f32_add_sse; - context->f32_fma = (MAT_PROC_LRA)f32_fma_sse; - context->f32_scalar_mul = f32_scalar_mul_sse; - context->f32_sub = (MAT_PROC_LRA)f32_sub_sse; - context->f32_transpose = (MAT_PROC_LA)f32_transpose_sse; + context->f32_kernel_add = KER_PROC(float, sse, add); + context->f32_panel_fma = KER_PROC(float, sse, fma); + context->f32_kernel_scalar_mul = KER_PROC(float, sse, scalar_mul); + context->f32_kernel_sub = KER_PROC(float, sse, sub); + context->f32_kernel_transpose = KER_PROC(float, sse, transpose); context->f32_normalize_3 = f32_normalize; } @@ -78,11 +78,11 @@ void load_intrinsic_avx(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; - context->f32_add = (MAT_PROC_LRA)f32_add_avx; - context->f32_fma = (MAT_PROC_LRA)f32_fma_avx; - context->f32_scalar_mul = f32_scalar_mul_avx; - context->f32_sub = (MAT_PROC_LRA)f32_sub_avx; - context->f32_transpose = (MAT_PROC_LA)f32_transpose_avx; + context->f32_kernel_add = KER_PROC(float, avx, add); + context->f32_panel_fma = KER_PROC(float, sse, fma); + context->f32_kernel_scalar_mul = KER_PROC(float, avx, scalar_mul); + context->f32_kernel_sub = KER_PROC(float, avx, sub); + context->f32_kernel_transpose = KER_PROC(float, avx, transpose); context->f32_normalize_3 = f32_normalize; } @@ -91,10 +91,10 @@ void load_intrinsic_512(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; - context->f32_add = (MAT_PROC_LRA)f32_add_512; - context->f32_fma = (MAT_PROC_LRA)f32_fma_512; - context->f32_scalar_mul = f32_scalar_mul_512; - context->f32_sub = (MAT_PROC_LRA)f32_sub_512; - context->f32_transpose = (MAT_PROC_LA)f32_transpose_512; + context->f32_kernel_add = KER_PROC(float, 512, add); + context->f32_panel_fma = KER_PROC(float, sse, fma); + context->f32_kernel_scalar_mul = KER_PROC(float, 512, scalar_mul); + context->f32_kernel_sub = KER_PROC(float, 512, sub); + context->f32_kernel_transpose = KER_PROC(float, 512, transpose); context->f32_normalize_3 = f32_normalize; } diff --git a/StgSharpDebug/RenderStream/Partical.cs b/StgSharpDebug/RenderStream/Partical.cs index 27fff12b..324b0d81 100644 --- a/StgSharpDebug/RenderStream/Partical.cs +++ b/StgSharpDebug/RenderStream/Partical.cs @@ -25,12 +25,10 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- -using StgSharp.Common.Graphics; using StgSharp.Controls; using StgSharp.Geometries; using StgSharp.Graphics; using StgSharp.Graphics.OpenGL; -using StgSharp.Internal.OpenGL; using StgSharp.Mathematics; using StgSharp.Mathematics.Graphic; using StgSharp.Timing;