youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

macro gymnastics to implement storage kernel, buffer op and complex compute panel

Browse Source
This commit is contained in:
Youmu Liu
2025-11-13 19:09:27 +08:00
parent 2dfb4ae86a
commit 1417785726
18 changed files with 373 additions and 383 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/Controlling/KeyEnum.cs
-1
View File
@@ -27,7 +27,6 @@
// -----------------------------------------------------------------------
using StgSharp.Graphics;
using StgSharp.Graphics.OpenGL;
using StgSharp.Internal.OpenGL;
using System;
StgSharp.Common/Global.cs
+1 -1
View File
@@ -27,7 +27,7 @@
// -----------------------------------------------------------------------
global using StgSharp.Internal;
global using StgSharp.Internal.Intrinsic;
global using StgSharp.Internal.OpenGL;
global
using System.Diagnostics.CodeAnalysis;
using System.Runtime.CompilerServices;
StgSharp.Common/Graphics/OpenGL/Buffer/EBO.cs
+1 -1
View File
@@ -27,7 +27,7 @@
// -----------------------------------------------------------------------
using StgSharp.Graphics;
using StgSharp.Graphics.OpenGL;
using StgSharp.Internal.OpenGL;
using StgSharp.Mathematics;
using System;
StgSharp.Common/Graphics/OpenGL/glType.cs
+1 -1
View File
@@ -31,7 +31,7 @@ using System.Collections.ObjectModel;
using System.Runtime.InteropServices;
using System.Text;
namespace StgSharp.Internal.OpenGL
namespace StgSharp.Graphics.OpenGL
{
internal delegate void GLVULKANPROCNV();
StgSharp.Common/HighPerformance/M512.cs
+5 -5
View File
@@ -41,20 +41,20 @@ namespace StgSharp.HighPerformance
public unsafe struct M512 : IRegisterPresentation
{
[FieldOffset(0)]private fixed byte _buffer[64];
[FieldOffset(0)]private Vector512<float> Vec;
[FieldOffset(0)] private fixed byte _buffer[64];
[FieldOffset(0)] private Vector512<float> Vec;
public ref T AsRef<T>() where T: unmanaged, INumber<T>
public ref T AsRef<T>() where T : unmanaged, INumber<T>
{
return ref Unsafe.As<byte, T>(ref _buffer[0]);
}
public void BroadCastFrom<T>(T value) where T: unmanaged, INumber<T>
public void BroadCastFrom<T>(T value) where T : unmanaged, INumber<T>
{
Vec = Vector512.Create(value).AsSingle();
}
public ref T Member<T>(int index) where T: unmanaged, INumber<T>
public ref T Member<T>(int index) where T : unmanaged, INumber<T>
{
return ref Unsafe.As<byte, T>(ref _buffer[0]);
}
StgSharp.Common/HighPerformance/IMatrixPanel.cs → StgSharp.Common/HighPerformance/MatrixPanel.cs
+14 -64
View File
@@ -1,6 +1,6 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="IMatrixPanel"
// file="MatrixPanel"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
@@ -35,11 +35,7 @@ namespace StgSharp.HighPerformance
public interface IMatrixPanel<T> : IDisposable where T: unmanaged,INumber<T>
{
static abstract int KernelColumnCount { get; }
static abstract int KernelRowCount { get; }
static abstract int ElementCount { get; }
static abstract int MaxElementCount { get; }
static abstract nuint Size { get; }
@@ -54,52 +50,26 @@ namespace StgSharp.HighPerformance
/// <typeparam name="T">
/// Type of matrix element.
/// </typeparam>
[InlineArray(16)]
public unsafe struct MatrixPanelSSE<T> : IMatrixPanel<T> where T: unmanaged, INumber<T>
{
private MatrixKernel<T> _buffer;
private M128 vector;
[Obsolete("Compute kernel should not be created from new(), use Create instead", true)]
public MatrixPanelSSE() { }
public static int KernelColumnCount
public static int MaxElementCount
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get
{
int size = sizeof(T);
return size < 4 ? 2 : 1;
return 16 * 16 / size;
}
}
public static int KernelRowCount
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get
{
int size = sizeof(T);
return size < 4 ? 2 : 1;
}
}
public static int ElementCount
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get
{
int size = sizeof(T);
return size < 4 ? 64 / size : 16;
}
}
public static unsafe nuint Size
{
get
{
nuint size = (nuint)sizeof(T);
return size < 4 ? 64 : 16 * size;
}
}
public static unsafe nuint Size { get; } = 16 * 16;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static MatrixPanelSSE<T>* Create()
@@ -130,15 +100,16 @@ namespace StgSharp.HighPerformance
/// <typeparam name="T">
/// Type of matrix element.
/// </typeparam>
[InlineArray(16)]
public unsafe struct MatrixPanelAVX<T> : IMatrixPanel<T> where T: unmanaged, INumber<T>
{
private M512 _buffer;
private M256 _buffer;
[Obsolete("Compute kernel should not be created from new(), use Create instead", true)]
public MatrixPanelAVX() { }
public static unsafe int ElementCount
public static unsafe int MaxElementCount
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get
@@ -148,18 +119,7 @@ namespace StgSharp.HighPerformance
}
}
public static int KernelColumnCount => 2;
public static int KernelRowCount => 2;
public static unsafe nuint Size
{
get
{
nuint size = (nuint)sizeof(T);
return size < 4 ? 128 : 16 * size;
}
}
public static unsafe nuint Size { get; } = 32 * 16;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static MatrixPanelAVX<T>* Create()
@@ -190,6 +150,7 @@ namespace StgSharp.HighPerformance
/// <typeparam name="T">
/// Type of matrix element.
/// </typeparam>
[InlineArray(16)]
public unsafe struct MatrixComputeKernelAVX512<T> : IMatrixPanel<T> where T: unmanaged, INumber<T>
{
@@ -198,7 +159,7 @@ namespace StgSharp.HighPerformance
[Obsolete("Compute kernel should not be created from new(), use Create instead", true)]
public MatrixComputeKernelAVX512() { }
public readonly unsafe int ElementCount
public static unsafe int MaxElementCount
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get
@@ -208,18 +169,7 @@ namespace StgSharp.HighPerformance
}
}
public static int KernelColumnCount { get; }
public static int KernelRowCount { get; }
public static unsafe nuint Size
{
get
{
nuint size = (nuint)sizeof(T);
return size < 4 ? 128 : 16 * size;
}
}
public static unsafe nuint Size { get; } = 32 * 16;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static MatrixComputeKernelAVX512<T>* Create()
StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs
+30 -30
View File
@@ -1,6 +1,6 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="MatrixParallelHandle"
// file="MatrixParallelTaskPackage"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
@@ -117,8 +117,8 @@ namespace StgSharp.Mathematics.Numeric
#endregion
}
[StructLayout(LayoutKind.Sequential)]
internal unsafe struct MatrixParallelTaskPackage<T> where T: unmanaged, INumber<T>
[StructLayout(LayoutKind.Explicit, Size =128)]
internal unsafe struct MatrixParallelTaskPackage<T> where T : unmanaged, INumber<T>
{
public MatrixParallelTaskPackage()
@@ -136,69 +136,69 @@ namespace StgSharp.Mathematics.Numeric
#region source
// Group 1: Left/Right matrix kernel base address
public MatrixKernel<T>* Left; // 8
public MatrixKernel<T>* Right; // 8
[FieldOffset(0)]public MatrixKernel<T>* Left; // 8
[FieldOffset(8)] public MatrixKernel<T>* Right; // 8
#endregion
#region ans
// Group 2: Result kernel base address + reserved
public MatrixKernel<T>* Result; // 8
public int ElementSize = sizeof(T);
public int ReservedPtr0; // 4 (extension/reserved)
[FieldOffset(16)] public MatrixKernel<T>* Result; // 8
[FieldOffset(24)] public int ElementSize = sizeof(T);
[FieldOffset(28)] public int ReservedPtr0; // 4 (extension/reserved)
#endregion
#region left enum
public int LeftPrimOffset; // 4
public int LeftPrimStride; // 4
public int LeftSecOffset; // 4
public int LeftSecStride; // 4
[FieldOffset(32)]public int LeftPrimOffset; // 4
[FieldOffset(36)]public int LeftPrimStride; // 4
[FieldOffset(40)]public int LeftSecOffset; // 4
[FieldOffset(44)] public int LeftSecStride; // 4
#endregion
#region right enum
public int RightPrimOffset; // 4
public int RightPrimStride; // 4
public int RightSecOffset; // 4
public int RightSecStride; // 4
[FieldOffset(48)]public int RightPrimOffset; // 4
[FieldOffset(52)]public int RightPrimStride; // 4
[FieldOffset(56)]public int RightSecOffset; // 4
[FieldOffset(60)]public int RightSecStride; // 4
#endregion
#region ans enum
public int ResultPrimOffset; // 4
public int ResultPrimStride; // 4
public int ResultSecOffset; // 4
public int ResultSecStride; // 4
[FieldOffset(64)]public int ResultPrimOffset; // 4
[FieldOffset(68)]public int ResultPrimStride; // 4
[FieldOffset(72)]public int ResultSecOffset; // 4
[FieldOffset(76)] public int ResultSecStride; // 4
#endregion
#region scalar/compute
public ScalarPacket* Scalar; // 8
public IntPtr ComputeHandle; // 8
[FieldOffset(80)] public ScalarPacket* Scalar; // 8
[FieldOffset(88)]public IntPtr ComputeHandle; // 8
#endregion
#region global profile
public int PrimCount; // 4
public int ComputeMode; // 4
public int SecCount; // 4
private readonly int ReservedInt0; // 4
[FieldOffset(96)]public int PrimCount; // 4
[FieldOffset(100)]public int ComputeMode; // 4
[FieldOffset(104)]public int SecCount; // 4
[FieldOffset(108)] private readonly int ReservedInt0; // 4
#endregion
#region count offset
public int PrimTileOffset; // 4
public int SecTileOffset; // 4
public int PrimColumnCountInTile; // 4
public int SecColumnCountInTile; // 4
[FieldOffset(112)]public int PrimTileOffset; // 4
[FieldOffset(116)]public int SecTileOffset; // 4
[FieldOffset(120)]public int PrimColumnCountInTile; // 4
[FieldOffset(124)]public int SecColumnCountInTile; // 4
#endregion
}
StgSharp.Native/StgSharp.Native.vcxproj
+1
View File
@@ -104,6 +104,7 @@
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="k_bufop.c" />
<ClCompile Include="sn-src\k_basicop.c" />
<ClCompile Include="sn-src\dllmain.c" />
<ClCompile Include="sn-src\str_factorial.c" />
StgSharp.Native/StgSharp.Native.vcxproj.filters
+3
View File
@@ -51,6 +51,9 @@
<ClCompile Include="sn-src\load_intrinsic.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="k_bufop.c">
<Filter>源文件</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="sn-head\gl.h">
StgSharp.Native/k_bufop.c
+29
View File
@@ -0,0 +1,29 @@
#include "sn_intrinsic.h"
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill)
{
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].xmm[0] = vec;
ans[i].xmm[1] = vec;
ans[i].xmm[2] = vec;
ans[i].xmm[3] = vec;
}
}
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
ans[i].ymm[0] = vec;
ans[i].ymm[1] = vec;
}
}
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].zmm[0] = vec;
}
}
StgSharp.Native/sn-head/sn_internal.h
+6 -3
View File
@@ -1,6 +1,3 @@
#ifdef _MSC_VER
#pragma once
#endif
#ifndef SN_INTERNAL
#define SN_INTERNAL
@@ -24,6 +21,12 @@
#define FORCEINLINE
#endif
#if defined(_MSC_VER)
#define SN_ALIGNAS(n) __declspec(align(n))
#else
#define SN_ALIGNAS(n) __attribute__((aligned(n)))
#endif
#define SN_APIDEF
#define PRIVATE static
StgSharp.Native/sn-head/sn_intrinsic.h
+163 -52
View File
@@ -33,71 +33,179 @@ typedef union uint64_u {
typedef struct mat_kernel mat_kernel;
typedef struct mat_panel mat_panel;
typedef union scalar_pack {
uint64_t data[8];
} scalar_pack;
/* Mirrors StgSharp.Mathematics.Numeric.MatrixParallelTaskPackage<T> (128 bytes). */
#define MAT_TASK(T) \
struct { \
MAT_KERNEL(T) * left; /* Offset 0: left matrix kernel base */ \
MAT_KERNEL(T) * right; /* Offset 8: right matrix kernel base */ \
MAT_KERNEL(T) * result; /* Offset 16: result matrix kernel base */ \
int32_t element_size; /* Offset 24: sizeof(T) */ \
int32_t reserved_ptr0; /* Offset 28: reserved/extension slot */ \
int32_t left_prim_offset; /* Offset 32 */ \
int32_t left_prim_stride; /* Offset 36 */ \
int32_t left_sec_offset; /* Offset 40 */ \
int32_t left_sec_stride; /* Offset 44 */ \
int32_t right_prim_offset; /* Offset 48 */ \
int32_t right_prim_stride; /* Offset 52 */ \
int32_t right_sec_offset; /* Offset 56 */ \
int32_t right_sec_stride; /* Offset 60 */ \
int32_t result_prim_offset; /* Offset 64 */ \
int32_t result_prim_stride; /* Offset 68 */ \
int32_t result_sec_offset; /* Offset 72 */ \
int32_t result_sec_stride; /* Offset 76 */ \
scalar_pack *scalar; /* Offset 80 */ \
void *compute_handle; /* Offset 88 */ \
int32_t prim_count; /* Offset 96 */ \
int32_t compute_mode; /* Offset 100 */ \
int32_t sec_count; /* Offset 104 */ \
int32_t reserved_int0; /* Offset 108 */ \
int32_t prim_tile_offset; /* Offset 112 */ \
int32_t sec_tile_offset; /* Offset 116 */ \
int32_t prim_cols_in_tile; /* Offset 120 */ \
int32_t sec_cols_in_tile; /* Offset 124 */ \
}
typedef struct matrix_task matrix_task;
#if defined(__cplusplus)
static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mismatch");
#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
_Static_assert(sizeof(MAT_TASK(float)) == 128, "matrix_parallel_task_package size mismatch");
#endif
#pragma region matrix buffer operation function type
typedef void(SN_DECL *BUF_PROC_AS)(mat_kernel *ans, scalar_pack *scalar, int count);
typedef void(SN_DECL *BUF_PROC_RAS)(mat_kernel *right, mat_kernel *ans, scalar_pack *scalar,
int count);
typedef void(SN_DECL *BUF_PROC_LRAS)(mat_kernel *left, mat_kernel *right, mat_kernel *ans,
scalar_pack *scalar, int count);
typedef void(SN_DECL *BUF_PROC_RA)(mat_kernel *right, mat_kernel *ans, int count);
typedef void(SN_DECL *BUF_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans,
int count);
#define BUF_PROC(T_type, T_arch, T_op_name) T_type##_buf_##T_op_name##_##T_arch
#define DECLARE_BUF_PROC_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, int count)
#define DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
scalar_pack const *scalar, int count)
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, int count)
#define DECLARE_BUF_PROC_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, int count)
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, int count)
#pragma endregion
#pragma region matrix panel function type
typedef void(SN_DECL *PNL_PROC_LRA)(mat_panel *left, mat_panel *right, mat_panel *ans);
typedef void(SN_DECL *PNL_PROC_RA)(mat_panel *right, mat_panel *ans);
typedef void(SN_DECL *PNL_PROC_LRAS)(mat_panel *left, mat_panel *right, mat_panel *ans,
scalar_pack *scalar);
typedef void(SN_DECL *PNL_PROC_RAS)(mat_panel *right, mat_panel *ans, scalar_pack *scalar);
#define PNL_PROC(T_type, T_arch, T_op_name) T_type##_pnl_##T_op_name##_##T_arch
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)
#pragma endregion
#pragma region matrix kernel function type
typedef void(SN_DECL *KER_PROC_AS)(mat_kernel *ans, scalar_pack *scalar);
typedef void(SN_DECL *KER_PROC_RA)(mat_kernel *left, mat_kernel *ans);
typedef void(SN_DECL *KER_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans);
typedef void(SN_DECL *KER_PROC_LAS)(mat_kernel *matrix, mat_kernel *ans, scalar_pack *scalar);
#define KER_PROC(T_type, T_arch, T_op_name) T_type##_kernel_##T_op_name##_##T_arch
#define DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar)
#define DECLARE_KER_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
scalar_pack const *scalar)
#define DECLARE_KER_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans)
#define DECLARE_KER_PROC_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans)
#pragma endregion
typedef void(SN_DECL *VECTORNORMALIZEPROC)(__m128 *source, __m128 *target);
typedef void(SN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans);
typedef void(SN_DECL *PANEL_PROC_AS)(MAT_PANEL_SSE(int));
typedef void(SN_DECL *MAT_PROC_AS)(mat_kernel *ans, scalar_pack *scalar);
typedef void(SN_DECL *MAT_PROC_LA)(mat_kernel *left, mat_kernel *ans);
typedef void(SN_DECL *MAT_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans);
typedef void(SN_DECL *MAT_PROC_LSA)(mat_kernel *matrix, scalar_pack *scalar, mat_kernel *ans);
typedef uint64_t(SN_DECL *FACTORIALROC)(int n);
#pragma region matix function
INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);
INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_add_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_sub_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_transpose_sse(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target);
INTERNAL void SN_DECL f32_transpose_avx(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target);
INTERNAL void SN_DECL f32_transpose_512(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target);
INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans,
scalar_pack const *scalar);
INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans,
scalar_pack const *scalar);
INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans,
scalar_pack const *scalar);
INTERNAL void SN_DECL f32_fill_sse(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler);
INTERNAL void SN_DECL f32_fill_avx(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler);
INTERNAL void SN_DECL f32_fill_512(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, add);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, add);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, add);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, sub);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, sub);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, sub);
DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose);
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul);
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul);
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul);
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill);
#pragma endregion
#pragma region fma
INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_fma_avx(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_fma_512(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, fma);
#pragma endregion
#pragma region scaler_simd
INTERNAL uint64_t SN_DECL factorial_simd_sse(int n);
#pragma endregion
#pragma region dot_product
@@ -115,21 +223,24 @@ INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 cons
#pragma region string
typedef int(SN_DECL *HASH)(byte const *str, int length);
typedef int(SN_DECL *INDEX_PAIR)(short const *stram, uint32_t target, int length);
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int length);
INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int length);
#pragma endregion
typedef struct sn_intrinsic {
int (*city_hash_simplify)(byte *str, int length);
HASH city_hash_simplify;
FACTORIALROC factorial_simd;
int (*index_pair)(short const *str, uint32_t target, int length);
MAT_PROC_LRA f32_add;
MAT_PROC_AS f32_fill;
MAT_PROC_LRA f32_fma;
MAT_PROC_LRA f32_scalar_mul;
MAT_PROC_LRA f32_sub;
MAT_PROC_LA f32_transpose;
INDEX_PAIR index_pair;
KER_PROC_LRA f32_kernel_add;
KER_PROC_AS f32_kernel_fill;
PNL_PROC_LRA f32_panel_fma;
KER_PROC_LRA f32_kernel_scalar_mul;
KER_PROC_LRA f32_kernel_sub;
KER_PROC_RA f32_kernel_transpose;
VECTORNORMALIZEPROC f32_normalize_3;
} sn_intrinsic;
StgSharp.Native/sn-head/sn_matkernel.h
+47 -87
View File
@@ -26,98 +26,58 @@
#endif
#include <stddef.h>
#include <stdint.h>
/* Helpers (all compile-time): */
#define SN_LANES(T_VEC, T) (sizeof(T_VEC) / sizeof(T))
#define SN_MAX(a, b) (((a)) > ((b)) ? (a) : (b))
#define SN_CEIL_DIV(a, b) (((a) + (b) - 1) / (b))
/* align decoration */
/* Panel geometry helpers (all compile-time): */
#define PANEL_Q(T, T_VEC, MINQ) SN_MAX(SN_LANES(T_VEC, T), (MINQ))
#define PANEL_CHUNKS(T, T_VEC, MINQ) SN_CEIL_DIV(PANEL_Q(T, T_VEC, MINQ), SN_LANES(T_VEC, T))
#if __cplusplus && defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
#define ML_ALIGNAS(n) _Alignas(n)
#elif defined(_MSC_VER)
#define ML_ALIGNAS(n) __declspec(align(n))
#else
#define ML_ALIGNAS(n) __attribute__((aligned(n)))
#endif
/* Round up to the alignment boundary in bytes */
#ifndef ML_ALIGN_UP
#define ML_ALIGN_UP(x, a) (((size_t)(x) + ((size_t)(a) - 1)) & ~((size_t)(a) - 1))
#endif
/* Vector register width in bytes */
#define ML_BYTES_XMM 16u
#define ML_BYTES_YMM 32u
#define ML_BYTES_ZMM 64u
/* Compute the aligned leading-direction element stride (LEp) */
#define ML_PANEL_STRIDE_ELEMS(T, LE, ALIGN) \
(ML_ALIGN_UP((size_t)(LE) * sizeof(T), (size_t)(ALIGN)) / sizeof(T))
/* Total element count and total byte count */
#define ML_PANEL_TOT_ELEMS(T, LE, MINOR, ALIGN) \
((size_t)ML_PANEL_STRIDE_ELEMS(T, (LE), (ALIGN)) * (size_t)(MINOR))
#define ML_TOT_BYTES(T, ELEMS) ((size_t)(ELEMS) * sizeof(T))
/* Number of SIMD view slots (ceil by total bytes) */
#define ML_XMM_COUNT(TOT_BYTES) (((size_t)(TOT_BYTES) + ML_BYTES_XMM - 1) / ML_BYTES_XMM)
#define ML_YMM_COUNT(TOT_BYTES) (((size_t)(TOT_BYTES) + ML_BYTES_YMM - 1) / ML_BYTES_YMM)
#define ML_ZMM_COUNT(TOT_BYTES) (((size_t)(TOT_BYTES) + ML_BYTES_ZMM - 1) / ML_BYTES_ZMM)
/* ========== Unified "panel" union (no enum/descriptor fields) ==========
Usage:
MAT_PANEL(float, 8, 32, 32) P; // LE=8, MINOR=32, aligned to 32 bytes
Views:
P.xmm[] / P.ymm[] / P.zmm[] // raw SIMD blocks
P.blk[MINOR][LEp] // 2D scalar view (includes padding for alignment)
P.flat[MINOR*LEp] // flat array
Notes:
- LE/MINOR/ALIGN must be compile-time constants
- LEp = ML_PANEL_STRIDE_ELEMS(T, LE, ALIGN)
- Layout/semantics defined by the calling convention (e.g. blk[k][r] is element r of block k)
/*
Column-major square panel with minimum side Q:
- Let R = lanes per vector = sizeof(T_VEC)/sizeof(T).
- Side length Q = max(R, MINQ) to enforce the requested minimum extent.
- Each column is stored contiguously via CH = ceil(Q/R) vector chunks.
- Memory layout:
vec[col][chunk] is a T_VEC holding up to R scalars of column 'col'
spanning rows chunk*R .. chunk*R + (R - 1).
flat[] mirrors the same column-major order (column stride = Q).
- SIMD view remains integer-typed; cast inside micro-kernels to the
appropriate floating-point/integer vector type on demand.
*/
#define MAT_PANEL(T, LE, MINOR, ALIGN) \
ML_ALIGNAS(ALIGN) union { \
/* Raw SIMD view: cast to __m256/__m512d inside the micro-kernel on demand */ \
__m128i xmm[ML_XMM_COUNT( \
ML_TOT_BYTES(T, ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))))]; \
__m256i ymm[ML_YMM_COUNT( \
ML_TOT_BYTES(T, ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))))]; \
__m512i zmm[ML_ZMM_COUNT( \
ML_TOT_BYTES(T, ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))))]; \
/* Scalar 2D view: minor �� LEp (LE dimension contains alignment padding) */ \
T blk[(MINOR)][ML_PANEL_STRIDE_ELEMS(T, (LE), (ALIGN))]; \
/* Flat view */ \
T flat[ML_PANEL_TOT_ELEMS(T, (LE), (MINOR), (ALIGN))]; \
#define MAT_PANEL_SQ_MIN(T, T_VEC, MINQ) \
SN_ALIGNAS(sizeof(T_VEC)) union { \
T_VEC vec[PANEL_Q(T, T_VEC, MINQ)][PANEL_CHUNKS(T, T_VEC, MINQ)]; \
T flat[PANEL_Q(T, T_VEC, MINQ) * PANEL_Q(T, T_VEC, MINQ)]; \
}
/* ---------- SSE preset ---------- */
#undef PANEL_LE
#undef PANEL_MINOR
#undef PANEL_ALIGN
#define PANEL_LE 4
#define PANEL_MINOR 32
#define PANEL_ALIGN 16
/* ======== SSE preset ========
4 x 32, 16B aligned
/* ISA presets with minimum square side:
- SSE : MINQ=4 (avoid 2x2 for double)
- AVX2 : MINQ=8
- AVX-512: MINQ=8 (16 for float/i32 naturally, so this is a no-op there)
*/
#define MAT_PANEL_SSE(T) MAT_PANEL(T, PANEL_LE, PANEL_MINOR, PANEL_ALIGN)
/* ---------- AVX2 preset ---------- */
#undef PANEL_LE
#undef PANEL_MINOR
#undef PANEL_ALIGN
#define PANEL_LE 8
#define PANEL_MINOR 64
#define PANEL_ALIGN 32
#define MAT_PANEL_AVX(T) MAT_PANEL(T, PANEL_LE, PANEL_MINOR, PANEL_ALIGN) /* 8 x 64, 32B aligned */
/* SSE panel layout: column-major buffer holding 4 logical columns with MINQ=4.
Columns: each column packs up to four scalars so ALIGN equals sizeof(__m128).
Alignment: every column boundary lands on a 16-byte multiple for XMM loads. */
#define MAT_PANEL_SSE(T) MAT_PANEL_SQ_MIN(T, __m128, 4)
/* AVX2 panel layout: column-major buffer holding 8 logical columns with MINQ=8.
Columns: each column stores eight elements via two 256-bit chunks when T is 32-bit.
Alignment: column starts are 32-byte aligned so AVX2 loads/stores remain contiguous. */
#define MAT_PANEL_AVX(T) MAT_PANEL_SQ_MIN(T, __m256, 8)
/* AVX-512 panel layout: column-major buffer holding 8 or more columns depending on T.
Columns: each column expands to ceil(Q/lanes) ZMM chunks; default MINQ=8 keeps double-wide tiles.
Alignment: column starts sit on 64-byte boundaries to match AVX-512 streaming access. */
#define MAT_PANEL_512(T) MAT_PANEL_SQ_MIN(T, __m512, 8)
/* ---------- AVX-512 preset ---------- */
#undef PANEL_LE
#undef PANEL_MINOR
#undef PANEL_ALIGN
#define PANEL_LE 16
#define PANEL_MINOR 64
#define PANEL_ALIGN 64
#define MAT_PANEL_512(T) \
MAT_PANEL(T, PANEL_LE, PANEL_MINOR, PANEL_ALIGN); /* 16 x 64, 64B aligned */
/* Access helpers (all compile-time friendly):
Given Q = PANEL_Q(...), CH = PANEL_CHUNKS(...), R = lanes per vector:
- Column c occupies vec[c][0..CH-1].
- Row r inside column c is located at vec[c][r / R] with lane (r % R).
*/
#define PANEL_COL_VEC(panel_obj, c, chunk) ((panel_obj).vec[(int)(c)][(int)(chunk)])
/* Scalar flat index helper for column-major addressing */
#define PANEL_ELEM(panel_obj, T, T_VEC, MINQ, r, c) \
((panel_obj).flat[(int)(c) * PANEL_Q(T, T_VEC, MINQ) + (int)(r)])
StgSharp.Native/sn-src/dllmain.c
+4 -13
View File
@@ -3,16 +3,9 @@
#ifdef _WIN32
// clang-format off
BOOL APIENTRY DllMain(
HMODULE hModule,
DWORD ul_reason_for_call,
LPVOID lpReserved
)
BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved)
{
switch (ul_reason_for_call)
{
switch (ul_reason_for_call) {
case DLL_PROCESS_ATTACH:
return 1;
case DLL_THREAD_ATTACH:
@@ -21,9 +14,9 @@ BOOL APIENTRY DllMain(
return 1;
case DLL_PROCESS_DETACH:
return 1;
break;
break;
}
return TRUE;
return TRUE;
}
#else
attribute((constructor)) void DllMain()
@@ -31,5 +24,3 @@ attribute((constructor)) void DllMain()
}
#endif // _WIN32
// clang-format on
StgSharp.Native/sn-src/k_basicop.c
+52 -58
View File
@@ -1,29 +1,34 @@
#include "sn_intrinsic.h"
INTERNAL void SN_DECL f32_fill_sse(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler)
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill)
{
register __m128 vec = _mm_set_ps1(((float *)(scaler->data))[0]);
dest->xmm[0] = vec;
dest->xmm[1] = vec;
dest->xmm[2] = vec;
dest->xmm[3] = vec;
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].xmm[0] = vec;
ans[i].xmm[1] = vec;
ans[i].xmm[2] = vec;
ans[i].xmm[3] = vec;
}
}
INTERNAL void SN_DECL f32_fill_avx(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler)
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scaler->data)));
dest->ymm[0] = vec;
dest->ymm[1] = vec;
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
ans[i].ymm[0] = vec;
ans[i].ymm[1] = vec;
}
}
INTERNAL void SN_DECL f32_fill_512(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler)
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scaler->data))[0]);
dest->zmm[0] = vec;
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].zmm[0] = vec;
}
}
INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, add)
{
register __m128 c0 = _mm_add_ps(left->xmm[0], right->xmm[0]);
register __m128 c1 = _mm_add_ps(left->xmm[1], right->xmm[1]);
@@ -36,8 +41,7 @@ INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(floa
ans->xmm[3] = c3;
}
INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, add)
{
register __m256 c0 = _mm256_add_ps(left->ymm[0], right->ymm[0]);
register __m256 c1 = _mm256_add_ps(left->ymm[1], right->ymm[1]);
@@ -45,15 +49,13 @@ INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(floa
ans->ymm[1] = c1;
}
INTERNAL void SN_DECL f32_add_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, add)
{
register __m512 c0 = _mm512_add_ps(left->zmm[0], right->zmm[0]);
ans->zmm[0] = c0;
}
INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, sub)
{
register __m128 c0 = _mm_sub_ps(left->xmm[0], right->xmm[0]);
register __m128 c1 = _mm_sub_ps(left->xmm[1], right->xmm[1]);
@@ -66,8 +68,7 @@ INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(floa
ans->xmm[3] = c3;
}
INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, sub)
{
register __m256 c0 = _mm256_sub_ps(left->ymm[0], right->ymm[0]);
register __m256 c1 = _mm256_sub_ps(left->ymm[1], right->ymm[1]);
@@ -75,37 +76,34 @@ INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(floa
ans->ymm[1] = c1;
}
INTERNAL void SN_DECL f32_sub_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, sub)
{
register __m512 c0 = _mm512_sub_ps(left->zmm[0], right->zmm[0]);
ans->zmm[0] = c0;
}
// Matrix transpose implementations
INTERNAL void SN_DECL f32_transpose_sse(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target)
DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose)
{
register __m128 col0 = source->xmm[0];
register __m128 col1 = source->xmm[1];
register __m128 col2 = source->xmm[2];
register __m128 col3 = source->xmm[3];
register __m128 col0 = right->xmm[0];
register __m128 col1 = right->xmm[1];
register __m128 col2 = right->xmm[2];
register __m128 col3 = right->xmm[3];
register __m128 tmp0 = _mm_unpacklo_ps(col0, col1);
register __m128 tmp1 = _mm_unpackhi_ps(col0, col1);
register __m128 tmp2 = _mm_unpacklo_ps(col2, col3);
register __m128 tmp3 = _mm_unpackhi_ps(col2, col3);
target->xmm[0] = _mm_movelh_ps(tmp0, tmp2);
target->xmm[1] = _mm_movehl_ps(tmp2, tmp0);
target->xmm[2] = _mm_movelh_ps(tmp1, tmp3);
target->xmm[3] = _mm_movehl_ps(tmp3, tmp1);
ans->xmm[0] = _mm_movelh_ps(tmp0, tmp2);
ans->xmm[1] = _mm_movehl_ps(tmp2, tmp0);
ans->xmm[2] = _mm_movelh_ps(tmp1, tmp3);
ans->xmm[3] = _mm_movehl_ps(tmp3, tmp1);
}
INTERNAL void SN_DECL f32_transpose_avx(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target)
DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose)
{
register __m256 col_low = source->ymm[0];
register __m256 col_high = source->ymm[1];
register __m256 col_low = right->ymm[0];
register __m256 col_high = right->ymm[1];
register __m256 unpack_low = _mm256_unpacklo_ps(col_low, col_high);
register __m256 unpack_high = _mm256_unpackhi_ps(col_low, col_high);
@@ -113,30 +111,28 @@ INTERNAL void SN_DECL f32_transpose_avx(MAT_KERNEL(float) const *source,
col_low = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x20);
col_high = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x31);
target->ymm[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0));
target->ymm[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
ans->ymm[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0));
ans->ymm[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
}
INTERNAL void SN_DECL f32_transpose_512(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target)
DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose)
{
register __m512 matrix = source->zmm[0];
register __m512 matrix = right->zmm[0];
const __m512i transpose_indices =
_mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
target->zmm[0] = _mm512_permutexvar_ps(transpose_indices, matrix);
ans->zmm[0] = _mm512_permutexvar_ps(transpose_indices, matrix);
}
INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans, scalar_pack const *scalar)
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul)
{
register __m128 scalar_vec = ((__m128 *)(scalar->data))[0];
register __m128 c0 = _mm_mul_ps(matrix->xmm[0], scalar_vec);
register __m128 c1 = _mm_mul_ps(matrix->xmm[1], scalar_vec);
register __m128 c2 = _mm_mul_ps(matrix->xmm[2], scalar_vec);
register __m128 c3 = _mm_mul_ps(matrix->xmm[3], scalar_vec);
register __m128 c0 = _mm_mul_ps(right->xmm[0], scalar_vec);
register __m128 c1 = _mm_mul_ps(right->xmm[1], scalar_vec);
register __m128 c2 = _mm_mul_ps(right->xmm[2], scalar_vec);
register __m128 c3 = _mm_mul_ps(right->xmm[3], scalar_vec);
ans->xmm[0] = c0;
ans->xmm[1] = c1;
@@ -144,24 +140,22 @@ INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix,
ans->xmm[3] = c3;
}
INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans, scalar_pack const *scalar)
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul)
{
register __m256 scalar_vec = ((__m256 *)(scalar->data))[0];
register __m256 c0 = _mm256_mul_ps(matrix->ymm[0], scalar_vec);
register __m256 c1 = _mm256_mul_ps(matrix->ymm[1], scalar_vec);
register __m256 c0 = _mm256_mul_ps(right->ymm[0], scalar_vec);
register __m256 c1 = _mm256_mul_ps(right->ymm[1], scalar_vec);
ans->ymm[0] = c0;
ans->ymm[1] = c1;
}
INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans, scalar_pack const *scalar)
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul)
{
register __m512 scalar_vec = ((__m512 *)(scalar->data))[0];
register __m512 c0 = _mm512_mul_ps(matrix->zmm[0], scalar_vec);
register __m512 c0 = _mm512_mul_ps(right->zmm[0], scalar_vec);
ans->zmm[0] = c0;
}
StgSharp.Native/sn-src/k_fma.c
+1 -50
View File
@@ -1,14 +1,8 @@
#include "sn_intrinsic.h"
INTERNAL void SN_DECL f32_fma_sse(MAT_PANEL_SSE(int) * left)
{
}
// ans = left * right + ans
// right is not transposed
INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, fma)
{
register __m128 source0 = _mm_load_ps(&left->xmm[0]);
register __m128 source1 = _mm_load_ps(&left->xmm[1]);
@@ -39,46 +33,3 @@ INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left,
_mm_store_ps(&ans->xmm[2], ans2);
_mm_store_ps(&ans->xmm[3], ans3);
}
// ans = left * right + ans
INTERNAL void SN_DECL f32_fma_avx(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans)
{
register __m256 source0 = _mm256_load_ps(&left->ymm[0]);
register __m256 source1 = _mm256_load_ps(&left->ymm[1]);
register __m256 ans0 = _mm256_load_ps(&ans->ymm[0]);
register __m256 ans1 = _mm256_load_ps(&ans->ymm[1]);
register __m128 bs0, bs1;
register __m256 broadcast;
for (size_t i = 0; i < 4; i++) {
bs0 = _mm_load_ps(&right->xmm[i]);
__m128 lo = _mm_shuffle_ps(bs0, bs0, 0x00);
__m128 hi = _mm_shuffle_ps(bs0, bs0, 0x55);
broadcast = _mm256_castps128_ps256(lo);
broadcast = _mm256_insertf128_ps(broadcast, bs0, 1);
ans0 = _mm256_add_ps(ans0, _mm256_mul_ps(source0, broadcast));
lo = _mm_shuffle_ps(bs0, bs0, 0xAA);
hi = _mm_shuffle_ps(bs0, bs0, 0xFF);
broadcast = _mm256_castps128_ps256(lo);
broadcast = _mm256_insertf128_ps(broadcast, bs0, 1);
ans1 = _mm256_add_ps(ans1, _mm256_mul_ps(source1, broadcast));
}
_mm256_store_ps(&ans->ymm[0], ans0);
_mm256_store_ps(&ans->ymm[1], ans1);
}
// ans = left * right + ans
INTERNAL void SN_DECL f32_fma_512(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans)
{
/*
* to be implemented
*/
}
StgSharp.Native/sn-src/load_intrinsic.c
+15 -15
View File
@@ -65,11 +65,11 @@ void load_intrinsic_sse(sn_intrinsic *context)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->f32_add = (MAT_PROC_LRA)f32_add_sse;
context->f32_fma = (MAT_PROC_LRA)f32_fma_sse;
context->f32_scalar_mul = f32_scalar_mul_sse;
context->f32_sub = (MAT_PROC_LRA)f32_sub_sse;
context->f32_transpose = (MAT_PROC_LA)f32_transpose_sse;
context->f32_kernel_add = KER_PROC(float, sse, add);
context->f32_panel_fma = KER_PROC(float, sse, fma);
context->f32_kernel_scalar_mul = KER_PROC(float, sse, scalar_mul);
context->f32_kernel_sub = KER_PROC(float, sse, sub);
context->f32_kernel_transpose = KER_PROC(float, sse, transpose);
context->f32_normalize_3 = f32_normalize;
}
@@ -78,11 +78,11 @@ void load_intrinsic_avx(sn_intrinsic *context)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->f32_add = (MAT_PROC_LRA)f32_add_avx;
context->f32_fma = (MAT_PROC_LRA)f32_fma_avx;
context->f32_scalar_mul = f32_scalar_mul_avx;
context->f32_sub = (MAT_PROC_LRA)f32_sub_avx;
context->f32_transpose = (MAT_PROC_LA)f32_transpose_avx;
context->f32_kernel_add = KER_PROC(float, avx, add);
context->f32_panel_fma = KER_PROC(float, sse, fma);
context->f32_kernel_scalar_mul = KER_PROC(float, avx, scalar_mul);
context->f32_kernel_sub = KER_PROC(float, avx, sub);
context->f32_kernel_transpose = KER_PROC(float, avx, transpose);
context->f32_normalize_3 = f32_normalize;
}
@@ -91,10 +91,10 @@ void load_intrinsic_512(sn_intrinsic *context)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->f32_add = (MAT_PROC_LRA)f32_add_512;
context->f32_fma = (MAT_PROC_LRA)f32_fma_512;
context->f32_scalar_mul = f32_scalar_mul_512;
context->f32_sub = (MAT_PROC_LRA)f32_sub_512;
context->f32_transpose = (MAT_PROC_LA)f32_transpose_512;
context->f32_kernel_add = KER_PROC(float, 512, add);
context->f32_panel_fma = KER_PROC(float, sse, fma);
context->f32_kernel_scalar_mul = KER_PROC(float, 512, scalar_mul);
context->f32_kernel_sub = KER_PROC(float, 512, sub);
context->f32_kernel_transpose = KER_PROC(float, 512, transpose);
context->f32_normalize_3 = f32_normalize;
}
StgSharpDebug/RenderStream/Partical.cs
-2
View File
@@ -25,12 +25,10 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Common.Graphics;
using StgSharp.Controls;
using StgSharp.Geometries;
using StgSharp.Graphics;
using StgSharp.Graphics.OpenGL;
using StgSharp.Internal.OpenGL;
using StgSharp.Mathematics;
using StgSharp.Mathematics.Graphic;
using StgSharp.Timing;