From 67fbbde4494fa8d232b90576ef9a3b1e257c6577 Mon Sep 17 00:00:00 2001 From: Nitload Date: Mon, 2 Mar 2026 00:24:15 +0800 Subject: [PATCH] mat proc kernel std tamplate is ok, merging old decl --- StgSharp.Graphics/Pixels.cs | 187 ++++++++++-------- .../sn-head/sn_intrinsic.context.x86.h | 147 +++++++------- StgSharp.Native/sn-head/sn_intrinsic.std.h | 65 +++--- StgSharp.Native/sn-src/m.basic.x86.c | 57 +----- StgSharp.Native/sn-src/m.bufop.x86.c | 51 ++--- StgSharp.Native/sn-src/m.fma.x86.c | 35 +++- StgSharp.Native/sn-src/m.fma_tile.x86.c | 16 +- StgSharp.Native/sn-src/m.solve.x86.c | 11 +- 8 files changed, 279 insertions(+), 290 deletions(-) diff --git a/StgSharp.Graphics/Pixels.cs b/StgSharp.Graphics/Pixels.cs index 12c5318d..e33f4495 100644 --- a/StgSharp.Graphics/Pixels.cs +++ b/StgSharp.Graphics/Pixels.cs @@ -1,83 +1,104 @@ -//----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -// file="Pixels" -// Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. -// -// ----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -using System; -using System.Collections.Generic; -using System.Runtime.InteropServices; -using System.Text; - -namespace StgSharp.Graphics -{ - public interface IPixel - { - - public float Red { get; set; } - - public float Green { get; set; } - - public float Blue { get; set; } - - public float Alpha { get; set; } - - internal int Size { get; } - - } - - [StructLayout(LayoutKind.Explicit, Size = 32)] - public unsafe struct Pixel32RGBA : IPixel - { - - [FieldOffset(0)] private unsafe fixed byte data[4]; - - public int Size => 4; - - float IPixel.Alpha - { - get => data[3] / 256 ; - set => data[3] = (byte)(value * 256); - } - - float IPixel.Blue - { - get => data[2] / 256 ; - set => data[2] = (byte)(value * 256); - } - - float IPixel.Green - { - get => data[1] / 256 ; - set => data[1] = (byte)(value * 256); - } - - float IPixel.Red - { - get => data[0] / 256 ; - set => data[0] = (byte)(value * 256); - } - - } -} +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="Pixels" +// Project: StgSharp +// AuthorGroup: Nitload Space +// Copyright (c) Nitload Space. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Text; + +namespace StgSharp.Graphics +{ + public interface IPixel + { + + public float Red { get; set; } + + public float Green { get; set; } + + public float Blue { get; set; } + + public float Alpha { get; set; } + + internal int Size { get; } + + } + + [StructLayout(LayoutKind.Explicit, Size = 32)] + public unsafe struct Pixel32RGBA : IPixel + { + + [FieldOffset(0)] private unsafe fixed byte data[4]; + [FieldOffset(0)] private readonly int mask; + + public readonly int Size => 4; + + public override bool Equals(object obj) + { + return obj is not null && obj is Pixel32RGBA pixel && this == pixel; + } + + public override readonly int GetHashCode() + { + return mask.GetHashCode(); + } + + public static bool operator !=(Pixel32RGBA left, Pixel32RGBA right) + { + return !(left == right); + } + + public static bool operator ==(Pixel32RGBA left, Pixel32RGBA right) + { + return left.mask == right.mask; + } + + float IPixel.Alpha + { + get => data[3] / 256; + set => data[3] = (byte)(value * 256); + } + + float IPixel.Blue + { + get => data[2] / 256; + set => data[2] = (byte)(value * 256); + } + + float IPixel.Green + { + get => data[1] / 256; + set => data[1] = (byte)(value * 256); + } + + float IPixel.Red + { + get => data[0] / 256; + set => data[0] = (byte)(value * 256); + } + + } +} diff --git a/StgSharp.Native/sn-head/sn_intrinsic.context.x86.h b/StgSharp.Native/sn-head/sn_intrinsic.context.x86.h index 1d400287..91555b44 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.context.x86.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.context.x86.h @@ -3,6 +3,7 @@ #if SN_IS_ARCH(SN_ARCH_X86_64) #include "sn_intrinsic.h" +#include "sn_intrinsic.std.h" // ----------------------------------------------------------------------------- // SIMDID layout helpers (see SIMDID.x86.md) @@ -57,39 +58,39 @@ #define SIMDID_MAIN_LVL_AVX10 0x05u #ifndef SIMDID_AVX512_BASE -#define SIMDID_AVX512_FP16 U << 13) relative bit 13 within [31..16] */ -#define SIMDID_AVX512_BF16 U << 12) -#define SIMDID_AVX512_VNNI U << 11) -#define SIMDID_AVX512_VBMI2 U << 10) -#define SIMDID_AVX512_VBMI U << 9) -#define SIMDID_AVX512_AI_RESERVED U << 8) -#define SIMDID_AVX512_BASE U << 4) -#define SIMDID_AVX512_IMPL_SHIFT -#define SIMDID_AVX512_IMPL_MASK x3U << SIMDID_AVX512_IMPL_SHIFT) -#define SIMDID_AVX512_IMPL_UNKNOWN x0U << SIMDID_AVX512_IMPL_SHIFT) -#define SIMDID_AVX512_IMPL_DUAL x1U << SIMDID_AVX512_IMPL_SHIFT) -#define SIMDID_AVX512_IMPL_NATIVE x2U << SIMDID_AVX512_IMPL_SHIFT) +#define SIMDID_AVX512_FP16 (1U << 13) /* relative bit 13 within [31..16] */ +#define SIMDID_AVX512_BF16 (1U << 12) +#define SIMDID_AVX512_VNNI (1U << 11) +#define SIMDID_AVX512_VBMI2 (1U << 10) +#define SIMDID_AVX512_VBMI (1U << 9) +#define SIMDID_AVX512_AI_RESERVED (1U << 8) +#define SIMDID_AVX512_BASE (1U << 4) +#define SIMDID_AVX512_IMPL_SHIFT 6 +#define SIMDID_AVX512_IMPL_MASK (0x3U << SIMDID_AVX512_IMPL_SHIFT) +#define SIMDID_AVX512_IMPL_UNKNOWN (0x0U << SIMDID_AVX512_IMPL_SHIFT) +#define SIMDID_AVX512_IMPL_DUAL (0x1U << SIMDID_AVX512_IMPL_SHIFT) +#define SIMDID_AVX512_IMPL_NATIVE (0x2U << SIMDID_AVX512_IMPL_SHIFT) #endif #ifndef SIMDID_AVX_BASE -#define SIMDID_AVX U << 0) relative bit 0 within [47..40] */ -#define SIMDID_AVX2 U << 1) -#define SIMDID_AVX_FMA U << 2) +#define SIMDID_AVX (1U << 0) /* relative bit 0 within [47..40] */ +#define SIMDID_AVX2 (1U << 1) +#define SIMDID_AVX_FMA (1U << 2) #define SIMDID_AVX_F16C (1U << 3) -#define SIMDID_AVX_DUAL ( \ - U << 4) glued/split-lane YMM (two 128-bit units; Zen1, Hygon, VIA/Zhaoxin) */ +#define SIMDID_AVX_DUAL \ + (1U << 4) /* glued/split-lane YMM (two 128-bit units; Zen1, Hygon, VIA/Zhaoxin) */ #endif #ifndef SIMDID_AMX_TILE -#define SIMDID_AMX_TILE (1U << 0) relative bit 0 within [39..32] */ +#define SIMDID_AMX_TILE (1U << 0) /* relative bit 0 within [39..32] */ #define SIMDID_AMX_INT8 (1U << 1) #define SIMDID_AMX_BF16 (1U << 2) #endif #ifndef SIMDID_AVX10_BASE -#define SIMDID_AVX10 U << 0) /* relative bit 0 within [55..48] */ -#define SIMDID_AVX10_V1 U << 1) -#define SIMDID_AVX10_V2 U << 2) +#define SIMDID_AVX10 (1U << 0) /* relative bit 0 within [55..48] */ +#define SIMDID_AVX10_V1 (1U << 1) +#define SIMDID_AVX10_V2 (1U << 2) #define SIMDID_AVX10_256W (1U << 3) #define SIMDID_AVX10_512W (1U << 4) #endif @@ -102,10 +103,10 @@ * 10 = P-core (performance/big core in a hybrid CPU) * 11 = unified (confirmed non-hybrid; numerically strongest) * Higher numeric value == stronger / higher-capability topology. */ -#define SIMDID_UARCH_HYBRID_E << 0) / E-core (efficiency/small) in a hybrid topology */ -#define SIMDID_UARCH_HYBRID_P << 1) / P-core (performance/big) in a hybrid topology */ -#define SIMDID_UARCH_SIMD_DUAL ( \ - << 2) / SIMD is glued/dual (summary: AVX_DUAL or AVX512_IMPL=DUAL) */ +#define SIMDID_UARCH_HYBRID_E (1U << 0) /* E-core (efficiency/small) in a hybrid topology */ +#define SIMDID_UARCH_HYBRID_P (1U << 1) /* P-core (performance/big) in a hybrid topology */ +#define SIMDID_UARCH_SIMD_DUAL \ + (1U << 2) /* SIMD is glued/dual (summary: AVX_DUAL or AVX512_IMPL=DUAL) */ #endif // Pack helpers (define locally to avoid missing macros when included standalone) @@ -133,71 +134,63 @@ INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target); -STD_MK_PROC_DECL(float, sse, , add); -STD_MK_PROC_DECL(float, avx, , add); -STD_MK_PROC_DECL(float, 512, , add); +SN_MK_PROC_DECL_STD(float, sse, , add); +SN_MK_PROC_DECL_STD(float, avx, , add); +SN_MK_PROC_DECL_STD(float, 512, , add); -STD_MK_PROC_DECL(double, sse, , add); -STD_MK_PROC_DECL(double, avx, , add); -STD_MK_PROC_DECL(double, 512, , add); +SN_MK_PROC_DECL_STD(double, sse, , add); +SN_MK_PROC_DECL_STD(double, avx, , add); +SN_MK_PROC_DECL_STD(double, 512, , add); -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub); -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub); -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub); +SN_MK_PROC_DECL_STD(float, sse, , sub); +SN_MK_PROC_DECL_STD(float, avx, , sub); +SN_MK_PROC_DECL_STD(float, 512, , sub); -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub); -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub); -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub); +SN_MK_PROC_DECL_STD(double, sse, , sub); +SN_MK_PROC_DECL_STD(double, avx, , sub); +SN_MK_PROC_DECL_STD(double, 512, , sub); -DECLARE_KER_PROC_RIGHT_ANS(float, sse, , transpose); -DECLARE_KER_PROC_RIGHT_ANS(float, avx, , transpose); -DECLARE_KER_PROC_RIGHT_ANS(float, 512, , transpose); +SN_MK_PROC_DECL_STD(float, sse, , transpose); +SN_MK_PROC_DECL_STD(float, avx, , transpose); +SN_MK_PROC_DECL_STD(float, 512, , transpose); -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul); -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul); -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul); +SN_MK_PROC_DECL_STD(float, sse, , scalar_mul); +SN_MK_PROC_DECL_STD(float, avx, , scalar_mul); +SN_MK_PROC_DECL_STD(float, 512, , scalar_mul); -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul); -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul); -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, , scalar_mul); +SN_MK_PROC_DECL_STD(double, sse, , scalar_mul); +SN_MK_PROC_DECL_STD(double, avx, , scalar_mul); +SN_MK_PROC_DECL_STD(double, 512, , scalar_mul); -DECLARE_BUF_PROC_ANS_SCALAR(float, sse, , fill); -DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill); -DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill); +SN_MK_PROC_DECL_STD(float, sse, , fill); +SN_MK_PROC_DECL_STD(float, avx, , fill); +SN_MK_PROC_DECL_STD(float, 512, , fill); -DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill); -DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill); -DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill); +SN_MK_PROC_DECL_STD(double, sse, , fill); +SN_MK_PROC_DECL_STD(double, avx, , fill); +SN_MK_PROC_DECL_STD(double, 512, , fill); -DECLARE_PNL_PROC_ANS(float, sse, , clear); -DECLARE_PNL_PROC_ANS(float, avx, , clear); -DECLARE_PNL_PROC_ANS(float, 512, , clear); +SN_MK_PROC_DECL_STD(float, sse, , quality); -DECLARE_PNL_PROC_ANS(double, sse, , clear); -DECLARE_PNL_PROC_ANS(double, avx, , clear); -DECLARE_PNL_PROC_ANS(double, 512, , clear); +SN_MK_PROC_DECL_STD(float, sse, , fma); +SN_MK_PROC_DECL_STD(float, avx, , fma); +SN_MK_PROC_DECL_STD(float, avx, _fma, fma); +SN_MK_PROC_DECL_STD(float, 512, , fma); -DECLARE_KER_PROC_RIGHT_ANS(float, sse, , quality); +SN_MK_PROC_DECL_STD(double, sse, , fma); +SN_MK_PROC_DECL_STD(double, avx, , fma); +SN_MK_PROC_DECL_STD(double, avx, _fma, fma); +SN_MK_PROC_DECL_STD(double, 512, , fma); -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma); -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma); -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma); -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma); +SN_MK_PROC_DECL_STD(float, sse, , fma); +SN_MK_PROC_DECL_STD(float, avx, , fma); +SN_MK_PROC_DECL_STD(float, avx, _fma, fma); +SN_MK_PROC_DECL_STD(float, 512, , fma); -DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma); -DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma); -DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma); -DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma); - -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, sse, , fma); -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, , fma); -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma); -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma); - -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma); -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, , fma); -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma); -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, 512, , fma); +SN_MK_PROC_DECL_STD(double, sse, , fma); +SN_MK_PROC_DECL_STD(double, avx, , fma); +SN_MK_PROC_DECL_STD(double, avx, _fma, fma); +SN_MK_PROC_DECL_STD(double, 512, , fma); #pragma endregion diff --git a/StgSharp.Native/sn-head/sn_intrinsic.std.h b/StgSharp.Native/sn-head/sn_intrinsic.std.h index 9266fa42..5b089c11 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.std.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.std.h @@ -8,11 +8,14 @@ #define static_assert _Static_assert #endif +#define mat_task_local ((_mat_task_local const *)task) + #define STD_MK_PROC_NAME(T_type, T_arch, T_feature, T_op) \ T_type##_mat_proc_##T_op##_##T_arch##_##T_feature -#define STD_MK_PROC_DECL(T_type, T_arch, T_feature, T_op) \ +#define SN_MK_PROC_DECL(T_type, P_name) INTERNAL void SN_DECL P_name(MAT_TASK(T_type) const *task) +#define SN_MK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \ INTERNAL void SN_DECL STD_MK_PROC_NAME(T_type, T_arch, T_feature, \ - T_op)(MAT_TASK(T_type) * task); + T_op)(MAT_TASK(T_type) const *task) #pragma region param styles @@ -22,33 +25,32 @@ #define _SN_MATPROC_RIGHT_ANS_SCALAR 1 #define _SN_MATPROC_ANS_SCALAR 1 -#define _SN_MK_EXPAND_RA(T) \ - MAT_KERNEL(T) const *right = (MAT_TASK(T) *)task->mat_1; \ - MAT_KERNEL(T) *restrict ans = (MAT_TASK(T) *)task->mat_2; +#define _SN_MK_EXPAND_RIGHT_ANS(T) \ + MAT_KERNEL(T) const *right = mat_task_local->mat_1; \ + MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2; -#define _SN_MK_EXPAND_LRA(T) \ - MAT_KERNEL(T) const *left = (MAT_TASK(T) *)task->mat_0; \ - MAT_KERNEL(T) const *right = (MAT_TASK(T) *)task->mat_1; \ - MAT_KERNEL(T) *restrict ans = (MAT_TASK(T) *)task->mat_2; +#define _SN_MK_EXPAND_LEFT_RIGHT_ANS(T) \ + MAT_KERNEL(T) const *left = mat_task_local->mat_0; \ + MAT_KERNEL(T) const *right = mat_task_local->mat_1; \ + MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2; -#define _SN_MK_EXPAND_RAS(T) \ - MAT_KERNEL(T) const *right = (MAT_TASK(T) *)task->mat_1; \ - MAT_KERNEL(T) *restrict ans = (MAT_TASK(T) *)task->mat_2; \ - void *restrict scalar = (MAT_TASK(T) *)task->scalar; +#define _SN_MK_EXPAND_RIGHT_ANS_SCALAR(T) \ + MAT_KERNEL(T) const *right = mat_task_local->mat_1; \ + MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2; \ + void *restrict scalar = mat_task_local->scalar; -#define _SN_MK_EXPAND_LRAS(T) \ - MAT_KERNEL(T) const *left = (MAT_KERNEL(T) *)task->mat_0; \ - MAT_KERNEL(T) const *right = (MAT_KERNEL(T) *)task->mat_1; \ - MAT_KERNEL(T) *restrict ans = (MAT_KERNEL(T) *)task->mat_2; \ - void *restrict scalar = (MAT_TASK(T) *)task->scalar; +#define _SN_MK_EXPAND_LEFT_RIGHT_ANS_SCALAR(T) \ + MAT_KERNEL(T) const *left = mat_task_local->mat_0; \ + MAT_KERNEL(T) const *right = mat_task_local->mat_1; \ + MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2; \ + void *restrict scalar = mat_task_local->scalar; -#define _SN_MK_EXPAND_AS(T) \ - MAT_KERNEL(T) *restrict ans = (MAT_KERNEL(T) *)task->mat_2; \ - void *restrict scalar = (scalar_pack const *)task->scalar; +#define _SN_MK_EXPAND_ANS_SCALAR(T) \ + MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2; \ + scalar_pack *restrict scalar = &(mat_task_local->padding); #define _SN_PARAM_STYLE(T) _SN_MATPROC_##T - -#define _SN_MK_EXPAND_PARAM(style, T) _SN_MK_EXPAND_##style(T) +#define _SN_MK_EXPAND_PARAM(style, T) _SN_MK_EXPAND_##style##(T) #pragma endregion @@ -60,23 +62,22 @@ #define _MK_MKPROC_EXPAND_INDEX_BUFFER \ size_t offset = UNSAFE_AS(size_t, task->padding.data[0]); \ - size_t count = UNSAFE_AS(size_t, task->padding[1]); + size_t count = UNSAFE_AS(size_t, task->padding.data[1]); #define _MK_MKPROC_EXPAND_INDEX_KERNEL \ size_t common_i = UNSAFE_AS(size_t, task->common_i); \ size_t common_j = UNSAFE_AS(size_t, task->common_j); \ size_t common_k = UNSAFE_AS(size_t, task->common_k); -#define _SN_MK_EXPAND_MEM(mem) _SN_MK_EXPAND_##mem +#define _SN_MK_EXPAND_MEM(mem) _MK_MKPROC_EXPAND_INDEX_##mem #define _SN_MEM_LAYOUT(T) _SN_MATPROC_LAYOUT_##T #pragma endregion -#define __mk_param(mem, param, T) \ - static_assert((_SN_PARAM_STYLE(param)) - 1 == 0, \ - "EXPAND_PARAM: invalid param style '" #param "'"); \ - static_assert((_SN_MEM_LAYOUT(mem)) - 1 == 0, \ - "EXPAND_PARAM: invalid mem layout '" #mem "'"); \ - _SN_MK_EXPAND_PARAM(style, T) \ - _SN_MK_EXPAND_MEM(layout) +#define __mk_param_std(mem, param, T) \ + static_assert((_SN_PARAM_STYLE(param)) == 1, \ + "EXPAND_PARAM: invalid param style '" #param "'"); \ + static_assert((_SN_MEM_LAYOUT(mem)) == 1, "EXPAND_PARAM: invalid mem layout '" #mem "'"); \ + typedef MAT_TASK(T) _mat_task_local; \ + _SN_MK_EXPAND_PARAM(param, T) _SN_MK_EXPAND_MEM(mem) #endif // !SN_INTRINSIC_PARAM_EXPAND diff --git a/StgSharp.Native/sn-src/m.basic.x86.c b/StgSharp.Native/sn-src/m.basic.x86.c index ea45f062..f28f7adf 100644 --- a/StgSharp.Native/sn-src/m.basic.x86.c +++ b/StgSharp.Native/sn-src/m.basic.x86.c @@ -3,59 +3,12 @@ #if SN_IS_ARCH(SN_ARCH_X86_64) #include "sn_intrinsic.h" -#include "sn_intrincontext.x86.h" +#include "sn_intrinsic.context.x86.h" +#include "sn_intrinsic.std.h" -STD_MK_PROC_DECL(float, sse, , clear) -{ - register __m128 zero = _mm_setzero_ps(); - for (size_t i = 0; i < 4; i++) { - PANEL_COL_VEC((*ans), i, 0) = zero; - } -} - -STD_MK_PROC_DECL(float, avx, , clear) -{ - register __m256 zero = _mm256_setzero_ps(); - for (size_t i = 0; i < 8; i++) { - PANEL_COL_VEC((*ans), i, 0) = zero; - } -} - -STD_MK_PROC_DECL(float, 512, , clear) -{ - register __m512 zero = _mm512_setzero_ps(); - for (size_t i = 0; i < 16; i++) { - PANEL_COL_VEC((*ans), i, 0) = zero; - } -} - -STD_MK_PROC_DECL(double, sse, , clear) -{ - register __m128 zero = _mm_setzero_ps(); - PANEL_COL_VEC((*ans), 0, 0) = zero; - PANEL_COL_VEC((*ans), 1, 0) = zero; - PANEL_COL_VEC((*ans), 2, 0) = zero; - PANEL_COL_VEC((*ans), 3, 0) = zero; -} - -STD_MK_PROC_DECL(double, avx, , clear) -{ - register __m256 zero = _mm256_setzero_ps(); - for (size_t i = 0; i < 8; i++) { - PANEL_COL_VEC((*ans), i, 0) = zero; - } -} - -STD_MK_PROC_DECL(double, 512, , clear) -{ - register __m512 zero = _mm512_setzero_ps(); - for (size_t i = 0; i < 16; i++) { - PANEL_COL_VEC((*ans), i, 0) = zero; - } -} // Matrix transpose implementations -STD_MK_PROC_DECL(float, sse, , transpose) +SN_MK_PROC_DECL_STD(float, sse, , transpose) { register __m128 col0 = right->f32_x[0]; register __m128 col1 = right->f32_x[1]; @@ -72,7 +25,7 @@ STD_MK_PROC_DECL(float, sse, , transpose) ans->f32_x[3] = _mm_movehl_ps(tmp3, tmp1); } -STD_MK_PROC_DECL(float, avx, , transpose) +SN_MK_PROC_DECL_STD(float, avx, , transpose) { register __m256 col_low = right->f32_y[0]; register __m256 col_high = right->f32_y[1]; @@ -87,7 +40,7 @@ STD_MK_PROC_DECL(float, avx, , transpose) ans->f32_y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1)); } -STD_MK_PROC_DECL(float, 512, , transpose) +SN_MK_PROC_DECL_STD(float, 512, , transpose) { register __m512 matrix = right->f32_z[0]; diff --git a/StgSharp.Native/sn-src/m.bufop.x86.c b/StgSharp.Native/sn-src/m.bufop.x86.c index 27005876..a42b5266 100644 --- a/StgSharp.Native/sn-src/m.bufop.x86.c +++ b/StgSharp.Native/sn-src/m.bufop.x86.c @@ -6,8 +6,9 @@ #include "sn_intrinsic.std.h" #include "sn_intrinsic.context.x86.h" -STD_MK_RPOC_DECL(float, sse, , fill) +SN_MK_PROC_DECL_STD(float, sse, , fill) { + __mk_param_std(BUFFER, ANS_SCALAR, float); register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]); for (size_t i = 0; i < count; i++) { ans[i].f32_x[0] = vec; @@ -17,7 +18,7 @@ STD_MK_RPOC_DECL(float, sse, , fill) } } -DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill) +SN_MK_PROC_DECL_STD(float, avx, , fill) { register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data))); for (size_t i = 0; i < count; i++) { @@ -26,7 +27,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill) } } -DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill) +SN_MK_PROC_DECL_STD(float, 512, , fill) { register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]); for (size_t i = 0; i < count; i++) { @@ -34,7 +35,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill) } } -DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill) +SN_MK_PROC_DECL_STD(double, sse, , fill) { register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]); for (size_t i = 0; i < count; i++) { @@ -49,7 +50,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill) } } -DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill) +SN_MK_PROC_DECL_STD(double, avx, , fill) { register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data))); for (size_t i = 0; i < count; i++) { @@ -60,7 +61,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill) } } -DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill) +SN_MK_PROC_DECL_STD(double, 512, , fill) { register __m512d vec = _mm512_broadcastsd_pd(_mm_loadu_pd(scalar)); for (size_t i = 0; i < count; i++) { @@ -69,9 +70,9 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill) } } -STD_MK_PROC_DECL(float, sse, , add) +SN_MK_PROC_DECL_STD(float, sse, , add) { - __mk_param(BUFFER, LEFT_RIGHT_ANS, float); + __mk_param_std(BUFFER, LEFT_RIGHT_ANS, float); for (size_t i = 0; i < count; i++) { register __m128 c0 = _mm_add_ps(left[i].f32_x[0], right[i].f32_x[0]); @@ -86,7 +87,7 @@ STD_MK_PROC_DECL(float, sse, , add) } } -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , add) +SN_MK_PROC_DECL_STD(float, avx, , add) { for (size_t i = 0; i < count; i++) { size_t s = sizeof(MAT_KERNEL(float)); @@ -97,14 +98,14 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , add) } } -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , add) +SN_MK_PROC_DECL_STD(float, 512, , add) { for (size_t i = 0; i < count; i++) { register __m512 c0 = _mm512_add_ps(left[i].f32_z[0], right[i].f32_z[0]); ans[i].f32_z[0] = c0; } } -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , add) +SN_MK_PROC_DECL_STD(double, sse, , add) { for (size_t i = 0; i < count; i++) { register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[0]), @@ -135,7 +136,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , add) } } -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , add) +SN_MK_PROC_DECL_STD(double, avx, , add) { for (size_t i = 0; i < count; i++) { size_t s = sizeof(MAT_KERNEL(float)); @@ -150,7 +151,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , add) } } -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , add) +SN_MK_PROC_DECL_STD(double, 512, , add) { for (size_t i = 0; i < count; i++) { register __m512d c0 = _mm512_add_pd(left[i].f64_z[0], right[i].f64_z[0]); @@ -160,7 +161,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , add) } } -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub) +SN_MK_PROC_DECL_STD(float, sse, , sub) { for (size_t i = 0; i < count; i++) { register __m128 c0 = _mm_sub_ps(left[i].f32_x[0], right[i].f32_x[0]); @@ -175,7 +176,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub) } } -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub) +SN_MK_PROC_DECL_STD(float, avx, , sub) { for (size_t i = 0; i < count; i++) { register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]); @@ -185,7 +186,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub) } } -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub) +SN_MK_PROC_DECL_STD(float, 512, , sub) { for (size_t i = 0; i < count; i++) { register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]); @@ -193,7 +194,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub) } } -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub) +SN_MK_PROC_DECL_STD(double, sse, , sub) { for (size_t i = 0; i < count; i++) { register __m128d c0 = _mm_sub_pd(left[i].f64_x[0], right[i].f64_x[0]); @@ -216,7 +217,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub) } } -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub) +SN_MK_PROC_DECL_STD(double, avx, , sub) { for (size_t i = 0; i < count; i++) { register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]); @@ -226,7 +227,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub) } } -DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub) +SN_MK_PROC_DECL_STD(double, 512, , sub) { for (size_t i = 0; i < count; i++) { register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]); @@ -234,7 +235,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub) } } -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul) +SN_MK_PROC_DECL_STD(float, sse, , scalar_mul) { register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]); for (size_t i = 0; i < count; i++) { @@ -250,7 +251,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul) } } -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul) +SN_MK_PROC_DECL_STD(float, avx, , scalar_mul) { register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data))); for (size_t i = 0; i < count; i++) { @@ -262,7 +263,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul) } } -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul) +SN_MK_PROC_DECL_STD(float, 512, , scalar_mul) { register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]); for (size_t i = 0; i < count; i++) { @@ -272,7 +273,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul) } } -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul) +SN_MK_PROC_DECL_STD(double, sse, , scalar_mul) { register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]); for (size_t i = 0; i < count; i++) { @@ -296,7 +297,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul) } } -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul) +SN_MK_PROC_DECL_STD(double, avx, , scalar_mul) { register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data))); for (size_t i = 0; i < count; i++) { @@ -312,7 +313,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul) } } -DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, , scalar_mul) +SN_MK_PROC_DECL_STD(double, 512, , scalar_mul) { register __m512d vec = _mm512_broadcastsd_pd(_mm_load_pd(scalar)); for (size_t i = 0; i < count; i++) { diff --git a/StgSharp.Native/sn-src/m.fma.x86.c b/StgSharp.Native/sn-src/m.fma.x86.c index f4da4d0b..66dd136a 100644 --- a/StgSharp.Native/sn-src/m.fma.x86.c +++ b/StgSharp.Native/sn-src/m.fma.x86.c @@ -3,14 +3,17 @@ #if SN_IS_ARCH(SN_ARCH_X86_64) #include "sn_intrinsic.h" -#include "sn_intrincontext.x86.h" +#include "sn_intrinsic.context.x86.h" +#include "sn_intrinsic.std.h" #define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i) // float -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma) +SN_MK_PROC_DECL_STD(float, sse, , fma) { + __mk_param_std(KERNEL, LEFT_RIGHT_ANS, float); + const __m128 a0 = _mm_load_ps((float *)&left->f32_x[0]); const __m128 a1 = _mm_load_ps((float *)&left->f32_x[1]); const __m128 a2 = _mm_load_ps((float *)&left->f32_x[2]); @@ -57,8 +60,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma) _mm_store_ps((float *)&ans->f32_x[3], c3); } -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma) +SN_MK_PROC_DECL_STD(float, avx, , fma) { + __mk_param_std(KERNEL, LEFT_RIGHT_ANS, float); + static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } }; static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } }; @@ -103,8 +108,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma) ans->f32_y[1] = c23; } -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma) +SN_MK_PROC_DECL_STD(float, avx, _fma, fma) { + __mk_param_std(KERNEL, LEFT_RIGHT_ANS, float); + static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } }; static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } }; @@ -149,8 +156,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma) ans->f32_y[1] = c23; } -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma) +SN_MK_PROC_DECL_STD(float, 512, , fma) { + __mk_param_std(KERNEL, LEFT_RIGHT_ANS, float); + static const __m512i b_shift = { .m512i_i32 = { // shuff order 0, 1, 2, 3, // col0 @@ -185,8 +194,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma) // double -DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma) +SN_MK_PROC_DECL_STD(double, sse, , fma) { + __mk_param_std(KERNEL, LEFT_RIGHT_ANS, float); + for (int col = 0; col < 4; ++col) { /* current answer column (two chunks: rows 0-1 and 2-3) */ register __m128d c_lo = _mm_load_pd(ans->f64_x + (col * 2 + 0)); @@ -229,8 +240,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma) } } -DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma) +SN_MK_PROC_DECL_STD(double, avx, , fma) { + __mk_param_std(KERNEL, LEFT_RIGHT_ANS, float); + const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]); const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]); const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]); @@ -273,8 +286,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma) _mm256_store_pd((double *)&ans->f64_y[3], ccol); } -DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma) +SN_MK_PROC_DECL_STD(double, avx, _fma, fma) { + __mk_param_std(KERNEL, LEFT_RIGHT_ANS, float); + const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]); const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]); const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]); @@ -317,8 +332,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma) _mm256_store_pd((double *)&ans->f64_y[3], ccol); } -DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma) +SN_MK_PROC_DECL_STD(double, 512, , fma) { + __mk_param_std(KERNEL, LEFT_RIGHT_ANS, float); + static const __m512i b01_shift = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } }; static const __m512i b23_shift = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } }; diff --git a/StgSharp.Native/sn-src/m.fma_tile.x86.c b/StgSharp.Native/sn-src/m.fma_tile.x86.c index 03a2d9ff..2a067d50 100644 --- a/StgSharp.Native/sn-src/m.fma_tile.x86.c +++ b/StgSharp.Native/sn-src/m.fma_tile.x86.c @@ -11,7 +11,7 @@ // float -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, sse, , fma) +SN_MK_PROC_DECL_STD(float, sse, , fma) { int i = enumeration->m128_i32[0]; int j = enumeration->m128_i32[1]; @@ -73,7 +73,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, sse, , fma) _mm_store_ps((float *)&ans_cur->f32_x[3], c3); } -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, , fma) +SN_MK_PROC_DECL_STD(float, avx, , fma) { static const __m256i b01_shift_source = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } }; static const __m256i b23_shift_source = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } }; @@ -136,7 +136,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, , fma) _mm256_store_ps((float *)&ans_cur->f32_y[1], c23); } -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma) +SN_MK_PROC_DECL_STD(float, avx, _fma, fma) { static const __m256i b01_shift_source = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } }; static const __m256i b23_shift_source = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } }; @@ -199,7 +199,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma) _mm256_store_ps((float *)&ans_cur->f32_y[1], c23); } -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma) +SN_MK_PROC_DECL_STD(float, 512, , fma) { static const __m512i b_shift_source = { .m512i_i32 = { // shuff order @@ -279,7 +279,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma) c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ } while (0) -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma) +SN_MK_PROC_DECL_STD(double, sse, , fma) { int i = enumeration->m128_i32[0]; int j = enumeration->m128_i32[1]; @@ -319,7 +319,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma) _mm_store_pd(ans->f64_x + (3 * 2 + 1), c3_hi); } -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, , fma) +SN_MK_PROC_DECL_STD(double, avx, , fma) { int i = enumeration->m128_i32[0]; int j = enumeration->m128_i32[1]; @@ -397,7 +397,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, , fma) _mm256_store_pd((double *)&ans_cur->f64_y[3], c3); } -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma) +SN_MK_PROC_DECL_STD(double, avx, _fma, fma) { int i = enumeration->m128_i32[0]; int j = enumeration->m128_i32[1]; @@ -459,7 +459,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma) _mm256_store_pd((double *)&ans_cur->f64_y[3], c3); } -DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, 512, , fma) +SN_MK_PROC_DECL_STD(double, 512, , fma) { static const __m512i b01_shift_source = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } }; static const __m512i b23_shift_source = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } }; diff --git a/StgSharp.Native/sn-src/m.solve.x86.c b/StgSharp.Native/sn-src/m.solve.x86.c index c9cec00f..373511d2 100644 --- a/StgSharp.Native/sn-src/m.solve.x86.c +++ b/StgSharp.Native/sn-src/m.solve.x86.c @@ -1,13 +1,14 @@ -#include "sn_target.h" +#include "sn_target.h" #if SN_IS_ARCH(SN_ARCH_X86_64) #include "sn_intrinsic.h" +#include "sn_intrinsic.std.h" #include static const float quality_threshold = 1e-6f; -DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(float, sse, , try_plu) +SN_MK_PROC_DECL_STD(float, sse, , try_plu) { /* Aliases */ MAT_KERNEL(float) *upper = left; /* U */ @@ -96,7 +97,7 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(float, sse, , try_plu) __m128 _t = _mm_shuffle_ps(_p, _p, _MM_SHUFFLE(2, 3, 0, 1)); \ _p = _mm_add_ps(_p, _t); \ _t = _mm_movehl_ps(_t, _p); \ - ans = _mm_cvtss_f32(_mm_add_ss(_p, _t)); \ + ans = _mm_cvtss_f32(_mm_add_ss(_p, _t)); \ } while (0) #define FIND_MAX_F32_SSE(a, b) \ @@ -111,8 +112,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(float, sse, , try_plu) * layout of scalarpack: * */ -DECLARE_KER_PROC_RIGHT_ANS(float, sse, , quality) +SN_MK_PROC_DECL_STD(float, sse, , quality) { + __mk_param_std(KERNEL, RIGHT_ANS, float); + static const __m128 tiny_source = { .m128_f32 = { 1e-6f, 1e-6f, 1e-6f, 1e-6f } }; register __m128 tiny = _mm_load_ps(tiny_source.m128_f32);