diff --git a/.gitignore b/.gitignore index 58660116..c133eea4 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ obj /cmake_build */.cache +.cache diff --git a/StgSharp.Common/HighPerformance/ProcessorAbstraction/LogicalCore.cs b/StgSharp.Common/HighPerformance/ProcessorAbstraction/LogicalCore.cs index ba3487ac..d5ca9b22 100644 --- a/StgSharp.Common/HighPerformance/ProcessorAbstraction/LogicalCore.cs +++ b/StgSharp.Common/HighPerformance/ProcessorAbstraction/LogicalCore.cs @@ -44,5 +44,21 @@ namespace StgSharp.Common.HighPerformance.ProcessorAbstraction /// public readonly uint NumaNodeId; + internal LogicalCore( + uint globalCpuId, + uint numaGroupId, + uint numaNodeId + ) + { + GlobalCpuId = globalCpuId; + NumaGroupId = numaGroupId; + NumaNodeId = numaNodeId; + } + + [Obsolete( + "Do not manually create instances of LogicalCore. Use NumaNode to obtain LogicalCore instances.", + true)] + public LogicalCore() { } + } } diff --git a/StgSharp.Common/HighPerformance/ProcessorAbstraction/Numa.cs b/StgSharp.Common/HighPerformance/ProcessorAbstraction/Numa.cs index 0501fa82..cb774710 100644 --- a/StgSharp.Common/HighPerformance/ProcessorAbstraction/Numa.cs +++ b/StgSharp.Common/HighPerformance/ProcessorAbstraction/Numa.cs @@ -47,9 +47,9 @@ namespace StgSharp.Common.HighPerformance.ProcessorAbstraction [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] public static partial int GetNumaEach( - int numaId, - Span coreCountInNuma, - Span globalCpuIdInNuma + uint numaId, + Span CpuIndexInNuma, + Span globalCpuIdInNuma ); [LibraryImport(Native.LibName, EntryPoint = "sn_get_numa_overall")] @@ -68,12 +68,22 @@ namespace StgSharp.Common.HighPerformance.ProcessorAbstraction throw new InvalidOperationException("Failed to get NUMA overall information."); } _numaNodes = new NumaNode[numaNodeCount]; - for (int i = 0; i < numaNodeCount; i++) - { - NumaNode node = new(coreCountInNuma[i]); - GetNumaEach(i, coreCountInNuma, node.GlobalCpuIds); + for (uint i = 0; i < numaNodeCount; i++) { + _numaNodes[i] = InitNumaNode(i, coreCountInNuma[(int)i]); } } + [MethodImpl(MethodImplOptions.NoInlining)] + private static NumaNode InitNumaNode( + uint numaId, + int count + ) + { + Span globalIndex = stackalloc uint[count]; + Span groupIndex = stackalloc ulong[count]; + _ = GetNumaEach(numaId, groupIndex, globalIndex); + return new(numaId, count, globalIndex); + } + } } diff --git a/StgSharp.Common/HighPerformance/ProcessorAbstraction/NumaNode.cs b/StgSharp.Common/HighPerformance/ProcessorAbstraction/NumaNode.cs index df888c63..8b7343c6 100644 --- a/StgSharp.Common/HighPerformance/ProcessorAbstraction/NumaNode.cs +++ b/StgSharp.Common/HighPerformance/ProcessorAbstraction/NumaNode.cs @@ -39,11 +39,17 @@ namespace StgSharp.Common.HighPerformance.ProcessorAbstraction private LogicalCore[] _logicalCores; - public NumaNode( - int count + internal NumaNode( + uint numaId, + int count, + Span logicalCoreIds + ) { _logicalCores = new LogicalCore[count]; + for (int i = 0; i < count; i++) { + _logicalCores[i] = new LogicalCore(1, numaId, logicalCoreIds[i]); + } } public int LogicalCoreCount => _logicalCores.Length; diff --git a/StgSharp.Native/.cache/clangd/index/StgSharpNative.h.4995CE7BF2BDDAFD.idx b/StgSharp.Native/.cache/clangd/index/StgSharpNative.h.4995CE7BF2BDDAFD.idx deleted file mode 100644 index 3174fc25..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/StgSharpNative.h.4995CE7BF2BDDAFD.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/dllmain.c.693656BDB16A2A66.idx b/StgSharp.Native/.cache/clangd/index/dllmain.c.693656BDB16A2A66.idx deleted file mode 100644 index 51c639b3..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/dllmain.c.693656BDB16A2A66.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/gl.h.872C02B44AACC9D0.idx b/StgSharp.Native/.cache/clangd/index/gl.h.872C02B44AACC9D0.idx deleted file mode 100644 index 7b5db758..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/gl.h.872C02B44AACC9D0.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/glfw3.h.E74F1A4E96981153.idx b/StgSharp.Native/.cache/clangd/index/glfw3.h.E74F1A4E96981153.idx deleted file mode 100644 index bd31d8b5..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/glfw3.h.E74F1A4E96981153.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/glfw_function.h.E7F62B4F36471C88.idx b/StgSharp.Native/.cache/clangd/index/glfw_function.h.E7F62B4F36471C88.idx deleted file mode 100644 index 507305ba..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/glfw_function.h.E7F62B4F36471C88.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/khrplatform.h.E0CC86B40468D38F.idx b/StgSharp.Native/.cache/clangd/index/khrplatform.h.E0CC86B40468D38F.idx deleted file mode 100644 index af53799e..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/khrplatform.h.E0CC86B40468D38F.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/load_glfw.c.8EEC02B2FC35B602.idx b/StgSharp.Native/.cache/clangd/index/load_glfw.c.8EEC02B2FC35B602.idx deleted file mode 100644 index adcedb13..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/load_glfw.c.8EEC02B2FC35B602.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/load_intrinsic.x86.c.F85DE0985CFEAFC7.idx b/StgSharp.Native/.cache/clangd/index/load_intrinsic.x86.c.F85DE0985CFEAFC7.idx deleted file mode 100644 index d874d5ea..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/load_intrinsic.x86.c.F85DE0985CFEAFC7.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/m.bufop.x86.c.D02B9612A86F3381.idx b/StgSharp.Native/.cache/clangd/index/m.bufop.x86.c.D02B9612A86F3381.idx deleted file mode 100644 index 202fb343..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/m.bufop.x86.c.D02B9612A86F3381.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/m.solve.x86.c.A740CE0EA3BB371C.idx b/StgSharp.Native/.cache/clangd/index/m.solve.x86.c.A740CE0EA3BB371C.idx deleted file mode 100644 index ccf8e9a2..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/m.solve.x86.c.A740CE0EA3BB371C.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/m.transpose.x86.c.E7D20DF7D1F53278.idx b/StgSharp.Native/.cache/clangd/index/m.transpose.x86.c.E7D20DF7D1F53278.idx deleted file mode 100644 index 4e3c9c2f..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/m.transpose.x86.c.E7D20DF7D1F53278.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/mk.fma.x86.c.46ABCA8FA7C19AAA.idx b/StgSharp.Native/.cache/clangd/index/mk.fma.x86.c.46ABCA8FA7C19AAA.idx deleted file mode 100644 index 31be940b..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/mk.fma.x86.c.46ABCA8FA7C19AAA.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/s.factorial.x86.c.829C3043BB9B0AB5.idx b/StgSharp.Native/.cache/clangd/index/s.factorial.x86.c.829C3043BB9B0AB5.idx deleted file mode 100644 index 992e0baa..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/s.factorial.x86.c.829C3043BB9B0AB5.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/s.hash.x86.c.A842A8040CA6DDDD.idx b/StgSharp.Native/.cache/clangd/index/s.hash.x86.c.A842A8040CA6DDDD.idx deleted file mode 100644 index 50f1825d..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/s.hash.x86.c.A842A8040CA6DDDD.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/simdid.x86.c.A5AE07ED8BFC4CE3.idx b/StgSharp.Native/.cache/clangd/index/simdid.x86.c.A5AE07ED8BFC4CE3.idx deleted file mode 100644 index 68ddce26..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/simdid.x86.c.A5AE07ED8BFC4CE3.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/simdid_compare.x86.c.A1B317399F224C45.idx b/StgSharp.Native/.cache/clangd/index/simdid_compare.x86.c.A1B317399F224C45.idx deleted file mode 100644 index a3b80aa9..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/simdid_compare.x86.c.A1B317399F224C45.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/sk.fma.x86.c.0814D9474B842FD2.idx b/StgSharp.Native/.cache/clangd/index/sk.fma.x86.c.0814D9474B842FD2.idx deleted file mode 100644 index bf9fde5f..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/sk.fma.x86.c.0814D9474B842FD2.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/sn_internal.h.3EF94CD61A762AF6.idx b/StgSharp.Native/.cache/clangd/index/sn_internal.h.3EF94CD61A762AF6.idx deleted file mode 100644 index 94656591..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/sn_internal.h.3EF94CD61A762AF6.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/sn_intrinsic.context.x86.h.34F603A805649DB0.idx b/StgSharp.Native/.cache/clangd/index/sn_intrinsic.context.x86.h.34F603A805649DB0.idx deleted file mode 100644 index 4888e54a..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/sn_intrinsic.context.x86.h.34F603A805649DB0.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/sn_intrinsic.h.E09549401BC3B590.idx b/StgSharp.Native/.cache/clangd/index/sn_intrinsic.h.E09549401BC3B590.idx deleted file mode 100644 index 7d407489..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/sn_intrinsic.h.E09549401BC3B590.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/sn_intrinsic.matkernel.h.161A9E0A54109A22.idx b/StgSharp.Native/.cache/clangd/index/sn_intrinsic.matkernel.h.161A9E0A54109A22.idx deleted file mode 100644 index 2fca90ce..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/sn_intrinsic.matkernel.h.161A9E0A54109A22.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/sn_intrinsic.std.h.7B815CC90AC635F2.idx b/StgSharp.Native/.cache/clangd/index/sn_intrinsic.std.h.7B815CC90AC635F2.idx deleted file mode 100644 index c19b484f..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/sn_intrinsic.std.h.7B815CC90AC635F2.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/sn_native.c.C654B32BC7BCDB73.idx b/StgSharp.Native/.cache/clangd/index/sn_native.c.C654B32BC7BCDB73.idx deleted file mode 100644 index 2691557d..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/sn_native.c.C654B32BC7BCDB73.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/sn_target.h.11D7D85B43E0081F.idx b/StgSharp.Native/.cache/clangd/index/sn_target.h.11D7D85B43E0081F.idx deleted file mode 100644 index 849f59ca..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/sn_target.h.11D7D85B43E0081F.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/stb_image.h.6F1AD7E8D3072049.idx b/StgSharp.Native/.cache/clangd/index/stb_image.h.6F1AD7E8D3072049.idx deleted file mode 100644 index 3d5b5a8b..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/stb_image.h.6F1AD7E8D3072049.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/thread.linux.c.125C6245463F4735.idx b/StgSharp.Native/.cache/clangd/index/thread.linux.c.125C6245463F4735.idx deleted file mode 100644 index 7726415c..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/thread.linux.c.125C6245463F4735.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/thread.windows.c.38B8C120DD62E7C8.idx b/StgSharp.Native/.cache/clangd/index/thread.windows.c.38B8C120DD62E7C8.idx deleted file mode 100644 index fe9cf19f..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/thread.windows.c.38B8C120DD62E7C8.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/vector.c.D485BA7D87050634.idx b/StgSharp.Native/.cache/clangd/index/vector.c.D485BA7D87050634.idx deleted file mode 100644 index ba322fe0..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/vector.c.D485BA7D87050634.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/vector_dot.c.9A6B648BA7B62E97.idx b/StgSharp.Native/.cache/clangd/index/vector_dot.c.9A6B648BA7B62E97.idx deleted file mode 100644 index 6825c4f2..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/vector_dot.c.9A6B648BA7B62E97.idx and /dev/null differ diff --git a/StgSharp.Native/.cache/clangd/index/wgl.h.A57E64BCBB5F3CE2.idx b/StgSharp.Native/.cache/clangd/index/wgl.h.A57E64BCBB5F3CE2.idx deleted file mode 100644 index 520365f4..00000000 Binary files a/StgSharp.Native/.cache/clangd/index/wgl.h.A57E64BCBB5F3CE2.idx and /dev/null differ diff --git a/StgSharp.Native/StgSharp.Native.vcxproj b/StgSharp.Native/StgSharp.Native.vcxproj index 1d8460d9..03d1bb29 100644 --- a/StgSharp.Native/StgSharp.Native.vcxproj +++ b/StgSharp.Native/StgSharp.Native.vcxproj @@ -185,7 +185,8 @@ - + + diff --git a/StgSharp.Native/StgSharp.Native.vcxproj.filters b/StgSharp.Native/StgSharp.Native.vcxproj.filters index b2ea2a86..3e9133e3 100644 --- a/StgSharp.Native/StgSharp.Native.vcxproj.filters +++ b/StgSharp.Native/StgSharp.Native.vcxproj.filters @@ -60,10 +60,13 @@ 源文件 - + 源文件 - + + 源文件 + + 源文件 diff --git a/StgSharp.Native/sn-head/StgSharpNative.h b/StgSharp.Native/sn-head/StgSharpNative.h index e783452e..c2b98863 100644 --- a/StgSharp.Native/sn-head/StgSharpNative.h +++ b/StgSharp.Native/sn-head/StgSharpNative.h @@ -56,4 +56,6 @@ SN_API SIMDID SN_DECL sn_get_unite_simd(SIMDID left, SIMDID right); SN_API int SN_DECL sn_compare_simd(SIMDID left, SIMDID right); SN_API void SN_DECL sn_set_thread_affinity(int core_id); +SN_API int SN_DECL sn_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global); +SN_API int SN_DECL sn_get_numa_overall(int *out_numa_count, int *count_of_each_numa); #endif diff --git a/StgSharp.Native/sn-src/mk.fma.x86.c b/StgSharp.Native/sn-src/mk.fma.x86.c index ef4c949d..dc7b5ebe 100644 --- a/StgSharp.Native/sn-src/mk.fma.x86.c +++ b/StgSharp.Native/sn-src/mk.fma.x86.c @@ -2,7 +2,8 @@ /* clang-format off */ #if SN_IS_ARCH(SN_ARCH_X86_64) -#include "sn_intrinsic.h" +// #include "sn_intrinsic.h" +#include "sn_template.fma.x86.h" #include "sn_intrinsic.std.h" #include "sn_intrinsic.context.x86.h" @@ -136,17 +137,6 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma) _mm256_store_ps((float *)&ans_cur->f32_y[1], c23); } -#define SN_MK_FMA_F32_512_PHASE(i)\ - do { \ - ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\ - c_0 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\ - ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\ - c_1 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\ - ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\ - c_2 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\ - ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\ - c_3 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\ - }while(0) SN_MK_PROC_DECL_STD(float, 512, , fma) { @@ -187,39 +177,6 @@ SN_MK_PROC_DECL_STD(float, 512, , fma) _mm512_store_ps((float *)ans_cur, c_0); } -// double -#define DOUBLE_KERNEL_FMA_CYCLE(i) \ - do { \ - /* k = 0 */ \ - register __m128d b_lo = _mm_load_pd(right_values + (2 * i + 0)); \ - register __m128d b_hi = _mm_load_pd(right_values + (2 * i + 1)); \ - register __m128d a_lo = _mm_load_pd(left_values + (0 * 2 + 0)); \ - register __m128d a_hi = _mm_load_pd(left_values + (0 * 2 + 1)); \ - register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \ - c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ - c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ - \ - /* group = 1 */ \ - a_lo = _mm_load_pd(left_values + (1 * 2 + 0)); \ - a_hi = _mm_load_pd(left_values + (1 * 2 + 1)); \ - b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \ - c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ - c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ - \ - /* groupp = 2 */ \ - a_lo = _mm_load_pd(left_values + (2 * 2 + 0)); \ - a_hi = _mm_load_pd(left_values + (2 * 2 + 1)); \ - b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \ - c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ - c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ - \ - /* froup = 3 */ \ - a_lo = _mm_load_pd(left_values + (3 * 2 + 0)); \ - a_hi = _mm_load_pd(left_values + (3 * 2 + 1)); \ - b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \ - c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ - c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ - } while (0) SN_MK_PROC_DECL_STD(double, sse, , fma) { @@ -243,13 +200,13 @@ SN_MK_PROC_DECL_STD(double, sse, , fma) const double *right_values = &right_cur->m[0][0]; const double *left_values = &left_cur->m[0][0]; - DOUBLE_KERNEL_FMA_CYCLE(0); + SN_MK_FMA_F64_SSE_PHASE(0); - DOUBLE_KERNEL_FMA_CYCLE(1); + SN_MK_FMA_F64_SSE_PHASE(1); - DOUBLE_KERNEL_FMA_CYCLE(2); + SN_MK_FMA_F64_SSE_PHASE(2); - DOUBLE_KERNEL_FMA_CYCLE(3); + SN_MK_FMA_F64_SSE_PHASE(3); } ans_cur->f64_x[0] = c0_lo; diff --git a/StgSharp.Native/sn-src/thread.windows.c b/StgSharp.Native/sn-src/thread.windows.c index 85d6ba96..27a0326e 100644 --- a/StgSharp.Native/sn-src/thread.windows.c +++ b/StgSharp.Native/sn-src/thread.windows.c @@ -48,7 +48,7 @@ SN_API int SN_DECL sn_get_numa_overall(int *out_numa_count, int *count_of_each_n return 1; } -SN_API int SN_DECL sn_get_numa_each(int numa_id, uint64_t *internal, uint64_t *global) +SN_API int SN_DECL sn_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global) { ULONGLONG mask = 0; if (!GetNumaNodeProcessorMask((UCHAR)numa_id, &mask)) @@ -56,9 +56,9 @@ SN_API int SN_DECL sn_get_numa_each(int numa_id, uint64_t *internal, uint64_t *g int intra = 0; while (mask) { - unsigned long idx; + uint32_t idx; _BitScanForward64(&idx, mask); - internal[intra] = intra; + int _ = (internal == NULL) ? intra : (internal[intra] = intra); global[intra] = (int)idx; intra++; mask &= mask - 1; diff --git a/StgSharp.Native/sn-template/sn_template.fma.x86.h b/StgSharp.Native/sn-template/sn_template.fma.x86.h new file mode 100644 index 00000000..53918585 --- /dev/null +++ b/StgSharp.Native/sn-template/sn_template.fma.x86.h @@ -0,0 +1,53 @@ +#include "sn_target.h" + +#if SN_IS_ARCH(SN_ARCH_X86_64) + + +#define SN_MK_FMA_F32_512_PHASE(i)\ + do { \ + ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\ + c_0 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\ + ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\ + c_1 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\ + ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\ + c_2 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\ + ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\ + c_3 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\ + }while(0) + + +// double +#define SN_MK_FMA_F64_SSE_PHASE(i) \ + do { \ + /* k = 0 */ \ + register __m128d b_lo = _mm_load_pd(right_values + (2 * i + 0)); \ + register __m128d b_hi = _mm_load_pd(right_values + (2 * i + 1)); \ + register __m128d a_lo = _mm_load_pd(left_values + (0 * 2 + 0)); \ + register __m128d a_hi = _mm_load_pd(left_values + (0 * 2 + 1)); \ + register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \ + c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ + c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ + \ + /* group = 1 */ \ + a_lo = _mm_load_pd(left_values + (1 * 2 + 0)); \ + a_hi = _mm_load_pd(left_values + (1 * 2 + 1)); \ + b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \ + c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ + c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ + \ + /* group = 2 */ \ + a_lo = _mm_load_pd(left_values + (2 * 2 + 0)); \ + a_hi = _mm_load_pd(left_values + (2 * 2 + 1)); \ + b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \ + c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ + c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ + \ + /* group = 3 */ \ + a_lo = _mm_load_pd(left_values + (3 * 2 + 0)); \ + a_hi = _mm_load_pd(left_values + (3 * 2 + 1)); \ + b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \ + c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ + c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ + } while (0) + +#endif \ No newline at end of file