diff --git a/StgSharp.Common/Collections/CapacityFixedStack.cs b/StgSharp.Common/Collections/CapacityFixedStack.cs index e8389de7..6c3bc5a8 100644 --- a/StgSharp.Common/Collections/CapacityFixedStack.cs +++ b/StgSharp.Common/Collections/CapacityFixedStack.cs @@ -91,7 +91,7 @@ namespace StgSharp.Collections } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ReadOnlySpan PopRange(int count) + public T[] PopRange(int count) { int i = int.Min(count, _index) - 1; return _values[0..i]; diff --git a/StgSharp.Common/HighPerformance/IRegisterType.cs b/StgSharp.Common/HighPerformance/IRegisterType.cs index 9579b965..3e854b3a 100644 --- a/StgSharp.Common/HighPerformance/IRegisterType.cs +++ b/StgSharp.Common/HighPerformance/IRegisterType.cs @@ -34,12 +34,65 @@ using System.Threading.Tasks; namespace StgSharp.HighPerformance { - public interface IRegisterType + /// + /// Abstraction for a wide SIMD register or equivalent packed numeric storage. Provides raw + /// reference access, broadcast initialization and element level access. Typical + /// implementations map to 128/256/512-bit hardware registers or software emulated blocks. + /// + + public interface IRegisterPresentation { + /// + /// Gets a reference to the first element of the register reinterpreted as type . + /// + /// + /// Unmanaged numeric element type implementing . + /// + /// + /// Writable reference to element 0 (lowest lane / lowest bits). + /// + /// + /// Intended for treating the register as a contiguous block of elements. The returned reference + /// must remain valid for the lifetime of the register and be identical to with index 0. Caller must choose a type layout compatible with the + /// actual register representation. + /// ref T AsRef() where T: unmanaged, INumber; + /// + /// Fills all elements (lanes) of the register with the provided scalar . + /// + /// + /// Unmanaged numeric element type implementing . + /// + /// + /// Scalar value to replicate to every element. + /// + /// + /// Should map to a single broadcast/shuffle instruction where possible. After completion all + /// lanes read back the same value. + /// + void BroadCastFrom(T value) where T: unmanaged, INumber; + + /// + /// Gets a writable reference to the element at the specified . + /// + /// + /// Unmanaged numeric element type implementing . + /// + /// + /// Zero-based lane index (range defined by implementation). + /// + /// + /// Writable reference to the lane. + /// + /// + /// Enables scalar lane edits without unpacking the entire register. Implementations may omit + /// bounds checks for performance. Caller is responsible for passing a valid index. + /// ref T Member(int index) where T: unmanaged, INumber; } -} +} \ No newline at end of file diff --git a/StgSharp.Common/HighPerformance/M128.cs b/StgSharp.Common/HighPerformance/M128.cs index 690372d1..53d1b251 100644 --- a/StgSharp.Common/HighPerformance/M128.cs +++ b/StgSharp.Common/HighPerformance/M128.cs @@ -68,6 +68,11 @@ namespace StgSharp.HighPerformance return Vec.AsVector4(); } + public void BroadCastFrom(T value) where T: unmanaged, INumber + { + Vec = Vector128.CreateScalarUnsafe(value).As(); + } + public override readonly bool Equals([NotNullWhen(true)] object obj) { return obj is M128 other && this == other; diff --git a/StgSharp.Common/HighPerformance/M256.cs b/StgSharp.Common/HighPerformance/M256.cs index 4314ce27..4683700c 100644 --- a/StgSharp.Common/HighPerformance/M256.cs +++ b/StgSharp.Common/HighPerformance/M256.cs @@ -31,6 +31,7 @@ using System.Linq; using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; using System.Text; using System.Threading.Tasks; @@ -41,6 +42,7 @@ namespace StgSharp.HighPerformance { [FieldOffset(0)] internal fixed byte Buffer[32]; + [FieldOffset(0)] internal Vector256 Vec; [MethodImpl(MethodImplOptions.AggressiveInlining)] public ref T AsRef() where T: unmanaged, INumber @@ -48,6 +50,11 @@ namespace StgSharp.HighPerformance return ref Unsafe.As(ref Buffer[0]); } + public void BroadCastFrom(T value) where T: unmanaged, INumber + { + Vec = Vector256.CreateScalarUnsafe(value).As(); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public ref T Member(int index) where T: unmanaged, INumber { diff --git a/StgSharp.Common/HighPerformance/M512.cs b/StgSharp.Common/HighPerformance/M512.cs index 57fd920e..e68cf449 100644 --- a/StgSharp.Common/HighPerformance/M512.cs +++ b/StgSharp.Common/HighPerformance/M512.cs @@ -31,6 +31,7 @@ using System.Linq; using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; using System.Text; using System.Threading.Tasks; @@ -41,12 +42,18 @@ namespace StgSharp.HighPerformance { [FieldOffset(0)]private fixed byte _buffer[64]; + [FieldOffset(0)]private Vector512 Vec; public ref T AsRef() where T: unmanaged, INumber { return ref Unsafe.As(ref _buffer[0]); } + public void BroadCastFrom(T value) where T: unmanaged, INumber + { + Vec = Vector512.CreateScalarUnsafe(value).As(); + } + public ref T Member(int index) where T: unmanaged, INumber { return ref Unsafe.As(ref _buffer[0]); diff --git a/StgSharp.Common/HighPerformance/IRegisterPresentation.cs b/StgSharp.Common/HighPerformance/ScalarPacket.cs similarity index 66% rename from StgSharp.Common/HighPerformance/IRegisterPresentation.cs rename to StgSharp.Common/HighPerformance/ScalarPacket.cs index 9f92daf5..cc03fecd 100644 --- a/StgSharp.Common/HighPerformance/IRegisterPresentation.cs +++ b/StgSharp.Common/HighPerformance/ScalarPacket.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="IRegisterPresentation" +// file="ScalarPacket" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -29,17 +29,36 @@ using System; using System.Collections.Generic; using System.Linq; using System.Numerics; +using System.Runtime.CompilerServices; using System.Text; using System.Threading.Tasks; namespace StgSharp.HighPerformance { - public interface IRegisterPresentation + public unsafe struct ScalarPacket { - public ref T AsRef() where T: unmanaged, INumber; + private fixed ulong data[8]; - public ref T Member(int index) where T: unmanaged, INumber; + public void BroadCast(int slotIndex, TScalar value) + where TScalar: unmanaged, INumber + where TRegister: unmanaged, IRegisterPresentation + { + if (slotIndex is < 0 or >= 8) { + throw new IndexOutOfRangeException(); + } + ref TRegister register = ref Unsafe.As(ref data[slotIndex]); + register.BroadCastFrom(value); + } + + public ref T Data(int index) where T: unmanaged, INumber + { + if (index is < 0 or >= 8) { + throw new IndexOutOfRangeException(); + } + + return ref Unsafe.As(ref data[index]); + } } } diff --git a/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs b/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs index 9600d3d2..ed08ad24 100644 --- a/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs +++ b/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs @@ -26,6 +26,7 @@ // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- using StgSharp.HighPerformance; +using System; using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -38,8 +39,9 @@ namespace StgSharp.Internal.Intrinsic public delegate* unmanaged[Cdecl] city_hash_simplify; public delegate* unmanaged[Cdecl] f32_add; + public delegate* unmanaged[Cdecl] f32_fill; public delegate* unmanaged[Cdecl] f32_fma; - public delegate* unmanaged[Cdecl] f32_scalar_mul; + public delegate* unmanaged[Cdecl] f32_scalar_mul; public delegate* unmanaged[Cdecl] f32_sub; public delegate* unmanaged[Cdecl] f32_transpose; public delegate* unmanaged[Cdecl] factorial_simd; @@ -52,4 +54,4 @@ namespace StgSharp.Internal.Intrinsic } } -} +} \ No newline at end of file diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs index 5dd527ac..b7599d40 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs @@ -31,6 +31,7 @@ using StgSharp.Internal; using System; using System.Collections.Generic; using System.Linq; +using System.Numerics; using System.Security.AccessControl; using System.Text; using System.Threading.Tasks; @@ -59,23 +60,21 @@ namespace StgSharp.Mathematics.Numeric } if (count <= 1024) { - ReadOnlySpan pool = MatrixParallel.LeadParallel(1); - if (pool.Length == 0) - { - /* - * wait - */ - } + MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1); + MatrixKernel* leftPtr = left.Buffer; MatrixKernel* rightPtr = right.Buffer; MatrixKernel* ansPtr = ans.Buffer; for (int i = 0; i < count; i++) { NativeIntrinsic.Intrinsic.f32_add(leftPtr + i, rightPtr + i, ansPtr + i); } + MatrixParallel.ReturnParallelResources(ref pool); return ans; } - MatrixParallelHandle taskGroup = MatrixParallel.PublicTask(left.GetColumnEnumeration(), right.GetColumnEnumeration(), ans.GetColumnEnumeration(), - (delegate**, MatrixKernel*, MatrixKernel*, void>)NativeIntrinsic.Intrinsic.f32_add); + MatrixParallelHandle taskGroup = MatrixParallel.PublicTask(left.GetColumnEnumeration(), + right.GetColumnEnumeration(), + ans.GetColumnEnumeration(), + (delegate**, MatrixKernel*, MatrixKernel*, void>)NativeIntrinsic.Intrinsic.f32_add); MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); /* @@ -85,5 +84,37 @@ namespace StgSharp.Mathematics.Numeric return ans; } + public static unsafe void Fill(Matrix left, T value) where T: unmanaged, INumber + { + long count = (long)left.KernelColumnLength * left.KernelRowLength; + ScalarPacket* scaler = MatrixParallelFactory.CreateScalarPacket(); + scaler->BroadCast(0, value); + if (count <= 256) + { + MatrixKernel* leftPtr = left.Buffer; + for (int i = 0; i < count; i++) { + NativeIntrinsic.Intrinsic.f32_fill(leftPtr + i, scaler); + } + } + if (count <= 1024) + { + MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1); + + MatrixKernel* leftPtr = left.Buffer; + for (int i = 0; i < count; i++) { + NativeIntrinsic.Intrinsic.f32_fill(leftPtr + i, scaler); + } + MatrixParallel.ReturnParallelResources(ref pool); + } + MatrixParallelHandle taskGroup = MatrixParallel.PublicTask(left.GetColumnEnumeration(), + (delegate**, MatrixKernel*, MatrixKernel*, void>)NativeIntrinsic.Intrinsic.f32_add); + + MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + /* + * run parallel + */ + taskGroup.Dispose(); + } + } } diff --git a/StgSharp.Mathematics/Numeric/MatrixParallel.Await.cs b/StgSharp.Mathematics/Numeric/MatrixParallel.Await.cs index f8dab3b0..0f8ed03f 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallel.Await.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallel.Await.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="MatrixParallel.Await" // Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -56,7 +56,16 @@ namespace StgSharp.Mathematics.Numeric min, recommendCount ); - _awaitingLeaderHandle.Enqueue(handle); + do + { + if (Interlocked.Exchange(ref _sortState, 1) == 0) + { + _awaitingLeaderHandle.Enqueue(handle); + _ = Interlocked.Exchange(ref _sortState, 0); + Interlocked.Increment(ref _createGeneration); + break; + } + } while (true); int cycleGeneration = _createGeneration; do { @@ -66,7 +75,7 @@ namespace StgSharp.Mathematics.Numeric { if (Interlocked.Exchange(ref _sortState, 1) == 0) { - if (_awaitingLeaderHandle.TryPopIfRefEqual(cycleGeneration, ref handle)) + if (_awaitingLeaderHandle.TryPopIfRefEqual(cycleGeneration, in handle)) { // Sort is implemented in TryPopIfRefEqual ret = _threads.PopRange(handle.BestCount); @@ -101,7 +110,16 @@ namespace StgSharp.Mathematics.Numeric public static void ReturnParallelResources(ref MatrixParallelThread[] pool) { - _threads.PushRange(pool); + while (true) + { + if (Interlocked.Exchange(ref _sortState, 1) == 0) + { + _threads.PushRange(pool); + Interlocked.Exchange(ref _sortState, 0); + break; + } + } + pool = null!; } diff --git a/StgSharp.Mathematics/Numeric/MatrixParallel.AwaitingQueue.cs b/StgSharp.Mathematics/Numeric/MatrixParallel.AwaitingQueue.cs index bc4b6cfa..3988ea09 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallel.AwaitingQueue.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallel.AwaitingQueue.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="MatrixParallel.AwaitingQueue" // Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -47,7 +47,6 @@ namespace StgSharp.Mathematics.Numeric private Element _quickElements = new(); private volatile int _count; - private volatile int _lastGeneration; private readonly Queue _fallback = []; public AwaitingQueue() { } @@ -68,22 +67,16 @@ namespace StgSharp.Mathematics.Numeric { if (_count == 0) { - _lastGeneration = generation; handle = null!; return false; } - if (_lastGeneration == generation) - { - for (int i = 0; i < _count; i++) { - _quickElements[i].UpdatePriority(generation); - } - Element.Sort(_quickElements); - } else - { - _lastGeneration = generation; + for (int i = 0; i < _count; i++) { + _quickElements[i].UpdatePriority(generation); } + Element.Sort(ref _quickElements); handle = _quickElements[0]; _count--; + _quickElements[0] = default!; if (_count > 7) { _ = _fallback.TryDequeue(out _quickElements[0]!); } @@ -94,40 +87,31 @@ namespace StgSharp.Mathematics.Numeric { if (_count == 0) { - _lastGeneration = generation; handle = null!; return false; } for (int i = 0; i < _count; i++) { _quickElements[i].UpdatePriority(generation); } - Element.Sort(_quickElements); - _lastGeneration = generation; + + Element.Sort(ref _quickElements); handle = _quickElements[0]; - _count--; - if (_count > 7) { - _ = _fallback.TryDequeue(out _quickElements[0]!); - } return true; } - public bool TryPopIfRefEqual(int generation, ref WaitHandle handle) + public bool TryPopIfRefEqual(int generation, in WaitHandle handle) { - if (_count == 0) - { - handle = null!; + if (_count == 0) { return false; } for (int i = 0; i < _count; i++) { _quickElements[i].UpdatePriority(generation); } Element.Sort(ref _quickElements); - _lastGeneration = generation; - handle = _quickElements[0]; - _count--; if (!ReferenceEquals(_quickElements[0], handle)) { return false; } + _count--; _quickElements[0] = null!; if (_count > 7) { _ = _fallback.TryDequeue(out _quickElements[0]!); @@ -147,7 +131,7 @@ namespace StgSharp.Mathematics.Numeric { int maxIdx = 0; WaitHandle first = e[0]; - int maxPr = first?.Priority ?? int.MaxValue; + int maxPr = first?.Priority ?? int.MinValue; for (int i = 1; i < 8; i++) { diff --git a/StgSharp.Mathematics/Numeric/MatrixParallel.cs b/StgSharp.Mathematics/Numeric/MatrixParallel.cs index 5b830baa..2d48becf 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallel.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallel.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="MatrixParallel" // Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -40,7 +40,7 @@ namespace StgSharp.Mathematics.Numeric private static bool _meaningful; - private static ConcurrentCapacityFixedStack _threads; + private static CapacityFixedStack _threads; private static volatile int RemainThreadCount = 0; private static readonly object _lock = new(); diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelHandle.cs b/StgSharp.Mathematics/Numeric/MatrixParallelHandle.cs index ade862d8..2de055e7 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelHandle.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelHandle.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="MatrixParallelTask" +// file="MatrixParallelHandle" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -34,22 +34,6 @@ using System.Runtime.InteropServices; namespace StgSharp.Mathematics.Numeric { - public unsafe struct ScalarPacket - { - - private fixed ulong data[8]; - - public ref T Data(int index) where T: unmanaged,INumber - { - if (index is < 0 or >= 8) { - throw new IndexOutOfRangeException(); - } - - return ref Unsafe.As(ref data[index]); - } - - } - [StructLayout(LayoutKind.Sequential)] internal unsafe struct MatrixParallelTaskPackageNonGeneric { @@ -59,11 +43,8 @@ namespace StgSharp.Mathematics.Numeric MatrixParallelTaskPackageNonGeneric* source, MatrixParallelTaskPackageNonGeneric* target) { - Buffer.MemoryCopy( - source, - target, - sizeof(MatrixParallelTaskPackageNonGeneric), - sizeof(MatrixParallelTaskPackageNonGeneric)); + Buffer.MemoryCopy(source, target, sizeof(MatrixParallelTaskPackageNonGeneric), + sizeof(MatrixParallelTaskPackageNonGeneric)); } #region source @@ -148,11 +129,8 @@ namespace StgSharp.Mathematics.Numeric [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe void Copy(MatrixParallelTaskPackage* source, MatrixParallelTaskPackage* target) { - Buffer.MemoryCopy( - source, - target, - sizeof(MatrixParallelTaskPackage), - sizeof(MatrixParallelTaskPackage)); + Buffer.MemoryCopy(source, target, sizeof(MatrixParallelTaskPackage), + sizeof(MatrixParallelTaskPackage)); } #region source diff --git a/StgSharp.Native/StgSharp.Native.vcxproj b/StgSharp.Native/StgSharp.Native.vcxproj index a52249ca..63c9665c 100644 --- a/StgSharp.Native/StgSharp.Native.vcxproj +++ b/StgSharp.Native/StgSharp.Native.vcxproj @@ -109,7 +109,9 @@ - + + CppCode + diff --git a/StgSharp.Native/StgSharp.Native.vcxproj.filters b/StgSharp.Native/StgSharp.Native.vcxproj.filters index e71f202f..79996cf6 100644 --- a/StgSharp.Native/StgSharp.Native.vcxproj.filters +++ b/StgSharp.Native/StgSharp.Native.vcxproj.filters @@ -24,9 +24,6 @@ 源文件 - - 源文件 - 源文件 @@ -51,6 +48,9 @@ 源文件 + + 源文件 + diff --git a/StgSharp.Native/sn-head/sn_intrinsic.h b/StgSharp.Native/sn-head/sn_intrinsic.h index cd6f263f..9444c8ce 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.h @@ -31,12 +31,7 @@ typedef union uint64_u { uint32_t u32[2]; } uint64_u; -typedef union mat_kernel { - __m128 xmm[4]; - __m256 ymm[2]; - __m512 zmm[1]; - float m[4][4]; -} mat_kernel; +typedef struct mat_kernel mat_kernel; typedef union scalar_pack { uint64_t data[8]; @@ -44,13 +39,10 @@ typedef union scalar_pack { typedef void(SN_DECL *VECTORNORMALIZEPROC)(__m128 *source, __m128 *target); typedef void(SN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans); -typedef void(SN_DECL *MATRIXTRANSPOSEPROC)(void *source, void *target); -typedef void(SN_DECL *MATRIXDETPROC)(void *mat, void *transpose); -typedef void(SN_DECL *MATARITHMETIC)(void *left, void *right, void *ans); -typedef void(SN_DECL *SCALERMATARITHMETIC)(void *left, void *right, void *ans); -typedef void(SN_DECL *MATFMAPROC)(mat_kernel *left, mat_kernel *right, mat_kernel *ans); -typedef void(SN_DECL *SCALARMATARITHMETIC)(mat_kernel *matrix, float scalar, mat_kernel *ans); -typedef void(SN_DECL *MATTRANSPOSEPROC)(mat_kernel *source, mat_kernel *target); +typedef void(SN_DECL *MAT_PROC_AS)(mat_kernel *ans, scalar_pack *scalar); +typedef void(SN_DECL *MAT_PROC_LA)(mat_kernel *left, mat_kernel *ans); +typedef void(SN_DECL *MAT_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans); +typedef void(SN_DECL *MAT_PROC_LSA)(mat_kernel *matrix, scalar_pack *scalar, mat_kernel *ans); typedef uint64_t(SN_DECL *FACTORIALROC)(int n); #pragma region matix function @@ -75,12 +67,18 @@ INTERNAL void SN_DECL f32_transpose_avx(MAT_KERNEL(float) const *source, MAT_KERNEL(float) *restrict target); INTERNAL void SN_DECL f32_transpose_512(MAT_KERNEL(float) const *source, MAT_KERNEL(float) *restrict target); -INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix, float const scalar, - MAT_KERNEL(float) *restrict ans); -INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix, float const scalar, - MAT_KERNEL(float) *restrict ans); -INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix, float const scalar, - MAT_KERNEL(float) *restrict ans); +INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix, + MAT_KERNEL(float) *restrict ans, + scalar_pack const *scalar); +INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix, + MAT_KERNEL(float) *restrict ans, + scalar_pack const *scalar); +INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix, + MAT_KERNEL(float) *restrict ans, + scalar_pack const *scalar); +INTERNAL void SN_DECL f32_fill_sse(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler); +INTERNAL void SN_DECL f32_fill_avx(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler); +INTERNAL void SN_DECL f32_fill_512(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler); #pragma endregion #pragma region fma @@ -88,6 +86,12 @@ INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix, float INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left, MAT_KERNEL(float) const *restrict right, MAT_KERNEL(float) *restrict ans); +INTERNAL void SN_DECL f32_fma_avx(MAT_KERNEL(float) const *restrict left, + MAT_KERNEL(float) const *restrict right, + MAT_KERNEL(float) *restrict ans); +INTERNAL void SN_DECL f32_fma_512(MAT_KERNEL(float) const *restrict left, + MAT_KERNEL(float) const *restrict right, + MAT_KERNEL(float) *restrict ans); #pragma endregion @@ -119,11 +123,12 @@ typedef struct sn_intrinsic { int (*city_hash_simplify)(byte *str, int length); FACTORIALROC factorial_simd; int (*index_pair)(short const *str, uint32_t target, int length); - MATARITHMETIC f32_add; - void (*f32_fma)(void *left, void *right, void *ans); - SCALARMATARITHMETIC f32_scalar_mul; - MATARITHMETIC f32_sub; - MATTRANSPOSEPROC f32_transpose; + MAT_PROC_LRA f32_add; + MAT_PROC_AS f32_fill; + MAT_PROC_LRA f32_fma; + MAT_PROC_LRA f32_scalar_mul; + MAT_PROC_LRA f32_sub; + MAT_PROC_LA f32_transpose; VECTORNORMALIZEPROC f32_normalize_3; } sn_intrinsic; diff --git a/StgSharp.Native/sn-src/k_basicop.c b/StgSharp.Native/sn-src/k_basicop.c index 51c484fa..31a7cda8 100644 --- a/StgSharp.Native/sn-src/k_basicop.c +++ b/StgSharp.Native/sn-src/k_basicop.c @@ -1,5 +1,27 @@ #include "sn_intrinsic.h" +INTERNAL void SN_DECL f32_fill_sse(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler) +{ + register __m128 vec = _mm_set_ps1(((float *)(scaler->data))[0]); + dest->xmm[0] = vec; + dest->xmm[1] = vec; + dest->xmm[2] = vec; + dest->xmm[3] = vec; +} + +INTERNAL void SN_DECL f32_fill_avx(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler) +{ + register __m256 vec = _mm256_broadcast_ss(((float *)(scaler->data))); + dest->ymm[0] = vec; + dest->ymm[1] = vec; +} + +INTERNAL void SN_DECL f32_fill_512(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler) +{ + register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scaler->data))[0]); + dest->zmm[0] = vec; +} + INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, MAT_KERNEL(float) *restrict ans) { @@ -106,11 +128,10 @@ INTERNAL void SN_DECL f32_transpose_512(MAT_KERNEL(float) const *source, target->zmm[0] = _mm512_permutexvar_ps(transpose_indices, matrix); } -INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix, float const scalar, - MAT_KERNEL(float) *restrict ans) +INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix, + MAT_KERNEL(float) *restrict ans, scalar_pack const *scalar) { - register __m128 temp = _mm_load_ss(&scalar); - register __m128 scalar_vec = _mm_shuffle_ps(temp, temp, _MM_SHUFFLE(0, 0, 0, 0)); + register __m128 scalar_vec = ((__m128 *)(scalar->data))[0]; register __m128 c0 = _mm_mul_ps(matrix->xmm[0], scalar_vec); register __m128 c1 = _mm_mul_ps(matrix->xmm[1], scalar_vec); @@ -123,10 +144,10 @@ INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix, float ans->xmm[3] = c3; } -INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix, float const scalar, - MAT_KERNEL(float) *restrict ans) +INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix, + MAT_KERNEL(float) *restrict ans, scalar_pack const *scalar) { - register __m256 scalar_vec = _mm256_broadcast_ss(&scalar); + register __m256 scalar_vec = ((__m256 *)(scalar->data))[0]; register __m256 c0 = _mm256_mul_ps(matrix->ymm[0], scalar_vec); register __m256 c1 = _mm256_mul_ps(matrix->ymm[1], scalar_vec); @@ -135,10 +156,10 @@ INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix, float ans->ymm[1] = c1; } -INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix, float const scalar, - MAT_KERNEL(float) *restrict ans) +INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix, + MAT_KERNEL(float) *restrict ans, scalar_pack const *scalar) { - __m512 scalar_vec = _mm512_broadcastss_ps(_mm_load_ss(&scalar)); + register __m512 scalar_vec = ((__m512 *)(scalar->data))[0]; register __m512 c0 = _mm512_mul_ps(matrix->zmm[0], scalar_vec); diff --git a/StgSharp.Native/sn-src/k_fma.c b/StgSharp.Native/sn-src/k_fma.c index efed0682..2f9fd22a 100644 --- a/StgSharp.Native/sn-src/k_fma.c +++ b/StgSharp.Native/sn-src/k_fma.c @@ -1,6 +1,7 @@ #include "sn_intrinsic.h" // ans = left * right + ans +// right is not transposed INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left, MAT_KERNEL(float) const *restrict right, MAT_KERNEL(float) *restrict ans) @@ -10,34 +11,47 @@ INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left, register __m128 source2 = _mm_load_ps(&left->xmm[2]); register __m128 source3 = _mm_load_ps(&left->xmm[3]); + register __m128 ans0 = _mm_load_ps(&ans->xmm[0]); + register __m128 ans1 = _mm_load_ps(&ans->xmm[1]); + register __m128 ans2 = _mm_load_ps(&ans->xmm[2]); + register __m128 ans3 = _mm_load_ps(&ans->xmm[3]); + register __m128 broadcast; - - register __m128 ans0 = _mm_setzero_ps(); - register __m128 ans1 = _mm_setzero_ps(); - register __m128 ans2 = _mm_setzero_ps(); - register __m128 ans3 = _mm_setzero_ps(); - for (size_t i = 0; i < 3; i++) { + for (size_t i = 0; i < 4; i++) { broadcast = _mm_load_ps(&right->xmm[i]); - _mm_add_ps(ans0, _mm_mul_ps(source0, broadcast)); - _mm_add_ps(ans1, _mm_mul_ps(source1, broadcast)); - _mm_add_ps(ans2, _mm_mul_ps(source2, broadcast)); - _mm_add_ps(ans3, _mm_mul_ps(source3, broadcast)); - for (size_t t = 0; t < 2; t++) { - broadcast = _mm_shuffle_ps(broadcast, broadcast, _MM_SHUFFLE(2, 1, 0, 3)); - _mm_add_ps(ans0, _mm_mul_ps(source0, broadcast)); - _mm_add_ps(ans1, _mm_mul_ps(source1, broadcast)); - _mm_add_ps(ans2, _mm_mul_ps(source2, broadcast)); - _mm_add_ps(ans3, _mm_mul_ps(source3, broadcast)); - } + ans0 = _mm_add_ps(ans0, + _mm_mul_ps(source0, _mm_shuffle_ps(broadcast, broadcast, 0x00))); + ans1 = _mm_add_ps(ans1, + _mm_mul_ps(source1, _mm_shuffle_ps(broadcast, broadcast, 0x55))); + ans2 = _mm_add_ps(ans2, + _mm_mul_ps(source2, _mm_shuffle_ps(broadcast, broadcast, 0xAA))); + ans3 = _mm_add_ps(ans3, + _mm_mul_ps(source3, _mm_shuffle_ps(broadcast, broadcast, 0xFF))); } - source0 = _mm_load_ps(&ans->xmm[0]); - source1 = _mm_load_ps(&ans->xmm[1]); - source2 = _mm_load_ps(&ans->xmm[2]); - source3 = _mm_load_ps(&ans->xmm[3]); - _mm_store_ps(&ans->xmm[0], _mm_add_ps(source0, ans0)); - _mm_store_ps(&ans->xmm[1], _mm_add_ps(source1, ans1)); - _mm_store_ps(&ans->xmm[2], _mm_add_ps(source2, ans2)); - _mm_store_ps(&ans->xmm[3], _mm_add_ps(source3, ans3)); + _mm_store_ps(&ans->xmm[0], ans0); + _mm_store_ps(&ans->xmm[1], ans1); + _mm_store_ps(&ans->xmm[2], ans2); + _mm_store_ps(&ans->xmm[3], ans3); +} + +// ans = left * right + ans +INTERNAL void SN_DECL f32_fma_avx(MAT_KERNEL(float) const *restrict left, + MAT_KERNEL(float) const *restrict right, + MAT_KERNEL(float) *restrict ans) +{ + /* + * to be implemented + */ +} + +// ans = left * right + ans +INTERNAL void SN_DECL f32_fma_512(MAT_KERNEL(float) const *restrict left, + MAT_KERNEL(float) const *restrict right, + MAT_KERNEL(float) *restrict ans) +{ + /* + * to be implemented + */ } diff --git a/StgSharp.Native/sn-src/load_intrinsic.c b/StgSharp.Native/sn-src/load_intrinsic.c index aad0134e..ffb88195 100644 --- a/StgSharp.Native/sn-src/load_intrinsic.c +++ b/StgSharp.Native/sn-src/load_intrinsic.c @@ -65,11 +65,11 @@ void load_intrinsic_sse(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; - context->f32_add = (MATARITHMETIC)f32_add_sse; - context->f32_fma = (void (*)(void *, void *, void *))f32_fma_sse; + context->f32_add = (MAT_PROC_LRA)f32_add_sse; + context->f32_fma = (MAT_PROC_LRA)f32_fma_sse; context->f32_scalar_mul = f32_scalar_mul_sse; - context->f32_sub = (MATARITHMETIC)f32_sub_sse; - context->f32_transpose = (MATTRANSPOSEPROC)f32_transpose_sse; + context->f32_sub = (MAT_PROC_LRA)f32_sub_sse; + context->f32_transpose = (MAT_PROC_LA)f32_transpose_sse; context->f32_normalize_3 = f32_normalize; } @@ -78,11 +78,11 @@ void load_intrinsic_avx(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; - context->f32_add = (MATARITHMETIC)f32_add_avx; - context->f32_fma = (void (*)(void *, void *, void *))f32_fma_sse; + context->f32_add = (MAT_PROC_LRA)f32_add_avx; + context->f32_fma = (MAT_PROC_LRA)f32_fma_avx; context->f32_scalar_mul = f32_scalar_mul_avx; - context->f32_sub = (MATARITHMETIC)f32_sub_avx; - context->f32_transpose = (MATTRANSPOSEPROC)f32_transpose_avx; + context->f32_sub = (MAT_PROC_LRA)f32_sub_avx; + context->f32_transpose = (MAT_PROC_LA)f32_transpose_avx; context->f32_normalize_3 = f32_normalize; } @@ -91,10 +91,10 @@ void load_intrinsic_512(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; - context->f32_add = (MATARITHMETIC)f32_add_512; - context->f32_fma = (void (*)(void *, void *, void *))f32_fma_sse; + context->f32_add = (MAT_PROC_LRA)f32_add_512; + context->f32_fma = (MAT_PROC_LRA)f32_fma_512; context->f32_scalar_mul = f32_scalar_mul_512; - context->f32_sub = (MATARITHMETIC)f32_sub_512; - context->f32_transpose = (MATTRANSPOSEPROC)f32_transpose_512; + context->f32_sub = (MAT_PROC_LRA)f32_sub_512; + context->f32_transpose = (MAT_PROC_LA)f32_transpose_512; context->f32_normalize_3 = f32_normalize; }