youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fma code added

Browse Source
This commit is contained in:
Youmu Liu
2025-12-09 18:07:53 +08:00
parent 6acc8629b8
commit 7bcb460a39
7 changed files with 230 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/Mathematics/Graphic/Camera.cs
+2 -2
View File
@@ -153,10 +153,10 @@ namespace StgSharp.Mathematics.Graphic
if (direction.GetLength() == 0) if (direction.GetLength() == 0)
{ {
// deadlock // deadlock
throw new ArgumentException(); throw new ArgumentException("Direction bust is not zero.");
} }
if (Vec3.IsParallel(direction, up)) { if (Vec3.IsParallel(direction, up)) {
throw new ArgumentException(); throw new ArgumentException("Direction and UP is on on same way");
} }
_target = direction; _target = direction;
this.up = up; this.up = up;
StgSharp.Common/Mathematics/Graphic/Matrix44.cs
+25 -65
View File
@@ -36,29 +36,24 @@ using System.Runtime.InteropServices;
namespace StgSharp.Mathematics.Graphic namespace StgSharp.Mathematics.Graphic
{ {
[CollectionBuilder(builderType:typeof(Matrix44), methodName: nameof(Matrix44.FromSpan))]
[StructLayout(LayoutKind.Explicit, Size = (2 * 16 * sizeof(float)) + sizeof(bool), Pack = 16)] [StructLayout(LayoutKind.Explicit, Size = (2 * 16 * sizeof(float)) + sizeof(bool), Pack = 16)]
public unsafe struct Matrix44 : IEquatable<Matrix44>, IMatrix<float>, IEnumerable<Vec4> public unsafe struct Matrix44 : IEquatable<Matrix44>, IMatrix<float>, IEnumerable<Vec4>
{ {
[FieldOffset(2 * 64 * sizeof(float))] internal bool isTransposed;
[FieldOffset(0 * sizeof(float))] internal Column column; [FieldOffset(0 * sizeof(float))] internal Column column;
[FieldOffset(0)] internal ColumnSet4 mat; [FieldOffset(0)] internal ColumnSet4 mat;
[FieldOffset(16 * sizeof(float))] internal ColumnSet4 transpose;
[FieldOffset(0)] internal MatrixKernel<float> kernel; [FieldOffset(0)] internal MatrixKernel<float> kernel;
internal Matrix44(ColumnSet4 mat) internal Matrix44(ColumnSet4 mat)
{ {
Unsafe.SkipInit(out this); Unsafe.SkipInit(out this);
isTransposed = false;
this.mat = mat; this.mat = mat;
} }
internal Matrix44(Vector4 vec0, Vector4 vec1, Vector4 vec2, Vector4 vec3) internal Matrix44(Vector4 vec0, Vector4 vec1, Vector4 vec2, Vector4 vec3)
{ {
Unsafe.SkipInit(out this); Unsafe.SkipInit(out this);
isTransposed = false;
mat.colum0 = vec0; mat.colum0 = vec0;
mat.colum1 = vec1; mat.colum1 = vec1;
mat.colum2 = vec2; mat.colum2 = vec2;
@@ -68,7 +63,6 @@ namespace StgSharp.Mathematics.Graphic
public Matrix44() public Matrix44()
{ {
Unsafe.SkipInit(out this); Unsafe.SkipInit(out this);
isTransposed = false;
} }
public unsafe Matrix44( public unsafe Matrix44(
@@ -90,7 +84,6 @@ namespace StgSharp.Mathematics.Graphic
float a33) float a33)
{ {
Unsafe.SkipInit(out this); Unsafe.SkipInit(out this);
isTransposed = false;
mat.colum0 = new Vector4(a00, a10, a20, a30); mat.colum0 = new Vector4(a00, a10, a20, a30);
mat.colum1 = new Vector4(a01, a11, a21, a31); mat.colum1 = new Vector4(a01, a11, a21, a31);
mat.colum2 = new Vector4(a02, a12, a22, a32); mat.colum2 = new Vector4(a02, a12, a22, a32);
@@ -110,12 +103,7 @@ namespace StgSharp.Mathematics.Graphic
throw new ArgumentOutOfRangeException(nameof(columNum)); throw new ArgumentOutOfRangeException(nameof(columNum));
} }
#endif #endif
InternalTranspose(); return kernel[rowNum + 4 * columNum];
fixed (float* p = &transpose.m00)
{
ulong pBit = (ulong)p + (ulong)sizeof(Vector4) * (ulong)rowNum + sizeof(float) * (ulong)columNum;
return *(float*)pBit;
}
} }
set set
{ {
@@ -128,13 +116,7 @@ namespace StgSharp.Mathematics.Graphic
throw new ArgumentOutOfRangeException(nameof(columNum)); throw new ArgumentOutOfRangeException(nameof(columNum));
} }
#endif #endif
InternalTranspose(); kernel[rowNum + 4 * columNum] = value;
fixed (float* p = &transpose.m00)
{
ulong pbit = (ulong)p + (ulong)sizeof(Vector4) * (ulong)rowNum + sizeof(float) * (ulong)columNum;
*(float*)pbit = value;
}
isTransposed = false;
} }
} }
@@ -142,8 +124,9 @@ namespace StgSharp.Mathematics.Graphic
{ {
get get
{ {
InternalTranspose(); Matrix44 transpose = new();
return new Matrix44(transpose); InternalTranspose(ref transpose);
return transpose;
} }
} }
@@ -176,42 +159,18 @@ namespace StgSharp.Mathematics.Graphic
return mat.Equals(other.mat); return mat.Equals(other.mat);
} }
public static Matrix44 FromSpan(ReadOnlySpan<Vec4> rowSpan)
{
Matrix44 mat = new Matrix44();
mat.transpose.colum0 = rowSpan[0].vec;
mat.transpose.colum1 = rowSpan[1].vec;
mat.transpose.colum2 = rowSpan[2].vec;
mat.transpose.colum3 = rowSpan[3].vec;
NativeIntrinsic.Intrinsic.f32_buffer_transpose(&mat.transpose, &mat.mat);
mat.isTransposed = false;
return mat;
}
public override int GetHashCode() public override int GetHashCode()
{ {
return HashCode.Combine(mat.colum0.GetHashCode(), mat.colum1.GetHashCode(), return HashCode.Combine(mat.colum0.GetHashCode(), mat.colum1.GetHashCode(),
mat.colum2.GetHashCode(), mat.colum3.GetHashCode()); mat.colum2.GetHashCode(), mat.colum3.GetHashCode());
} }
public override string ToString()
{
InternalTranspose();
return $"{transpose.colum0}{transpose.colum1}{transpose.colum2}{transpose.colum3}";
}
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
internal unsafe void InternalTranspose() internal unsafe void InternalTranspose(ref Matrix44 transpose)
{ {
if (!isTransposed) fixed (MatrixKernel<float>* source = &this.kernel, target = &transpose.kernel) {
{
fixed (ColumnSet4* source = &mat, target = &transpose) {
NativeIntrinsic.Intrinsic.f32_buffer_transpose(source, target); NativeIntrinsic.Intrinsic.f32_buffer_transpose(source, target);
} }
isTransposed = true;
}
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -237,23 +196,23 @@ namespace StgSharp.Mathematics.Graphic
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Matrix44 operator *(Matrix44 left, Matrix44 right) public static Matrix44 operator *(Matrix44 left, Matrix44 right)
{ {
left.InternalTranspose(); Matrix44 transpose = left.Transpose;
return new Matrix44(Vector4.Dot(left.transpose.colum0, right.mat.colum0), return new Matrix44(Vector4.Dot(transpose.mat.colum0, right.mat.colum0),
Vector4.Dot(left.transpose.colum0, right.mat.colum1), Vector4.Dot(transpose.mat.colum0, right.mat.colum1),
Vector4.Dot(left.transpose.colum0, right.mat.colum2), Vector4.Dot(transpose.mat.colum0, right.mat.colum2),
Vector4.Dot(left.transpose.colum0, right.mat.colum3), Vector4.Dot(transpose.mat.colum0, right.mat.colum3),
Vector4.Dot(left.transpose.colum1, right.mat.colum0), Vector4.Dot(transpose.mat.colum1, right.mat.colum0),
Vector4.Dot(left.transpose.colum1, right.mat.colum1), Vector4.Dot(transpose.mat.colum1, right.mat.colum1),
Vector4.Dot(left.transpose.colum1, right.mat.colum2), Vector4.Dot(transpose.mat.colum1, right.mat.colum2),
Vector4.Dot(left.transpose.colum1, right.mat.colum3), Vector4.Dot(transpose.mat.colum1, right.mat.colum3),
Vector4.Dot(left.transpose.colum2, right.mat.colum0), Vector4.Dot(transpose.mat.colum2, right.mat.colum0),
Vector4.Dot(left.transpose.colum2, right.mat.colum1), Vector4.Dot(transpose.mat.colum2, right.mat.colum1),
Vector4.Dot(left.transpose.colum2, right.mat.colum2), Vector4.Dot(transpose.mat.colum2, right.mat.colum2),
Vector4.Dot(left.transpose.colum2, right.mat.colum3), Vector4.Dot(transpose.mat.colum2, right.mat.colum3),
Vector4.Dot(left.transpose.colum3, right.mat.colum0), Vector4.Dot(transpose.mat.colum3, right.mat.colum0),
Vector4.Dot(left.transpose.colum3, right.mat.colum1), Vector4.Dot(transpose.mat.colum3, right.mat.colum1),
Vector4.Dot(left.transpose.colum3, right.mat.colum2), Vector4.Dot(transpose.mat.colum3, right.mat.colum2),
Vector4.Dot(left.transpose.colum3, right.mat.colum3)); Vector4.Dot(transpose.mat.colum3, right.mat.colum3));
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -291,3 +250,4 @@ namespace StgSharp.Mathematics.Graphic
} }
} }
StgSharp.Common/Mathematics/Graphic/Vec4.cs
+12 -10
View File
@@ -140,13 +140,13 @@ namespace StgSharp.Mathematics.Graphic
public static Vec4 FromSpan(ReadOnlySpan<float> span) public static Vec4 FromSpan(ReadOnlySpan<float> span)
{ {
return new Vec4 if (span.Length < 4) {
{ throw new ArgumentException("Span length must be at least 4.", nameof(span));
X = span[0], }
Y = span[1],
Z = span[2], // Read 4 floats as a Vector4 in one operation without per-element copy.
W = span[3] Vector4 v = MemoryMarshal.Read<Vector4>(MemoryMarshal.AsBytes(span));
}; return new Vec4(v);
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -185,9 +185,11 @@ namespace StgSharp.Mathematics.Graphic
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vec4 operator *(Matrix44 mat, Vec4 vec) public static Vec4 operator *(Matrix44 mat, Vec4 vec)
{ {
mat.InternalTranspose(); Matrix44 transpose = mat.Transpose;
return new Vec4(Vector4.Dot(mat.transpose.colum0, vec.vec), Vector4.Dot(mat.transpose.colum1, vec.vec), return new Vec4(Vector4.Dot(transpose.mat.colum0, vec.vec),
Vector4.Dot(mat.transpose.colum2, vec.vec), Vector4.Dot(mat.transpose.colum3, vec.vec)); Vector4.Dot(transpose.mat.colum1, vec.vec),
Vector4.Dot(transpose.mat.colum2, vec.vec),
Vector4.Dot(transpose.mat.colum3, vec.vec));
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
StgSharp.Native/sn-head/sn_intrinsic.h
+16 -11
View File
@@ -142,21 +142,25 @@ typedef void(SN_DECL *PNL_PROC_RAS)(mat_panel *right, mat_panel *ans, scalar_pac
#define PNL_PROC(T_type, T_arch, T_op_name) T_type##_pnl_##T_op_name##_##T_arch #define PNL_PROC(T_type, T_arch, T_op_name) T_type##_pnl_##T_op_name##_##T_arch
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \ #define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \ INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \ T_op_name)(MAT_PANEL_##T_arch##(T_type) const *left, \
MAT_PANEL##T_arch##(T_type) *restrict ans) MAT_PANEL_##T_arch##(T_type) const *right, \
MAT_PANEL_##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \ #define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \ INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \ T_op_name)(MAT_PANEL_##T_arch##(T_type) const *left, \
MAT_PANEL##T_arch##(T_type) *restrict ans) MAT_PANEL_##T_arch##(T_type) const *right, \
MAT_PANEL_##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ #define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \ INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \ MAT_PANEL_##T_arch##(T_type) const *left, \
MAT_PANEL##T_arch##(T_type) *restrict ans, scalar_pack const *scalar) MAT_PANEL_##T_arch##(T_type) const *right, \
MAT_PANEL_##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ #define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \ INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \ MAT_PANEL_##T_arch##(T_type) const *left, \
MAT_PANEL##T_arch##(T_type) *restrict ans, scalar_pack const *scalar) MAT_PANEL_##T_arch##(T_type) const *right, \
MAT_PANEL_##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)
#pragma endregion #pragma endregion
@@ -219,7 +223,8 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill);
#pragma region fma #pragma region fma
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, fma); DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma);
#pragma endregion #pragma endregion
StgSharp.Native/sn-head/sn_matkernel.h
+9 -5
View File
@@ -67,13 +67,17 @@ typedef MAT_KERNEL(uint64_t) sn_zero_kernel_u64;
/* SSE panel layout: column-major buffer holding 4 logical columns with MINQ=4. /* SSE panel layout: column-major buffer holding 4 logical columns with MINQ=4.
Columns: each column packs up to four scalars so ALIGN equals sizeof(__m128). Columns: each column packs up to four scalars so ALIGN equals sizeof(__m128).
Alignment: every column boundary lands on a 16-byte multiple for XMM loads. */ Alignment: every column boundary lands on a 16-byte multiple for XMM loads. */
#define MAT_PANEL_SSE(T) SN_ALIGNAS(16) MAT_PANEL_SQ_MIN_NOALIGN(T, __m128, 4) #define MAT_PANEL_sse(T) SN_ALIGNAS(16) MAT_PANEL_SQ_MIN_NOALIGN(T, __m128, 4)
/* AVX2 panel layout: column-major buffer holding 8 logical columns with MINQ=8. /* AVX2 panel layout: column-major buffer holding 8 logical columns with MINQ=8.
Columns: each column stores eight elements via two 256-bit chunks when T is 32-bit. Columns: each column stores eight elements via two 256-bit chunks when T is 32-bit.
Alignment: column starts are 32-byte aligned so AVX2 loads/stores remain contiguous. */ Alignment: column starts are 32-byte aligned so AVX2 loads/stores remain contiguous. */
#define MAT_PANEL_AVX(T) SN_ALIGNAS(32) MAT_PANEL_SQ_MIN_NOALIGN(T, __m256, 8) #define MAT_PANEL_avx(T) SN_ALIGNAS(32) MAT_PANEL_SQ_MIN_NOALIGN(T, __m256, 8)
/* AVX2 panel layout: column-major buffer holding 8 logical columns with MINQ=8.
Columns: each column stores eight elements via two 256-bit chunks when T is 32-bit.
Alignment: column starts are 32-byte aligned so AVX2 loads/stores remain contiguous. */
#define MAT_PANEL_avx_fma(T) SN_ALIGNAS(32) MAT_PANEL_SQ_MIN_NOALIGN(T, __m256, 8)
/* AVX-512 panel layout: column-major buffer holding 8 or more columns depending on T. /* AVX-512 panel layout: column-major buffer holding 8 or more columns depending on T.
Columns: each column expands to ceil(Q/lanes) ZMM chunks; default MINQ=8 keeps double-wide tiles. Columns: each column expands to ceil(Q/lanes) ZMM chunks; default MINQ=8 keeps doublse-wide tiles.
Alignment: column starts sit on 64-byte boundaries to match AVX-512 streaming access. */ Alignment: column starts sit on 64-byte boundaries to match AVX-512 streaming access. */
#define MAT_PANEL_512(T) SN_ALIGNAS(64) MAT_PANEL_SQ_MIN_NOALIGN(T, __m512, 8) #define MAT_PANEL_512(T) SN_ALIGNAS(64) MAT_PANEL_SQ_MIN_NOALIGN(T, __m512, 8)
@@ -113,7 +117,7 @@ extern __declspec(align(64)) sn_zero_kernel_u64 SN_ZERO_KERNEL;
int col_index, int row_index) int col_index, int row_index)
/* Declarations for AVX float variant */ /* Declarations for AVX float variant */
DECLARE_BUILD_PANEL(AVX, float); DECLARE_BUILD_PANEL(avx, float);
DECLARE_STORE_PANEL(AVX, float); DECLARE_STORE_PANEL(avx, float);
#endif /* SN_MATKERNEL */ #endif /* SN_MATKERNEL */
StgSharp.Native/sn-src/k_fma.c
+151 -24
View File
@@ -1,35 +1,162 @@
#include "sn_intrinsic.h" #include "sn_intrinsic.h"
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
// ans = left * right + ans // ans = left * right + ans
// right is not transposed // right is not transposed
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, fma) DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma)
{ {
register __m128 source0 = _mm_load_ps(&left->xmm[0]); const __m128 a0 = PANEL_COL_VEC((*left), 0, 0);
register __m128 source1 = _mm_load_ps(&left->xmm[1]); const __m128 a1 = PANEL_COL_VEC((*left), 1, 0);
register __m128 source2 = _mm_load_ps(&left->xmm[2]); const __m128 a2 = PANEL_COL_VEC((*left), 2, 0);
register __m128 source3 = _mm_load_ps(&left->xmm[3]); const __m128 a3 = PANEL_COL_VEC((*left), 3, 0);
register __m128 ans0 = _mm_load_ps(&ans->xmm[0]); __m128 c0 = PANEL_COL_VEC((*ans), 0, 0);
register __m128 ans1 = _mm_load_ps(&ans->xmm[1]); __m128 c1 = PANEL_COL_VEC((*ans), 1, 0);
register __m128 ans2 = _mm_load_ps(&ans->xmm[2]); __m128 c2 = PANEL_COL_VEC((*ans), 2, 0);
register __m128 ans3 = _mm_load_ps(&ans->xmm[3]); __m128 c3 = PANEL_COL_VEC((*ans), 3, 0);
register __m128 broadcast; __m128 bcol;
for (size_t i = 0; i < 4; i++) {
broadcast = _mm_load_ps(&right->xmm[i]);
ans0 = _mm_add_ps(ans0, /* Column 0 */
_mm_mul_ps(source0, _mm_shuffle_ps(broadcast, broadcast, 0x00))); bcol = PANEL_COL_VEC((*right), 0, 0);
ans1 = _mm_add_ps(ans1, c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
_mm_mul_ps(source1, _mm_shuffle_ps(broadcast, broadcast, 0x55))); c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
ans2 = _mm_add_ps(ans2, c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
_mm_mul_ps(source2, _mm_shuffle_ps(broadcast, broadcast, 0xAA))); c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
ans3 = _mm_add_ps(ans3,
_mm_mul_ps(source3, _mm_shuffle_ps(broadcast, broadcast, 0xFF))); /* Column 1 */
bcol = PANEL_COL_VEC((*right), 1, 0);
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = PANEL_COL_VEC((*right), 2, 0);
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = PANEL_COL_VEC((*right), 3, 0);
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
PANEL_COL_VEC((*ans), 0, 0) = c0;
PANEL_COL_VEC((*ans), 1, 0) = c1;
PANEL_COL_VEC((*ans), 2, 0) = c2;
PANEL_COL_VEC((*ans), 3, 0) = c3;
} }
_mm_store_ps(&ans->xmm[0], ans0); // ans = left * right + ans
_mm_store_ps(&ans->xmm[1], ans1); // right is not transposed
_mm_store_ps(&ans->xmm[2], ans2); DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma)
_mm_store_ps(&ans->xmm[3], ans3); {
static const __m256i idx0 = { 1, 1, 1, 1, 1, 1, 1, 1 };
for (int col = 0; col < 8; ++col) {
register __m256 bcol = _mm256_load_ps((float *)&right->vec[col][0]);
register __m256 ccol = _mm256_load_ps((float *)&ans->vec[col][0]);
register __m256i idx = _mm256_setzero_si256();
register __m256i increament = _mm256_load_si256(&idx0);
register __m256 a0 = _mm256_load_ps((float *)&left->vec[0][0]);
register __m256 a1 = _mm256_load_ps((float *)&left->vec[1][0]);
register __m256 a2 = _mm256_load_ps((float *)&left->vec[2][0]);
register __m256 a3 = _mm256_load_ps((float *)&left->vec[3][0]);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a0, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a1, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a2, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a3, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
a0 = _mm256_load_ps((float *)&left->vec[4][0]);
a1 = _mm256_load_ps((float *)&left->vec[5][0]);
a2 = _mm256_load_ps((float *)&left->vec[6][0]);
a3 = _mm256_load_ps((float *)&left->vec[7][0]);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a0, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a1, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a2, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a3, _mm256_permutevar8x32_ps(bcol, idx)));
_mm256_store_ps((float *)&ans->vec[col][0], ccol);
}
}
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, fma)
{
const __m512i ones = _mm512_set1_epi32(1);
for (int col = 0; col < 8; ++col) {
register __m512 bcol = _mm512_load_ps((float *)&right->vec[col][0]);
register __m512 ccol = _mm512_load_ps((float *)&ans->vec[col][0]);
register __m512i idx = _mm512_setzero_si512();
register __m512i increament = _mm512_load_si512(&ones);
register __m512 a0 = _mm512_load_ps((float *)&left->vec[0][0]);
register __m512 a1 = _mm512_load_ps((float *)&left->vec[1][0]);
register __m512 a2 = _mm512_load_ps((float *)&left->vec[2][0]);
register __m512 a3 = _mm512_load_ps((float *)&left->vec[3][0]);
register __m512 a4 = _mm512_load_ps((float *)&left->vec[4][0]);
register __m512 a5 = _mm512_load_ps((float *)&left->vec[5][0]);
register __m512 a6 = _mm512_load_ps((float *)&left->vec[6][0]);
register __m512 a7 = _mm512_load_ps((float *)&left->vec[7][0]);
ccol = _mm512_fmadd_ps(a0, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a1, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a2, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a3, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a4, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a5, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a6, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a7, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
a0 = _mm512_load_ps((float *)&left->vec[8][0]);
a1 = _mm512_load_ps((float *)&left->vec[9][0]);
a2 = _mm512_load_ps((float *)&left->vec[10][0]);
a3 = _mm512_load_ps((float *)&left->vec[11][0]);
a4 = _mm512_load_ps((float *)&left->vec[12][0]);
a5 = _mm512_load_ps((float *)&left->vec[13][0]);
a6 = _mm512_load_ps((float *)&left->vec[14][0]);
a7 = _mm512_load_ps((float *)&left->vec[15][0]);
ccol = _mm512_fmadd_ps(a0, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a1, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a2, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a3, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a4, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a5, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a6, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a7, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
_mm512_store_ps((float *)&ans->vec[col][0], ccol);
}
} }
StgSharp.Native/sn-src/k_panel.c
+2 -2
View File
@@ -1,6 +1,6 @@
#include "sn_matkernel.h" #include "sn_matkernel.h"
DECLARE_BUILD_PANEL(AVX, float) DECLARE_BUILD_PANEL(avx, float)
{ {
const int Q = 8; /* fixed for AVX float panel */ const int Q = 8; /* fixed for AVX float panel */
@@ -41,7 +41,7 @@ DECLARE_BUILD_PANEL(AVX, float)
} }
} }
DECLARE_STORE_PANEL(AVX, float) DECLARE_STORE_PANEL(avx, float)
{ {
const int Q = 8; /* fixed for AVX float panel */ const int Q = 8; /* fixed for AVX float panel */