youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fma code added

Browse Source
This commit is contained in:
Youmu Liu
2025-12-09 18:07:53 +08:00
parent 6acc8629b8
commit 7bcb460a39
7 changed files with 230 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/Mathematics/Graphic/Camera.cs
+2 -2
View File
@@ -153,10 +153,10 @@ namespace StgSharp.Mathematics.Graphic
if (direction.GetLength() == 0)
{
// deadlock
throw new ArgumentException();
throw new ArgumentException("Direction bust is not zero.");
}
if (Vec3.IsParallel(direction, up)) {
throw new ArgumentException();
throw new ArgumentException("Direction and UP is on on same way");
}
_target = direction;
this.up = up;
StgSharp.Common/Mathematics/Graphic/Matrix44.cs
+30 -70
View File
@@ -36,29 +36,24 @@ using System.Runtime.InteropServices;
namespace StgSharp.Mathematics.Graphic
{
[CollectionBuilder(builderType:typeof(Matrix44), methodName: nameof(Matrix44.FromSpan))]
[StructLayout(LayoutKind.Explicit, Size = (2 * 16 * sizeof(float)) + sizeof(bool), Pack = 16)]
public unsafe struct Matrix44 : IEquatable<Matrix44>, IMatrix<float>, IEnumerable<Vec4>
{
[FieldOffset(2 * 64 * sizeof(float))] internal bool isTransposed;
[FieldOffset(0 * sizeof(float))] internal Column column;
[FieldOffset(0)] internal ColumnSet4 mat;
[FieldOffset(16 * sizeof(float))] internal ColumnSet4 transpose;
[FieldOffset(0)] internal MatrixKernel<float> kernel;
internal Matrix44(ColumnSet4 mat)
{
Unsafe.SkipInit(out this);
isTransposed = false;
this.mat = mat;
}
internal Matrix44(Vector4 vec0, Vector4 vec1, Vector4 vec2, Vector4 vec3)
{
Unsafe.SkipInit(out this);
isTransposed = false;
mat.colum0 = vec0;
mat.colum1 = vec1;
mat.colum2 = vec2;
@@ -68,7 +63,6 @@ namespace StgSharp.Mathematics.Graphic
public Matrix44()
{
Unsafe.SkipInit(out this);
isTransposed = false;
}
public unsafe Matrix44(
@@ -90,7 +84,6 @@ namespace StgSharp.Mathematics.Graphic
float a33)
{
Unsafe.SkipInit(out this);
isTransposed = false;
mat.colum0 = new Vector4(a00, a10, a20, a30);
mat.colum1 = new Vector4(a01, a11, a21, a31);
mat.colum2 = new Vector4(a02, a12, a22, a32);
@@ -101,7 +94,7 @@ namespace StgSharp.Mathematics.Graphic
{
get
{
#if DEBUG
#if DEBUG
if (rowNum is > 3 or < 0)
{
throw new ArgumentOutOfRangeException(nameof(columNum));
@@ -109,17 +102,12 @@ namespace StgSharp.Mathematics.Graphic
if (columNum is > 3 or < 0) {
throw new ArgumentOutOfRangeException(nameof(columNum));
}
#endif
InternalTranspose();
fixed (float* p = &transpose.m00)
{
ulong pBit = (ulong)p + (ulong)sizeof(Vector4) * (ulong)rowNum + sizeof(float) * (ulong)columNum;
return *(float*)pBit;
}
#endif
return kernel[rowNum + 4 * columNum];
}
set
{
#if DEBUG
#if DEBUG
if (rowNum is > 3 or < 0)
{
throw new ArgumentOutOfRangeException(nameof(columNum));
@@ -127,14 +115,8 @@ namespace StgSharp.Mathematics.Graphic
if (columNum is > 3 or < 0) {
throw new ArgumentOutOfRangeException(nameof(columNum));
}
#endif
InternalTranspose();
fixed (float* p = &transpose.m00)
{
ulong pbit = (ulong)p + (ulong)sizeof(Vector4) * (ulong)rowNum + sizeof(float) * (ulong)columNum;
*(float*)pbit = value;
}
isTransposed = false;
#endif
kernel[rowNum + 4 * columNum] = value;
}
}
@@ -142,8 +124,9 @@ namespace StgSharp.Mathematics.Graphic
{
get
{
InternalTranspose();
return new Matrix44(transpose);
Matrix44 transpose = new();
InternalTranspose(ref transpose);
return transpose;
}
}
@@ -176,41 +159,17 @@ namespace StgSharp.Mathematics.Graphic
return mat.Equals(other.mat);
}
public static Matrix44 FromSpan(ReadOnlySpan<Vec4> rowSpan)
{
Matrix44 mat = new Matrix44();
mat.transpose.colum0 = rowSpan[0].vec;
mat.transpose.colum1 = rowSpan[1].vec;
mat.transpose.colum2 = rowSpan[2].vec;
mat.transpose.colum3 = rowSpan[3].vec;
NativeIntrinsic.Intrinsic.f32_buffer_transpose(&mat.transpose, &mat.mat);
mat.isTransposed = false;
return mat;
}
public override int GetHashCode()
{
return HashCode.Combine(mat.colum0.GetHashCode(), mat.colum1.GetHashCode(),
mat.colum2.GetHashCode(), mat.colum3.GetHashCode());
}
public override string ToString()
{
InternalTranspose();
return $"{transpose.colum0}{transpose.colum1}{transpose.colum2}{transpose.colum3}";
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal unsafe void InternalTranspose()
internal unsafe void InternalTranspose(ref Matrix44 transpose)
{
if (!isTransposed)
{
fixed (ColumnSet4* source = &mat, target = &transpose) {
NativeIntrinsic.Intrinsic.f32_buffer_transpose(source, target);
}
isTransposed = true;
fixed (MatrixKernel<float>* source = &this.kernel, target = &transpose.kernel) {
NativeIntrinsic.Intrinsic.f32_buffer_transpose(source, target);
}
}
@@ -237,23 +196,23 @@ namespace StgSharp.Mathematics.Graphic
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Matrix44 operator *(Matrix44 left, Matrix44 right)
{
left.InternalTranspose();
return new Matrix44(Vector4.Dot(left.transpose.colum0, right.mat.colum0),
Vector4.Dot(left.transpose.colum0, right.mat.colum1),
Vector4.Dot(left.transpose.colum0, right.mat.colum2),
Vector4.Dot(left.transpose.colum0, right.mat.colum3),
Vector4.Dot(left.transpose.colum1, right.mat.colum0),
Vector4.Dot(left.transpose.colum1, right.mat.colum1),
Vector4.Dot(left.transpose.colum1, right.mat.colum2),
Vector4.Dot(left.transpose.colum1, right.mat.colum3),
Vector4.Dot(left.transpose.colum2, right.mat.colum0),
Vector4.Dot(left.transpose.colum2, right.mat.colum1),
Vector4.Dot(left.transpose.colum2, right.mat.colum2),
Vector4.Dot(left.transpose.colum2, right.mat.colum3),
Vector4.Dot(left.transpose.colum3, right.mat.colum0),
Vector4.Dot(left.transpose.colum3, right.mat.colum1),
Vector4.Dot(left.transpose.colum3, right.mat.colum2),
Vector4.Dot(left.transpose.colum3, right.mat.colum3));
Matrix44 transpose = left.Transpose;
return new Matrix44(Vector4.Dot(transpose.mat.colum0, right.mat.colum0),
Vector4.Dot(transpose.mat.colum0, right.mat.colum1),
Vector4.Dot(transpose.mat.colum0, right.mat.colum2),
Vector4.Dot(transpose.mat.colum0, right.mat.colum3),
Vector4.Dot(transpose.mat.colum1, right.mat.colum0),
Vector4.Dot(transpose.mat.colum1, right.mat.colum1),
Vector4.Dot(transpose.mat.colum1, right.mat.colum2),
Vector4.Dot(transpose.mat.colum1, right.mat.colum3),
Vector4.Dot(transpose.mat.colum2, right.mat.colum0),
Vector4.Dot(transpose.mat.colum2, right.mat.colum1),
Vector4.Dot(transpose.mat.colum2, right.mat.colum2),
Vector4.Dot(transpose.mat.colum2, right.mat.colum3),
Vector4.Dot(transpose.mat.colum3, right.mat.colum0),
Vector4.Dot(transpose.mat.colum3, right.mat.colum1),
Vector4.Dot(transpose.mat.colum3, right.mat.colum2),
Vector4.Dot(transpose.mat.colum3, right.mat.colum3));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -291,3 +250,4 @@ namespace StgSharp.Mathematics.Graphic
}
}
StgSharp.Common/Mathematics/Graphic/Vec4.cs
+12 -10
View File
@@ -140,13 +140,13 @@ namespace StgSharp.Mathematics.Graphic
public static Vec4 FromSpan(ReadOnlySpan<float> span)
{
return new Vec4
{
X = span[0],
Y = span[1],
Z = span[2],
W = span[3]
};
if (span.Length < 4) {
throw new ArgumentException("Span length must be at least 4.", nameof(span));
}
// Read 4 floats as a Vector4 in one operation without per-element copy.
Vector4 v = MemoryMarshal.Read<Vector4>(MemoryMarshal.AsBytes(span));
return new Vec4(v);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -185,9 +185,11 @@ namespace StgSharp.Mathematics.Graphic
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vec4 operator *(Matrix44 mat, Vec4 vec)
{
mat.InternalTranspose();
return new Vec4(Vector4.Dot(mat.transpose.colum0, vec.vec), Vector4.Dot(mat.transpose.colum1, vec.vec),
Vector4.Dot(mat.transpose.colum2, vec.vec), Vector4.Dot(mat.transpose.colum3, vec.vec));
Matrix44 transpose = mat.Transpose;
return new Vec4(Vector4.Dot(transpose.mat.colum0, vec.vec),
Vector4.Dot(transpose.mat.colum1, vec.vec),
Vector4.Dot(transpose.mat.colum2, vec.vec),
Vector4.Dot(transpose.mat.colum3, vec.vec));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
StgSharp.Native/sn-head/sn_intrinsic.h
+22 -17
View File
@@ -141,22 +141,26 @@ typedef void(SN_DECL *PNL_PROC_RAS)(mat_panel *right, mat_panel *ans, scalar_pac
#define PNL_PROC(T_type, T_arch, T_op_name) T_type##_pnl_##T_op_name##_##T_arch
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, \
T_op_name)(MAT_PANEL_##T_arch##(T_type) const *left, \
MAT_PANEL_##T_arch##(T_type) const *right, \
MAT_PANEL_##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, \
T_op_name)(MAT_PANEL_##T_arch##(T_type) const *left, \
MAT_PANEL_##T_arch##(T_type) const *right, \
MAT_PANEL_##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL_##T_arch##(T_type) const *left, \
MAT_PANEL_##T_arch##(T_type) const *right, \
MAT_PANEL_##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL_##T_arch##(T_type) const *left, \
MAT_PANEL_##T_arch##(T_type) const *right, \
MAT_PANEL_##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)
#pragma endregion
@@ -219,7 +223,8 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill);
#pragma region fma
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma);
#pragma endregion
StgSharp.Native/sn-head/sn_matkernel.h
+10 -6
View File
@@ -67,15 +67,19 @@ typedef MAT_KERNEL(uint64_t) sn_zero_kernel_u64;
/* SSE panel layout: column-major buffer holding 4 logical columns with MINQ=4.
Columns: each column packs up to four scalars so ALIGN equals sizeof(__m128).
Alignment: every column boundary lands on a 16-byte multiple for XMM loads. */
#define MAT_PANEL_SSE(T) SN_ALIGNAS(16) MAT_PANEL_SQ_MIN_NOALIGN(T, __m128, 4)
#define MAT_PANEL_sse(T) SN_ALIGNAS(16) MAT_PANEL_SQ_MIN_NOALIGN(T, __m128, 4)
/* AVX2 panel layout: column-major buffer holding 8 logical columns with MINQ=8.
Columns: each column stores eight elements via two 256-bit chunks when T is 32-bit.
Alignment: column starts are 32-byte aligned so AVX2 loads/stores remain contiguous. */
#define MAT_PANEL_AVX(T) SN_ALIGNAS(32) MAT_PANEL_SQ_MIN_NOALIGN(T, __m256, 8)
#define MAT_PANEL_avx(T) SN_ALIGNAS(32) MAT_PANEL_SQ_MIN_NOALIGN(T, __m256, 8)
/* AVX2 panel layout: column-major buffer holding 8 logical columns with MINQ=8.
Columns: each column stores eight elements via two 256-bit chunks when T is 32-bit.
Alignment: column starts are 32-byte aligned so AVX2 loads/stores remain contiguous. */
#define MAT_PANEL_avx_fma(T) SN_ALIGNAS(32) MAT_PANEL_SQ_MIN_NOALIGN(T, __m256, 8)
/* AVX-512 panel layout: column-major buffer holding 8 or more columns depending on T.
Columns: each column expands to ceil(Q/lanes) ZMM chunks; default MINQ=8 keeps double-wide tiles.
Columns: each column expands to ceil(Q/lanes) ZMM chunks; default MINQ=8 keeps doublse-wide tiles.
Alignment: column starts sit on 64-byte boundaries to match AVX-512 streaming access. */
#define MAT_PANEL_512(T) SN_ALIGNAS(64) MAT_PANEL_SQ_MIN_NOALIGN(T, __m512, 8)
#define MAT_PANEL_512(T) SN_ALIGNAS(64) MAT_PANEL_SQ_MIN_NOALIGN(T, __m512, 8)
/* Global zero kernel declaration (defined in dllmain.c), aligned to 64 bytes */
#ifdef __cplusplus
@@ -113,7 +117,7 @@ extern __declspec(align(64)) sn_zero_kernel_u64 SN_ZERO_KERNEL;
int col_index, int row_index)
/* Declarations for AVX float variant */
DECLARE_BUILD_PANEL(AVX, float);
DECLARE_STORE_PANEL(AVX, float);
DECLARE_BUILD_PANEL(avx, float);
DECLARE_STORE_PANEL(avx, float);
#endif /* SN_MATKERNEL */
StgSharp.Native/sn-src/k_fma.c
+152 -25
View File
@@ -1,35 +1,162 @@
#include "sn_intrinsic.h"
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
// ans = left * right + ans
// right is not transposed
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, fma)
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma)
{
register __m128 source0 = _mm_load_ps(&left->xmm[0]);
register __m128 source1 = _mm_load_ps(&left->xmm[1]);
register __m128 source2 = _mm_load_ps(&left->xmm[2]);
register __m128 source3 = _mm_load_ps(&left->xmm[3]);
const __m128 a0 = PANEL_COL_VEC((*left), 0, 0);
const __m128 a1 = PANEL_COL_VEC((*left), 1, 0);
const __m128 a2 = PANEL_COL_VEC((*left), 2, 0);
const __m128 a3 = PANEL_COL_VEC((*left), 3, 0);
register __m128 ans0 = _mm_load_ps(&ans->xmm[0]);
register __m128 ans1 = _mm_load_ps(&ans->xmm[1]);
register __m128 ans2 = _mm_load_ps(&ans->xmm[2]);
register __m128 ans3 = _mm_load_ps(&ans->xmm[3]);
__m128 c0 = PANEL_COL_VEC((*ans), 0, 0);
__m128 c1 = PANEL_COL_VEC((*ans), 1, 0);
__m128 c2 = PANEL_COL_VEC((*ans), 2, 0);
__m128 c3 = PANEL_COL_VEC((*ans), 3, 0);
register __m128 broadcast;
for (size_t i = 0; i < 4; i++) {
broadcast = _mm_load_ps(&right->xmm[i]);
__m128 bcol;
ans0 = _mm_add_ps(ans0,
_mm_mul_ps(source0, _mm_shuffle_ps(broadcast, broadcast, 0x00)));
ans1 = _mm_add_ps(ans1,
_mm_mul_ps(source1, _mm_shuffle_ps(broadcast, broadcast, 0x55)));
ans2 = _mm_add_ps(ans2,
_mm_mul_ps(source2, _mm_shuffle_ps(broadcast, broadcast, 0xAA)));
ans3 = _mm_add_ps(ans3,
_mm_mul_ps(source3, _mm_shuffle_ps(broadcast, broadcast, 0xFF)));
}
/* Column 0 */
bcol = PANEL_COL_VEC((*right), 0, 0);
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
_mm_store_ps(&ans->xmm[0], ans0);
_mm_store_ps(&ans->xmm[1], ans1);
_mm_store_ps(&ans->xmm[2], ans2);
_mm_store_ps(&ans->xmm[3], ans3);
/* Column 1 */
bcol = PANEL_COL_VEC((*right), 1, 0);
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = PANEL_COL_VEC((*right), 2, 0);
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = PANEL_COL_VEC((*right), 3, 0);
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
PANEL_COL_VEC((*ans), 0, 0) = c0;
PANEL_COL_VEC((*ans), 1, 0) = c1;
PANEL_COL_VEC((*ans), 2, 0) = c2;
PANEL_COL_VEC((*ans), 3, 0) = c3;
}
// ans = left * right + ans
// right is not transposed
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma)
{
static const __m256i idx0 = { 1, 1, 1, 1, 1, 1, 1, 1 };
for (int col = 0; col < 8; ++col) {
register __m256 bcol = _mm256_load_ps((float *)&right->vec[col][0]);
register __m256 ccol = _mm256_load_ps((float *)&ans->vec[col][0]);
register __m256i idx = _mm256_setzero_si256();
register __m256i increament = _mm256_load_si256(&idx0);
register __m256 a0 = _mm256_load_ps((float *)&left->vec[0][0]);
register __m256 a1 = _mm256_load_ps((float *)&left->vec[1][0]);
register __m256 a2 = _mm256_load_ps((float *)&left->vec[2][0]);
register __m256 a3 = _mm256_load_ps((float *)&left->vec[3][0]);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a0, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a1, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a2, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a3, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
a0 = _mm256_load_ps((float *)&left->vec[4][0]);
a1 = _mm256_load_ps((float *)&left->vec[5][0]);
a2 = _mm256_load_ps((float *)&left->vec[6][0]);
a3 = _mm256_load_ps((float *)&left->vec[7][0]);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a0, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a1, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a2, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a3, _mm256_permutevar8x32_ps(bcol, idx)));
_mm256_store_ps((float *)&ans->vec[col][0], ccol);
}
}
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, fma)
{
const __m512i ones = _mm512_set1_epi32(1);
for (int col = 0; col < 8; ++col) {
register __m512 bcol = _mm512_load_ps((float *)&right->vec[col][0]);
register __m512 ccol = _mm512_load_ps((float *)&ans->vec[col][0]);
register __m512i idx = _mm512_setzero_si512();
register __m512i increament = _mm512_load_si512(&ones);
register __m512 a0 = _mm512_load_ps((float *)&left->vec[0][0]);
register __m512 a1 = _mm512_load_ps((float *)&left->vec[1][0]);
register __m512 a2 = _mm512_load_ps((float *)&left->vec[2][0]);
register __m512 a3 = _mm512_load_ps((float *)&left->vec[3][0]);
register __m512 a4 = _mm512_load_ps((float *)&left->vec[4][0]);
register __m512 a5 = _mm512_load_ps((float *)&left->vec[5][0]);
register __m512 a6 = _mm512_load_ps((float *)&left->vec[6][0]);
register __m512 a7 = _mm512_load_ps((float *)&left->vec[7][0]);
ccol = _mm512_fmadd_ps(a0, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a1, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a2, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a3, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a4, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a5, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a6, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a7, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
a0 = _mm512_load_ps((float *)&left->vec[8][0]);
a1 = _mm512_load_ps((float *)&left->vec[9][0]);
a2 = _mm512_load_ps((float *)&left->vec[10][0]);
a3 = _mm512_load_ps((float *)&left->vec[11][0]);
a4 = _mm512_load_ps((float *)&left->vec[12][0]);
a5 = _mm512_load_ps((float *)&left->vec[13][0]);
a6 = _mm512_load_ps((float *)&left->vec[14][0]);
a7 = _mm512_load_ps((float *)&left->vec[15][0]);
ccol = _mm512_fmadd_ps(a0, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a1, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a2, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a3, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a4, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a5, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a6, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a7, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
_mm512_store_ps((float *)&ans->vec[col][0], ccol);
}
}
StgSharp.Native/sn-src/k_panel.c
+2 -2
View File
@@ -1,6 +1,6 @@
#include "sn_matkernel.h"
DECLARE_BUILD_PANEL(AVX, float)
DECLARE_BUILD_PANEL(avx, float)
{
const int Q = 8; /* fixed for AVX float panel */
@@ -41,7 +41,7 @@ DECLARE_BUILD_PANEL(AVX, float)
}
}
DECLARE_STORE_PANEL(AVX, float)
DECLARE_STORE_PANEL(avx, float)
{
const int Q = 8; /* fixed for AVX float panel */