From 62cf2430f4abc1f9341e7b7995f30268dd616b19 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Fri, 19 Sep 2025 20:38:44 +0800 Subject: [PATCH] mat kernel turns to be generic --- StgSharp.Native/sn-head/sn_intrinsic.h | 50 ++++---- StgSharp.Native/sn-src/StgSharpC.c | 6 +- StgSharp.Native/sn-src/k_basicop.c | 45 ++++---- StgSharp.Native/sn-src/k_fma.c | 2 +- StgSharp.Native/sn-src/load_intrinsic.c | 30 ++--- StgSharp.Native/sn-src/sn_native.c | 6 +- StgSharp.Native/sn-src/str_string.c | 2 +- StgSharp.Native/sn-src/vector.c | 2 +- StgSharp.Native/sn-src/vector_dot.c | 146 ++++++------------------ 9 files changed, 111 insertions(+), 178 deletions(-) diff --git a/StgSharp.Native/sn-head/sn_intrinsic.h b/StgSharp.Native/sn-head/sn_intrinsic.h index ac06c550..9ff16f3b 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.h @@ -62,37 +62,37 @@ typedef uint64_t(SN_DECL *FACTORIALROC)(int n); INTERNAL void normalize(__m128 *source, __m128 *target); INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) * ans); + MAT_KERNEL(float) *restrict ans); INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) * ans); + MAT_KERNEL(float) *restrict ans); INTERNAL void SN_DECL f32_add_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) * ans); + MAT_KERNEL(float) *restrict ans); INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) * ans); + MAT_KERNEL(float) *restrict ans); INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) * ans); + MAT_KERNEL(float) *restrict ans); INTERNAL void SN_DECL f32_sub_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, - MAT_KERNEL(float) * ans); + MAT_KERNEL(float) *restrict ans); INTERNAL void SN_DECL f32_transpose_sse(MAT_KERNEL(float) const *source, - MAT_KERNEL(float) * target); + MAT_KERNEL(float) *restrict target); INTERNAL void SN_DECL f32_transpose_avx(MAT_KERNEL(float) const *source, - MAT_KERNEL(float) * target); + MAT_KERNEL(float) *restrict target); INTERNAL void SN_DECL f32_transpose_512(MAT_KERNEL(float) const *source, - MAT_KERNEL(float) * target); + MAT_KERNEL(float) *restrict target); INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix, float const scalar, - MAT_KERNEL(float) * ans); + MAT_KERNEL(float) *restrict ans); INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix, float const scalar, - MAT_KERNEL(float) * ans); + MAT_KERNEL(float) *restrict ans); INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix, float const scalar, - MAT_KERNEL(float) * ans); + MAT_KERNEL(float) *restrict ans); #pragma endregion #pragma region fma -INTERNAL void SN_DECL kernel_fma_sse(MAT_KERNEL(float) const *restrict left, - MAT_KERNEL(float) const *restrict right, - MAT_KERNEL(float) *restrict ans); +INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left, + MAT_KERNEL(float) const *restrict right, + MAT_KERNEL(float) *restrict ans); #pragma endregion @@ -100,6 +100,16 @@ INTERNAL void SN_DECL kernel_fma_sse(MAT_KERNEL(float) const *restrict left, INTERNAL uint64_t SN_DECL factorial_simd_sse(int n); #pragma endregion +#pragma region dot_product + +INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose, + __m128 const *vector, __m128 *ans); +INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans); +INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans); +INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans); + +#pragma endregion + #pragma region string INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int length); @@ -111,11 +121,11 @@ typedef struct sn_intrinsic { int (*city_hash_simplify)(byte *str, int length); FACTORIALROC factorial_simd; int (*index_pair)(short const *str, uint32_t target, int length); - MATARITHMETIC kernel_add; - void (*kernel_fma)(void *left, void *right, void *ans); - SCALARMATARITHMETIC kernel_scalar_mul; - MATARITHMETIC kernel_sub; - MATTRANSPOSEPROC kernel_transpose; + MATARITHMETIC f32_add; + void (*f32_fma)(void *left, void *right, void *ans); + SCALARMATARITHMETIC f32_scalar_mul; + MATARITHMETIC f32_sub; + MATTRANSPOSEPROC f32_transpose; VECTORNORMALIZEPROC normalize_3; } sn_intrinsic; diff --git a/StgSharp.Native/sn-src/StgSharpC.c b/StgSharp.Native/sn-src/StgSharpC.c index 310a71e3..3e9a0fe6 100644 --- a/StgSharp.Native/sn-src/StgSharpC.c +++ b/StgSharp.Native/sn-src/StgSharpC.c @@ -27,7 +27,7 @@ SSCAPI void SSCDECL initGL(int majorVersion, int minorVersion) glfwWindowHint(GLFW_RESIZABLE, GLFW_TRUE); } -SSCAPI GLFWglproc SSCDECL loadGlfuncDefault(char *procName) +SSCAPI GLFWglproc SSCDECL loadGlfuncDefault(char const *procName) { void *ret = glfwGetProcAddress(procName); // printf("%llu\n", (uint64_t)ret); @@ -55,7 +55,7 @@ SSCAPI unsigned int linkShaderProgram(GladGLContext *context, GLuint shaderProgr } } -SSCAPI void SSCDECL loadImageData(char *location, Image *out, imageLoader loader) +SSCAPI void SSCDECL loadImageData(char const *location, Image *out, imageLoader loader) { if (loader == NULL) { loader = stbi_load; @@ -63,7 +63,7 @@ SSCAPI void SSCDECL loadImageData(char *location, Image *out, imageLoader loader out->pixelPtr = loader(location, &(out->width), &(out->height), &(out->channel), 0); } -SSCAPI void SSCDECL unloadImageData(Image *out) +SSCAPI void SSCDECL unloadImageData(Image *restrict out) { free(out->pixelPtr); } diff --git a/StgSharp.Native/sn-src/k_basicop.c b/StgSharp.Native/sn-src/k_basicop.c index 17f47261..51c484fa 100644 --- a/StgSharp.Native/sn-src/k_basicop.c +++ b/StgSharp.Native/sn-src/k_basicop.c @@ -1,7 +1,7 @@ #include "sn_intrinsic.h" -INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) * left, MAT_KERNEL(float) * right, - MAT_KERNEL(float) * ans) +INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, + MAT_KERNEL(float) *restrict ans) { register __m128 c0 = _mm_add_ps(left->xmm[0], right->xmm[0]); register __m128 c1 = _mm_add_ps(left->xmm[1], right->xmm[1]); @@ -14,8 +14,8 @@ INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) * left, MAT_KERNEL(float) * ans->xmm[3] = c3; } -INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) * left, MAT_KERNEL(float) * right, - MAT_KERNEL(float) * ans) +INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, + MAT_KERNEL(float) *restrict ans) { register __m256 c0 = _mm256_add_ps(left->ymm[0], right->ymm[0]); register __m256 c1 = _mm256_add_ps(left->ymm[1], right->ymm[1]); @@ -23,15 +23,15 @@ INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) * left, MAT_KERNEL(float) * ans->ymm[1] = c1; } -INTERNAL void SN_DECL f32_add_512(MAT_KERNEL(float) * left, MAT_KERNEL(float) * right, - MAT_KERNEL(float) * ans) +INTERNAL void SN_DECL f32_add_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, + MAT_KERNEL(float) *restrict ans) { register __m512 c0 = _mm512_add_ps(left->zmm[0], right->zmm[0]); ans->zmm[0] = c0; } -INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) * left, MAT_KERNEL(float) * right, - MAT_KERNEL(float) * ans) +INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, + MAT_KERNEL(float) *restrict ans) { register __m128 c0 = _mm_sub_ps(left->xmm[0], right->xmm[0]); register __m128 c1 = _mm_sub_ps(left->xmm[1], right->xmm[1]); @@ -44,8 +44,8 @@ INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) * left, MAT_KERNEL(float) * ans->xmm[3] = c3; } -INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) * left, MAT_KERNEL(float) * right, - MAT_KERNEL(float) * ans) +INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, + MAT_KERNEL(float) *restrict ans) { register __m256 c0 = _mm256_sub_ps(left->ymm[0], right->ymm[0]); register __m256 c1 = _mm256_sub_ps(left->ymm[1], right->ymm[1]); @@ -53,15 +53,16 @@ INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) * left, MAT_KERNEL(float) * ans->ymm[1] = c1; } -INTERNAL void SN_DECL f32_sub_512(MAT_KERNEL(float) * left, MAT_KERNEL(float) * right, - MAT_KERNEL(float) * ans) +INTERNAL void SN_DECL f32_sub_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right, + MAT_KERNEL(float) *restrict ans) { register __m512 c0 = _mm512_sub_ps(left->zmm[0], right->zmm[0]); ans->zmm[0] = c0; } // Matrix transpose implementations -INTERNAL void SN_DECL kernel_transpose_sse(MAT_KERNEL(float) * source, MAT_KERNEL(float) * target) +INTERNAL void SN_DECL f32_transpose_sse(MAT_KERNEL(float) const *source, + MAT_KERNEL(float) *restrict target) { register __m128 col0 = source->xmm[0]; register __m128 col1 = source->xmm[1]; @@ -78,7 +79,8 @@ INTERNAL void SN_DECL kernel_transpose_sse(MAT_KERNEL(float) * source, MAT_KERNE target->xmm[3] = _mm_movehl_ps(tmp3, tmp1); } -INTERNAL void SN_DECL kernel_transpose_avx(MAT_KERNEL(float) * source, MAT_KERNEL(float) * target) +INTERNAL void SN_DECL f32_transpose_avx(MAT_KERNEL(float) const *source, + MAT_KERNEL(float) *restrict target) { register __m256 col_low = source->ymm[0]; register __m256 col_high = source->ymm[1]; @@ -93,7 +95,8 @@ INTERNAL void SN_DECL kernel_transpose_avx(MAT_KERNEL(float) * source, MAT_KERNE target->ymm[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1)); } -INTERNAL void SN_DECL kernel_transpose_512(MAT_KERNEL(float) * source, MAT_KERNEL(float) * target) +INTERNAL void SN_DECL f32_transpose_512(MAT_KERNEL(float) const *source, + MAT_KERNEL(float) *restrict target) { register __m512 matrix = source->zmm[0]; @@ -103,8 +106,8 @@ INTERNAL void SN_DECL kernel_transpose_512(MAT_KERNEL(float) * source, MAT_KERNE target->zmm[0] = _mm512_permutexvar_ps(transpose_indices, matrix); } -INTERNAL void SN_DECL kernel_scalar_mul_sse(MAT_KERNEL(float) * matrix, float scalar, - MAT_KERNEL(float) * ans) +INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix, float const scalar, + MAT_KERNEL(float) *restrict ans) { register __m128 temp = _mm_load_ss(&scalar); register __m128 scalar_vec = _mm_shuffle_ps(temp, temp, _MM_SHUFFLE(0, 0, 0, 0)); @@ -120,8 +123,8 @@ INTERNAL void SN_DECL kernel_scalar_mul_sse(MAT_KERNEL(float) * matrix, float sc ans->xmm[3] = c3; } -INTERNAL void SN_DECL kernel_scalar_mul_avx(MAT_KERNEL(float) * matrix, float scalar, - MAT_KERNEL(float) * ans) +INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix, float const scalar, + MAT_KERNEL(float) *restrict ans) { register __m256 scalar_vec = _mm256_broadcast_ss(&scalar); @@ -132,8 +135,8 @@ INTERNAL void SN_DECL kernel_scalar_mul_avx(MAT_KERNEL(float) * matrix, float sc ans->ymm[1] = c1; } -INTERNAL void SN_DECL kernel_scalar_mul_512(MAT_KERNEL(float) * matrix, float scalar, - MAT_KERNEL(float) * ans) +INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix, float const scalar, + MAT_KERNEL(float) *restrict ans) { __m512 scalar_vec = _mm512_broadcastss_ps(_mm_load_ss(&scalar)); diff --git a/StgSharp.Native/sn-src/k_fma.c b/StgSharp.Native/sn-src/k_fma.c index b6c8ab7d..96a67e0f 100644 --- a/StgSharp.Native/sn-src/k_fma.c +++ b/StgSharp.Native/sn-src/k_fma.c @@ -2,7 +2,7 @@ // ans = left * right(T) + ans INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left, - mat_kernel const *restrict right, mat_kernel *restrict ans) + MAT_KERNEL(float) const *restrict right, MAT_KERNEL(float) *restrict ans) { register __m128 source0 = _mm_load_ps(&left->xmm[0]); register __m128 source1 = _mm_load_ps(&left->xmm[1]); diff --git a/StgSharp.Native/sn-src/load_intrinsic.c b/StgSharp.Native/sn-src/load_intrinsic.c index 525440f9..169e1af4 100644 --- a/StgSharp.Native/sn-src/load_intrinsic.c +++ b/StgSharp.Native/sn-src/load_intrinsic.c @@ -66,11 +66,11 @@ void load_intrinsic_sse(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; - context->kernel_add = (MATARITHMETIC)f32_add_sse; - context->kernel_fma = (void(*)(void*, void*, void*))kernel_fma_sse; - context->kernel_scalar_mul = kernel_scalar_mul_sse; - context->kernel_sub = (MATARITHMETIC)f32_sub_sse; - context->kernel_transpose = (MATTRANSPOSEPROC)kernel_transpose_sse; + context->f32_add = (MATARITHMETIC)f32_add_sse; + context->f32_fma = (void (*)(void *, void *, void *))f32_fma_sse; + context->f32_scalar_mul = f32_scalar_mul_sse; + context->f32_sub = (MATARITHMETIC)f32_sub_sse; + context->f32_transpose = (MATTRANSPOSEPROC)f32_transpose_sse; context->normalize_3 = normalize; } @@ -79,11 +79,11 @@ void load_intrinsic_avx(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; - context->kernel_add = (MATARITHMETIC)f32_add_avx; - context->kernel_fma = (void(*)(void*, void*, void*))kernel_fma_sse; - context->kernel_scalar_mul = kernel_scalar_mul_avx; - context->kernel_sub = (MATARITHMETIC)f32_sub_avx; - context->kernel_transpose = (MATTRANSPOSEPROC)kernel_transpose_avx; + context->f32_add = (MATARITHMETIC)f32_add_avx; + context->f32_fma = (void(*)(void*, void*, void*))f32_fma_sse; + context->f32_scalar_mul = f32_scalar_mul_avx; + context->f32_sub = (MATARITHMETIC)f32_sub_avx; + context->f32_transpose = (MATTRANSPOSEPROC)f32_transpose_avx; context->normalize_3 = normalize; } @@ -92,10 +92,10 @@ void load_intrinsic_512(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; - context->kernel_add = (MATARITHMETIC)f32_add_512; - context->kernel_fma = (void(*)(void*, void*, void*))kernel_fma_sse; - context->kernel_scalar_mul = kernel_scalar_mul_512; - context->kernel_sub = (MATARITHMETIC)f32_sub_512; - context->kernel_transpose = (MATTRANSPOSEPROC)kernel_transpose_512; + context->f32_add = (MATARITHMETIC)f32_add_512; + context->f32_fma = (void(*)(void*, void*, void*))f32_fma_sse; + context->f32_scalar_mul = f32_scalar_mul_512; + context->f32_sub = (MATARITHMETIC)f32_sub_512; + context->f32_transpose = (MATTRANSPOSEPROC)f32_transpose_512; context->normalize_3 = normalize; } diff --git a/StgSharp.Native/sn-src/sn_native.c b/StgSharp.Native/sn-src/sn_native.c index 987a86dc..4ee2b33d 100644 --- a/StgSharp.Native/sn-src/sn_native.c +++ b/StgSharp.Native/sn-src/sn_native.c @@ -27,7 +27,7 @@ SN_API void SN_DECL initGL(int majorVersion, int minorVersion) glfwWindowHint(GLFW_RESIZABLE, GLFW_TRUE); } -SN_API GLFWglproc SN_DECL loadGlfuncDefault(char *procName) +SN_API GLFWglproc SN_DECL loadGlfuncDefault(char const *procName) { void *ret = glfwGetProcAddress(procName); // printf("%llu\n", (uint64_t)ret); @@ -55,7 +55,7 @@ SN_API unsigned int linkShaderProgram(GladGLContext *context, GLuint shaderProgr } } -SN_API void SN_DECL loadImageData(char *location, Image *out, imageLoader loader) +SN_API void SN_DECL loadImageData(char const *location, Image *out, imageLoader loader) { if (loader == NULL) { loader = stbi_load; @@ -63,7 +63,7 @@ SN_API void SN_DECL loadImageData(char *location, Image *out, imageLoader loader out->pixelPtr = loader(location, &(out->width), &(out->height), &(out->channel), 0); } -SN_API void SN_DECL unloadImageData(Image *out) +SN_API void SN_DECL unloadImageData(Image *restrict out) { free(out->pixelPtr); } diff --git a/StgSharp.Native/sn-src/str_string.c b/StgSharp.Native/sn-src/str_string.c index 21efc77e..d3995444 100644 --- a/StgSharp.Native/sn-src/str_string.c +++ b/StgSharp.Native/sn-src/str_string.c @@ -1,6 +1,6 @@ #include "sn_intrinsic.h" -int index_pair_sse(short const *str, uint32_t target, int length) +INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int length) { int index = 0, result = 0, size = length, end = size - 8; __m128i s = { diff --git a/StgSharp.Native/sn-src/vector.c b/StgSharp.Native/sn-src/vector.c index 1c249685..65744302 100644 --- a/StgSharp.Native/sn-src/vector.c +++ b/StgSharp.Native/sn-src/vector.c @@ -1,7 +1,7 @@ #include "StgSharpNative.h" #include "sn_internal.h" -SN_API void SN_DECL normalize(__m128 *source, __m128 *target) +SN_API void SN_DECL normalize(__m128 *restrict source, __m128 *restrict target) { __m128 a_length_sq = _mm_mul_ps(*source, *source); a_length_sq = _mm_add_ps(a_length_sq, diff --git a/StgSharp.Native/sn-src/vector_dot.c b/StgSharp.Native/sn-src/vector_dot.c index 3d1163e6..4d00414d 100644 --- a/StgSharp.Native/sn-src/vector_dot.c +++ b/StgSharp.Native/sn-src/vector_dot.c @@ -3,145 +3,65 @@ static const int avxdot_indices[] = { 0, 4, 1, 5, 2, 3, 6, 7 }; -INTERNAL void FORCEINLINE SN_DECL dot_41_sse(__4_columnset *transpose, __m128 *vector, __m128 *ans) +INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose, + __m128 const *vector, __m128 *ans) { - *ans = _mm_dp_ps(transpose->column[0], *vector, 0B11110001); - *ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], *vector, 0B11110001), + *ans = _mm_dp_ps(transpose->xmm[0], *vector, 0B11110001); + *ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[1], *vector, 0B11110001), SSE_INSERT_SIMPLE(1, 1)); - *ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], *vector, 0B11110001), + *ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[2], *vector, 0B11110001), SSE_INSERT_SIMPLE(1, 3)); - *ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[3], *vector, 0B11110001), + *ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[3], *vector, 0B11110001), SSE_INSERT_SIMPLE(1, 4)); } -INTERNAL void SN_DECL dot_41_avx(__4_columnset *transpose, __m128 *vector, __m128 *ans) +INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans) { __m256 _double = _mm256_set_m128(vector[0], vector[0]); - __m256 tmp = _mm256_dp_ps(transpose->half[0], _double, 0B11110001); - tmp = _mm256_blend_ps(tmp, _mm256_dp_ps(transpose->half[0], _double, 0B11110010), + __m256 tmp = _mm256_dp_ps(transpose->ymm[0], _double, 0B11110001); + tmp = _mm256_blend_ps(tmp, _mm256_dp_ps(transpose->ymm[0], _double, 0B11110010), 0B00100010); *ans = _mm256_castps256_ps128( _mm256_permutevar8x32_ps(tmp, _mm256_loadu_epi32(avxdot_indices))); } -INTERNAL void FORCEINLINE SN_DECL dot_31_sse(__3_columnset *transpose, __m128 *vector, __m128 *ans) +INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans) { - *ans = _mm_dp_ps(transpose->column[0], *vector, 0B11110001); - *ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], *vector, 0B11110001), - SSE_INSERT_SIMPLE(1, 2)); - *ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], *vector, 0B11110001), - SSE_INSERT_SIMPLE(1, 3)); -} - -INTERNAL void FORCEINLINE SN_DECL dot_31_avx(__3_columnset *transpose, __m128 *vector, __m128 *ans) -{ - __m256 _double = _mm256_set_m128(vector[0], vector[0]); - __m256 tmp = _mm256_dp_ps(transpose->part, _double, 0B11110001); - *ans = _mm256_castps256_ps128( - _mm256_permutevar8x32_ps(tmp, _mm256_loadu_epi32(avxdot_indices))); - *ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], *vector, 0B11110001), - SSE_INSERT_SIMPLE(1, 3)); -} - -INTERNAL void SN_DECL dot_42_sse(__4_columnset *transpose, __m128 *vector, __m128 *ans) -{ - ans[0] = _mm_dp_ps(transpose->column[0], vector[0], 0B11110001); - ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], vector[0], 0B11110001), + ans[0] = _mm_dp_ps(transpose->xmm[0], vector[0], 0B11110001); + ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[1], vector[0], 0B11110001), SSE_INSERT_SIMPLE(1, 2)); - ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], vector[0], 0B11110001), + ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[2], vector[0], 0B11110001), SSE_INSERT_SIMPLE(1, 3)); - ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[3], vector[0], 0B11110001), + ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[3], vector[0], 0B11110001), SSE_INSERT_SIMPLE(1, 4)); - ans[1] = _mm_dp_ps(transpose->column[0], vector[1], 0B11110001); - ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], vector[1], 0B11110001), + ans[1] = _mm_dp_ps(transpose->xmm[0], vector[1], 0B11110001); + ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[1], vector[1], 0B11110001), SSE_INSERT_SIMPLE(1, 2)); - ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], vector[1], 0B11110001), + ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[2], vector[1], 0B11110001), SSE_INSERT_SIMPLE(1, 3)); - ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[3], vector[1], 0B11110001), + ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[3], vector[1], 0B11110001), SSE_INSERT_SIMPLE(1, 4)); } -INTERNAL void SN_DECL dot_32_sse(__3_columnset *transpose, __m128 *vector, __m128 *ans) +INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans) { - ans[0] = _mm_dp_ps(transpose->column[0], vector[0], 0B11110001); - ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], vector[0], 0B11110001), - SSE_INSERT_SIMPLE(1, 2)); - ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], vector[0], 0B11110001), - SSE_INSERT_SIMPLE(1, 3)); + __m256 temp = transpose->ymm[0]; + __m256 vector_stream = _mm256_set_m128(vector[1], vector[0]); - ans[1] = _mm_dp_ps(transpose->column[0], vector[1], 0B11110001); - ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], vector[1], 0B11110001), - SSE_INSERT_SIMPLE(1, 2)); - ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], vector[1], 0B11110001), - SSE_INSERT_SIMPLE(1, 3)); -} - -INTERNAL void SN_DECL dot_42_avx(__4_columnset *transpose, __m128 *vector, __m128 *ans) -{ - __2_columnset *ans_ptr = (__2_columnset *)ans, *right_ptr = (__2_columnset *)vector; - __m256 temp = ALIGN256(transpose->half[0]); - - ans_ptr->stream = _mm256_dp_ps(temp, right_ptr->stream, 0B11110001); + __m256 result = _mm256_dp_ps(temp, vector_stream, 0B11110001); temp = _mm256_permute2f128_ps(temp, temp, 0B00000001); - ans_ptr->stream = _mm256_blend_ps( - ans_ptr->stream, _mm256_dp_ps(temp, right_ptr->stream, 0B11110010), 0B00100010); + result = _mm256_blend_ps( + result, _mm256_dp_ps(temp, vector_stream, 0B11110010), 0B00100010); - temp = ALIGN256(transpose->half[1]); - ans_ptr->stream = _mm256_blend_ps( - ans_ptr->stream, _mm256_dp_ps(temp, right_ptr->stream, 0B11110100), 0B01000100); + temp = transpose->ymm[1]; + result = _mm256_blend_ps( + result, _mm256_dp_ps(temp, vector_stream, 0B11110100), 0B01000100); temp = _mm256_permute2f128_ps(temp, temp, 0B00000001); - ans_ptr->stream = _mm256_blend_ps( - ans_ptr->stream, _mm256_dp_ps(temp, right_ptr->stream, 0B11111000), 0B10001000); + result = _mm256_blend_ps( + result, _mm256_dp_ps(temp, vector_stream, 0B11111000), 0B10001000); - ans[1] = _mm_shuffle_ps(ALIGN(ans[1]), ALIGN(ans[1]), _MM_SHUFFLE(2, 3, 0, 1)); -} - -INTERNAL void SN_DECL dot_32_avx(__3_columnset *transpose, __m128 *vector, __m128 *ans) -{ - __2_columnset *ans_ptr = (__2_columnset *)ans, *right_ptr = (__2_columnset *)vector; - __m256 temp = transpose->part; - - ans_ptr->stream = _mm256_dp_ps(temp, right_ptr->stream, 0B11110001); - temp = _mm256_permute2f128_ps(temp, temp, 0B00000001); - ans_ptr->stream = _mm256_blend_ps( - ans_ptr->stream, _mm256_dp_ps(temp, right_ptr->stream, 0B11110010), 0B00100010); - - temp = _mm256_set_m128(transpose->column[2], transpose->column[2]); - ans_ptr->stream = _mm256_blend_ps( - ans_ptr->stream, _mm256_dp_ps(temp, right_ptr->stream, 0B11110100), 0B01000100); - - ans[1] = _mm_shuffle_ps(ALIGN(ans[1]), ALIGN(ans[1]), _MM_SHUFFLE(2, 2, 0, 1)); -} - -INTERNAL void SN_DECL dot_43_sse(__4_columnset *transpose, __m128 *vector, __m128 *ans) -{ - dot_41_sse(transpose, vector, ans); - dot_41_sse(transpose, vector + 1, ans + 1); - dot_41_sse(transpose, vector + 2, ans + 2); -} - -INTERNAL void SN_DECL dot_43_avx(__4_columnset *transpose, __m128 *vector, __m128 *ans) -{ - __3_columnset *ans_ptr = (__3_columnset *)ans, *right_ptr = (__3_columnset *)vector; - __m256 temp = ALIGN256(transpose->half[0]); - __m256 third = _mm256_set_m128(right_ptr->column[2], right_ptr->column[2]); - - ans_ptr->part = _mm256_dp_ps(temp, right_ptr->part, 0B11110001); - __m256 ans_third = _mm256_dp_ps(temp, third, 0B11110001); - temp = _mm256_permute2f128_ps(temp, temp, 0B00000001); - ans_ptr->part = _mm256_blend_ps( - ans_ptr->part, _mm256_dp_ps(temp, right_ptr->part, 0B11110010), 0B00100010); - - temp = ALIGN256(transpose->half[1]); - ans_ptr->part = _mm256_blend_ps( - ans_ptr->part, _mm256_dp_ps(temp, right_ptr->part, 0B11110100), 0B01000100); - ans_third = _mm256_blend_ps(ans_third, _mm256_dp_ps(temp, third, 0B11110010), 0B00100010); - temp = _mm256_permute2f128_ps(temp, temp, 0B00000001); - ans_ptr->part = _mm256_blend_ps( - ans_ptr->part, _mm256_dp_ps(temp, right_ptr->part, 0B11111000), 0B10001000); - - ans[1] = _mm_shuffle_ps(ALIGN(ans[1]), ALIGN(ans[1]), _MM_SHUFFLE(2, 3, 0, 1)); - ans[2] = _mm256_castps256_ps128( - _mm256_permutevar8x32_ps(ans_third, _mm256_loadu_epi32(avxdot_indices))); -} + ans[0] = _mm256_castps256_ps128(result); + ans[1] = _mm256_extractf128_ps(result, 1); + ans[1] = _mm_shuffle_ps(ans[1], ans[1], _MM_SHUFFLE(2, 3, 0, 1)); +} \ No newline at end of file