youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'giga' of ssh://ssh.github.com:443/Nitload-NSI/CCD_Github into local

Browse Source 
merge back to main device
This commit is contained in:
Nitload
2025-09-20 20:53:30 +08:00
parent aded199bc6 62cf2430f4
commit 0404ecf044
9 changed files with 111 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-head/sn_intrinsic.h
+30 -20
View File
@@ -62,37 +62,37 @@ typedef uint64_t(SN_DECL *FACTORIALROC)(int n);
INTERNAL void normalize(__m128 *source, __m128 *target);
INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) * ans);
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) * ans);
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_add_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) * ans);
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) * ans);
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) * ans);
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_sub_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) * ans);
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_transpose_sse(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) * target);
MAT_KERNEL(float) *restrict target);
INTERNAL void SN_DECL f32_transpose_avx(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) * target);
MAT_KERNEL(float) *restrict target);
INTERNAL void SN_DECL f32_transpose_512(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) * target);
MAT_KERNEL(float) *restrict target);
INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix, float const scalar,
MAT_KERNEL(float) * ans);
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix, float const scalar,
MAT_KERNEL(float) * ans);
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix, float const scalar,
MAT_KERNEL(float) * ans);
MAT_KERNEL(float) *restrict ans);
#pragma endregion
#pragma region fma
INTERNAL void SN_DECL kernel_fma_sse(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans);
#pragma endregion
@@ -100,6 +100,16 @@ INTERNAL void SN_DECL kernel_fma_sse(MAT_KERNEL(float) const *restrict left,
INTERNAL uint64_t SN_DECL factorial_simd_sse(int n);
#pragma endregion
#pragma region dot_product
INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
__m128 const *vector, __m128 *ans);
INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans);
INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans);
INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans);
#pragma endregion
#pragma region string
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int length);
@@ -111,11 +121,11 @@ typedef struct sn_intrinsic {
int (*city_hash_simplify)(byte *str, int length);
FACTORIALROC factorial_simd;
int (*index_pair)(short const *str, uint32_t target, int length);
MATARITHMETIC kernel_add;
void (*kernel_fma)(void *left, void *right, void *ans);
SCALARMATARITHMETIC kernel_scalar_mul;
MATARITHMETIC kernel_sub;
MATTRANSPOSEPROC kernel_transpose;
MATARITHMETIC f32_add;
void (*f32_fma)(void *left, void *right, void *ans);
SCALARMATARITHMETIC f32_scalar_mul;
MATARITHMETIC f32_sub;
MATTRANSPOSEPROC f32_transpose;
VECTORNORMALIZEPROC normalize_3;
} sn_intrinsic;
StgSharp.Native/sn-src/StgSharpC.c
+3 -3
View File
@@ -27,7 +27,7 @@ SSCAPI void SSCDECL initGL(int majorVersion, int minorVersion)
glfwWindowHint(GLFW_RESIZABLE, GLFW_TRUE);
}
SSCAPI GLFWglproc SSCDECL loadGlfuncDefault(char *procName)
SSCAPI GLFWglproc SSCDECL loadGlfuncDefault(char const *procName)
{
void *ret = glfwGetProcAddress(procName);
// printf("%llu\n", (uint64_t)ret);
@@ -55,7 +55,7 @@ SSCAPI unsigned int linkShaderProgram(GladGLContext *context, GLuint shaderProgr
}
}
SSCAPI void SSCDECL loadImageData(char *location, Image *out, imageLoader loader)
SSCAPI void SSCDECL loadImageData(char const *location, Image *out, imageLoader loader)
{
if (loader == NULL) {
loader = stbi_load;
@@ -63,7 +63,7 @@ SSCAPI void SSCDECL loadImageData(char *location, Image *out, imageLoader loader
out->pixelPtr = loader(location, &(out->width), &(out->height), &(out->channel), 0);
}
SSCAPI void SSCDECL unloadImageData(Image *out)
SSCAPI void SSCDECL unloadImageData(Image *restrict out)
{
free(out->pixelPtr);
}
StgSharp.Native/sn-src/k_basicop.c
+24 -21
View File
@@ -1,7 +1,7 @@
#include "sn_intrinsic.h"
INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) * left, MAT_KERNEL(float) * right,
MAT_KERNEL(float) * ans)
INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans)
{
register __m128 c0 = _mm_add_ps(left->xmm[0], right->xmm[0]);
register __m128 c1 = _mm_add_ps(left->xmm[1], right->xmm[1]);
@@ -14,8 +14,8 @@ INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) * left, MAT_KERNEL(float) *
ans->xmm[3] = c3;
}
INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) * left, MAT_KERNEL(float) * right,
MAT_KERNEL(float) * ans)
INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans)
{
register __m256 c0 = _mm256_add_ps(left->ymm[0], right->ymm[0]);
register __m256 c1 = _mm256_add_ps(left->ymm[1], right->ymm[1]);
@@ -23,15 +23,15 @@ INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) * left, MAT_KERNEL(float) *
ans->ymm[1] = c1;
}
INTERNAL void SN_DECL f32_add_512(MAT_KERNEL(float) * left, MAT_KERNEL(float) * right,
MAT_KERNEL(float) * ans)
INTERNAL void SN_DECL f32_add_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans)
{
register __m512 c0 = _mm512_add_ps(left->zmm[0], right->zmm[0]);
ans->zmm[0] = c0;
}
INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) * left, MAT_KERNEL(float) * right,
MAT_KERNEL(float) * ans)
INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans)
{
register __m128 c0 = _mm_sub_ps(left->xmm[0], right->xmm[0]);
register __m128 c1 = _mm_sub_ps(left->xmm[1], right->xmm[1]);
@@ -44,8 +44,8 @@ INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) * left, MAT_KERNEL(float) *
ans->xmm[3] = c3;
}
INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) * left, MAT_KERNEL(float) * right,
MAT_KERNEL(float) * ans)
INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans)
{
register __m256 c0 = _mm256_sub_ps(left->ymm[0], right->ymm[0]);
register __m256 c1 = _mm256_sub_ps(left->ymm[1], right->ymm[1]);
@@ -53,15 +53,16 @@ INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) * left, MAT_KERNEL(float) *
ans->ymm[1] = c1;
}
INTERNAL void SN_DECL f32_sub_512(MAT_KERNEL(float) * left, MAT_KERNEL(float) * right,
MAT_KERNEL(float) * ans)
INTERNAL void SN_DECL f32_sub_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans)
{
register __m512 c0 = _mm512_sub_ps(left->zmm[0], right->zmm[0]);
ans->zmm[0] = c0;
}
// Matrix transpose implementations
INTERNAL void SN_DECL kernel_transpose_sse(MAT_KERNEL(float) * source, MAT_KERNEL(float) * target)
INTERNAL void SN_DECL f32_transpose_sse(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target)
{
register __m128 col0 = source->xmm[0];
register __m128 col1 = source->xmm[1];
@@ -78,7 +79,8 @@ INTERNAL void SN_DECL kernel_transpose_sse(MAT_KERNEL(float) * source, MAT_KERNE
target->xmm[3] = _mm_movehl_ps(tmp3, tmp1);
}
INTERNAL void SN_DECL kernel_transpose_avx(MAT_KERNEL(float) * source, MAT_KERNEL(float) * target)
INTERNAL void SN_DECL f32_transpose_avx(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target)
{
register __m256 col_low = source->ymm[0];
register __m256 col_high = source->ymm[1];
@@ -93,7 +95,8 @@ INTERNAL void SN_DECL kernel_transpose_avx(MAT_KERNEL(float) * source, MAT_KERNE
target->ymm[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
}
INTERNAL void SN_DECL kernel_transpose_512(MAT_KERNEL(float) * source, MAT_KERNEL(float) * target)
INTERNAL void SN_DECL f32_transpose_512(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target)
{
register __m512 matrix = source->zmm[0];
@@ -103,8 +106,8 @@ INTERNAL void SN_DECL kernel_transpose_512(MAT_KERNEL(float) * source, MAT_KERNE
target->zmm[0] = _mm512_permutexvar_ps(transpose_indices, matrix);
}
INTERNAL void SN_DECL kernel_scalar_mul_sse(MAT_KERNEL(float) * matrix, float scalar,
MAT_KERNEL(float) * ans)
INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix, float const scalar,
MAT_KERNEL(float) *restrict ans)
{
register __m128 temp = _mm_load_ss(&scalar);
register __m128 scalar_vec = _mm_shuffle_ps(temp, temp, _MM_SHUFFLE(0, 0, 0, 0));
@@ -120,8 +123,8 @@ INTERNAL void SN_DECL kernel_scalar_mul_sse(MAT_KERNEL(float) * matrix, float sc
ans->xmm[3] = c3;
}
INTERNAL void SN_DECL kernel_scalar_mul_avx(MAT_KERNEL(float) * matrix, float scalar,
MAT_KERNEL(float) * ans)
INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix, float const scalar,
MAT_KERNEL(float) *restrict ans)
{
register __m256 scalar_vec = _mm256_broadcast_ss(&scalar);
@@ -132,8 +135,8 @@ INTERNAL void SN_DECL kernel_scalar_mul_avx(MAT_KERNEL(float) * matrix, float sc
ans->ymm[1] = c1;
}
INTERNAL void SN_DECL kernel_scalar_mul_512(MAT_KERNEL(float) * matrix, float scalar,
MAT_KERNEL(float) * ans)
INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix, float const scalar,
MAT_KERNEL(float) *restrict ans)
{
__m512 scalar_vec = _mm512_broadcastss_ps(_mm_load_ss(&scalar));
StgSharp.Native/sn-src/k_fma.c
+1 -1
View File
@@ -2,7 +2,7 @@
// ans = left * right(T) + ans
INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left,
mat_kernel const *restrict right, mat_kernel *restrict ans)
MAT_KERNEL(float) const *restrict right, MAT_KERNEL(float) *restrict ans)
{
register __m128 source0 = _mm_load_ps(&left->xmm[0]);
register __m128 source1 = _mm_load_ps(&left->xmm[1]);
StgSharp.Native/sn-src/load_intrinsic.c
+15 -15
View File
@@ -66,11 +66,11 @@ void load_intrinsic_sse(sn_intrinsic *context)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->kernel_add = (MATARITHMETIC)f32_add_sse;
context->kernel_fma = (void(*)(void*, void*, void*))kernel_fma_sse;
context->kernel_scalar_mul = kernel_scalar_mul_sse;
context->kernel_sub = (MATARITHMETIC)f32_sub_sse;
context->kernel_transpose = (MATTRANSPOSEPROC)kernel_transpose_sse;
context->f32_add = (MATARITHMETIC)f32_add_sse;
context->f32_fma = (void (*)(void *, void *, void *))f32_fma_sse;
context->f32_scalar_mul = f32_scalar_mul_sse;
context->f32_sub = (MATARITHMETIC)f32_sub_sse;
context->f32_transpose = (MATTRANSPOSEPROC)f32_transpose_sse;
context->normalize_3 = normalize;
}
@@ -79,11 +79,11 @@ void load_intrinsic_avx(sn_intrinsic *context)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->kernel_add = (MATARITHMETIC)f32_add_avx;
context->kernel_fma = (void(*)(void*, void*, void*))kernel_fma_sse;
context->kernel_scalar_mul = kernel_scalar_mul_avx;
context->kernel_sub = (MATARITHMETIC)f32_sub_avx;
context->kernel_transpose = (MATTRANSPOSEPROC)kernel_transpose_avx;
context->f32_add = (MATARITHMETIC)f32_add_avx;
context->f32_fma = (void(*)(void*, void*, void*))f32_fma_sse;
context->f32_scalar_mul = f32_scalar_mul_avx;
context->f32_sub = (MATARITHMETIC)f32_sub_avx;
context->f32_transpose = (MATTRANSPOSEPROC)f32_transpose_avx;
context->normalize_3 = normalize;
}
@@ -92,10 +92,10 @@ void load_intrinsic_512(sn_intrinsic *context)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->kernel_add = (MATARITHMETIC)f32_add_512;
context->kernel_fma = (void(*)(void*, void*, void*))kernel_fma_sse;
context->kernel_scalar_mul = kernel_scalar_mul_512;
context->kernel_sub = (MATARITHMETIC)f32_sub_512;
context->kernel_transpose = (MATTRANSPOSEPROC)kernel_transpose_512;
context->f32_add = (MATARITHMETIC)f32_add_512;
context->f32_fma = (void(*)(void*, void*, void*))f32_fma_sse;
context->f32_scalar_mul = f32_scalar_mul_512;
context->f32_sub = (MATARITHMETIC)f32_sub_512;
context->f32_transpose = (MATTRANSPOSEPROC)f32_transpose_512;
context->normalize_3 = normalize;
}
StgSharp.Native/sn-src/sn_native.c
+3 -3
View File
@@ -27,7 +27,7 @@ SN_API void SN_DECL initGL(int majorVersion, int minorVersion)
glfwWindowHint(GLFW_RESIZABLE, GLFW_TRUE);
}
SN_API GLFWglproc SN_DECL loadGlfuncDefault(char *procName)
SN_API GLFWglproc SN_DECL loadGlfuncDefault(char const *procName)
{
void *ret = glfwGetProcAddress(procName);
// printf("%llu\n", (uint64_t)ret);
@@ -55,7 +55,7 @@ SN_API unsigned int linkShaderProgram(GladGLContext *context, GLuint shaderProgr
}
}
SN_API void SN_DECL loadImageData(char *location, Image *out, imageLoader loader)
SN_API void SN_DECL loadImageData(char const *location, Image *out, imageLoader loader)
{
if (loader == NULL) {
loader = stbi_load;
@@ -63,7 +63,7 @@ SN_API void SN_DECL loadImageData(char *location, Image *out, imageLoader loader
out->pixelPtr = loader(location, &(out->width), &(out->height), &(out->channel), 0);
}
SN_API void SN_DECL unloadImageData(Image *out)
SN_API void SN_DECL unloadImageData(Image *restrict out)
{
free(out->pixelPtr);
}
StgSharp.Native/sn-src/str_string.c
+1 -1
View File
@@ -1,6 +1,6 @@
#include "sn_intrinsic.h"
int index_pair_sse(short const *str, uint32_t target, int length)
INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int length)
{
int index = 0, result = 0, size = length, end = size - 8;
__m128i s = {
StgSharp.Native/sn-src/vector.c
+1 -1
View File
@@ -1,7 +1,7 @@
#include "StgSharpNative.h"
#include "sn_internal.h"
SN_API void SN_DECL normalize(__m128 *source, __m128 *target)
SN_API void SN_DECL normalize(__m128 *restrict source, __m128 *restrict target)
{
__m128 a_length_sq = _mm_mul_ps(*source, *source);
a_length_sq = _mm_add_ps(a_length_sq,
StgSharp.Native/sn-src/vector_dot.c
+32 -112
View File
@@ -3,145 +3,65 @@
static const int avxdot_indices[] = { 0, 4, 1, 5, 2, 3, 6, 7 };
INTERNAL void FORCEINLINE SN_DECL dot_41_sse(__4_columnset *transpose, __m128 *vector, __m128 *ans)
INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
__m128 const *vector, __m128 *ans)
{
*ans = _mm_dp_ps(transpose->column[0], *vector, 0B11110001);
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], *vector, 0B11110001),
*ans = _mm_dp_ps(transpose->xmm[0], *vector, 0B11110001);
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[1], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 1));
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], *vector, 0B11110001),
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[2], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[3], *vector, 0B11110001),
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[3], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 4));
}
INTERNAL void SN_DECL dot_41_avx(__4_columnset *transpose, __m128 *vector, __m128 *ans)
INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans)
{
__m256 _double = _mm256_set_m128(vector[0], vector[0]);
__m256 tmp = _mm256_dp_ps(transpose->half[0], _double, 0B11110001);
tmp = _mm256_blend_ps(tmp, _mm256_dp_ps(transpose->half[0], _double, 0B11110010),
__m256 tmp = _mm256_dp_ps(transpose->ymm[0], _double, 0B11110001);
tmp = _mm256_blend_ps(tmp, _mm256_dp_ps(transpose->ymm[0], _double, 0B11110010),
0B00100010);
*ans = _mm256_castps256_ps128(
_mm256_permutevar8x32_ps(tmp, _mm256_loadu_epi32(avxdot_indices)));
}
INTERNAL void FORCEINLINE SN_DECL dot_31_sse(__3_columnset *transpose, __m128 *vector, __m128 *ans)
INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans)
{
*ans = _mm_dp_ps(transpose->column[0], *vector, 0B11110001);
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
}
INTERNAL void FORCEINLINE SN_DECL dot_31_avx(__3_columnset *transpose, __m128 *vector, __m128 *ans)
{
__m256 _double = _mm256_set_m128(vector[0], vector[0]);
__m256 tmp = _mm256_dp_ps(transpose->part, _double, 0B11110001);
*ans = _mm256_castps256_ps128(
_mm256_permutevar8x32_ps(tmp, _mm256_loadu_epi32(avxdot_indices)));
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
}
INTERNAL void SN_DECL dot_42_sse(__4_columnset *transpose, __m128 *vector, __m128 *ans)
{
ans[0] = _mm_dp_ps(transpose->column[0], vector[0], 0B11110001);
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], vector[0], 0B11110001),
ans[0] = _mm_dp_ps(transpose->xmm[0], vector[0], 0B11110001);
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[1], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], vector[0], 0B11110001),
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[2], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[3], vector[0], 0B11110001),
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[3], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 4));
ans[1] = _mm_dp_ps(transpose->column[0], vector[1], 0B11110001);
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], vector[1], 0B11110001),
ans[1] = _mm_dp_ps(transpose->xmm[0], vector[1], 0B11110001);
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[1], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], vector[1], 0B11110001),
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[2], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[3], vector[1], 0B11110001),
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[3], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 4));
}
INTERNAL void SN_DECL dot_32_sse(__3_columnset *transpose, __m128 *vector, __m128 *ans)
INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans)
{
ans[0] = _mm_dp_ps(transpose->column[0], vector[0], 0B11110001);
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
__m256 temp = transpose->ymm[0];
__m256 vector_stream = _mm256_set_m128(vector[1], vector[0]);
ans[1] = _mm_dp_ps(transpose->column[0], vector[1], 0B11110001);
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
}
INTERNAL void SN_DECL dot_42_avx(__4_columnset *transpose, __m128 *vector, __m128 *ans)
{
__2_columnset *ans_ptr = (__2_columnset *)ans, *right_ptr = (__2_columnset *)vector;
__m256 temp = ALIGN256(transpose->half[0]);
ans_ptr->stream = _mm256_dp_ps(temp, right_ptr->stream, 0B11110001);
__m256 result = _mm256_dp_ps(temp, vector_stream, 0B11110001);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
ans_ptr->stream = _mm256_blend_ps(
ans_ptr->stream, _mm256_dp_ps(temp, right_ptr->stream, 0B11110010), 0B00100010);
result = _mm256_blend_ps(
result, _mm256_dp_ps(temp, vector_stream, 0B11110010), 0B00100010);
temp = ALIGN256(transpose->half[1]);
ans_ptr->stream = _mm256_blend_ps(
ans_ptr->stream, _mm256_dp_ps(temp, right_ptr->stream, 0B11110100), 0B01000100);
temp = transpose->ymm[1];
result = _mm256_blend_ps(
result, _mm256_dp_ps(temp, vector_stream, 0B11110100), 0B01000100);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
ans_ptr->stream = _mm256_blend_ps(
ans_ptr->stream, _mm256_dp_ps(temp, right_ptr->stream, 0B11111000), 0B10001000);
result = _mm256_blend_ps(
result, _mm256_dp_ps(temp, vector_stream, 0B11111000), 0B10001000);
ans[1] = _mm_shuffle_ps(ALIGN(ans[1]), ALIGN(ans[1]), _MM_SHUFFLE(2, 3, 0, 1));
}
INTERNAL void SN_DECL dot_32_avx(__3_columnset *transpose, __m128 *vector, __m128 *ans)
{
__2_columnset *ans_ptr = (__2_columnset *)ans, *right_ptr = (__2_columnset *)vector;
__m256 temp = transpose->part;
ans_ptr->stream = _mm256_dp_ps(temp, right_ptr->stream, 0B11110001);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
ans_ptr->stream = _mm256_blend_ps(
ans_ptr->stream, _mm256_dp_ps(temp, right_ptr->stream, 0B11110010), 0B00100010);
temp = _mm256_set_m128(transpose->column[2], transpose->column[2]);
ans_ptr->stream = _mm256_blend_ps(
ans_ptr->stream, _mm256_dp_ps(temp, right_ptr->stream, 0B11110100), 0B01000100);
ans[1] = _mm_shuffle_ps(ALIGN(ans[1]), ALIGN(ans[1]), _MM_SHUFFLE(2, 2, 0, 1));
}
INTERNAL void SN_DECL dot_43_sse(__4_columnset *transpose, __m128 *vector, __m128 *ans)
{
dot_41_sse(transpose, vector, ans);
dot_41_sse(transpose, vector + 1, ans + 1);
dot_41_sse(transpose, vector + 2, ans + 2);
}
INTERNAL void SN_DECL dot_43_avx(__4_columnset *transpose, __m128 *vector, __m128 *ans)
{
__3_columnset *ans_ptr = (__3_columnset *)ans, *right_ptr = (__3_columnset *)vector;
__m256 temp = ALIGN256(transpose->half[0]);
__m256 third = _mm256_set_m128(right_ptr->column[2], right_ptr->column[2]);
ans_ptr->part = _mm256_dp_ps(temp, right_ptr->part, 0B11110001);
__m256 ans_third = _mm256_dp_ps(temp, third, 0B11110001);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
ans_ptr->part = _mm256_blend_ps(
ans_ptr->part, _mm256_dp_ps(temp, right_ptr->part, 0B11110010), 0B00100010);
temp = ALIGN256(transpose->half[1]);
ans_ptr->part = _mm256_blend_ps(
ans_ptr->part, _mm256_dp_ps(temp, right_ptr->part, 0B11110100), 0B01000100);
ans_third = _mm256_blend_ps(ans_third, _mm256_dp_ps(temp, third, 0B11110010), 0B00100010);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
ans_ptr->part = _mm256_blend_ps(
ans_ptr->part, _mm256_dp_ps(temp, right_ptr->part, 0B11111000), 0B10001000);
ans[1] = _mm_shuffle_ps(ALIGN(ans[1]), ALIGN(ans[1]), _MM_SHUFFLE(2, 3, 0, 1));
ans[2] = _mm256_castps256_ps128(
_mm256_permutevar8x32_ps(ans_third, _mm256_loadu_epi32(avxdot_indices)));
ans[0] = _mm256_castps256_ps128(result);
ans[1] = _mm256_extractf128_ps(result, 1);
ans[1] = _mm_shuffle_ps(ans[1], ans[1], _MM_SHUFFLE(2, 3, 0, 1));
}