youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MulAvailable

Browse Source
This commit is contained in:
Nitload
2025-12-16 23:13:17 +08:00
parent 28ea5d1d1e
commit 3486375470
2637 changed files with 23787 additions and 838773 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-src/dllmain.c
-35
View File
@@ -1,35 +0,0 @@
#include "StgSharpC.h"
#ifdef _WIN32
// clang-format off
BOOL APIENTRY DllMain(
HMODULE hModule,
DWORD ul_reason_for_call,
LPVOID lpReserved
)
{
switch (ul_reason_for_call)
{
case DLL_PROCESS_ATTACH:
return 1;
case DLL_THREAD_ATTACH:
return 1;
case DLL_THREAD_DETACH:
return 1;
case DLL_PROCESS_DETACH:
return 1;
break;
}
return TRUE;
}
#else
attribute((constructor)) void DllMain()
{
}
#endif // _WIN32
// clang-format on
StgSharp.Native/sn-src/intrinsic.c
-143
View File
@@ -1,143 +0,0 @@
#include "StgSharpC.h"
#include "ssc_intrinsic.h"
#if defined(_MSC_VER) && !defined(__clang__)
#include <intrin.h>
#include <isa_availability.h>
#else
#include <cpuid.h>
#endif // _MSC_VER
PRIVATE void load_intrinsic_sse(ssc_intrinsic *context);
PRIVATE void load_intrinsic_avx(ssc_intrinsic *context);
PRIVATE void load_intrinsic_512(ssc_intrinsic *context);
PRIVATE int check_isa(most_advanced_instruction instruction_family)
{
if (instruction_family != NEON) {
#if defined(_MSC_VER) && !defined(__clang__)
return __check_isa_support(instruction_family, 0);
#else
switch (instruction_family) {
case SSE:
return __builtin_cpu_supports("sse4.2");
case AVX256:
return __builtin_cpu_supports("avx2");
case AVX512:
return __builtin_cpu_supports("avx512f");
default:
return 0;
}
#endif // _MSC_VER
}
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
return 1;
#else
return 0;
#endif
}
SSCAPI int load_intrinsic_function(void *intrinsic_context)
{
ssc_null_assert(intrinsic_context);
ssc_intrinsic *context = (ssc_intrinsic *)intrinsic_context;
if (check_isa(AVX512)) {
load_intrinsic_512(context);
return 3;
} else if (check_isa(AVX256)) {
load_intrinsic_avx(context);
return 2;
} else if (check_isa(SSE)) {
load_intrinsic_sse(context);
return 1;
} else if (check_isa(NEON)) {
return -1;
}
return 0;
}
void load_intrinsic_sse(ssc_intrinsic *context)
{
context->add_m2 = addmatrix2_sse;
context->add_m3 = addmatrix3_sse;
context->add_m4 = addmatrix4_sse;
context->transpose_23 = transpose23;
context->transpose_24 = transpose24;
context->transpose_32 = transpose32;
context->transpose_33 = transpose33;
context->transpose_34 = transpose34;
context->transpose_42 = transpose42;
context->transpose_43 = transpose43;
context->transpose_44 = transpose44;
context->normalize_3 = normalize;
context->det_matrix33 = det_mat3;
context->det_matrix44 = det_mat4;
context->dot_31 = dot_31_sse;
context->dot_32 = dot_32_sse;
context->dot_41 = dot_41_sse;
context->dot_42 = dot_42_sse;
context->dot_43 = dot_43_sse;
context->city_hash_simplify = city_hash_simplify_sse;
context->index_pair = index_pair_sse;
context->sub_m2 = submatrix2_sse;
context->sub_m3 = submatrix3_sse;
context->sub_m4 = submatrix4_sse;
context->factorial_simd = factorial_simd_sse;
}
void load_intrinsic_avx(ssc_intrinsic *context)
{
context->add_m2 = addmatrix2_avx;
context->add_m3 = addmatrix3_avx;
context->add_m4 = addmatrix4_avx;
context->transpose_23 = transpose23;
context->transpose_24 = transpose24;
context->transpose_32 = transpose32;
context->transpose_33 = transpose33;
context->transpose_34 = transpose34;
context->transpose_42 = transpose42;
context->transpose_43 = transpose43;
context->transpose_44 = transpose44;
context->normalize_3 = normalize;
context->det_matrix33 = det_mat3;
context->det_matrix44 = det_mat4;
context->dot_31 = dot_31_avx;
context->dot_32 = dot_32_avx;
context->dot_41 = dot_41_avx;
context->dot_42 = dot_42_avx;
context->dot_43 = dot_43_avx;
context->city_hash_simplify = city_hash_simplify_sse;
context->index_pair = index_pair_sse;
context->factorial_simd = factorial_simd_sse;
}
void load_intrinsic_512(ssc_intrinsic *context)
{
context->add_m2 = addmatrix2_avx;
context->add_m3 = addmatrix3_512;
context->add_m4 = addmatrix4_512;
context->transpose_23 = transpose23;
context->transpose_24 = transpose24;
context->transpose_32 = transpose32;
context->transpose_33 = transpose33;
context->transpose_34 = transpose34;
context->transpose_42 = transpose42;
context->transpose_43 = transpose43;
context->transpose_44 = transpose44;
context->normalize_3 = normalize;
context->det_matrix33 = det_mat3;
context->det_matrix44 = det_mat4;
context->dot_31 = dot_31_avx;
context->dot_32 = dot_32_avx;
context->dot_41 = dot_41_avx;
context->dot_42 = dot_42_avx;
context->dot_43 = dot_43_avx;
context->city_hash_simplify = city_hash_simplify_sse;
context->index_pair = index_pair_sse;
context->factorial_simd = factorial_simd_sse;
}
StgSharp.Native/sn-src/k_basicop.c
+71
View File
@@ -0,0 +1,71 @@
#include "sn_intrinsic.h"
DECLARE_PNL_PROC_ANS(float, sse, clear)
{
register __m128 zero = _mm_setzero_ps();
PANEL_COL_VEC((*ans), 0, 0) = zero;
PANEL_COL_VEC((*ans), 1, 0) = zero;
PANEL_COL_VEC((*ans), 2, 0) = zero;
PANEL_COL_VEC((*ans), 3, 0) = zero;
}
DECLARE_PNL_PROC_ANS(float, avx, clear)
{
register __m256 zero = _mm256_setzero_ps();
PANEL_COL_VEC((*ans), 0, 0) = zero;
PANEL_COL_VEC((*ans), 1, 0) = zero;
PANEL_COL_VEC((*ans), 2, 0) = zero;
PANEL_COL_VEC((*ans), 3, 0) = zero;
}
DECLARE_PNL_PROC_ANS(float, 512, clear)
{
register __m512 zero = _mm512_setzero_ps();
PANEL_COL_VEC((*ans), 0, 0) = zero;
PANEL_COL_VEC((*ans), 1, 0) = zero;
PANEL_COL_VEC((*ans), 2, 0) = zero;
PANEL_COL_VEC((*ans), 3, 0) = zero;
}
// Matrix transpose implementations
DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose)
{
register __m128 col0 = right->xmm[0];
register __m128 col1 = right->xmm[1];
register __m128 col2 = right->xmm[2];
register __m128 col3 = right->xmm[3];
register __m128 tmp0 = _mm_unpacklo_ps(col0, col1);
register __m128 tmp1 = _mm_unpackhi_ps(col0, col1);
register __m128 tmp2 = _mm_unpacklo_ps(col2, col3);
register __m128 tmp3 = _mm_unpackhi_ps(col2, col3);
ans->xmm[0] = _mm_movelh_ps(tmp0, tmp2);
ans->xmm[1] = _mm_movehl_ps(tmp2, tmp0);
ans->xmm[2] = _mm_movelh_ps(tmp1, tmp3);
ans->xmm[3] = _mm_movehl_ps(tmp3, tmp1);
}
DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose)
{
register __m256 col_low = right->ymm[0];
register __m256 col_high = right->ymm[1];
register __m256 unpack_low = _mm256_unpacklo_ps(col_low, col_high);
register __m256 unpack_high = _mm256_unpackhi_ps(col_low, col_high);
col_low = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x20);
col_high = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x31);
ans->ymm[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0));
ans->ymm[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
}
DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose)
{
register __m512 matrix = right->zmm[0];
const __m512i transpose_indices =
_mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
ans->zmm[0] = _mm512_permutexvar_ps(transpose_indices, matrix);
}
StgSharp.Native/sn-src/k_bufop.c
+134
View File
@@ -0,0 +1,134 @@
#include "sn_intrinsic.h"
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill)
{
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].xmm[0] = vec;
ans[i].xmm[1] = vec;
ans[i].xmm[2] = vec;
ans[i].xmm[3] = vec;
}
}
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
ans[i].ymm[0] = vec;
ans[i].ymm[1] = vec;
}
}
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].zmm[0] = vec;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, add)
{
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_add_ps(left[i].xmm[0], right[i].xmm[0]);
register __m128 c1 = _mm_add_ps(left[i].xmm[1], right[i].xmm[1]);
register __m128 c2 = _mm_add_ps(left[i].xmm[2], right[i].xmm[2]);
register __m128 c3 = _mm_add_ps(left[i].xmm[3], right[i].xmm[3]);
ans[i].xmm[0] = c0;
ans[i].xmm[1] = c1;
ans[i].xmm[2] = c2;
ans[i].xmm[3] = c3;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, add)
{
for (size_t i = 0; i < count; i++) {
size_t s = sizeof(MAT_KERNEL(float));
register __m256 c0 = _mm256_add_ps(left[i].ymm[0], right[i].ymm[0]);
register __m256 c1 = _mm256_add_ps(left[i].ymm[1], right[i].ymm[1]);
ans[i].ymm[0] = c0;
ans[i].ymm[1] = c1;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, add)
{
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_add_ps(left[i].zmm[0], right[i].zmm[0]);
ans[i].zmm[0] = c0;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, sub)
{
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_sub_ps(left[i].xmm[0], right[i].xmm[0]);
register __m128 c1 = _mm_sub_ps(left[i].xmm[1], right[i].xmm[1]);
register __m128 c2 = _mm_sub_ps(left[i].xmm[2], right[i].xmm[2]);
register __m128 c3 = _mm_sub_ps(left[i].xmm[3], right[i].xmm[3]);
ans[i].xmm[0] = c0;
ans[i].xmm[1] = c1;
ans[i].xmm[2] = c2;
ans[i].xmm[3] = c3;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, sub)
{
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].ymm[0], right[i].ymm[0]);
register __m256 c1 = _mm256_sub_ps(left[i].ymm[1], right[i].ymm[1]);
ans[i].ymm[0] = c0;
ans[i].ymm[1] = c1;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, sub)
{
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].zmm[0], right[i].zmm[0]);
ans[i].zmm[0] = c0;
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul)
{
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_mul_ps(right[i].xmm[0], vec);
register __m128 c1 = _mm_mul_ps(right[i].xmm[1], vec);
register __m128 c2 = _mm_mul_ps(right[i].xmm[2], vec);
register __m128 c3 = _mm_mul_ps(right[i].xmm[3], vec);
ans[i].xmm[0] = c0;
ans[i].xmm[1] = c1;
ans[i].xmm[2] = c2;
ans[i].xmm[3] = c3;
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_mul_ps(right[i].ymm[0], vec);
register __m256 c1 = _mm256_mul_ps(right[i].ymm[1], vec);
ans[i].ymm[0] = c0;
ans[i].ymm[1] = c1;
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_mul_ps(right[i].zmm[0], vec);
ans[i].zmm[0] = c0;
}
}
StgSharp.Native/sn-src/k_fma.c
+205
View File
@@ -0,0 +1,205 @@
#include "sn_intrinsic.h"
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
// ans = left * right + ans
// right is not transposed
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma)
{
const __m128 a0 = PANEL_COL_VEC((*left), 0, 0);
const __m128 a1 = PANEL_COL_VEC((*left), 1, 0);
const __m128 a2 = PANEL_COL_VEC((*left), 2, 0);
const __m128 a3 = PANEL_COL_VEC((*left), 3, 0);
__m128 c0 = PANEL_COL_VEC((*ans), 0, 0);
__m128 c1 = PANEL_COL_VEC((*ans), 1, 0);
__m128 c2 = PANEL_COL_VEC((*ans), 2, 0);
__m128 c3 = PANEL_COL_VEC((*ans), 3, 0);
__m128 bcol;
/* Column 0 */
bcol = PANEL_COL_VEC((*right), 0, 0);
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = PANEL_COL_VEC((*right), 1, 0);
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = PANEL_COL_VEC((*right), 2, 0);
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = PANEL_COL_VEC((*right), 3, 0);
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
PANEL_COL_VEC((*ans), 0, 0) = c0;
PANEL_COL_VEC((*ans), 1, 0) = c1;
PANEL_COL_VEC((*ans), 2, 0) = c2;
PANEL_COL_VEC((*ans), 3, 0) = c3;
}
// ans = left * right + ans
// right is not transposed
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx_fma, fma)
{
static const __m256i idx0 = { 1, 1, 1, 1, 1, 1, 1, 1 };
for (int col = 0; col < 8; ++col) {
register __m256 bcol = _mm256_load_ps((float *)&right->vec[col][0]);
register __m256 ccol = _mm256_load_ps((float *)&ans->vec[col][0]);
register __m256i idx = _mm256_setzero_si256();
register __m256i increament = _mm256_load_si256(&idx0);
register __m256 a0 = _mm256_load_ps((float *)&left->vec[0][0]);
register __m256 a1 = _mm256_load_ps((float *)&left->vec[1][0]);
register __m256 a2 = _mm256_load_ps((float *)&left->vec[2][0]);
register __m256 a3 = _mm256_load_ps((float *)&left->vec[3][0]);
ccol = _mm256_fmadd_ps(a0, _mm256_permutevar8x32_ps(bcol, idx), ccol);
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_fmadd_ps(a1, _mm256_permutevar8x32_ps(bcol, idx), ccol);
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_fmadd_ps(a2, _mm256_permutevar8x32_ps(bcol, idx), ccol);
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_fmadd_ps(a3, _mm256_permutevar8x32_ps(bcol, idx), ccol);
idx = _mm256_add_epi32(idx, increament);
a0 = _mm256_load_ps((float *)&left->vec[4][0]);
a1 = _mm256_load_ps((float *)&left->vec[5][0]);
a2 = _mm256_load_ps((float *)&left->vec[6][0]);
a3 = _mm256_load_ps((float *)&left->vec[7][0]);
ccol = _mm256_fmadd_ps(a0, _mm256_permutevar8x32_ps(bcol, idx), ccol);
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_fmadd_ps(a1, _mm256_permutevar8x32_ps(bcol, idx), ccol);
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_fmadd_ps(a2, _mm256_permutevar8x32_ps(bcol, idx), ccol);
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_fmadd_ps(a3, _mm256_permutevar8x32_ps(bcol, idx), ccol);
_mm256_store_ps((float *)&ans->vec[col][0], ccol);
}
}
// ans = left * right + ans
// right is not transposed
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma)
{
static const __m256i idx0 = { 1, 1, 1, 1, 1, 1, 1, 1 };
for (int col = 0; col < 8; ++col) {
register __m256 bcol = _mm256_load_ps((float *)&right->vec[col][0]);
register __m256 ccol = _mm256_load_ps((float *)&ans->vec[col][0]);
register __m256i idx = _mm256_setzero_si256();
register __m256i increament = _mm256_load_si256(&idx0);
register __m256 a0 = _mm256_load_ps((float *)&left->vec[0][0]);
register __m256 a1 = _mm256_load_ps((float *)&left->vec[1][0]);
register __m256 a2 = _mm256_load_ps((float *)&left->vec[2][0]);
register __m256 a3 = _mm256_load_ps((float *)&left->vec[3][0]);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a0, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a1, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a2, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a3, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
a0 = _mm256_load_ps((float *)&left->vec[4][0]);
a1 = _mm256_load_ps((float *)&left->vec[5][0]);
a2 = _mm256_load_ps((float *)&left->vec[6][0]);
a3 = _mm256_load_ps((float *)&left->vec[7][0]);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a0, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a1, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a2, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a3, _mm256_permutevar8x32_ps(bcol, idx)));
_mm256_store_ps((float *)&ans->vec[col][0], ccol);
}
}
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, fma)
{
const __m512i ones = _mm512_set1_epi32(1);
for (int col = 0; col < 8; ++col) {
register __m512 bcol = _mm512_load_ps((float *)&right->vec[col][0]);
register __m512 ccol = _mm512_load_ps((float *)&ans->vec[col][0]);
register __m512i idx = _mm512_setzero_si512();
register __m512i increament = _mm512_load_si512(&ones);
register __m512 a0 = _mm512_load_ps((float *)&left->vec[0][0]);
register __m512 a1 = _mm512_load_ps((float *)&left->vec[1][0]);
register __m512 a2 = _mm512_load_ps((float *)&left->vec[2][0]);
register __m512 a3 = _mm512_load_ps((float *)&left->vec[3][0]);
register __m512 a4 = _mm512_load_ps((float *)&left->vec[4][0]);
register __m512 a5 = _mm512_load_ps((float *)&left->vec[5][0]);
register __m512 a6 = _mm512_load_ps((float *)&left->vec[6][0]);
register __m512 a7 = _mm512_load_ps((float *)&left->vec[7][0]);
ccol = _mm512_fmadd_ps(a0, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a1, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a2, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a3, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a4, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a5, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a6, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a7, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
a0 = _mm512_load_ps((float *)&left->vec[8][0]);
a1 = _mm512_load_ps((float *)&left->vec[9][0]);
a2 = _mm512_load_ps((float *)&left->vec[10][0]);
a3 = _mm512_load_ps((float *)&left->vec[11][0]);
a4 = _mm512_load_ps((float *)&left->vec[12][0]);
a5 = _mm512_load_ps((float *)&left->vec[13][0]);
a6 = _mm512_load_ps((float *)&left->vec[14][0]);
a7 = _mm512_load_ps((float *)&left->vec[15][0]);
ccol = _mm512_fmadd_ps(a0, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a1, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a2, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a3, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a4, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a5, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a6, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a7, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
_mm512_store_ps((float *)&ans->vec[col][0], ccol);
}
}
StgSharp.Native/sn-src/k_panel.c
+189
View File
@@ -0,0 +1,189 @@
#include "sn_matkernel.h"
DECLARE_BUILD_PANEL(avx, float)
{
const int Q = 8; /* fixed for AVX float panel */
/* Neighbor kernels; substitute global zero kernel where out-of-bounds */
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
const MAT_KERNEL(float) *const k_lt = matrix + (col_index * row_length + row_index);
const MAT_KERNEL(float) *const k_rt =
has_right ? (matrix + ((col_index + 1) * row_length + row_index)) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) *const k_lb =
has_bottom ? (matrix + (col_index * row_length + (row_index + 1))) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((col_index + 1) * row_length + (row_index + 1))) :
(const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
register __m128 top128, bot128;
register __m256 col256;
for (int pc = 0; pc < Q; ++pc) {
const int is_right_half = (pc >= 4);
const int kc_col = pc & 3; /* column index inside 4x4 kernel */
const MAT_KERNEL(float) *src_top = is_right_half ? k_rt : k_lt;
const MAT_KERNEL(float) *src_bot = is_right_half ? k_rb : k_lb;
/* Vector loads: each kernel column is a compact __m128 */
top128 = src_top->xmm[kc_col];
bot128 = src_bot->xmm[kc_col];
/* Combine into an AVX column: low 128 = top, high 128 = bottom */
col256 = _mm256_set_m128(bot128, top128);
PANEL_COL_VEC((*panel), pc, 0) = col256;
}
}
DECLARE_BUILD_PANEL(sse, float)
{
/* Single kernel -> 4 columns, each is one __m128 column vector. */
const MAT_KERNEL(float) *src = matrix + (col_index * row_length + row_index);
for (int pc = 0; pc < 4; ++pc) {
PANEL_COL_VEC((*panel), pc, 0) = src->xmm[pc];
}
}
DECLARE_BUILD_PANEL(512, float)
{
const MAT_KERNEL(float) *const zero = (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL;
const MAT_KERNEL(float) *k[4][4];
for (int kc = 0; kc < 4; ++kc) {
for (int kr = 0; kr < 4; ++kr) {
const int col = col_index + kc;
const int row = row_index + kr;
k[kc][kr] = (col < col_length && row < row_length) ?
(matrix + (col * row_length + row)) :
zero;
}
}
/* Each panel column corresponds to a kernel column (0..3) within a block col (0..3). */
for (int pc = 0; pc < 16; ++pc) {
const int blk_col = pc >> 2; /* which kernel column block (0..3) */
const int kc_col = pc & 3; /* column inside a 4x4 kernel */
const __m128 r0 = k[blk_col][0]->xmm[kc_col];
const __m128 r1 = k[blk_col][1]->xmm[kc_col];
const __m128 r2 = k[blk_col][2]->xmm[kc_col];
const __m128 r3 = k[blk_col][3]->xmm[kc_col];
const __m256 top = _mm256_set_m128(r1, r0);
const __m256 bot = _mm256_set_m128(r3, r2);
__m512 col = _mm512_castps256_ps512(top);
col = _mm512_insertf32x8(col, bot, 1);
PANEL_COL_VEC((*panel), pc, 0) = col;
}
}
DECLARE_STORE_PANEL(avx, float)
{
const int Q = 8; /* fixed for AVX float panel */
/* Local dummy kernel to swallow out-of-bounds writes; no init needed */
MAT_KERNEL(float) zero;
const int has_right = (col_index + 1) < col_length;
const int has_bottom = (row_index + 1) < row_length;
MAT_KERNEL(float) *const k_lt = (col_index < col_length && row_index < row_length) ?
(matrix + (col_index * row_length + row_index)) :
&zero;
MAT_KERNEL(float) *const k_rt =
has_right && (row_index < row_length) ?
(matrix + ((col_index + 1) * row_length + row_index)) :
&zero;
MAT_KERNEL(float) *const k_lb =
has_bottom && (col_index < col_length) ?
(matrix + (col_index * row_length + (row_index + 1))) :
&zero;
MAT_KERNEL(float) *const k_rb =
(has_right && has_bottom) ?
(matrix + ((col_index + 1) * row_length + (row_index + 1))) :
&zero;
register __m128 top128, bot128;
int i = 0;
for (; i < Q / 2; i++) {
const int kc_col = i; /* column index inside 4x4 kernel */
/* Load the panel column vector */
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
/* Split into top/bottom halves */
top128 = _mm256_castps256_ps128(col256); /* low 128 */
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
/* Store back into kernels directly (out-of-bounds writes go to dummy) */
k_rt->xmm[kc_col] = top128;
k_rb->xmm[kc_col] = bot128;
}
for (; i < Q; i++) {
const int kc_col = i - 4; /* column index inside 4x4 kernel */
/* Load the panel column vector */
const __m256 col256 = PANEL_COL_VEC((*panel), i, 0);
/* Split into top/bottom halves */
top128 = _mm256_castps256_ps128(col256); /* low 128 */
bot128 = _mm256_extractf128_ps(col256, 1); /* high 128 */
/* Store back into kernels directly (out-of-bounds writes go to dummy) */
k_lt->xmm[kc_col] = top128;
k_lb->xmm[kc_col] = bot128;
}
}
DECLARE_STORE_PANEL(sse, float)
{
MAT_KERNEL(float) *dst = matrix + (col_index * row_length + row_index);
for (int pc = 0; pc < 4; ++pc) {
const __m128 col = PANEL_COL_VEC((*panel), pc, 0);
dst->xmm[pc] = col;
}
}
DECLARE_STORE_PANEL(512, float)
{
MAT_KERNEL(float) zero;
MAT_KERNEL(float) *k[4][4];
for (int kc = 0; kc < 4; ++kc) {
for (int kr = 0; kr < 4; ++kr) {
const int col = col_index + kc;
const int row = row_index + kr;
k[kc][kr] = (col < col_length && row < row_length) ?
(matrix + (col * row_length + row)) :
&zero;
}
}
for (int pc = 0; pc < 16; ++pc) {
const int blk_col = pc >> 2; /* which kernel column block (0..3) */
const int kc_col = pc & 3; /* column inside a 4x4 kernel */
const __m512 col = PANEL_COL_VEC((*panel), pc, 0);
const __m256 top = _mm512_castps512_ps256(col);
const __m256 bot = _mm512_extractf32x8_ps(col, 1);
const __m128 r0 = _mm256_castps256_ps128(top);
const __m128 r1 = _mm256_extractf128_ps(top, 1);
const __m128 r2 = _mm256_castps256_ps128(bot);
const __m128 r3 = _mm256_extractf128_ps(bot, 1);
k[blk_col][0]->xmm[kc_col] = r0;
k[blk_col][1]->xmm[kc_col] = r1;
k[blk_col][2]->xmm[kc_col] = r2;
k[blk_col][3]->xmm[kc_col] = r3;
}
}
StgSharp.Native/sn-src/load_glfw.c
+123
View File
@@ -0,0 +1,123 @@
#include "StgSharpNative.h"
#include "glfw_function.h"
#include <string.h>
SN_API void SN_DECL load_glfw_functions(GLFWFunctionTable *table)
{
table->glfwCreateCursor = glfwCreateCursor;
table->glfwCreateStandardCursor = glfwCreateStandardCursor;
table->glfwCreateWindow = glfwCreateWindow;
table->glfwDefaultWindowHints = glfwDefaultWindowHints;
table->glfwDestroyCursor = glfwDestroyCursor;
table->glfwDestroyWindow = glfwDestroyWindow;
table->glfwExtensionSupported = glfwExtensionSupported;
table->glfwFocusWindow = glfwFocusWindow;
table->glfwGetClipboardString = glfwGetClipboardString;
table->glfwGetCurrentContext = glfwGetCurrentContext;
table->glfwGetCursorPos = glfwGetCursorPos;
table->glfwGetError = glfwGetError;
table->glfwGetFrameBufferSize = glfwGetFramebufferSize;
table->glfwGetGamepadName = glfwGetGamepadName;
table->glfwGetGamepadState = glfwGetGamepadState;
table->glfwGetGammaRamp = glfwGetGammaRamp;
table->glfwGetInputMode = glfwGetInputMode;
table->glfwGetJoystickAxes = glfwGetJoystickAxes;
table->glfwGetJoystickButtons = glfwGetJoystickButtons;
table->glfwGetJoystickGUID = glfwGetJoystickGUID;
table->glfwGetJoystickHats = glfwGetJoystickHats;
table->glfwGetJoystickName = glfwGetJoystickName;
table->glfwGetJoystickUserPointer = glfwGetJoystickUserPointer;
table->glfwGetKey = glfwGetKey;
table->glfwGetKeyName = glfwGetKeyName;
table->glfwGetKeyScancode = glfwGetKeyScancode;
table->glfwGetMonitorContentScale = glfwGetMonitorContentScale;
table->glfwGetMonitorName = glfwGetMonitorName;
table->glfwGetMonitorPhysicalSize = glfwGetMonitorPhysicalSize;
table->glfwGetMonitorPos = glfwGetMonitorPos;
table->glfwGetMonitors = glfwGetMonitors;
table->glfwGetMonitorUserPointer = glfwGetMonitorUserPointer;
table->glfwGetMonitorWorkarea = glfwGetMonitorWorkarea;
table->glfwGetMouseButton = glfwGetMouseButton;
table->glfwGetPrimaryMonitor = glfwGetPrimaryMonitor;
table->glfwGetProcAddress = glfwGetProcAddress;
table->glfwGetRequiredInstanceExtensions = glfwGetRequiredInstanceExtensions;
table->glfwGetTime = glfwGetTime;
table->glfwGetTimerFrequency = glfwGetTimerFrequency;
table->glfwGetTimerValue = glfwGetTimerValue;
table->glfwGetVersion = glfwGetVersion;
table->glfwGetVersionString = glfwGetVersionString;
table->glfwGetVideoMode = glfwGetVideoMode;
table->glfwGetVideoModes = glfwGetVideoModes;
table->glfwGetWindowAttrib = glfwGetWindowAttrib;
table->glfwGetWindowContentScale = glfwGetWindowContentScale;
table->glfwGetWindowFrameSize = glfwGetWindowFrameSize;
table->glfwGetWindowMonitor = glfwGetWindowMonitor;
table->glfwGetWindowOpacity = glfwGetWindowOpacity;
table->glfwGetWindowPos = glfwGetWindowPos;
table->glfwGetWindowSize = glfwGetWindowSize;
table->glfwGetWindowUserPointer = glfwGetWindowUserPointer;
table->glfwHideWindow = glfwHideWindow;
table->glfwIconifyWindow = glfwIconifyWindow;
table->glfwInit = glfwInit;
table->glfwInitHint = glfwInitHint;
table->glfwJoystickIsGamePad = glfwJoystickIsGamepad;
table->glfwJoystickPresent = glfwJoystickPresent;
table->glfwMakeContextCurrent = glfwMakeContextCurrent;
table->glfwMaximizeWindow = glfwMaximizeWindow;
table->glfwPollEvents = glfwPollEvents;
table->glfwPostEmptyEvent = glfwPostEmptyEvent;
table->glfwRawMouseMotionSupported = glfwRawMouseMotionSupported;
table->glfwRequestWindowAttention = glfwRequestWindowAttention;
table->glfwRestoreWindow = glfwRestoreWindow;
table->glfwSetCharCallback = glfwSetCharCallback;
table->glfwSetCharModsCallback = glfwSetCharModsCallback;
table->glfwSetClipboardString = glfwSetClipboardString;
table->glfwSetCursor = glfwSetCursor;
table->glfwSetCursorEnterCallback = glfwSetCursorEnterCallback;
table->glfwSetCursorPos = glfwSetCursorPos;
table->glfwSetCursorPosCallback = glfwSetCursorPosCallback;
table->glfwSetDropCallback = glfwSetDropCallback;
table->glfwSetErrorCallback = glfwSetErrorCallback;
table->glfwSetFramebufferSizeCallback = glfwSetFramebufferSizeCallback;
table->glfwSetGamma = glfwSetGamma;
table->glfwSetGammaRamp = glfwSetGammaRamp;
table->glfwSetInputMode = glfwSetInputMode;
table->glfwSetJoystickCallback = glfwSetJoystickCallback;
table->glfwSetJoystickUserPointer = glfwSetJoystickUserPointer;
table->glfwSetKeyCallback = glfwSetKeyCallback;
table->glfwSetMonitorCallback = glfwSetMonitorCallback;
table->glfwSetMonitorUserPointer = glfwSetMonitorUserPointer;
table->glfwSetMouseButtonCallback = glfwSetMouseButtonCallback;
table->glfwSetScrollCallback = glfwSetScrollCallback;
table->glfwSetTime = glfwSetTime;
table->glfwSetWindowAspectRatio = glfwSetWindowAspectRatio;
table->glfwSetWindowAttrib = glfwSetWindowAttrib;
table->glfwSetWindowCloseCallback = glfwSetWindowCloseCallback;
table->glfwSetWindowContentScaleCallback = glfwSetWindowContentScaleCallback;
table->glfwSetWindowFocusCallback = glfwSetWindowFocusCallback;
table->glfwSetWindowIcon = glfwSetWindowIcon;
table->glfwSetWindowIconifyCallback = glfwSetWindowIconifyCallback;
table->glfwSetWindowMaximizeCallback = glfwSetWindowMaximizeCallback;
table->glfwSetWindowMonitor = glfwSetWindowMonitor;
table->glfwSetWindowOpacity = glfwSetWindowOpacity;
table->glfwSetWindowPos = glfwSetWindowPos;
table->glfwSetWindowPosCallback = glfwSetWindowPosCallback;
table->glfwSetWindowRefreshCallback = glfwSetWindowRefreshCallback;
table->glfwSetWindowShouldClose = glfwSetWindowShouldClose;
table->glfwSetWindowSize = glfwSetWindowSize;
table->glfwSetWindowSizeCallback = glfwSetWindowSizeCallback;
table->glfwSetWindowSizeLimits = glfwSetWindowSizeLimits;
table->glfwSetWindowTitle = glfwSetWindowTitle;
table->glfwSetWindowUserPointer = glfwSetWindowUserPointer;
table->glfwShowWindow = glfwShowWindow;
table->glfwSwapBuffers = glfwSwapBuffers;
table->glfwSwapInterval = glfwSwapInterval;
table->glfwTerminate = glfwTerminate;
table->glfwUpdateGamePadMappings = glfwUpdateGamepadMappings;
table->glfwVulkanSupported = glfwVulkanSupported;
table->glfwWaitEvents = glfwWaitEvents;
table->glfwWaitEventsTimeout = glfwWaitEventsTimeout;
table->glfwWindowHint = glfwWindowHint;
table->glfwWindowHintString = glfwWindowHintString;
table->glfwWindowShouldClose = glfwWindowShouldClose;
}
StgSharp.Native/sn-src/load_intrinsic.c
+112
View File
@@ -0,0 +1,112 @@
#include "StgSharpNative.h"
#include "sn_intrinsic.h"
#if defined(_MSC_VER) && !defined(__clang__)
#include <intrin.h>
#include <isa_availability.h>
#else
#include <cpuid.h>
#endif // _MSC_VER
PRIVATE void load_intrinsic_sse(sn_intrinsic *context);
PRIVATE void load_intrinsic_avx(sn_intrinsic *context);
PRIVATE void load_intrinsic_512(sn_intrinsic *context);
PRIVATE int check_isa(most_advanced_instruction instruction_family)
{
if (instruction_family != NEON) {
#if defined(_MSC_VER) && !defined(__clang__)
return __check_isa_support(instruction_family, 0);
#else
switch (instruction_family) {
case SSE:
return __builtin_cpu_supports("sse4.2");
case AVX256:
return __builtin_cpu_supports("avx2");
case AVX512:
return __builtin_cpu_supports("avx512f");
default:
return 0;
}
#endif // _MSC_VER
}
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
return 1;
#else
return 0;
#endif
}
SN_API int load_intrinsic_function(void *intrinsic_context)
{
sn_null_assert(intrinsic_context);
sn_intrinsic *context = (sn_intrinsic *)intrinsic_context;
if (check_isa(AVX512)) {
load_intrinsic_512(context);
return 3;
} else if (check_isa(AVX256)) {
load_intrinsic_avx(context);
return 2;
} else if (check_isa(SSE)) {
load_intrinsic_sse(context);
return 1;
} else if (check_isa(NEON)) {
return -1;
}
return 0;
}
void load_intrinsic_sse(sn_intrinsic *context)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->f32_clear_panel = PNL_PROC(float, sse, clear);
context->f32_build_panel = BUILD_PANEL(sse, float);
context->f32_buffer_add = BUF_PROC(float, sse, add);
context->f32_panel_fma = PNL_PROC(float, sse, fma);
context->f32_buffer_fill = BUF_PROC(float, sse, fill);
context->variant = BUF_PROC(float, sse, scalar_mul);
context->f32_buffer_sub = BUF_PROC(float, sse, sub);
context->f32_buffer_transpose = KER_PROC(float, sse, transpose);
context->f32_buffer_scalar_mul = BUF_PROC(float, sse, scalar_mul);
context->f32_normalize_3 = f32_normalize;
}
void load_intrinsic_avx(sn_intrinsic *context)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->f32_clear_panel = PNL_PROC(float, avx, clear);
context->f32_build_panel = BUILD_PANEL(avx, float);
context->f32_buffer_add = BUF_PROC(float, avx, add);
context->f32_panel_fma = PNL_PROC(float, avx, fma);
context->f32_buffer_fill = BUF_PROC(float, avx, fill);
context->variant = BUF_PROC(float, avx, scalar_mul);
context->f32_buffer_sub = BUF_PROC(float, avx, sub);
context->f32_buffer_transpose = KER_PROC(float, avx, transpose);
context->f32_buffer_scalar_mul = BUF_PROC(float, avx, scalar_mul);
context->f32_normalize_3 = f32_normalize;
}
void load_intrinsic_512(sn_intrinsic *context)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->f32_clear_panel = PNL_PROC(float, sse, clear);
context->f32_build_panel = BUILD_PANEL(512, float);
context->f32_buffer_add = BUF_PROC(float, 512, add);
context->f32_panel_fma = PNL_PROC(float, 512, fma);
context->f32_buffer_fill = BUF_PROC(float, 512, fill);
context->variant = BUF_PROC(float, 512, scalar_mul);
context->f32_buffer_sub = BUF_PROC(float, 512, sub);
context->f32_buffer_transpose = KER_PROC(float, 512, transpose);
context->f32_buffer_scalar_mul = BUF_PROC(float, 512, scalar_mul);
context->f32_normalize_3 = f32_normalize;
}
StgSharp.Native/sn-src/mat2.c
-57
View File
@@ -1,57 +0,0 @@
#include "StgSharpC.h"
#include "ssc_intrinsic.h"
INTERNAL void __cdecl transpose23(__2_columnset *source, __3_columnset *target)
{
__m128 t0 = _mm_shuffle_ps(ALIGN(source->column[0]), ALIGN(source->column[1]),
_MM_SHUFFLE(1, 0, 1, 0));
__m128 t1 = _mm_shuffle_ps(ALIGN(source->column[0]), ALIGN(source->column[1]),
_MM_SHUFFLE(3, 2, 3, 2));
target->column[0] = _mm_shuffle_ps(t0, zero_vec, _MM_SHUFFLE(2, 0, 2, 0));
target->column[1] = _mm_shuffle_ps(t0, zero_vec, _MM_SHUFFLE(3, 1, 3, 1));
target->column[2] = _mm_shuffle_ps(t1, zero_vec, _MM_SHUFFLE(2, 0, 2, 0));
}
INTERNAL void __cdecl transpose24(__2_columnset *source, __4_columnset *target)
{
__m128 t0 = _mm_shuffle_ps(ALIGN(source->column[0]), ALIGN(source->column[1]),
_MM_SHUFFLE(1, 0, 1, 0));
__m128 t1 = _mm_shuffle_ps(ALIGN(source->column[0]), ALIGN(source->column[1]),
_MM_SHUFFLE(3, 2, 3, 2));
target->column[0] = _mm_shuffle_ps(t0, zero_vec, _MM_SHUFFLE(2, 0, 2, 0));
target->column[1] = _mm_shuffle_ps(t0, zero_vec, _MM_SHUFFLE(3, 1, 3, 1));
target->column[2] = _mm_shuffle_ps(t1, zero_vec, _MM_SHUFFLE(2, 0, 2, 0));
target->column[3] = _mm_shuffle_ps(t1, zero_vec, _MM_SHUFFLE(3, 1, 3, 1));
}
INTERNAL void __cdecl addmatrix2_sse(__2_columnset *left, __2_columnset *right, __2_columnset *ans)
{
__m128 temp0 = _mm_add_ps(ALIGN(left->column[0]), ALIGN(right->column[0]));
__m128 temp1 = _mm_add_ps(ALIGN(left->column[1]), ALIGN(right->column[1]));
_mm_storeu_ps(&ans->m[0][0], temp0);
_mm_storeu_ps(&ans->m[0][1], temp1);
}
INTERNAL void __cdecl addmatrix2_avx(__2_columnset *left, __2_columnset *right, __2_columnset *ans)
{
__m256 temp = _mm256_add_ps(ALIGN256(left->column[0]), ALIGN256(right->column[0]));
_mm256_storeu_ps(&ans->m[0][0], temp);
}
INTERNAL void __cdecl submatrix2_sse(__2_columnset *left, __2_columnset *right, __2_columnset *ans)
{
__m128 temp0 = _mm_sub_ps(ALIGN(left->column[0]), ALIGN(right->column[0]));
__m128 temp1 = _mm_sub_ps(ALIGN(left->column[1]), ALIGN(right->column[1]));
_mm_storeu_ps(&ans->m[0][0], temp0);
_mm_storeu_ps(&ans->m[0][1], temp1);
}
INTERNAL void __cdecl submatrix2_avx(__2_columnset *left, __2_columnset *right, __2_columnset *ans)
{
__m256 temp = _mm256_sub_ps(ALIGN256(left->column[0]), ALIGN256(right->column[0]));
_mm256_storeu_ps(&ans->m[0][0], temp);
}
StgSharp.Native/sn-src/mat3.c
-130
View File
@@ -1,130 +0,0 @@
#include "StgSharpC.h"
#include "ssc_intrinsic.h"
#include <stdio.h>
INTERNAL void _cdecl transpose32(__3_columnset *source, __2_columnset *target)
{
__m128 t0 = _mm_shuffle_ps(ALIGN(source->column[0]), ALIGN(source->column[1]),
_MM_SHUFFLE(1, 0, 1, 0));
__m128 t2 = _mm_shuffle_ps(ALIGN(source->column[2]), zero_vec, _MM_SHUFFLE(1, 0, 1, 0));
target->column[0] = _mm_shuffle_ps(t0, t2, _MM_SHUFFLE(2, 0, 2, 0));
target->column[1] = _mm_shuffle_ps(t0, t2, _MM_SHUFFLE(3, 1, 3, 1));
}
INTERNAL void _cdecl transpose33(__3_columnset *source, __3_columnset *target)
{
__m128 t0 = _mm_shuffle_ps(ALIGN(source->column[0]), ALIGN(source->column[1]),
_MM_SHUFFLE(1, 0, 1, 0));
__m128 t1 = _mm_shuffle_ps(ALIGN(source->column[0]), ALIGN(source->column[1]),
_MM_SHUFFLE(3, 2, 3, 2));
__m128 t2 = _mm_shuffle_ps(ALIGN(source->column[2]), zero_vec, _MM_SHUFFLE(1, 0, 1, 0));
__m128 t3 = _mm_shuffle_ps(ALIGN(source->column[2]), zero_vec, _MM_SHUFFLE(3, 2, 3, 2));
target->column[0] = _mm_shuffle_ps(t0, t2, _MM_SHUFFLE(2, 0, 2, 0));
target->column[1] = _mm_shuffle_ps(t0, t2, _MM_SHUFFLE(3, 1, 3, 1));
target->column[2] = _mm_shuffle_ps(t1, t3, _MM_SHUFFLE(2, 0, 2, 0));
}
INTERNAL void _cdecl transpose34(__3_columnset *source, __4_columnset *target)
{
__m128 t0 = _mm_shuffle_ps(ALIGN(source->column[0]), ALIGN(source->column[1]),
_MM_SHUFFLE(1, 0, 1, 0));
__m128 t1 = _mm_shuffle_ps(ALIGN(source->column[0]), ALIGN(source->column[1]),
_MM_SHUFFLE(3, 2, 3, 2));
__m128 t2 = _mm_shuffle_ps(ALIGN(source->column[2]), zero_vec, _MM_SHUFFLE(1, 0, 1, 0));
__m128 t3 = _mm_shuffle_ps(ALIGN(source->column[2]), zero_vec, _MM_SHUFFLE(3, 2, 3, 2));
target->column[0] = _mm_shuffle_ps(t0, t2, _MM_SHUFFLE(2, 0, 2, 0));
target->column[1] = _mm_shuffle_ps(t0, t2, _MM_SHUFFLE(3, 1, 3, 1));
target->column[2] = _mm_shuffle_ps(t1, t3, _MM_SHUFFLE(2, 0, 2, 0));
target->column[3] = _mm_shuffle_ps(t1, t3, _MM_SHUFFLE(3, 1, 3, 1));
}
INTERNAL float _cdecl det_mat3_internal(__m128 const c0, __m128 const c1, __m128 const c2)
{
//let adding row aligned
__m128 addc1 = _mm_shuffle_ps(c1, c1, _MM_SHUFFLE(3, 0, 2, 1));
__m128 addc2 = _mm_shuffle_ps(c2, c2, _MM_SHUFFLE(3, 1, 0, 2));
//let sub row aligned
__m128 subc1 = _mm_shuffle_ps(c1, c1, _MM_SHUFFLE(3, 1, 0, 2));
__m128 subc2 = _mm_shuffle_ps(c2, c2, _MM_SHUFFLE(3, 0, 2, 1));
//multple all add row
__m128 cache = _mm_mul_ps(c0, addc1);
cache = _mm_mul_ps(cache, addc2);
//multiple all sub row
__m128 sub = _mm_mul_ps(c0, subc1);
sub = _mm_mul_ps(sub, subc2);
//main-corner sub sub-corner
cache = _mm_sub_ps(cache, sub);
__m128 ret = _mm_hadd_ps(cache, cache); //element 1 add element 2
ret = _mm_add_ps(ret, cache); //(e1 add e2) add element 3
return ret.m128_f32[2];
}
INTERNAL float _cdecl det_mat3(__3_columnset *source, __3_columnset *transpose)
{
return det_mat3_internal(source->column[0], source->column[1], source->column[2]);
}
#pragma region add
INTERNAL void __cdecl addmatrix3_sse(__3_columnset *left, __3_columnset *right, __3_columnset *ans)
{
__m128 temp0 = _mm_add_ps(ALIGN(left->column[0]), ALIGN(right->column[0]));
__m128 temp1 = _mm_add_ps(ALIGN(left->column[1]), ALIGN(right->column[1]));
__m128 temp2 = _mm_add_ps(ALIGN(left->column[2]), ALIGN(right->column[2]));
_mm_storeu_ps(&ans->m[0][0], temp0);
_mm_storeu_ps(&ans->m[0][1], temp1);
_mm_storeu_ps(&ans->m[0][2], temp2);
}
INTERNAL void __cdecl addmatrix3_avx(__3_columnset *left, __3_columnset *right, __3_columnset *ans)
{
__m256 temp0 = _mm256_add_ps(ALIGN256(left->part), ALIGN256(right->part));
__m128 temp1 = _mm_add_ps(ALIGN(left->column[2]), ALIGN(right->column[2]));
_mm256_storeu_ps(&ans->m[0][0], temp0);
_mm_storeu_ps(&ans->m[0][2], temp1);
}
INTERNAL void __cdecl addmatrix3_512(__3_columnset *left, __3_columnset *right, __3_columnset *ans)
{
__m512 temp = _mm512_add_ps(_mm512_loadu_ps(left), _mm512_loadu_ps(right));
_mm512_mask_store_ps(ans, 0x0fff, temp);
}
INTERNAL void __cdecl submatrix3_sse(__3_columnset *left, __3_columnset *right, __3_columnset *ans)
{
__m128 temp0 = _mm_sub_ps(ALIGN(left->column[0]), ALIGN(right->column[0]));
__m128 temp1 = _mm_sub_ps(ALIGN(left->column[1]), ALIGN(right->column[1]));
__m128 temp2 = _mm_sub_ps(ALIGN(left->column[2]), ALIGN(right->column[2]));
_mm_storeu_ps(&ans->m[0][0], temp0);
_mm_storeu_ps(&ans->m[0][1], temp1);
_mm_storeu_ps(&ans->m[0][2], temp2);
}
INTERNAL void __cdecl submatrix3_avx(__3_columnset *left, __3_columnset *right, __3_columnset *ans)
{
__m256 temp0 = _mm256_sub_ps(ALIGN256(left->part), ALIGN256(right->part));
__m128 temp1 = _mm_sub_ps(ALIGN(left->column[2]), ALIGN(right->column[2]));
_mm256_storeu_ps(&ans->m[0][0], temp0);
_mm_storeu_ps(&ans->m[0][2], temp1);
}
INTERNAL void __cdecl submatrix3_512(__3_columnset *left, __3_columnset *right, __3_columnset *ans)
{
__m512 temp = _mm512_sub_ps(_mm512_loadu_ps(left), _mm512_loadu_ps(right));
_mm512_mask_store_ps(ans, 0x0fff, temp);
}
#pragma endregion
StgSharp.Native/sn-src/mat4.c
-103
View File
@@ -1,103 +0,0 @@
#include "StgSharpC.h"
#include "ssc_intrinsic.h"
#include <stdio.h>
#include <pmmintrin.h>
INTERNAL void __cdecl transpose42(__4_columnset const *source, __2_columnset *target)
{
__m128 t0 = _mm_shuffle_ps(ALIGN(source->column[0]), ALIGN(source->column[1]),
_MM_SHUFFLE(1, 0, 1, 0));
__m128 t2 = _mm_shuffle_ps(ALIGN(source->column[2]), ALIGN(source->column[3]),
_MM_SHUFFLE(1, 0, 1, 0));
target->column[0] = _mm_shuffle_ps(t0, t2, _MM_SHUFFLE(2, 0, 2, 0));
target->column[1] = _mm_shuffle_ps(t0, t2, _MM_SHUFFLE(3, 1, 3, 1));
}
INTERNAL void __cdecl transpose43(__4_columnset const *source, __3_columnset *target)
{
__m128 t0 = _mm_shuffle_ps(ALIGN(source->column[0]), ALIGN(source->column[1]),
_MM_SHUFFLE(1, 0, 1, 0));
__m128 t1 = _mm_shuffle_ps(ALIGN(source->column[0]), ALIGN(source->column[1]),
_MM_SHUFFLE(3, 2, 3, 2));
__m128 t2 = _mm_shuffle_ps(ALIGN(source->column[2]), ALIGN(source->column[3]),
_MM_SHUFFLE(1, 0, 1, 0));
__m128 t3 = _mm_shuffle_ps(ALIGN(source->column[2]), ALIGN(source->column[3]),
_MM_SHUFFLE(3, 2, 3, 2));
target->column[0] = _mm_shuffle_ps(t0, t2, _MM_SHUFFLE(2, 0, 2, 0));
target->column[1] = _mm_shuffle_ps(t0, t2, _MM_SHUFFLE(3, 1, 3, 1));
target->column[2] = _mm_shuffle_ps(t1, t3, _MM_SHUFFLE(2, 0, 2, 0));
}
INTERNAL void __cdecl transpose44(__4_columnset const *source, __4_columnset *target)
{
__m128 t0 = _mm_shuffle_ps(ALIGN(source->column[0]), ALIGN(source->column[1]),
_MM_SHUFFLE(1, 0, 1, 0));
__m128 t1 = _mm_shuffle_ps(ALIGN(source->column[0]), ALIGN(source->column[1]),
_MM_SHUFFLE(3, 2, 3, 2));
__m128 t2 = _mm_shuffle_ps(ALIGN(source->column[2]), ALIGN(source->column[3]),
_MM_SHUFFLE(1, 0, 1, 0));
__m128 t3 = _mm_shuffle_ps(ALIGN(source->column[2]), ALIGN(source->column[3]),
_MM_SHUFFLE(3, 2, 3, 2));
target->column[0] = _mm_shuffle_ps(t0, t2, _MM_SHUFFLE(2, 0, 2, 0));
target->column[1] = _mm_shuffle_ps(t0, t2, _MM_SHUFFLE(3, 1, 3, 1));
target->column[2] = _mm_shuffle_ps(t1, t3, _MM_SHUFFLE(2, 0, 2, 0));
target->column[3] = _mm_shuffle_ps(t1, t3, _MM_SHUFFLE(3, 1, 3, 1));
}
INTERNAL float __cdecl det_mat4(__4_columnset const *mat_ptr, __4_columnset const *transpose)
{
__m128 mc0 = ALIGN(mat_ptr->column[0]);
__m128 mc1 = ALIGN(mat_ptr->column[1]);
__m128 mc2 = ALIGN(mat_ptr->column[2]);
__m128 mc3 = ALIGN(mat_ptr->column[3]);
__m128 cogeneration =
_mm_set_ps(det_mat3_internal(mc0, mc1, mc2), det_mat3_internal(mc0, mc1, mc3),
det_mat3_internal(mc0, mc2, mc3), det_mat3_internal(mc1, mc2, mc3));
__m128 retvec = _mm_mul_ps(ALIGN(transpose->column[3]), cogeneration);
retvec = _mm_hsub_ps(retvec, retvec);
retvec = _mm_hadd_ps(retvec, retvec);
return -retvec.m128_f32[0];
}
INTERNAL void SSCDECL addmatrix4_sse(__4_columnset *left, __4_columnset *right, __4_columnset *ans)
{
ans->column[0] = _mm_add_ps(ALIGN(left->column[0]), ALIGN(right->column[0]));
ans->column[1] = _mm_add_ps(ALIGN(left->column[1]), ALIGN(right->column[1]));
ans->column[2] = _mm_add_ps(ALIGN(left->column[2]), ALIGN(right->column[2]));
ans->column[3] = _mm_add_ps(ALIGN(left->column[3]), ALIGN(right->column[3]));
}
INTERNAL void SSCDECL addmatrix4_avx(__4_columnset *left, __4_columnset *right, __4_columnset *ans)
{
ans->half[0] = _mm256_add_ps(ALIGN256(left->half[0]), ALIGN256(right->half[0]));
ans->half[1] = _mm256_add_ps(ALIGN256(left->half[1]), ALIGN256(right->half[1]));
}
INTERNAL void SSCDECL addmatrix4_512(__4_columnset *left, __4_columnset *right, __4_columnset *ans)
{
ans->stream = _mm512_add_ps(ALIGN512(left->stream), ALIGN512(right->stream));
}
INTERNAL void SSCDECL submatrix4_sse(__4_columnset *left, __4_columnset *right, __4_columnset *ans)
{
ans->column[0] = _mm_sub_ps(ALIGN(left->column[0]), ALIGN(right->column[0]));
ans->column[1] = _mm_sub_ps(ALIGN(left->column[1]), ALIGN(right->column[1]));
ans->column[2] = _mm_sub_ps(ALIGN(left->column[2]), ALIGN(right->column[2]));
ans->column[3] = _mm_sub_ps(ALIGN(left->column[3]), ALIGN(right->column[3]));
}
INTERNAL void SSCDECL submatrix4_avx(__4_columnset *left, __4_columnset *right, __4_columnset *ans)
{
ans->half[0] = _mm256_sub_ps(ALIGN256(left->half[0]), ALIGN256(right->half[0]));
ans->half[1] = _mm256_sub_ps(ALIGN256(left->half[1]), ALIGN256(right->half[1]));
}
INTERNAL void SSCDECL submatrix4_512(__4_columnset *left, __4_columnset *right, __4_columnset *ans)
{
ans->stream = _mm512_sub_ps(ALIGN512(left->stream), ALIGN512(right->stream));
}
StgSharp.Native/sn-src/StgSharpC.c → StgSharp.Native/sn-src/sn_native.c
+13 -11
View File
@@ -1,13 +1,16 @@
#include "StgSharpC.h"
#ifndef STB_IMAGE_IMPLEMENTATION
#define STB_IMAGE_IMPLEMENTATION
#endif
#include "StgSharpNative.h"
#include "gl.h"
#include "glfw3.h"
#include "../lib/glfw/glfw3.h"
#include <stdio.h>
#include <string.h>
extern GladGLContext *currentContext;
extern char infolog[512];
SSCAPI int SSCDECL glCheckShaderStat(GladGLContext *context, uint64_t shaderHandle, int key,
SN_API int SN_DECL glCheckShaderStat(GladGLContext *context, uint64_t shaderHandle, int key,
char **logRef)
{
int stat = 0;
@@ -19,7 +22,7 @@ SSCAPI int SSCDECL glCheckShaderStat(GladGLContext *context, uint64_t shaderHand
return stat;
}
SSCAPI void SSCDECL initGL(int majorVersion, int minorVersion)
SN_API void SN_DECL initGL(int majorVersion, int minorVersion)
{
glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, majorVersion);
glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, minorVersion);
@@ -27,12 +30,12 @@ SSCAPI void SSCDECL initGL(int majorVersion, int minorVersion)
glfwWindowHint(GLFW_RESIZABLE, GLFW_TRUE);
}
SSCAPI GLFWglproc SSCDECL loadGlfuncDefault(char *procName)
SN_API GLFWglproc SN_DECL loadGlfuncDefault(char const *procName)
{
void *ret = glfwGetProcAddress(procName);
// printf("%llu\n", (uint64_t)ret);
if (ret == NULL) {
strcpy_s(infolog,30, "Failed to load OpenGL api:");
strcpy_s(infolog, 30, "Failed to load OpenGL api:");
for (size_t i = 0; i < sizeof(procName); i++) {
infolog[30 + i] = procName[i];
}
@@ -41,7 +44,7 @@ SSCAPI GLFWglproc SSCDECL loadGlfuncDefault(char *procName)
return (GLFWglproc)ret;
}
SSCAPI unsigned int linkShaderProgram(GladGLContext *context, GLuint shaderProgram)
SN_API unsigned int linkShaderProgram(GladGLContext *context, GLuint shaderProgram)
{
context->LinkProgram(shaderProgram);
@@ -55,20 +58,19 @@ SSCAPI unsigned int linkShaderProgram(GladGLContext *context, GLuint shaderProgr
}
}
SSCAPI void SSCDECL loadImageData(char *location, Image *out, imageLoader loader)
SN_API void SN_DECL loadImageData(char const *location, Image *out, imageLoader loader)
{
if (loader == NULL) {
loader = stbi_load;
}
out->pixelPtr = loader(location, &(out->width), &(out->height), &(out->channel), 0);
}
SSCAPI void SSCDECL unloadImageData(Image *out)
SN_API void SN_DECL unloadImageData(Image *out)
{
free(out->pixelPtr);
}
SSCAPI char *SSCDECL readLog()
SN_API char *SN_DECL readLog()
{
infolog[511] = '\0';
return infolog;
StgSharp.Native/sn-src/stb_image.c
-8475
View File
File diff suppressed because it is too large Load Diff
StgSharp.Native/sn-src/FactorialSIMD.c → StgSharp.Native/sn-src/str_factorial.c
+5 -5
View File
@@ -1,10 +1,10 @@
#include "ssc_internal.h"
#include "ssc_intrinsic.h"
#include "sn_internal.h"
#include "sn_intrinsic.h"
INTERNAL __m128i facIncrement = { 4, 4, 4, 4 };
INTERNAL __m128i facBase = { 1, 2, 3, 4 };
INTERNAL uint64_t SSCDECL factorial_simd_sse(int n)
INTERNAL uint64_t SN_DECL factorial_simd_sse(int n)
{
__m128i baseResult = facBase;
__m128i facBaseLocal = facBase;
@@ -17,7 +17,7 @@ INTERNAL uint64_t SSCDECL factorial_simd_sse(int n)
uint64_t result = (uint64_t)baseResult.m128i_i32[0] * (uint64_t)baseResult.m128i_i32[1] *
(uint64_t)baseResult.m128i_i32[2] * (uint64_t)baseResult.m128i_i32[3];
for (uint64_t i = rest; i < n; i++) {
result *= i ;
result *= i;
}
return result;
}
}
StgSharp.Native/sn-src/hash.c → StgSharp.Native/sn-src/str_hash.c
+3 -3
View File
@@ -1,7 +1,7 @@
#include "ssc_intrinsic.h"
#include "StgSharpC.h"
#include "sn_intrinsic.h"
#include "StgSharpNative.h"
INTERNAL int SSCDECL city_hash_simplify_sse(byte const *str, int const length)
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int const length)
{
uint32_t hash = length, size = length << 1;
byte *end = str + size, *ptr = str;
StgSharp.Native/sn-src/string.c → StgSharp.Native/sn-src/str_string.c
+3 -8
View File
@@ -1,14 +1,9 @@
#include "ssc_intrinsic.h"
#include "sn_intrinsic.h"
int index_pair_sse(short const *str, uint32_t target, int length)
INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int length)
{
int index = 0, result = 0, size = length, end = size - 8;
__m128i s = {
target,
target,
target,
target,
};
__m128i s = _mm_set1_epi32(target);
__m128i vec, com;
while (index < end) {
StgSharp.Native/sn-src/vector.c
-31
View File
@@ -1,31 +0,0 @@
#include "StgSharpC.h"
#include "ssc_internal.h"
SSCAPI void SSCDECL normalize(__m128 *source, __m128 *target)
{
__m128 a_length_sq = _mm_mul_ps(*source, *source);
a_length_sq = _mm_add_ps(a_length_sq,
_mm_shuffle_ps(a_length_sq, a_length_sq, _MM_SHUFFLE(2, 3, 0, 1)));
a_length_sq = _mm_add_ss(a_length_sq,
_mm_shuffle_ps(a_length_sq, a_length_sq, _MM_SHUFFLE(0, 0, 3, 2)));
__m128 a_inv_length = _mm_rsqrt_ps(a_length_sq);
*source = _mm_mul_ps(*source, a_inv_length);
__m128 dot_product = _mm_mul_ps(*target, *source);
dot_product = _mm_add_ps(dot_product,
_mm_shuffle_ps(dot_product, dot_product, _MM_SHUFFLE(2, 3, 0, 1)));
dot_product = _mm_add_ss(dot_product,
_mm_shuffle_ps(dot_product, dot_product, _MM_SHUFFLE(0, 0, 3, 2)));
__m128 b_orthogonal = _mm_sub_ps(*target, _mm_mul_ps(*source, dot_product));
__m128 b_length_sq = _mm_mul_ps(b_orthogonal, b_orthogonal);
b_length_sq = _mm_add_ps(b_length_sq,
_mm_shuffle_ps(b_length_sq, b_length_sq, _MM_SHUFFLE(2, 3, 0, 1)));
b_length_sq = _mm_add_ss(b_length_sq,
_mm_shuffle_ps(b_length_sq, b_length_sq, _MM_SHUFFLE(0, 0, 3, 2)));
__m128 b_inv_length = _mm_rsqrt_ps(b_length_sq);
*target = _mm_mul_ps(b_orthogonal, b_inv_length);
}
StgSharp.Native/sn-src/vector_dot.c
-147
View File
@@ -1,147 +0,0 @@
#include "ssc_intrinsic.h"
#include "SSC_internal.h"
static const int avxdot_indices[] = { 0, 4, 1, 5, 2, 3, 6, 7 };
INTERNAL void FORCEINLINE SSCDECL dot_41_sse(__4_columnset *transpose, __m128 *vector, __m128 *ans)
{
*ans = _mm_dp_ps(transpose->column[0], *vector, 0B11110001);
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 1));
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[3], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 4));
}
INTERNAL void SSCDECL dot_41_avx(__4_columnset *transpose, __m128 *vector, __m128 *ans)
{
__m256 _double = _mm256_set_m128(vector[0], vector[0]);
__m256 tmp = _mm256_dp_ps(transpose->half[0], _double, 0B11110001);
tmp = _mm256_blend_ps(tmp, _mm256_dp_ps(transpose->half[0], _double, 0B11110010),
0B00100010);
*ans = _mm256_castps256_ps128(
_mm256_permutevar8x32_ps(tmp, _mm256_loadu_epi32(avxdot_indices)));
}
INTERNAL void FORCEINLINE SSCDECL dot_31_sse(__3_columnset *transpose, __m128 *vector, __m128 *ans)
{
*ans = _mm_dp_ps(transpose->column[0], *vector, 0B11110001);
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
}
INTERNAL void FORCEINLINE SSCDECL dot_31_avx(__3_columnset *transpose, __m128 *vector, __m128 *ans)
{
__m256 _double = _mm256_set_m128(vector[0], vector[0]);
__m256 tmp = _mm256_dp_ps(transpose->part, _double, 0B11110001);
*ans = _mm256_castps256_ps128(
_mm256_permutevar8x32_ps(tmp, _mm256_loadu_epi32(avxdot_indices)));
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
}
INTERNAL void SSCDECL dot_42_sse(__4_columnset *transpose, __m128 *vector, __m128 *ans)
{
ans[0] = _mm_dp_ps(transpose->column[0], vector[0], 0B11110001);
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[3], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 4));
ans[1] = _mm_dp_ps(transpose->column[0], vector[1], 0B11110001);
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[3], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 4));
}
INTERNAL void SSCDECL dot_32_sse(__3_columnset *transpose, __m128 *vector, __m128 *ans)
{
ans[0] = _mm_dp_ps(transpose->column[0], vector[0], 0B11110001);
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
ans[1] = _mm_dp_ps(transpose->column[0], vector[1], 0B11110001);
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[1], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->column[2], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
}
INTERNAL void SSCDECL dot_42_avx(__4_columnset *transpose, __m128 *vector, __m128 *ans)
{
__2_columnset *ans_ptr = (__2_columnset *)ans, *right_ptr = (__2_columnset *)vector;
__m256 temp = ALIGN256(transpose->half[0]);
ans_ptr->stream = _mm256_dp_ps(temp, right_ptr->stream, 0B11110001);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
ans_ptr->stream = _mm256_blend_ps(
ans_ptr->stream, _mm256_dp_ps(temp, right_ptr->stream, 0B11110010), 0B00100010);
temp = ALIGN256(transpose->half[1]);
ans_ptr->stream = _mm256_blend_ps(
ans_ptr->stream, _mm256_dp_ps(temp, right_ptr->stream, 0B11110100), 0B01000100);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
ans_ptr->stream = _mm256_blend_ps(
ans_ptr->stream, _mm256_dp_ps(temp, right_ptr->stream, 0B11111000), 0B10001000);
ans[1] = _mm_shuffle_ps(ALIGN(ans[1]), ALIGN(ans[1]), _MM_SHUFFLE(2, 3, 0, 1));
}
INTERNAL void SSCDECL dot_32_avx(__3_columnset *transpose, __m128 *vector, __m128 *ans)
{
__2_columnset *ans_ptr = (__2_columnset *)ans, *right_ptr = (__2_columnset *)vector;
__m256 temp = transpose->part;
ans_ptr->stream = _mm256_dp_ps(temp, right_ptr->stream, 0B11110001);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
ans_ptr->stream = _mm256_blend_ps(
ans_ptr->stream, _mm256_dp_ps(temp, right_ptr->stream, 0B11110010), 0B00100010);
temp = _mm256_set_m128(transpose->column[2], transpose->column[2]);
ans_ptr->stream = _mm256_blend_ps(
ans_ptr->stream, _mm256_dp_ps(temp, right_ptr->stream, 0B11110100), 0B01000100);
ans[1] = _mm_shuffle_ps(ALIGN(ans[1]), ALIGN(ans[1]), _MM_SHUFFLE(2, 2, 0, 1));
}
INTERNAL void SSCDECL dot_43_sse(__4_columnset *transpose, __m128 *vector, __m128 *ans)
{
dot_41_sse(transpose, vector, ans);
dot_41_sse(transpose, vector + 1, ans + 1);
dot_41_sse(transpose, vector + 2, ans + 2);
}
INTERNAL void SSCDECL dot_43_avx(__4_columnset *transpose, __m128 *vector, __m128 *ans)
{
__3_columnset *ans_ptr = (__3_columnset *)ans, *right_ptr = (__3_columnset *)vector;
__m256 temp = ALIGN256(transpose->half[0]);
__m256 third = _mm256_set_m128(right_ptr->column[2], right_ptr->column[2]);
ans_ptr->part = _mm256_dp_ps(temp, right_ptr->part, 0B11110001);
__m256 ans_third = _mm256_dp_ps(temp, third, 0B11110001);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
ans_ptr->part = _mm256_blend_ps(
ans_ptr->part, _mm256_dp_ps(temp, right_ptr->part, 0B11110010), 0B00100010);
temp = ALIGN256(transpose->half[1]);
ans_ptr->part = _mm256_blend_ps(
ans_ptr->part, _mm256_dp_ps(temp, right_ptr->part, 0B11110100), 0B01000100);
ans_third = _mm256_blend_ps(ans_third, _mm256_dp_ps(temp, third, 0B11110010), 0B00100010);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
ans_ptr->part = _mm256_blend_ps(
ans_ptr->part, _mm256_dp_ps(temp, right_ptr->part, 0B11111000), 0B10001000);
ans[1] = _mm_shuffle_ps(ALIGN(ans[1]), ALIGN(ans[1]), _MM_SHUFFLE(2, 3, 0, 1));
ans[2] = _mm256_castps256_ps128(
_mm256_permutevar8x32_ps(ans_third, _mm256_loadu_epi32(avxdot_indices)));
}