youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new LU solvability for kernel,on dev

Browse Source
This commit is contained in:
Youmu Liu
2026-01-20 13:52:02 +08:00
parent b7697d0342
commit cd36d0962b
2 changed files with 102 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-src/m.fma_tile.x86.c
+39 -42
View File
@@ -267,36 +267,38 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
} }
// double // double
#define DOUBLE_KERNEL_FMA_CYCLE(i) \ #define DOUBLE_KERNEL_FMA_CYCLE(i) \
/* k = 0 */ \ do { \
register __m128d b_lo = _mm_load_pd(right_cur + (2 * i + 0)); \ /* k = 0 */ \
register __m128d b_hi = _mm_load_pd(right_cur + (2 * i + 1)); \ register __m128d b_lo = _mm_load_pd(right_cur + (2 * i + 0)); \
register __m128d a_lo = _mm_load_pd(left_cur + (0 * 2 + 0)); \ register __m128d b_hi = _mm_load_pd(right_cur + (2 * i + 1)); \
register __m128d a_hi = _mm_load_pd(left_cur + (0 * 2 + 1)); \ register __m128d a_lo = _mm_load_pd(left_cur + (0 * 2 + 0)); \
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \ register __m128d a_hi = _mm_load_pd(left_cur + (0 * 2 + 1)); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
\ c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
/* group = 1 */ \ \
a_lo = _mm_load_pd(left_cur + (1 * 2 + 0)); \ /* group = 1 */ \
a_hi = _mm_load_pd(left_cur + (1 * 2 + 1)); \ a_lo = _mm_load_pd(left_cur + (1 * 2 + 0)); \
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \ a_hi = _mm_load_pd(left_cur + (1 * 2 + 1)); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
\ c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
/* groupp = 2 */ \ \
a_lo = _mm_load_pd(left_cur + (2 * 2 + 0)); \ /* groupp = 2 */ \
a_hi = _mm_load_pd(left_cur + (2 * 2 + 1)); \ a_lo = _mm_load_pd(left_cur + (2 * 2 + 0)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \ a_hi = _mm_load_pd(left_cur + (2 * 2 + 1)); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
\ c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
/* froup = 3 */ \ \
a_lo = _mm_load_pd(left_cur + (3 * 2 + 0)); \ /* froup = 3 */ \
a_hi = _mm_load_pd(left_cur + (3 * 2 + 1)); \ a_lo = _mm_load_pd(left_cur + (3 * 2 + 0)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \ a_hi = _mm_load_pd(left_cur + (3 * 2 + 1)); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
} while (0);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma) DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
{ {
@@ -319,18 +321,13 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
const *left_cur = left + GET_INDEX(ansHeight, i, k), const *left_cur = left + GET_INDEX(ansHeight, i, k),
const *right_cur = right + GET_INDEX(commonK, j, k); // GetKernelAddressUnsafe const *right_cur = right + GET_INDEX(commonK, j, k); // GetKernelAddressUnsafe
{ DOUBLE_KERNEL_FMA_CYCLE(0)
DOUBLE_KERNEL_FMA_CYCLE(0)
} DOUBLE_KERNEL_FMA_CYCLE(1)
{
DOUBLE_KERNEL_FMA_CYCLE(1) DOUBLE_KERNEL_FMA_CYCLE(2)
}
{ DOUBLE_KERNEL_FMA_CYCLE(3)
DOUBLE_KERNEL_FMA_CYCLE(2)
}
{
DOUBLE_KERNEL_FMA_CYCLE(3)
}
} }
_mm_store_pd(ans->f64_x + (0 * 2 + 0), c0_lo); _mm_store_pd(ans->f64_x + (0 * 2 + 0), c0_lo);
StgSharp.Native/sn-src/m.solve.x86.c
+63 -12
View File
@@ -5,6 +5,8 @@
#include "sn_intrinsic.h" #include "sn_intrinsic.h"
#include <math.h> #include <math.h>
static const float quality_threshold = 1e-6f;
DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(float, sse, , try_plu) DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(float, sse, , try_plu)
{ {
/* Aliases */ /* Aliases */
@@ -88,10 +90,26 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(float, sse, , try_plu)
} }
} }
#define DOT_SSE(a, b, ans) \
do { \
__m128 _p = _mm_mul_ps((a), (b)); /* p0 p1 p2 p3 */ \
__m128 _t = _mm_shuffle_ps(_p, _p, _MM_SHUFFLE(2, 3, 0, 1)); \
_p = _mm_add_ps(_p, _t); \
_t = _mm_movehl_ps(_t, _p); \
ans = _mm_cvt_ss(_mm_add_ss(_p, _t)); \
} while (0)
/*
* layout of scalarpack:
*
*/
DECLARE_KER_PROC_RIGHT_ANS(float, sse, , quality) DECLARE_KER_PROC_RIGHT_ANS(float, sse, , quality)
{ {
static const __m128 abs_mask_source = { .m128_i32 = { 0x80000000, 0x80000000, 0x80000000, static const __m128 abs_mask_source = { .m128_i32 = { 0x80000000, 0x80000000, 0x80000000,
0x80000000 } }; 0x80000000 } };
static const __m128 tiny_source = { .m128_f32 = { 1e-6f, 1e-6f, 1e-6f, 1e-6f } };
register __m128 tiny = _mm_load_ps(tiny_source.m128_f32);
register __m128 abs_mask = _mm_load_ps(abs_mask_source.m128_i32); register __m128 abs_mask = _mm_load_ps(abs_mask_source.m128_i32);
register __m128 col0 = _mm_andnot_ps(abs_mask, _mm_load_ps((float *)&right->f32_x[0])); register __m128 col0 = _mm_andnot_ps(abs_mask, _mm_load_ps((float *)&right->f32_x[0]));
@@ -109,21 +127,54 @@ DECLARE_KER_PROC_RIGHT_ANS(float, sse, , quality)
max = _mm_max_ps(max, _mm_shuffle_ps(max, max, _MM_SHUFFLE(2, 3, 0, 1))); max = _mm_max_ps(max, _mm_shuffle_ps(max, max, _MM_SHUFFLE(2, 3, 0, 1)));
float norm = _mm_cvtss_f32(max); float norm = _mm_cvtss_f32(max);
ans->f32_x[0].m128_f32[0] = norm;
// min on cross // min on cross
col1 = _mm_shuffle_ps(col1, col1, _MM_SHUFFLE(0, 3, 2, 1)); register __m128 col_shuff_1 = _mm_shuffle_ps(col1, col1, _MM_SHUFFLE(0, 3, 2, 1));
col2 = _mm_shuffle_ps(col2, col2, _MM_SHUFFLE(1, 0, 3, 2)); register __m128 col_shuff_2 = _mm_shuffle_ps(col2, col2, _MM_SHUFFLE(1, 0, 3, 2));
col3 = _mm_shuffle_ps(col3, col3, _MM_SHUFFLE(2, 1, 0, 3)); register __m128 col_shuff_3 = _mm_shuffle_ps(col3, col3, _MM_SHUFFLE(2, 1, 0, 3));
max = _mm_min_ps(col0, col2); register __m128 min = _mm_min_ps(col0, col_shuff_1);
register __m128 max0 = _mm_min_ps(max, col1); register __m128 min0 = _mm_min_ps(col_shuff_2, col_shuff_3);
max = _mm_min_ps(max0, max); min = _mm_min_ps(min, min0);
float min_val = _mm_cvtss_f32(max); float quality = _mm_cvt_ss(_mm_div_ps(min, _mm_add_ps(max, tiny)));
ans->f32_x[0].m128_f32[1] = min_val;
float quality = min_val / (norm + 1e-20f);
ans->f32_x[0].m128_f32[2] = quality; ans->f32_x[0].m128_f32[2] = quality;
if (quality < quality_threshold) {
return;
}
__m128 self_dot_src = { 0 };
// try in reigister mix later
DOT_SSE(col0, col0, self_dot_src.m128_f32[0]);
DOT_SSE(col1, col1, self_dot_src.m128_f32[1]);
DOT_SSE(col2, col2, self_dot_src.m128_f32[2]);
DOT_SSE(col3, col3, self_dot_src.m128_f32[3]);
register __m128 self_dot = _mm_load_ps(self_dot_src.m128_f32);
__m128 cross_dot_src[2];
DOT_SSE(col0, col1, cross_dot_src[0].m128_f32[0]);
DOT_SSE(col0, col2, cross_dot_src[0].m128_f32[1]);
DOT_SSE(col0, col3, cross_dot_src[0].m128_f32[2]);
DOT_SSE(col1, col2, cross_dot_src[0].m128_f32[3]);
DOT_SSE(col1, col3, cross_dot_src[1].m128_f32[0]);
DOT_SSE(col2, col3, cross_dot_src[1].m128_f32[1]);
register __m128 cross_dot_0 = _mm_load_ps(cross_dot_src[0].m128_f32);
register __m128 cross_dot_1 = _mm_load_ps(cross_dot_src[1].m128_f32);
register __m128 base_0 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(1, 0, 0, 0));
register __m128 base_1 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(3, 3, 2, 1));
register __m128 relation =
_mm_div_ps(cross_dot_0, _mm_add_ps(_mm_mul_ps(base_0, base_1), tiny));
_mm_store_ps(ans->f32_x[1].m128_f32, relation);
base_0 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(2, 3, 2, 1));
base_1 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(3, 3, 3, 3));
relation = _mm_div_ps(cross_dot_0, _mm_add_ps(_mm_mul_ps(base_0, base_1), tiny));
_mm_store_ps(ans->f32_x[2].m128_f32, relation);
} }
#endif #endif