diff --git a/StgSharp.Native/sn-src/m.fma_tile.x86.c b/StgSharp.Native/sn-src/m.fma_tile.x86.c index 445cf9f6..120ff0cc 100644 --- a/StgSharp.Native/sn-src/m.fma_tile.x86.c +++ b/StgSharp.Native/sn-src/m.fma_tile.x86.c @@ -267,36 +267,38 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma) } // double -#define DOUBLE_KERNEL_FMA_CYCLE(i) \ - /* k = 0 */ \ - register __m128d b_lo = _mm_load_pd(right_cur + (2 * i + 0)); \ - register __m128d b_hi = _mm_load_pd(right_cur + (2 * i + 1)); \ - register __m128d a_lo = _mm_load_pd(left_cur + (0 * 2 + 0)); \ - register __m128d a_hi = _mm_load_pd(left_cur + (0 * 2 + 1)); \ - register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \ - c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ - c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ - \ - /* group = 1 */ \ - a_lo = _mm_load_pd(left_cur + (1 * 2 + 0)); \ - a_hi = _mm_load_pd(left_cur + (1 * 2 + 1)); \ - b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \ - c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ - c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ - \ - /* groupp = 2 */ \ - a_lo = _mm_load_pd(left_cur + (2 * 2 + 0)); \ - a_hi = _mm_load_pd(left_cur + (2 * 2 + 1)); \ - b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \ - c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ - c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ - \ - /* froup = 3 */ \ - a_lo = _mm_load_pd(left_cur + (3 * 2 + 0)); \ - a_hi = _mm_load_pd(left_cur + (3 * 2 + 1)); \ - b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \ - c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ - c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); +#define DOUBLE_KERNEL_FMA_CYCLE(i) \ + do { \ + /* k = 0 */ \ + register __m128d b_lo = _mm_load_pd(right_cur + (2 * i + 0)); \ + register __m128d b_hi = _mm_load_pd(right_cur + (2 * i + 1)); \ + register __m128d a_lo = _mm_load_pd(left_cur + (0 * 2 + 0)); \ + register __m128d a_hi = _mm_load_pd(left_cur + (0 * 2 + 1)); \ + register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \ + c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ + c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ + \ + /* group = 1 */ \ + a_lo = _mm_load_pd(left_cur + (1 * 2 + 0)); \ + a_hi = _mm_load_pd(left_cur + (1 * 2 + 1)); \ + b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \ + c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ + c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ + \ + /* groupp = 2 */ \ + a_lo = _mm_load_pd(left_cur + (2 * 2 + 0)); \ + a_hi = _mm_load_pd(left_cur + (2 * 2 + 1)); \ + b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \ + c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ + c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ + \ + /* froup = 3 */ \ + a_lo = _mm_load_pd(left_cur + (3 * 2 + 0)); \ + a_hi = _mm_load_pd(left_cur + (3 * 2 + 1)); \ + b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \ + c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ + c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ + } while (0); DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma) { @@ -319,18 +321,13 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma) const *left_cur = left + GET_INDEX(ansHeight, i, k), const *right_cur = right + GET_INDEX(commonK, j, k); // GetKernelAddressUnsafe - { - DOUBLE_KERNEL_FMA_CYCLE(0) - } - { - DOUBLE_KERNEL_FMA_CYCLE(1) - } - { - DOUBLE_KERNEL_FMA_CYCLE(2) - } - { - DOUBLE_KERNEL_FMA_CYCLE(3) - } + DOUBLE_KERNEL_FMA_CYCLE(0) + + DOUBLE_KERNEL_FMA_CYCLE(1) + + DOUBLE_KERNEL_FMA_CYCLE(2) + + DOUBLE_KERNEL_FMA_CYCLE(3) } _mm_store_pd(ans->f64_x + (0 * 2 + 0), c0_lo); diff --git a/StgSharp.Native/sn-src/m.solve.x86.c b/StgSharp.Native/sn-src/m.solve.x86.c index c4f2f602..40c98b1e 100644 --- a/StgSharp.Native/sn-src/m.solve.x86.c +++ b/StgSharp.Native/sn-src/m.solve.x86.c @@ -5,6 +5,8 @@ #include "sn_intrinsic.h" #include +static const float quality_threshold = 1e-6f; + DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(float, sse, , try_plu) { /* Aliases */ @@ -88,10 +90,26 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(float, sse, , try_plu) } } +#define DOT_SSE(a, b, ans) \ + do { \ + __m128 _p = _mm_mul_ps((a), (b)); /* p0 p1 p2 p3 */ \ + __m128 _t = _mm_shuffle_ps(_p, _p, _MM_SHUFFLE(2, 3, 0, 1)); \ + _p = _mm_add_ps(_p, _t); \ + _t = _mm_movehl_ps(_t, _p); \ + ans = _mm_cvt_ss(_mm_add_ss(_p, _t)); \ + } while (0) + +/* +* layout of scalarpack: +* +*/ DECLARE_KER_PROC_RIGHT_ANS(float, sse, , quality) { static const __m128 abs_mask_source = { .m128_i32 = { 0x80000000, 0x80000000, 0x80000000, 0x80000000 } }; + static const __m128 tiny_source = { .m128_f32 = { 1e-6f, 1e-6f, 1e-6f, 1e-6f } }; + + register __m128 tiny = _mm_load_ps(tiny_source.m128_f32); register __m128 abs_mask = _mm_load_ps(abs_mask_source.m128_i32); register __m128 col0 = _mm_andnot_ps(abs_mask, _mm_load_ps((float *)&right->f32_x[0])); @@ -109,21 +127,54 @@ DECLARE_KER_PROC_RIGHT_ANS(float, sse, , quality) max = _mm_max_ps(max, _mm_shuffle_ps(max, max, _MM_SHUFFLE(2, 3, 0, 1))); float norm = _mm_cvtss_f32(max); - ans->f32_x[0].m128_f32[0] = norm; - // min on cross - col1 = _mm_shuffle_ps(col1, col1, _MM_SHUFFLE(0, 3, 2, 1)); - col2 = _mm_shuffle_ps(col2, col2, _MM_SHUFFLE(1, 0, 3, 2)); - col3 = _mm_shuffle_ps(col3, col3, _MM_SHUFFLE(2, 1, 0, 3)); - max = _mm_min_ps(col0, col2); - register __m128 max0 = _mm_min_ps(max, col1); - max = _mm_min_ps(max0, max); + register __m128 col_shuff_1 = _mm_shuffle_ps(col1, col1, _MM_SHUFFLE(0, 3, 2, 1)); + register __m128 col_shuff_2 = _mm_shuffle_ps(col2, col2, _MM_SHUFFLE(1, 0, 3, 2)); + register __m128 col_shuff_3 = _mm_shuffle_ps(col3, col3, _MM_SHUFFLE(2, 1, 0, 3)); + register __m128 min = _mm_min_ps(col0, col_shuff_1); + register __m128 min0 = _mm_min_ps(col_shuff_2, col_shuff_3); + min = _mm_min_ps(min, min0); - float min_val = _mm_cvtss_f32(max); - ans->f32_x[0].m128_f32[1] = min_val; - - float quality = min_val / (norm + 1e-20f); + float quality = _mm_cvt_ss(_mm_div_ps(min, _mm_add_ps(max, tiny))); ans->f32_x[0].m128_f32[2] = quality; + if (quality < quality_threshold) { + return; + } + + __m128 self_dot_src = { 0 }; + // try in reigister mix later + DOT_SSE(col0, col0, self_dot_src.m128_f32[0]); + DOT_SSE(col1, col1, self_dot_src.m128_f32[1]); + DOT_SSE(col2, col2, self_dot_src.m128_f32[2]); + DOT_SSE(col3, col3, self_dot_src.m128_f32[3]); + + register __m128 self_dot = _mm_load_ps(self_dot_src.m128_f32); + + __m128 cross_dot_src[2]; + DOT_SSE(col0, col1, cross_dot_src[0].m128_f32[0]); + DOT_SSE(col0, col2, cross_dot_src[0].m128_f32[1]); + DOT_SSE(col0, col3, cross_dot_src[0].m128_f32[2]); + DOT_SSE(col1, col2, cross_dot_src[0].m128_f32[3]); + DOT_SSE(col1, col3, cross_dot_src[1].m128_f32[0]); + DOT_SSE(col2, col3, cross_dot_src[1].m128_f32[1]); + + register __m128 cross_dot_0 = _mm_load_ps(cross_dot_src[0].m128_f32); + register __m128 cross_dot_1 = _mm_load_ps(cross_dot_src[1].m128_f32); + + register __m128 base_0 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(1, 0, 0, 0)); + register __m128 base_1 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(3, 3, 2, 1)); + + register __m128 relation = + _mm_div_ps(cross_dot_0, _mm_add_ps(_mm_mul_ps(base_0, base_1), tiny)); + + _mm_store_ps(ans->f32_x[1].m128_f32, relation); + + base_0 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(2, 3, 2, 1)); + base_1 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(3, 3, 3, 3)); + + relation = _mm_div_ps(cross_dot_0, _mm_add_ps(_mm_mul_ps(base_0, base_1), tiny)); + + _mm_store_ps(ans->f32_x[2].m128_f32, relation); } #endif