keep working on matrix. key is to remove old matrix types

Signed-off-by: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com>
This commit is contained in:
Youmu Liu
2025-08-26 22:02:38 +08:00
parent e5e4b1dd36
commit 9f6e3fe13a
15 changed files with 152 additions and 812 deletions
+7
View File
@@ -50,9 +50,16 @@ namespace StgSharp.HighPerformance
internal M128(Vector4 v)
{
Unsafe.SkipInit(out this);
Vec = v.AsVector128();
}
internal M128(Vector128<float> v)
{
Unsafe.SkipInit(out this);
Vec = v;
}
public ref T AsRef<T>() where T: unmanaged, INumber<T>
{
return ref Unsafe.As<byte, T>(ref Buffer[0]);
@@ -61,6 +61,26 @@ namespace StgSharp.HighPerformance
Unsafe.SkipInit(out this);
}
public MatrixKernel(bool clear)
{
if (!clear) {
Unsafe.SkipInit(out this);
}
}
public static MatrixKernel Fp32Unit
{
get
{
MatrixKernel kernel = new(true);
kernel.FP32Buffer[0] = 1f;
kernel.FP32Buffer[3] = 1f;
kernel.FP32Buffer[8] = 1f;
kernel.FP32Buffer[15] = 1f;
return kernel;
}
}
public ref T UnsafeIndex<T>(int column, int row) where T: unmanaged, INumber<T>
{
ref M128 col = ref Unsafe.As<byte, M128>(ref Buffer[0]);
@@ -1,62 +0,0 @@
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
// file="UnmanagedMemoryHandle.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance
{
public ref struct UnmanagedMemoryHandle
{
public nuint Postion;
public Span<byte> Buffer;
public UnmanagedMemoryHandle() { }
public ref byte this[int index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => ref Buffer[index];
}
public Span<byte>.Enumerator GetEnumerator()
{
return Buffer.GetEnumerator();
}
}
}
@@ -1,144 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="UnmanagedMemoryPool.IndexNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection.Emit;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance
{
public unsafe partial class UnmanagedMemoryPool
{
private const int
LevelOf4096 = 4,
LevelOf1024 = 3,
LevelOf256 = 2,
LevelOf64 = 1;
internal readonly ref struct NextIndexNode
{
private readonly Span<M64> _layout;
public readonly nuint this[int level]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => Position - _layout[level].Member<uint>(0);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
set => _layout[level].Member<uint>(0) = (uint)(Position - value);
}
private readonly ref nuint Position
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => ref _layout[0].Member<nuint>(0);
}
}
internal readonly ref struct PreviousIndexNode
{
private readonly Span<M64> _layout;
public readonly nuint this[int level]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => Position - _layout[level].Member<uint>(1);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
set => _layout[level].Member<uint>(1) = (uint)(Position - value);
}
private readonly ref nuint Position
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => ref _layout[0].Member<nuint>(0);
}
}
[StructLayout(LayoutKind.Explicit)]
internal readonly ref struct IndexNode
{
private static readonly uint[] _sizeArr = [16, 64, 256, 1024, 4096];
[FieldOffset(0)] private readonly NextIndexNode nxt;
[FieldOffset(0)] private readonly PreviousIndexNode pre;
[FieldOffset(0)] private readonly Span<M64> _layout;
public IndexNode(ref byte span)
{
_layout = MemoryMarshal.CreateSpan(ref Unsafe.As<byte, M64>(ref span), 8);
}
public readonly ref nuint Position => ref _layout[0].Member<nuint>(0);
public readonly ref uint Size => ref _layout[5].Member<uint>(0);
public readonly ref int Level => ref _layout[5].Member<int>(1);
public readonly ref nuint Segment => ref _layout[6].Member<nuint>(0);
public readonly uint BlockSize => LevelToSize(Level);
public static uint LevelToSize(int level)
{
return level switch
{
< 0 or > 4 => throw new ArgumentOutOfRangeException(
nameof(level), "Level must be from 1 to 4."),
_ => _sizeArr[level]
};
}
#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
public readonly ref readonly NextIndexNode Next
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => ref nxt;
}
public readonly ref readonly PreviousIndexNode Previous
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => ref pre;
}
#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
}
}
}
@@ -1,187 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="UnmanagedMemoryPool.IndexOperation.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System.Runtime.CompilerServices;
namespace StgSharp.HighPerformance
{
public unsafe partial class UnmanagedMemoryPool
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ConnectToNodeAtLevel(
IndexNode previous,
IndexNode current,
int level,
bool isHead)
{
if (isHead)
{
_entrance[level] = current.Position;
current.Previous[level] = 0;
} else
{
previous.Next[level] = current.Previous[level] = (uint)(current.Position - previous.Position);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private IndexNode GetNode(nuint pos)
{
ref byte span = ref *(_buffer + pos);
IndexNode node = new(ref span)
{
Position = pos
};
return node;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool IsNodeBeginning(nuint pos, int level)
{
return _entrance[level] == pos;
}
private bool Rebase(IndexNode n, uint offset)
{
nuint pos = n.Position;
uint size = n.Size;
if (size <= offset + 64)
{
for (int i = 0; i <= n.Level; i++) {
RemoveLevelHead(n, i);
}
return false;
}
uint
newOffset = ((offset / 64) + 1) * 64,
remain = n.Size - newOffset,
c64 = remain % 4,
c256 = remain / 4,
c1024 = c256 / 4,
c4096 = c1024 / 4;
c256 %= 4;
c1024 %= 4;
IndexNode
/*
* level is not important here, higher level will get 0 Size,
* and ignored in coming logic
*/
p64 = GetNode(n.Position - n.Previous[1]),
p256 = GetNode(n.Position - n.Previous[2]),
p1024 = GetNode(n.Position - n.Previous[3]),
p4096 = GetNode(n.Position - n.Previous[4]),
n64 = GetNode(n.Position + n.Previous[1]),
n256 = GetNode(n.Position + n.Previous[2]),
n1024 = GetNode(n.Position + n.Previous[3]),
n4096 = GetNode(n.Position + n.Previous[4]);
IndexNode nTemp;
if (c64 != 0)
{
// set param to separated 64 bytes block and set next and previous offset to it.
size = c64 * 64;
pos += size;
nTemp = GetNode(pos);
nTemp.Position = pos;
nTemp.Size = size;
ConnectToNodeAtLevel(p64, nTemp, 1, n.Position == _entrance[1]);
p64 = nTemp;
}
if (c256 != 0)
{
size = c256 * 256;
pos += size;
nTemp = GetNode(pos);
nTemp.Position = pos;
nTemp.Size = size;
nTemp.Previous[2] = n.Previous[2] + size;
ConnectToNodeAtLevel(p64, nTemp, 1, p64.Position == _entrance[1]);
ConnectToNodeAtLevel(p256, nTemp, 2, n.Position == _entrance[2]);
p64 = p256 = nTemp;
}
if (c1024 != 0)
{
size = c1024 * 1024;
pos += size;
nTemp = GetNode(pos);
nTemp.Position = pos;
nTemp.Size = size;
nTemp.Previous[2] = n.Previous[2] + size;
// connect to previous node
ConnectToNodeAtLevel(p64, nTemp, 1, p64.Position == _entrance[1]);
ConnectToNodeAtLevel(p256, nTemp, 2, p256.Position == _entrance[2]);
ConnectToNodeAtLevel(p1024, nTemp, 3, n.Position == _entrance[3]);
p64 = p256 = p1024 = nTemp;
}
if (c4096 != 0)
{
size = c1024 * 1024;
pos += size;
nTemp = GetNode(pos);
nTemp.Position = pos;
nTemp.Size = size;
nTemp.Previous[2] = n.Previous[2] + size;
// connect to previous node
ConnectToNodeAtLevel(p64, nTemp, 1, p64.Position == _entrance[1]);
ConnectToNodeAtLevel(p256, nTemp, 2, p256.Position == _entrance[2]);
ConnectToNodeAtLevel(p1024, nTemp, 3, p1024.Position == _entrance[3]);
ConnectToNodeAtLevel(p4096, nTemp, 4, n.Position == _entrance[4]);
p64 = p256 = p1024 = p4096 = nTemp;
}
ConnectToNodeAtLevel(p64, n64, 1, false);
ConnectToNodeAtLevel(
p256, n256, 2, _entrance[2] == n.Position && n256.Position == n.Position);
ConnectToNodeAtLevel(
p1024, n1024, 3, _entrance[3] == n.Position && n1024.Position == n.Position);
ConnectToNodeAtLevel(
p4096, n4096, 4, _entrance[4] == n.Position && n4096.Position == n.Position);
return true;
}
private void RemoveLevelHead(IndexNode n, int level)
{
IndexNode nxt, pre;
if (n.Previous[level] == 0)
{
// head of usable LEVEL;
nxt = GetNode(n.Position + n.Next[level]);
_entrance[level] = nxt.Position;
}
pre = GetNode(n.Previous[level]);
nxt = GetNode(n.Next[level]);
nxt.Previous[level] = nxt.Previous[level] + n.Previous[level];
pre.Next[level] = pre.Next[level] + n.Next[level];
}
}
}
@@ -1,156 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="UnmanagedMemoryPool.MegaManage.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System.Collections.Generic;
using System.Runtime.CompilerServices;
namespace StgSharp.HighPerformance
{
public unsafe partial class UnmanagedMemoryPool
{
private readonly LinkedList<MegaBlockHead> _megaAlloc = new();
private readonly LinkedList<MegaBlockHead> _megaEmpty = new();
private MegaBlockHead _megaFirst;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void InsertMegaNear(MegaBlockHead previous, MegaBlockHead current)
{
MegaBlockHead pp = current.PreviousNear;
pp.NextNear = previous;
previous.PreviousNear = pp;
previous.NextNear = current;
current.PreviousNear = previous;
}
private static void InsertMegaSorted(LinkedList<MegaBlockHead> list, MegaBlockHead newItem)
{
LinkedListNode<MegaBlockHead>? node = list.First;
while (node != null && node.Value.Position < newItem.Position) {
node = node.Next;
}
_ = node != null ? list.AddBefore(node, newItem) : list.AddLast(newItem);
}
private bool TryGetMegaBlock(nuint size, out nuint position, int align = 1)
{
LinkedListNode<MegaBlockHead> node = _megaEmpty.First!;
while (node is not null)
{
MegaBlockHead block = node.Value;
if (block.Size >= size + (nuint)align)
{
position = block.Position;
nuint s = position + (nuint)(align - 1);
MegaBlockHead b;
if (block.Rebase(s))
{
b = new()
{
Position = position,
Size = position + (nuint)(align - 1),
Usage = MegaBlockUsage.MegaAlloc,
};
InsertMegaSorted(_megaAlloc, b);
InsertMegaNear(b, block);
} else
{
b = new()
{
Position = position,
Size = block.Size,
Usage = MegaBlockUsage.MegaAlloc,
};
InsertMegaSorted(_megaAlloc, b);
_megaEmpty.Remove(node);
InsertMegaNear(b, block.NextNear);
}
position = s & (nuint)~(align - 1);
return true;
}
node = node.Next!;
}
position = 0;
return false;
}
private unsafe class MegaBlockHead
{
private readonly ulong[] Heads = new ulong[4];
public MegaBlockHead NextNear { get; set; }
public MegaBlockHead PreviousNear { get; set; }
public MegaBlockUsage Usage { get; set; }
public nuint Position { get; set; }
public nuint Size { get; set; }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ref nuint DagHead(int level)
{
return ref Unsafe.As<ulong, nuint>(ref Heads[level]);
}
public bool Rebase(nuint size)
{
nuint s = this.Size - size;
if (s > 1024 * 1024 * 128)//space larger than 128MB is considered a new mega block
{
Position += size;
this.Size -= size;
Usage = MegaBlockUsage.EmptyDag;
return true;
}
MegaBlockHead pre = PreviousNear, nxt = NextNear;
pre.NextNear = nxt;
nxt.PreviousNear = pre;
return false;
}
}
private enum MegaBlockUsage
{
EmptyDag = 0,
UsedDag = 1,
MegaAlloc = 2
}
}
}
@@ -1,171 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="UnmanagedMemoryPool.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Concurrent;
using System.Net.Security;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
using System.Transactions;
namespace StgSharp.HighPerformance
{
public unsafe partial class UnmanagedMemoryPool
{
private readonly byte* _buffer;
// 4k,1k,256b,64b,16b
private nuint[] _entrance = new nuint[5];
private bool _disposedValue;
private readonly nuint _initialCapacity;
internal UnmanagedMemoryPool(nuint initialCapacity, int memoryAlign)
{
_initialCapacity = initialCapacity;
_buffer = (byte*)NativeMemory.AlignedAlloc(initialCapacity, (nuint)memoryAlign);
}
public UnmanagedMemoryHandle Allocate(uint size = 0)
{
if (size == 0) {
return new UnmanagedMemoryHandle
{
Postion = 0,
Buffer = new Span<byte>(_buffer, 0)
};
}
if (size > 0x8000_0000) {
throw new ArgumentOutOfRangeException(
"Size of buffer cannot be greater than 2147483648");
}
int level = GetLevelFromSize(size);
IndexNode allocator = GetNode(_entrance[level]);
nuint previous = allocator.Previous[level];
if (allocator.Size < size)
{
previous = allocator.Position;
allocator = GetNode(allocator.Next[level]);
}
bool isOverflow = false;
checked
{
try
{
isOverflow = (uint)(allocator.Position - previous + size) > 0xC0000000;
}
catch (OverflowException)
{
isOverflow = true;
}
}
if (isOverflow)
{
// offset is too large
}
UnmanagedMemoryHandle handle = new()
{
Postion = allocator.Position,
Buffer = new Span<byte>(_buffer + allocator.Position, (int)size),
};
Rebase(allocator, size);
return handle;
}
public static UnmanagedMemoryPool Create(nuint initialCapacity, int memoryAlign = 16)
{
UnmanagedMemoryPool pool = new(initialCapacity, memoryAlign);
IndexNode node = pool.GetNode(0);
for (int i = 4; i > 0; i--)
{
node.Previous[i] = 0;
node.Next[i] = IndexNode.LevelToSize(i);
}
node.Size = (uint)initialCapacity;
pool._megaFirst = new MegaBlockHead
{
Position = 0,
Size = initialCapacity,
Usage = MegaBlockUsage.EmptyDag
};
_ = pool._megaEmpty.AddLast(pool._megaFirst);
return pool;
}
public void Dispose()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
}
public void Free(UnmanagedMemoryHandle handle)
{
if (_buffer + handle.Postion != Unsafe.AsPointer(ref handle.Buffer[0])) {
throw new InvalidOperationException("Handle does not belong to pool");
}
nuint location = handle.Postion;
IndexNode root = GetNode(_entrance[4]);
while (root.Position > location) { }
}
protected virtual void Dispose(bool disposing)
{
if (!_disposedValue)
{
if (disposing) { }
NativeMemory.AlignedFree(_buffer);
_entrance = null!;
_disposedValue = true;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int GetLevelFromSize(uint size)
{
int level = 0;
uint temp = size / 8;
while (temp > 3)
{
level++;
temp >>= 2;
}
return level;
}
~UnmanagedMemoryPool()
{
Dispose(disposing:false);
}
}
}