youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

after merge

Browse Source
This commit is contained in:
Nitload
2026-05-04 17:25:01 +08:00
parent e1167bb071 8aa319a120
commit 156f6afe85
14 changed files with 364 additions and 397 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs
+67 -142
View File
@@ -26,6 +26,7 @@
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.IO;
using System.Numerics;
using System.Reflection.Metadata;
using hlsfHandle = StgSharp.HighPerformance.Memory.HlsfHandle;
@@ -39,49 +40,40 @@ namespace StgSharp.HighPerformance.Memory
nuint size
)
{
if (size > _maxSize) {
if (size > _maxAllocSize) {
throw new InvalidOperationException("Requested size exceeds maximum allocatable size.");
}
/*
int level = GetLevelFromSize(size);
int segmentIndex = DetermineSegmentIndex(_levelSizeArray, size, level);
int segmentIndex = GetSegmentFromLevel(size, level);
int i = level, s = segmentIndex;
Entry* handle;
while (!TryPopLevelForAllocation(i, s, out handle))
{
if (++s >= 3)
{
s = 0;
i++;
}
if (i > MaxLevel)
{
if (TryPopLevelForAllocation(10, 1, out handle))
{
break;
} else if (TryAllocateNew32MBBlock(out handle))
{
break;
}
throw new OverflowException("Out of memory: Unable to allocate new 16MB block or find suitable bucket for allocation.");
}
}
/**/
if (handle != null)
Entry* entry = null;
nuint requestedOffset = GetAllocSize(size);
(int level, int segment) = GetIndexPairOnce(size);
int index = (3 * level) + segment;
int i = index;
while ((i < _bucketHeads.Length) && !TryPopLevelForAllocation(index, out entry)) {
i++;
}
if ((i > _maxLevel) && !TryAllocateNewBlock(requestedOffset, out entry)) {
throw new OverflowException("Out of memory: Unable to allocate new 16MB block or find suitable bucket for allocation.");
}
if (entry != null)
{
int requestedOffset = (segmentIndex + 1) * ((int)_levelSizeArray[level]);
if (requestedOffset > (int)handle->Size) {
if (requestedOffset > entry->Size) {
throw new OverflowException("Out of memory: insufficient block size returned from spare memory.");
}
nuint remainder = handle->Size - ((nuint)requestedOffset);
SetEntryLevel(handle, handle->Position, (uint)requestedOffset);
handle->State = EntryState.Alloc;
nuint remainder = entry->Size - requestedOffset;
SetEntryLevel(entry, entry->Position, (uint)requestedOffset);
entry->State = EntryState.Alloc;
if (remainder > 0) {
SliceMemory(handle->NextNear, remainder);
OrganizeMemory(entry->NextNear, remainder);
}
}
return new(handle, size);
return new(entry, size);
}
public bool TryAlloc(
@@ -89,49 +81,40 @@ namespace StgSharp.HighPerformance.Memory
out HlsfHandle handle
)
{
if (size > _maxSize)
if (size > _maxAllocSize)
{
handle = default;
return false;
}
/*
int level = GetLevelFromSize(size);
int segmentIndex = DetermineSegmentIndex(_levelSizeArray, size, level);
int segmentIndex = GetSegmentFromLevel(size, level);
int i = level, s = segmentIndex;
Entry* entry;
while (!TryPopLevelForAllocation(i, s, out entry))
{
if (++s >= 3)
{
s = 0;
i++;
}
if (i > MaxLevel)
{
if (TryPopLevelForAllocation(10, 1, out entry))
{
break;
} else if (TryAllocateNew32MBBlock(out entry))
{
break;
}
handle = default;
return false;
}
/**/
Entry* entry = null;
nuint requestedOffset = GetAllocSize(size);
(int level, int segment) = GetIndexPairOnce(size);
int index = (3 * level) + segment;
int i = index;
while ((i < _bucketHeads.Length) && !TryPopLevelForAllocation(index, out entry)) {
i++;
}
if ((i > _maxLevel) && !TryAllocateNewBlock(requestedOffset, out entry)) {
throw new OverflowException("Out of memory: Unable to allocate new 16MB block or find suitable bucket for allocation.");
}
if (entry != null)
{
int requestedOffset = (segmentIndex + 1) * ((int)_levelSizeArray[level]);
if (requestedOffset > (int)entry->Size) {
if (requestedOffset > entry->Size) {
throw new OverflowException("Out of memory: insufficient block size returned from spare memory.");
}
nuint remainder = entry->Size - ((nuint)requestedOffset);
nuint remainder = entry->Size - requestedOffset;
SetEntryLevel(entry, entry->Position, (uint)requestedOffset);
entry->State = EntryState.Alloc;
if (remainder > 0) {
SliceMemory(entry->NextNear, remainder);
OrganizeMemory(entry->NextNear, remainder);
}
}
handle = new(entry, size);
@@ -139,16 +122,16 @@ namespace StgSharp.HighPerformance.Memory
}
/// <summary>
/// Slice the memory block into smaller blocks and add them to the appropriate buckets
/// Slice the memory block into smaller blocks and add them to the appropriate buckets.
/// Notice: this method is not responsible for freeing the original BucketNode
/// </summary>
/// <param name="e">
///
/// The next entry nearby the part to slice
/// </param>
/// <param name="sizeToSlice">
///
/// </param>
private void SliceMemory(
private void OrganizeMemory(
Entry* e,
nuint sizeToSlice
)
@@ -159,97 +142,22 @@ namespace StgSharp.HighPerformance.Memory
nuint end = e->Position - ((nuint)m_Buffer); // end offset of buffer to slice
int i = (BitOperations.TrailingZeroCount(offset) - 6) / 2;
i = int.Min(10, i);
for (; i < _levelSizeArray.Length; i++)
{
uint size = _levelSizeArray[i];
nuint upper = size * 4;
nuint edge = (((offset + upper) - 1U) / upper) * upper;
int count = (int)((nuint.Min(edge, end) - offset) / size);
if (count > 0)
{
nuint newSize = ((nuint)count) * size;
Entry* newEntry = (Entry*)_entries.Allocate();
InsertEntryBefore(newEntry, next);
SetEntryLevel(newEntry, offset + ((nuint)m_Buffer), newSize);
newEntry->State = EntryState.Empty;
BucketNode* b = (BucketNode*)newEntry->Position;
b->EntryRef = newEntry;
PushBucketToLevel(i, count - 1, b);
offset += newSize;
remain -= newSize;
}
if (remain <= 0) {
return;
}
}
if (i > MaxLevel)
{
uint size = _levelSizeArray[^1];
nuint upper = size * 4;
nuint edge = (end / upper) * upper;
int count = (int)((nuint.Min(edge, end) - offset) / upper);
if (count > 0)
{
nuint newSize = ((nuint)count) * upper;
Entry* newEntry = (Entry*)_entries.Allocate();
InsertEntryBefore(newEntry, next);
SetEntryLevel(newEntry, offset + ((nuint)m_Buffer), newSize);
newEntry->State = EntryState.Empty;
BucketNode* b = (BucketNode*)newEntry->Position;
b->EntryRef = newEntry;
PushBucketToLevel(i, count - 1, b);
offset += newSize;
remain -= newSize;
if (remain <= 0) {
return;
}
}
}
for (i--; i >= 0; i--)
{
uint size = _levelSizeArray[i];
nuint edge = (end / size) * size;
int count = (int)((nuint.Min(edge, end) - offset) / size);
if (count > 0)
{
nuint newSize = ((nuint)count) * size;
Entry* newEntry = (Entry*)_entries.Allocate();
InsertEntryBefore(newEntry, next);
SetEntryLevel(newEntry, offset + ((nuint)m_Buffer), newSize);
newEntry->State = EntryState.Empty;
BucketNode* b = (BucketNode*)newEntry->Position;
b->EntryRef = newEntry;
b->NextLevel = null;
b->PreviousLevel = null;
PushBucketToLevel(i, count - 1, b);
offset += newSize;
remain -= newSize;
}
if (remain <= 0) {
return;
}
}
}
/// <summary>
/// Try to allocate a new 16MB block when all buckets are empty
/// </summary>
/// <param name="size">
/// Size of memory block to be allocated. Notice: Size is assumed as times of 64.
/// </param>
/// <param name="handle">
/// Output handle to the allocated Entry
/// </param>
/// <returns>
/// True if allocation succeeded
/// </returns>
private bool TryAllocateNew32MBBlock(
private bool TryAllocateNewBlock(
nuint size,
out Entry* handle
)
{
@@ -260,7 +168,7 @@ namespace StgSharp.HighPerformance.Memory
handle = null;
return false;
}
nuint size = (remain > 32 * 1024 * 1024L) ? (32 * 1024 * 1024U) : remain;
size = nuint.Min(size, remain);
// build new entry
@@ -278,6 +186,23 @@ namespace StgSharp.HighPerformance.Memory
return true;
}
/// <summary>
/// Helper method for allocation - converts BucketNode to Entry
/// </summary>
private bool TryPopLevelForAllocation(
int bucketIndex,
out Entry* entry
)
{
if (TryPopBucket(bucketIndex, out BucketNode* bucketNode))
{
entry = bucketNode->EntryRef;
return entry != null;
}
entry = null;
return false;
}
/// <summary>
/// Helper method for allocation - converts BucketNode to Entry
/// </summary>
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs
+40 -14
View File
@@ -35,6 +35,7 @@ namespace StgSharp.HighPerformance.Memory
public unsafe partial class HybridLayerSegregatedFitAllocator
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void PushBucketToLevel(
int levelIndex,
int sizeIndex,
@@ -45,7 +46,16 @@ namespace StgSharp.HighPerformance.Memory
return;
}
Index index = (levelIndex > MaxLevel) ? (^1) : ((3 * levelIndex) + sizeIndex);
int index = (3 * levelIndex) + sizeIndex;
PushNodeToBucket(index, node);
}
private void PushNodeToBucket(
int index,
BucketNode* node
)
{
index = (index >= _bucketHeads.Length) ? _bucketHeads.Length : index;
if (_bucketHeads[index] is null)
{
_bucketHeads[index] = node;
@@ -79,10 +89,11 @@ namespace StgSharp.HighPerformance.Memory
BucketNode* node
)
{
Index index = (levelIndex > MaxLevel) ? (^1) : ((3 * levelIndex) + sizeIndex);
int index = (3 * levelIndex) + sizeIndex;
index = (index >= _bucketHeads.Length) ? _bucketHeads.Length : index;
if (_bucketHeads[index] == node)
{
if (TryPopLevel(levelIndex, sizeIndex, out _)) {
if (TryPopBucket(index, out _)) {
return;
}
}
@@ -102,14 +113,16 @@ namespace StgSharp.HighPerformance.Memory
return;
}
private bool TryPopLevel(
int levelIndex,
int sizeIndex,
private bool TryPopBucket(
int index,
out BucketNode* node
)
{
Index index = (levelIndex > MaxLevel) ? (^1) : ((3 * levelIndex) + sizeIndex);
/*
* Attention:
* This method is UNSAFE. Index may exceed range of _bucketHeads.
* Be careful when calling this method.
*/
BucketNode* head = _bucketHeads[index];
if (head == null)
@@ -126,24 +139,37 @@ namespace StgSharp.HighPerformance.Memory
_bucketHeads[index] = null;
node = head;
return true;
} else
{
next->PreviousLevel = null;
}
_bucketHeads[index] = head->NextLevel;
node = head;
node->IsInBucket = false;
node->NextLevel = null;
if (next != null) {
next->PreviousLevel = null;
}
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool TryPopLevel(
int levelIndex,
int sizeIndex,
out BucketNode* node
)
{
int index = (3 * levelIndex) + sizeIndex;
index = (index >= _bucketHeads.Length) ? (_bucketHeads.Length - 1) : index;
return TryPopBucket(index, out node);
}
private bool TryRemoveFromBucket(
int levelIndex,
int sizeIndex,
BucketNode* node
)
{
Index index = (levelIndex > 9) ? (^1) : ((3 * levelIndex) + sizeIndex);
int index = (3 * levelIndex) + sizeIndex;
index = (index >= _bucketHeads.Length) ? _bucketHeads.Length : index;
if (_bucketHeads[index] == node)
{
bool ok = TryPopLevel(levelIndex, sizeIndex, out _);
@@ -169,7 +195,7 @@ namespace StgSharp.HighPerformance.Memory
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// allocation area (user-writable region)
/// </summary>
[StructLayout(LayoutKind.Explicit)]
internal struct BucketNode
@@ -177,7 +203,7 @@ namespace StgSharp.HighPerformance.Memory
[FieldOffset(0)] internal Entry* EntryRef; // Pointer to corresponding Entry
// [FieldOffset(16)] internal ulong LockBuffer;
[FieldOffset(8)] internal BucketNode* NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal BucketNode* NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(24)] internal BucketNode* PreviousLevel; // Bucket stack linked list: previous pointer
[FieldOffset(32)] internal bool IsInBucket; // Whether in bucket
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Debug.cs
+16 -8
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="HLSFAllocator.Debug"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -36,7 +36,8 @@ namespace StgSharp.HighPerformance.Memory
public sealed class HLSFPositionChainBreakException : Exception
{
public HLSFPositionChainBreakException() : base("Position chain of an HLSF allocator is broken") { }
public HLSFPositionChainBreakException()
: base("Position chain of an HLSF allocator is broken") { }
}
@@ -45,7 +46,9 @@ namespace StgSharp.HighPerformance.Memory
private object _lock = new object();
public void DisplayEntryLayout(bool singleStep)
public void DisplayEntryLayout(
bool singleStep
)
{
lock (_lock)
{
@@ -61,7 +64,10 @@ namespace StgSharp.HighPerformance.Memory
}
}
public void DumpPositionChain(bool singleStep = false, int maxCount = 4096)
public void DumpPositionChain(
bool singleStep = false,
int maxCount = 4096
)
{
lock (_lock)
{
@@ -73,7 +79,7 @@ namespace StgSharp.HighPerformance.Memory
cur = cur->NextNear;
}
Console.WriteLine(EntryToString(_spareMemory));
while (cur != null && cur != _spareMemory && cnt < maxCount)
while ((cur != null) && (cur != _spareMemory) && (cnt < maxCount))
{
Console.WriteLine(EntryToString(cur));
cnt++;
@@ -112,11 +118,13 @@ namespace StgSharp.HighPerformance.Memory
}
}
private static string EntryToString(Entry* e)
private static string EntryToString(
Entry* e
)
{
BucketNode* b = (BucketNode*)e->Position;
return $"{(ulong)e->PreviousNear}<-prev- Position:{(ulong)e} Bucket:{{{(ulong)e->Position},{e->Level},{e->Size}}} -next->{(ulong)e->NextNear}";
}
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs
+1 -1
View File
@@ -79,7 +79,7 @@ namespace StgSharp.HighPerformance.Memory
cur = next;
} while ((cur != null) && (cur != _spareMemory) && (cur->State == EntryState.Empty));
if (size > 0) {
SliceMemory(cur, size);
OrganizeMemory(cur, size);
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Level.cs
+79 -47
View File
@@ -28,6 +28,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Security;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Text;
@@ -38,72 +39,103 @@ namespace StgSharp.HighPerformance.Memory
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private const int MaxLevel = 9;
private uint[] _levelSizeArray = [
64U,
256U,
1024U,
4096U,
16384U,
65536U,
262144U,
1048576U,
4194304U,
16777216U
];
public int AlignOfLevel(
int bucketLevel
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private nuint GetAllocSize(
nuint size
)
{
uint blockSize = _levelSizeArray[bucketLevel];
return Math.Min((int)blockSize, _align);
return (size + 63u) & (~63u);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetLevelFromSize(
nuint size
private int GetBucketIndexOnce(
nuint size
)
{
/*
* Level sizes: [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, ......]
* Pattern: 64 * 4^level = 2^6 * 2^(2*level) = 2^(6+2*level)
*
* For size S, we need: 2^(6+2*level) >= S
* So: 6+2*level >= log2(S)
* Therefore: level >= (log2(S) - 6) / 2
*
* We use hardware LZCNT instruction via BitOperations.LeadingZeroCount
* log2(S) = 63 - LeadingZeroCount(S) for 64-bit values
*/
int log2Size = 63 - BitOperations.LeadingZeroCount((ulong)size);
// Calculate level: level = max(0, (log2Size - 6) / 2)
int level = Math.Max(0, (log2Size - 6) / 2);
// caller is responsible for ensuring level is within bounds
return level;
level = Math.Min(level, _maxLevel);
uint levelSize = 1u << (log2Size & (~1));
int segment = ((int)(size / levelSize)) - 1;
return (level * 3) + segment;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetLevelFromSize(
uint size
private (int level, int segment) GetIndexPairOnce(
nuint size
)
{
return GetLevelFromSize(size);
int log2Size = 63 - BitOperations.LeadingZeroCount((ulong)size);
// Calculate level: level = max(0, (log2Size - 6) / 2)
int level = Math.Max(0, (log2Size - 6) / 2);
// caller is responsible for ensuring level is within bounds
level = Math.Min(level, _maxLevel);
uint levelSize = 1u << (log2Size & (~1));
int segment = ((int)(size / levelSize)) - 1;
return (level, segment);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int GetLevelFromSize(
nuint size
)
{
/*
* Level sizes: [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, ......]
* Pattern: 64 * 4^level = 2^6 * 2^(2*level) = 2^(6+2*level)
*
* For size S, we need: 2^(6+2*level) >= S
* So: 6+2*level >= log2(S)
* Therefore: level >= (log2(S) - 6) / 2
*
* We use hardware LZCNT instruction via BitOperations.LeadingZeroCount
* log2(S) = 63 - LeadingZeroCount(S) for 64-bit values
*/
int log2Size = 63 - BitOperations.LeadingZeroCount((ulong)size);
// Calculate level: level = max(0, (log2Size - 6) / 2)
int level = Math.Max(0, (log2Size - 6) / 2);
// caller is responsible for ensuring level is within bounds
return Math.Min(level, _maxLevel);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint GetLevelSizeFromLevel(
int level
)
{
return 1u << (6 + (2 * level));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint GetLevelSizeFromSize(
nuint size
)
{
int log2Size = 63 - BitOperations.LeadingZeroCount((ulong)size);
return 1u << (log2Size & (~1));
}
/// <summary>
/// Determine segment index based on size and originLevel. Input level value is regarded
/// safe.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int GetSegmentFromLevel(
ulong size,
int level
)
{
return ((int)(size / GetLevelSizeFromLevel(level))) - 1;
}
}
public enum CommonMemoryAlign : int
{
XMMLevel = 16,
YMMLevel = 32,
ZMMLevel = 64,
ArmNEONLevel = 16,
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs
+13 -41
View File
@@ -37,6 +37,9 @@ namespace StgSharp.HighPerformance.Memory
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
/// <returns>
/// True if two mem blocks are adjacent.
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool AssertAdjacentTo(
Entry* current,
@@ -48,36 +51,6 @@ namespace StgSharp.HighPerformance.Memory
(current->Position + current->Size == other->Position);
}
/// <summary>
/// check out if a bucket is within the same higher-level boundary as currentOffset
/// </summary>
private bool AssertBoundary(
Entry* previous,
Entry* current,
int level
)
{
level = int.Min(level, MaxLevel);
uint nextLevelSize = _levelSizeArray[level] * 4;
ulong prevHead = previous->Position - ((ulong)m_Buffer),
currHead = current->Position - ((ulong)m_Buffer);
return currHead / nextLevelSize == prevHead / nextLevelSize;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int DetermineSegmentIndex(
uint[] levelSizeArray,
ulong size,
int level
)
{
level = int.Min(level, MaxLevel);
return (int)((size - 1) / levelSizeArray[level]);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void InsertEntryAfter(
Entry* entry,
@@ -120,7 +93,7 @@ namespace StgSharp.HighPerformance.Memory
nuint size = entry->Size;
int originLevel = entry->Level;
int segment = DetermineSegmentIndex(_levelSizeArray, size, originLevel);
int segment = GetSegmentFromLevel(size, originLevel);
do
{
// previous direction merge
@@ -129,7 +102,7 @@ namespace StgSharp.HighPerformance.Memory
(p == _spareMemory) ||
(p->State != EntryState.Empty) ||
(p->Level != entry->Level) ||
!AssertBoundary(p, entry, originLevel))
!AssertAdjacentTo(p, entry))
{
break;
}
@@ -137,7 +110,7 @@ namespace StgSharp.HighPerformance.Memory
// Console.WriteLine($"{(ulong)p->Size} merged to {(ulong)entry->Size} previous");
entry->Size += p->Size;
entry->Position = p->Position;
RemoveFromBucket(originLevel, DetermineSegmentIndex(_levelSizeArray, p->Size, originLevel), (BucketNode*)p->Position);
RemoveFromBucket(originLevel, GetSegmentFromLevel(p->Size, originLevel), (BucketNode*)p->Position);
RemoveFromPositionChain(p);
} while (true);
@@ -149,19 +122,18 @@ namespace StgSharp.HighPerformance.Memory
(n == _spareMemory) ||
(n->State != EntryState.Empty) ||
(n->Level != entry->Level) ||
!AssertBoundary(n, entry, originLevel))
!AssertAdjacentTo(n, entry))
{
break;
}
// Console.WriteLine($"{(ulong)n->Size} merged to {(ulong)entry->Size} next");
entry->Size += n->Size;
RemoveFromBucket(originLevel, DetermineSegmentIndex(_levelSizeArray, n->Size, originLevel), (BucketNode*)n->Position);
RemoveFromBucket(originLevel, GetSegmentFromLevel(n->Size, originLevel), (BucketNode*)n->Position);
RemoveFromPositionChain(n);
} while (true);
originLevel = int.Min(9, originLevel);
entry->Level += (entry->Size >= _levelSizeArray[originLevel] * 4) ? 1 : 0;
entry->Level &= 15;
entry->Level = GetLevelFromSize(entry->Size);
BucketNode* bucket = (BucketNode*)entry->Position;
bucket->EntryRef = entry;
@@ -190,10 +162,10 @@ namespace StgSharp.HighPerformance.Memory
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SetEntryLevel(
Entry* entry,
nuint position,
nuint size
private void SetEntryLevel(
Entry* entry,
nuint position,
nuint size
)
{
entry ->Position = position;
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs
+7 -15
View File
@@ -30,7 +30,6 @@ using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using hlsfHandle = StgSharp.HighPerformance.Memory.HlsfHandle;
namespace StgSharp.HighPerformance.Memory
{
@@ -38,16 +37,17 @@ namespace StgSharp.HighPerformance.Memory
{
// Bucket system field - directly in main allocator class
private readonly BucketNode *[] _bucketHeads;
private readonly BucketNode*[] _bucketHeads;
// _spareMemory is also the beginning position of allocator
private Entry *_spareMemory;
private Entry* _spareMemory;
private readonly byte* m_Buffer;
private readonly Action<nuint> _freeHandle;
private bool disposedValue;
private readonly int _align;
private readonly nuint _maxSize;
private readonly int _maxLevel;
private readonly nuint _maxAllocSize;
private readonly nuint _size;
private readonly SlabAllocator<Entry> _entries;
@@ -88,17 +88,9 @@ namespace StgSharp.HighPerformance.Memory
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->Size = byteSize;
int levelCountOrigin = _levelSizeArray.Length;
int levelCount = GetLevelFromSize(maxBlockSize) + 1;
Array.Resize(ref _levelSizeArray, levelCount);
_bucketHeads = new BucketNode*[(levelCount * 3) + 1];
uint size = _levelSizeArray[^1];
for (int i = levelCountOrigin; i < levelCount; i++)
{
size *= 4;
_levelSizeArray[i] = size;
}
_maxSize = maxBlockSize;
_maxLevel = GetLevelFromSize(maxBlockSize) + 1;
_bucketHeads = new BucketNode*[(_maxLevel * 3) + 1];
_maxAllocSize = maxBlockSize;
_freeHandle = freeHandle ?? (static _ => { });
}
StgSharp.Common/HighPerformance/Memory/L4.cs
+1
View File
@@ -43,6 +43,7 @@ namespace StgSharp.HighPerformance.Memory
private const int PredictionCount = 2048;
private const int PredictionLeastAhead = 1024;
// private static readonly IIntrinsicKernel _intrinsic = IIntrinsicKernel.Instance;
private readonly CacheLineHead* _head;
private readonly MapArray* _mapCount;
private readonly PredictionArray* _predictCount;
StgSharp.Mathematics/Internal/IntrinsicContext.cs
+39 -9
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="IntrinsicContext"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -68,7 +68,9 @@ namespace StgSharp.Mathematics.Internal
[FieldOffset(0)] public readonly ushort Size;
private MatrixElementType(int mask)
private MatrixElementType(
int mask
)
{
Unsafe.SkipInit(out this);
Unsafe.SkipInit(out Size);
@@ -76,6 +78,24 @@ namespace StgSharp.Mathematics.Internal
this.Mask = mask;
}
[Obsolete("Do not use this constructor. Use pre-defined ones from static fields.", true)]
public MatrixElementType()
{
throw new NotSupportedException("Do not use this constructor. Use pre-defined ones from static fields.");
}
public override readonly bool Equals(
object? obj
)
{
return (obj is MatrixElementType type) && (Mask == type.Mask);
}
public override readonly int GetHashCode()
{
return Mask;
}
}
[StructLayout(LayoutKind.Sequential)]
@@ -105,14 +125,19 @@ namespace StgSharp.Mathematics.Internal
private nint handle; //f32_panel_mul
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public readonly void Call(MatrixIntrinsicHandle index, MatrixParallelTask* task)
public readonly void Call(
MatrixIntrinsicHandle index,
MatrixParallelTask* task
)
{
nint ptr = this[(int)index];
((delegate* unmanaged[Cdecl]<MatrixParallelTask*, void>)ptr)(task);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public readonly nint Get(MatrixIntrinsicHandle index)
public readonly nint Get(
MatrixIntrinsicHandle index
)
{
return this[(int)index];
}
@@ -143,15 +168,20 @@ namespace StgSharp.Mathematics.Internal
MatrixKernel* left,
MatrixKernel* right,
MatrixKernel* ans,
ulong scalar)
ulong scalar
)
{
nint ptr = this[(int)index];
((delegate* unmanaged[Cdecl]<MatrixKernel*, MatrixKernel*, MatrixKernel*, ulong, void>)ptr)(left, right,
ans, scalar);
((delegate* unmanaged[Cdecl]<MatrixKernel*, MatrixKernel*, MatrixKernel*, ulong, void>)ptr)(left,
right,
ans,
scalar);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public readonly nint Get(MatrixIntrinsicHandle index)
public readonly nint Get(
MatrixIntrinsicHandle index
)
{
return this[(int)index];
}
StgSharp.Mathematics/Numeric/Linear/MatrixCompute.cs
+1 -1
View File
@@ -71,7 +71,7 @@ namespace StgSharp.Mathematics.Numeric
op(task);
return;
}
MatrixParallelWrap taskWrap = MatrixParallel.ScheduleTask<BufferQueueLeftRightAns>(task);
MatrixParallelWrap taskWrap = MatrixParallel.ScheduleTask<BufferQueueLeftRightAns_NoNuma>(task);
taskWrap.LaunchParallel(SleepMode.DeepSleep);
taskWrap.Dispose();
}
StgSharp.Mathematics/Numeric/Runtime/MatrixParallelQueue.cs
+46 -113
View File
@@ -33,6 +33,7 @@ using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices.JavaScript;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
@@ -42,153 +43,85 @@ namespace StgSharp.Mathematics.Numeric
internal unsafe interface IMatrixParallelQueue<TSelf> where TSelf : IMatrixParallelQueue<TSelf>
{
static abstract TSelf Create(
MatrixParallelTask* baseTask,
L4 buffer,
int numaCount
bool InitNumaPredictor(
out IL4Predict predictor
);
void Return(
MatrixParallelTask* task,
in L4CacheLine left,
in L4CacheLine right,
in L4CacheLine ans
);
bool Steal(
bool RequestTaskResource(
MatrixParallelTask* task,
out L4CacheLine left,
out L4CacheLine right,
out L4CacheLine ans
);
void ReturnTaskResource(
MatrixParallelTask* task,
in L4CacheLine left,
in L4CacheLine right,
in L4CacheLine ans
);
}
internal sealed unsafe class BufferQueueLeftRightAns : IMatrixParallelQueue<BufferQueueLeftRightAns>
internal interface IBufferQueueLeftRightAns : IMatrixParallelQueue<IBufferQueueLeftRightAns> { }
internal sealed unsafe class BufferQueueLeftRightAns_NoNuma : IMatrixParallelQueue<IBufferQueueLeftRightAns>
{
private MatrixParallelTask* _baseTask;
private bool _needNuma;
private L4 _buffer;
private readonly nuint _byteSize;
private readonly nuint _count;
private volatile nuint _cursor;
private readonly nuint Count;
private BufferQueueLeftRightAns() { }
private volatile nuint _mat1;
private volatile nuint _mat2;
private volatile nuint _mat3;
private BufferQueueLeftRightAns(
MatrixParallelTask* baseTask
private BufferQueueLeftRightAns_NoNuma() { }
internal BufferQueueLeftRightAns_NoNuma(
MatrixParallelTask* baseTask
)
{
_baseTask = baseTask;
Count = ((nuint)_baseTask->CommonX) * ((nuint)_baseTask->CommonY);
_count = ((nuint)_baseTask->CommonX) * ((nuint)_baseTask->CommonY);
_byteSize = ((nuint)_baseTask->Mat1) + (((nuint)_baseTask->CommonX) * ((nuint)_baseTask->CommonY) * _baseTask->ElementType.Size);
_mat1 = (nuint)_baseTask->Mat1;
_mat2 = (nuint)_baseTask->Mat2;
_mat3 = (nuint)_baseTask->Mat3;
}
public static unsafe BufferQueueLeftRightAns Create(
MatrixParallelTask* baseTask,
L4 buffer,
int numaCount
public bool InitNumaPredictor(
out IL4Predict predictor
)
{
return new BufferQueueLeftRightAns(baseTask);
predictor = null;
return false;
}
public unsafe void Return(
MatrixParallelTask* task,
in L4CacheLine left,
in L4CacheLine right,
in L4CacheLine ans
)
{
if (left.Address != 0) {
left.Dispose();
}
if (right.Address != 0) {
right.Dispose();
}
if (ans.Address != 0) {
ans.Dispose();
}
}
public unsafe bool Steal(
public unsafe bool RequestTaskResource(
MatrixParallelTask* task,
out L4CacheLine left,
out L4CacheLine right,
out L4CacheLine ans
)
{
throw new NotImplementedException();
left = new L4CacheLine(0, ref Unsafe.AsRef<int>(null), ref Unsafe.AsRef<int>(null));
right = new L4CacheLine(0, ref Unsafe.AsRef<int>(null), ref Unsafe.AsRef<int>(null));
ans = new L4CacheLine(0, ref Unsafe.AsRef<int>(null), ref Unsafe.AsRef<int>(null));
return true;
}
private class Predictor(
BufferQueueLeftRightAns host
) : IL4Predict
public unsafe void ReturnTaskResource(
MatrixParallelTask* task,
in L4CacheLine left,
in L4CacheLine right,
in L4CacheLine ans
)
{
private readonly BufferQueueLeftRightAns _host;
public PredictResult Predict(
Span<CacheLineDescription> node,
out int count
)
{
nuint begin = _host._cursor;
nuint mat1 = (nuint)_host._baseTask->Mat1;
nuint mat2 = (nuint)_host._baseTask->Mat2;
nuint mat3 = (nuint)_host._baseTask->Mat3;
nuint _cur = begin * _host._baseTask->ElementType.Size;
nuint mat1_cur = ((nuint)_host._baseTask->Mat1) + _cur;
nuint mat2_cur = ((nuint)_host._baseTask->Mat2) + _cur;
nuint mat3_cur = ((nuint)_host._baseTask->Mat3) + _cur;
nuint end = ((nuint)_host._baseTask->Mat1) + (((nuint)_host._baseTask->CommonX) * ((nuint)_host._baseTask->CommonY) * _host._baseTask->ElementType.Size);
nuint mat1_end = ((nuint)_host._baseTask->Mat1) + end;
nuint mat2_end = ((nuint)_host._baseTask->Mat2) + end;
nuint mat3_end = ((nuint)_host._baseTask->Mat3) + end;
nuint size = 1024 * 16 * _host._baseTask->ElementType.Size;
for (int i = 0; i < node.Length; i += 3)
{
int cycle = i / 3;
ref CacheLineDescription prediction = ref node[i];
prediction.Address = mat1_cur + size;
}
count = 1;
return PredictResult.Success;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Prefetch(
nuint origin,
uint policy,
Span<byte> cache
)
{
MatrixCachePolicy pEnum = (MatrixCachePolicy)policy;
if ((pEnum & (MatrixCachePolicy.Ans | MatrixCachePolicy.Sequential)) != 0)
{
if ((pEnum & (MatrixCachePolicy.Left | MatrixCachePolicy.Right)) != 0)
{
byte* bPtr = (byte*)origin;
new Span<byte>(bPtr, cache.Length).CopyTo(cache);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteBack(
nuint origin,
uint policy,
ReadOnlySpan<byte> cache
)
{
MatrixCachePolicy pEnum = (MatrixCachePolicy)policy;
if ((pEnum & (MatrixCachePolicy.Ans | MatrixCachePolicy.Sequential)) != 0)
{
byte* bPtr = (byte*)origin;
cache.CopyTo(new Span<byte>(bPtr, cache.Length));
}
}
return;
}
}
StgSharp.Mathematics/Numeric/Runtime/MatrixParallelTask.cs
+6 -6
View File
@@ -52,20 +52,20 @@ namespace StgSharp.Mathematics.Numeric.Runtime
[FieldOffset(8)] public MatrixKernel* Mat2;
[FieldOffset(16)] public MatrixKernel* Mat3;
/// <summary>
/// Being offset of first kernel in a L4 cache line.
/// </summary>
[FieldOffset(24)] public nuint CacheIndex;
/// <summary>
/// Count of kernels to be calculated in a L4 cache line.
/// </summary>
[FieldOffset(28)] public int CacheLength;
[FieldOffset(32)] public int CommonK;
[FieldOffset(36)] public int CommonX;
[FieldOffset(40)] public int CommonY;
[FieldOffset(44)] public int SpanLength;
[FieldOffset(44)] public int VectorLength;
[FieldOffset(48)] public MatrixElementType ElementType;
[FieldOffset(52)] public MatrixIntrinsicHandle ComputeHandle;
[FieldOffset(56)] public nint Span;
/// <summary>
/// Beginning offset of first kernel in a L4 cache line.
/// </summary>
[FieldOffset(24)] public nuint Offset;
[FieldOffset(56)] public nuint Vector;
[FieldOffset(64)] public ScalarPacket Scalar;
public MatrixParallelTask()
StgSharp.Mathematics/Numeric/Runtime/QueueFactory.cs
+42
View File
@@ -0,0 +1,42 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="QueueFactory"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Mathematics.Numeric
{
internal interface IMatrixQueueFactory
{
IMatrixParallelQueue<IMatrixParallelQueue> CreateSequential();
}
}
StgSharp.Native/compile_commands.json
+6
View File
@@ -41,6 +41,12 @@
"file": "D:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/sn-src/mk.fma.x86.c",
"output": "D:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/build/cmake-clang-gnu/CMakeFiles/StgSharp.Native.dir/sn-src/mk.fma.x86.c.obj"
},
{
"directory": "D:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/build/cmake-clang-gnu",
"command": "C:\\PROGRA~1\\LLVM\\bin\\clang.exe -DSN_ARCH_TARGET=SN_ARCH_X86_64 -DSN_EXTERN_TARGET -DSN_OS_TARGET=SN_OS_WINDOWS -DStgSharp_Native_EXPORTS -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/sn-head -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/sn-template -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/lib -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/lib/glfw -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/lib/stbi -O0 -g -Xclang -gcodeview -std=gnu11 -D_DEBUG -D_DLL -D_MT -Xclang --dependent-lib=msvcrtd -mavx -mavx2 -mfma -mavx512f -mavx512vl -o CMakeFiles\\StgSharp.Native.dir\\sn-src\\mk.p_scan.x86.c.obj -c D:\\NitloadIndustry\\研究项目\\1-Flagship\\TH2020\\ProjectFile\\内核源代码\\StgSharp.Native\\sn-src\\mk.p_scan.x86.c",
"file": "D:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/sn-src/mk.p_scan.x86.c",
"output": "D:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/build/cmake-clang-gnu/CMakeFiles/StgSharp.Native.dir/sn-src/mk.p_scan.x86.c.obj"
},
{
"directory": "D:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/build/cmake-clang-gnu",
"command": "C:\\PROGRA~1\\LLVM\\bin\\clang.exe -DSN_ARCH_TARGET=SN_ARCH_X86_64 -DSN_EXTERN_TARGET -DSN_OS_TARGET=SN_OS_WINDOWS -DStgSharp_Native_EXPORTS -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/sn-head -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/sn-template -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/lib -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/lib/glfw -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/lib/stbi -O0 -g -Xclang -gcodeview -std=gnu11 -D_DEBUG -D_DLL -D_MT -Xclang --dependent-lib=msvcrtd -mavx -mavx2 -mfma -mavx512f -mavx512vl -o CMakeFiles\\StgSharp.Native.dir\\sn-src\\s.factorial.x86.c.obj -c D:\\NitloadIndustry\\研究项目\\1-Flagship\\TH2020\\ProjectFile\\内核源代码\\StgSharp.Native\\sn-src\\s.factorial.x86.c",