HLSf use nuint rather than long, L4 now uses hlsf rather than array to alloc cache line.

This commit is contained in:
Youmu Liu
2026-04-21 20:32:28 +08:00
parent 2e2d64a2ce
commit 4f3e206dcb
11 changed files with 151 additions and 133 deletions
@@ -212,10 +212,10 @@ namespace StgSharp.HighPerformance.Memory
[FieldOffset(0)] internal Entry* NextNear; // Position linked list: next pointer
[FieldOffset(8)] internal Entry* PreviousNear; // Position linked list: previous pointer
[FieldOffset(24)] internal int Level; // Level of the block in the allocator
[FieldOffset(32)] internal int Level; // Level of the block in the allocator
[FieldOffset(16)] internal nuint Size; // Block size information
[FieldOffset(36)] public EntryState State;
[FieldOffset(28)] public nuint Position; // Memory block position
[FieldOffset(24)] public nuint Position; // Memory block position
}
@@ -27,6 +27,8 @@
// -----------------------------------------------------------------------
using System.Runtime.InteropServices;
using Hlsf = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator;
namespace StgSharp.HighPerformance.Memory
{
public partial class L4
@@ -40,25 +42,33 @@ namespace StgSharp.HighPerformance.Memory
}
[StructLayout(LayoutKind.Sequential, Pack = 16)]
[StructLayout(LayoutKind.Explicit, Size = 64, Pack = 16)]
private unsafe struct CacheLineHead
{
internal nuint _address;
internal nuint _origin;
[FieldOffset(0)] internal Hlsf.Entry AllocatorEntry;
#region
#region reference counting and state information
internal int _refCount;
internal uint _size;
[FieldOffset(52)] internal int RefCount;
[FieldOffset(56)] internal int RemainingPredictionCount;
[FieldOffset(60)] internal int Id;
#endregion
#region
internal int _id;
internal short _predictor;
internal ushort _profile;
#region hlsf entry alias
[FieldOffset(24)] public readonly nuint Address;
[FieldOffset(16)] public readonly uint Size;
#endregion
#region cache line information
[FieldOffset(50)] internal short Predictor;
[FieldOffset(48)] internal ushort Profile;
[FieldOffset(40)] internal nuint Origin;
#endregion
}
@@ -35,6 +35,7 @@ using System.Text;
using System.Threading.Tasks;
using Hlsf = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator;
using HlsfEntry = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator.Entry;
namespace StgSharp.HighPerformance.Memory
{
@@ -76,14 +77,14 @@ namespace StgSharp.HighPerformance.Memory
int bitPos = BitOperations.TrailingZeroCount(evictMask);
int index = bitPos + i;
CacheLineHead* head = &_head[index];
if (Interlocked.CompareExchange(ref head->_refCount, int.MinValue, 0) == 0)
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
{
// EvictLine(i + index);
_ = _map.TryRemove(head->_origin, out _);
int id = head->_predictor;
_ = _map.TryRemove(head->Origin, out _);
int id = head->Predictor;
IL4Predict predict = pSpan[id] ??
throw new L4PredictUnregisteredException(id);
predict.WriteBack(head->_origin, head->_profile, new ReadOnlySpan<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan<byte>((byte*)head->Address, (int)head->Size));
RecycleLine(index);
remain--;
}
@@ -103,24 +104,24 @@ namespace StgSharp.HighPerformance.Memory
int id
)
{
Hlsf.Entry* entry = _entryManager._basePtr + id;
HlsfEntry* entry = (HlsfEntry*)(_entryManager._basePtr + id);
_bufferAllocator.Free(entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe bool TryGetEmptyLine(
nuint size,
out int index,
out HlsfHandle handle
out int index
)
{
if (!_bufferAllocator.TryAlloc(size, out handle))
if (!_bufferAllocator.TryAlloc(size, out HlsfHandle handle))
{
index = 0;
return false;
}
Hlsf.Entry* entry = handle.EntryHandle;
index = (int)(entry - _entryManager._basePtr);
index = (int)(entry - ((HlsfEntry*)_entryManager._basePtr));
return true;
}
@@ -28,6 +28,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
@@ -44,39 +45,9 @@ namespace StgSharp.HighPerformance.Memory
public interface IL4Predict
{
/*
* Planned optimization contract types:
* 1. internal interface IL4PredictOptimizationContract
* - Trusted extension point queried once during predictor registration.
* - Produces normalized compatibility tokens for a given map policy.
* 2. readonly struct L4PredictCompatibilityToken
* - Hot-path comparable descriptor cached by L4.
* - Should at least carry ResidentFamily, FlushFamily and AccessMode.
* 3. enum L4PredictAccessMode
* - Distinguishes read-only sources from write-back or write-through targets.
*
* L4 should only use these cached tokens in the prefetch path. Predictors without the
* internal contract stay on the conservative path: write back the old owner, then
* rematerialize the cache line for the new predictor.
*/
/// <summary>
/// Determines whether a cache line materialized by <paramref name="predict" /> under <paramref
/// name="policy" /> can be adopted by the current predictor without an intermediate
/// rematerialization.
/// </summary>
/// <remarks>
/// This is intentionally stronger than comparing raw policy bits for equality. A true result
/// means the resident bytes can continue to be interpreted safely after ownership is transferred
/// to the current predictor.
/// </remarks>
bool IsSamePolicy(
ushort policy,
IL4Predict predict
);
PredictResult Predict(
Span<CacheLinePrediction> node,
Span<CacheLineDescription> node,
out int count
);
@@ -119,20 +90,21 @@ namespace StgSharp.HighPerformance.Memory
}
public struct CacheLinePrediction : IEquatable<CacheLinePrediction>
[StructLayout(LayoutKind.Explicit)]
public struct CacheLineDescription : IEquatable<CacheLineDescription>
{
public override readonly bool Equals(
object? obj
)
{
return (obj is CacheLinePrediction prediction) &&
return (obj is CacheLineDescription prediction) &&
Address.Equals(prediction.Address) &&
(MapPolicy == prediction.MapPolicy);
}
public readonly bool Equals(
CacheLinePrediction other
CacheLineDescription other
)
{
return
@@ -141,29 +113,54 @@ namespace StgSharp.HighPerformance.Memory
public override readonly int GetHashCode()
{
return HashCode.Combine(Address, MapPolicy);
return (int)ComposePredictionKey64(Address, MapPolicy);
}
public readonly nuint GetHashCodeLong()
{
return ComposePredictionKey64(Address, MapPolicy);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint ComposePredictionKey64(
nuint address,
ushort mapPolicy
)
{
nuint x = address;
nuint y = mapPolicy;
unchecked
{
nuint mixed = x ^ (y * ((nuint)0x9e3779b97f4a7c15UL));
mixed ^= mixed >> 30;
mixed *= (nuint)0xbf58476d1ce4e5b9UL;
mixed ^= mixed >> 27;
mixed *= (nuint)0x94d049bb133111ebUL;
mixed ^= mixed >> 31;
return mixed;
}
}
public static bool operator !=(
CacheLinePrediction left,
CacheLinePrediction right
CacheLineDescription left,
CacheLineDescription right
)
{
return !(left == right);
}
public static bool operator ==(
CacheLinePrediction left,
CacheLinePrediction right
CacheLineDescription left,
CacheLineDescription right
)
{
return left.Equals(right);
}
#pragma warning disable CA1051
public nuint Address;
public uint Size;
public ushort MapPolicy;
internal short Predictor;
[FieldOffset(0)]public nuint Address;
[FieldOffset(8)]public uint Size;
[FieldOffset(12)]public ushort MapPolicy;
[FieldOffset(14)]internal short Predictor;
#pragma warning restore CA1051
}
}
@@ -44,7 +44,7 @@ namespace StgSharp.HighPerformance.Memory
}
IL4Predict predict = CurrentPredict;
int predictIndex = CurrentPredictIndex;
Span<CacheLinePrediction> span = new(_prediction, PredictionCount);
Span<CacheLineDescription> span = new(_prediction, PredictionCount);
PredictResult result = predict.Predict(span, out int actualCount);
if ((result & PredictResult.Exhausted) != 0) {
_exhausted = !TryGetNextPredict();
@@ -58,7 +58,7 @@ namespace StgSharp.HighPerformance.Memory
AgeAll();
for (int i = 0; i < actualCount; i++)
{
ref CacheLinePrediction handle = ref span[i];
ref CacheLineDescription handle = ref span[i];
nuint address = handle.Address;
if (!_map.TryGet(address, out _))
{
@@ -75,34 +75,34 @@ namespace StgSharp.HighPerformance.Memory
for (int i = 0; i < missIndex; i++)
{
CacheLineHead* head;
ref CacheLinePrediction pHandle = ref span[i];
ref CacheLineDescription pHandle = ref span[i];
nuint address = pHandle.Address;
int index;
if (_map.TryGet(address, out nuint handle))
{
head = (CacheLineHead*)handle;
index = head->_id;
index = head->Id;
(*_eviction)[index] = 0; // reset eviction counter
#region update predictor belonging
int predictId = head->_predictor;
int predictId = head->Predictor;
IL4Predict belong = predictorSpan[predictId];
// TODO 由于L4的缓存行改为“可变长度”缓存行,需要调整缓存行复用策略
// This branch performs resident-line adoption rather than a simple policy
// equality check. Returning true means the current predictor can keep using
// the existing cache bytes without a write-back plus re-prefetch cycle.
if (predict.IsSamePolicy(head->_profile, belong))
if (predict.IsSamePolicy(head->Profile, belong))
{
head->_predictor = (short)predictIndex;
head->Predictor = (short)predictIndex;
} else
{
// Compatibility failed, so the old owner must first flush its view of
// the line before the new predictor rematerializes the cache bytes.
belong.WriteBack(head->_origin, head->_profile, new ReadOnlySpan<byte>((byte*)head->_address, head->_size));
head->_profile = pHandle.MapPolicy;
predict.Prefetch(head->_origin, head->_profile, new Span<byte>((byte*)head->_address, head->_size));
belong.WriteBack(head->Origin, head->Profile, new ReadOnlySpan<byte>((byte*)head->Address, (int)head->Size));
head->Profile = pHandle.MapPolicy;
predict.Prefetch(head->Origin, head->Profile, new Span<byte>((byte*)head->Address, (int)head->Size));
}
#endregion
@@ -112,8 +112,7 @@ namespace StgSharp.HighPerformance.Memory
int retryCount = 0, remain = missIndex - i;
const int maxRetry = 1024;
HlsfHandle hlsfHandle;
while (!TryGetEmptyLine(pHandle.Size, out index, out hlsfHandle))
while (!TryGetEmptyLine(pHandle.Size, out index))
{
remain = Evict(remain);
retryCount++;
@@ -126,15 +125,13 @@ namespace StgSharp.HighPerformance.Memory
}
head = &_head[index];
head->_id = index;
head->_origin = address;
head->_profile = pHandle.MapPolicy;
head->_refCount = 0;
head->_predictor = (short)predictIndex;
head->_address = (nuint)hlsfHandle.Pointer;
head->_size = pHandle.Size;
head->Origin = address;
head->Profile = pHandle.MapPolicy;
head->RefCount = 0;
head->Predictor = (short)predictIndex;
// pre fetch them here
predict.Prefetch(head->_origin, head->_profile, new Span<byte>((byte*)head->_address, (int)head->_size));
predict.Prefetch(head->Origin, head->Profile, new Span<byte>((byte*)head->Address, (int)head->Size));
_ = _map.TryAddOrSet(address, (nuint)head, out _);
}
_ = Interlocked.Add(ref _aheadCount, actualCount);
@@ -46,20 +46,20 @@ namespace StgSharp.HighPerformance.Memory
private int _maxUnused;
private int _recycleCount;
internal readonly Hlsf.Entry* _basePtr;
internal readonly CacheLineHead* _basePtr;
internal readonly int* _indexReuse;
public Slab(
nuint ptr
)
{
_basePtr = (Hlsf.Entry*)ptr;
_basePtr = (CacheLineHead*)ptr;
_indexReuse = (int*)(_basePtr + Capacity);
_maxUnused = 0;
_recycleCount = 0;
}
public static nuint RequestedAllocation { get; } = ((nuint)(Unsafe.SizeOf<Hlsf.Entry>() + sizeof(int))) * Capacity;
public static nuint RequestedAllocation { get; } = ((nuint)(Unsafe.SizeOf<CacheLineHead>() + sizeof(int))) * Capacity;
public override nuint Allocate()
{
+11 -10
View File
@@ -46,7 +46,7 @@ namespace StgSharp.HighPerformance.Memory
private static readonly IIntrinsicKernel _intrinsic = IIntrinsicKernel.Instance;
private readonly EvictionArray* _eviction;
private readonly CacheLineHead* _head;
private readonly CacheLinePrediction* _prediction;
private readonly CacheLineDescription* _prediction;
private bool _disposed;
private bool _exhausted;
@@ -73,7 +73,7 @@ namespace StgSharp.HighPerformance.Memory
// init order is fallow by align requirement of each component.
nuint entrySize = Slab.RequestedAllocation; // SIMDID, no less than 16 byte bit
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLinePrediction>()) * PredictionCount; //16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>(); // 16 byte align
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
@@ -94,7 +94,7 @@ namespace StgSharp.HighPerformance.Memory
ptr += entrySize;
_head = (CacheLineHead*)ptr;
ptr += headSize;
_prediction = (CacheLinePrediction*)ptr;
_prediction = (CacheLineDescription*)ptr;
ptr += predictionSize;
_map = SwissTable.Create(ptr, _ => { });
ptr += mapSize;
@@ -140,7 +140,7 @@ namespace StgSharp.HighPerformance.Memory
// init order is fallow by align requirement of each component.
nuint entrySize = Slab.RequestedAllocation; // SIMDID, no less than 16 byte bit
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLinePrediction>()) * PredictionCount; //16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>(); // 16 byte align
nuint normalizedBufferSize = ByteCapacity; // align to 4096 byte, page size
@@ -150,7 +150,7 @@ namespace StgSharp.HighPerformance.Memory
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public L4CacheLine Map(
nuint origin
ref CacheLineDescription prediction
)
{
if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
@@ -173,19 +173,20 @@ namespace StgSharp.HighPerformance.Memory
}
while (true)
{
if (_map.TryGet(origin, out nuint entryHandle))
if (_map.TryGet(prediction.GetHashCodeLong(), out nuint entryHandle))
{
// cache hit
CacheLineHead* entry = (CacheLineHead*)entryHandle;
if (Interlocked.Increment(ref entry->_refCount) > 0)
if (Interlocked.Increment(ref entry->RefCount) > 0)
{
(*_eviction)[entry->_id] = 0;
(*_eviction)[entry->Id] = 0;
_ = Interlocked.Increment(ref _activeCount);
_ = Interlocked.Decrement(ref _aheadCount);
return new L4CacheLine(entry->_address, ref _activeCount, ref entry->_refCount);
_ = Interlocked.Decrement(ref entry->RemainingPredictionCount);
return new L4CacheLine(entry->Address, ref _activeCount, ref entry->RefCount);
} else
{
_ = Interlocked.Decrement(ref entry->_refCount);
_ = Interlocked.Decrement(ref entry->RefCount);
}
}
Thread.Sleep(0);
@@ -1,30 +1,27 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="MatrixEnumerator.cs"
// file="MatrixEnumerator"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
@@ -47,7 +44,8 @@ namespace StgSharp.Mathematics.Numeric.Runtime
}
public unsafe interface IMatrixCachePredictor<TSelf> : IL4Predict where TSelf: class, IMatrixCachePredictor<TSelf>
public unsafe interface IMatrixCachePredictor<TSelf> : IL4Predict
where TSelf : class, IMatrixCachePredictor<TSelf>
{
public static abstract PredictPolicy Policy { get; set; }
@@ -62,22 +60,36 @@ namespace StgSharp.Mathematics.Numeric.Runtime
public static PredictPolicy Policy { get; set; } = PredictPolicy.SquareRowMajor;
public bool IsSamePolicy(uint policy, IL4Predict predict)
public bool IsSamePolicy(
uint policy,
IL4Predict predict
)
{
throw new NotImplementedException();
}
public PredictResult Predict(Span<CacheLinePrediction> node, out int count)
public PredictResult Predict(
Span<CacheLineDescription> node,
out int count
)
{
throw new NotImplementedException();
}
public void Prefetch(nuint origin, uint policy, Span<byte> cache)
public void Prefetch(
nuint origin,
uint policy,
Span<byte> cache
)
{
throw new NotImplementedException();
}
public void WriteBack(nuint origin, uint policy, ReadOnlySpan<byte> cache)
public void WriteBack(
nuint origin,
uint policy,
ReadOnlySpan<byte> cache
)
{
throw new NotImplementedException();
}
+2 -2
View File
@@ -51,13 +51,13 @@ namespace StgSharp
}
public bool Predict(
Span<CacheLinePrediction> node,
Span<CacheLineDescription> node,
out int count
)
{
count = Math.Min(_addresses.Length, node.Length);
for (int i = 0; i < count; i++) {
node[i] = new CacheLinePrediction
node[i] = new CacheLineDescription
{
Address = _addresses[i],
MapPolicy = 0,
+5 -5
View File
@@ -56,13 +56,13 @@ namespace StgSharp
}
public bool Predict(
Span<CacheLinePrediction> node,
Span<CacheLineDescription> node,
out int count
)
{
count = Math.Min(_addresses.Length, node.Length);
for (int i = 0; i < count; i++) {
node[i] = new CacheLinePrediction
node[i] = new CacheLineDescription
{
Address = _addresses[i],
MapPolicy = 0,
@@ -308,7 +308,7 @@ namespace StgSharp
}
L4Assert(
evictedOrigin != nuint.Zero,
$"{$"WriteBack origin should be a batch1 address, "}{($"WriteBackLog=[{string.Join(", ", predictor2.WriteBackLog)}]")}");
$"{$"WriteBack origin should be a batch1 address, "}{$"WriteBackLog=[{string.Join(", ", predictor2.WriteBackLog)}]"}");
// 断言 c: SwissTable 中已不含该 origin
L4Assert(
@@ -323,11 +323,11 @@ namespace StgSharp
remapped.Dispose();
L4Assert(
refill == 0xCC,
$"{$"Re-mapped evicted line should be re-prefetched with 0xCC, "}{($"got 0x{refill:X2} (0xAA = stale data not evicted)")}");
$"{$"Re-mapped evicted line should be re-prefetched with 0xCC, "}{$"got 0x{refill:X2} (0xAA = stale data not evicted)"}");
if (_l4FailCount == 0) {
Console.WriteLine(
$"{($"PASS (evicted 0x{evictedOrigin:X}, remain={remain}, ")}{$"WriteBack×{predictor2.WriteBackLog.Count})"}");
$"{$"PASS (evicted 0x{evictedOrigin:X}, remain={remain}, "}{$"WriteBack×{predictor2.WriteBackLog.Count})"}");
}
}
catch (Exception ex)
+2 -2
View File
@@ -58,14 +58,14 @@ namespace StgSharp
}
public bool Predict(
Span<CacheLinePrediction> node,
Span<CacheLineDescription> node,
out int count
)
{
PredictCallCount++;
count = Math.Min(_addresses.Length, node.Length);
for (int i = 0; i < count; i++) {
node[i] = new CacheLinePrediction
node[i] = new CacheLineDescription
{
Address = _addresses[i],
MapPolicy = 0,