daily update on 20240419 by nitload youmu, SSD merged into main stream

This commit is contained in:
YoumuLiu
2024-04-20 02:36:36 +08:00
parent efeb4ba058
commit 7b67eb2f4d
81 changed files with 4568 additions and 33 deletions
@@ -0,0 +1,100 @@
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
// file="CodeCounter.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
using StgSharp.Data;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
namespace StgSharp.Data
{
public static class CodeSeparator
{
public static List<(T, ulong)> CountCodeFrequency<T>(T[] rawArray) where T : struct
{
Dictionary<T, ulong> frequencyCollection = new Dictionary<T, ulong>();
foreach (var item in rawArray)
{
if (frequencyCollection.ContainsKey(item))
{
frequencyCollection[item] += 1;
}
else
{
frequencyCollection.Add(item, 1);
}
}
return frequencyCollection.OrderByDescending(pair => pair.Value)
.Select(pair => (pair.Key, pair.Value)).ToList();
}
public static unsafe T[] SeparateCode<T>(string filePath) where T : struct
{
byte[] allByte = File.ReadAllBytes(filePath);
List<T> rawList = new List<T>();
fixed (byte* allBytePtr = allByte)
{
for (int i = 0; i < allByte.Length; i += Marshal.SizeOf<T>())
{
T t = new T();
t = *(T*)(allBytePtr + i);
rawList.Add(t);
}
}
return rawList.ToArray();
}
public static unsafe T[] SeparateCode<T>(byte[] stream) where T : struct
{
List<T> rawList = new List<T>();
fixed (byte* allBytePtr = stream)
{
for (int i = 0; i < stream.Length; i += Marshal.SizeOf<T>())
{
T t = new T();
t = *(T*)(allBytePtr + i);
rawList.Add(t);
}
}
return rawList.ToArray();
}
}
}
@@ -0,0 +1,82 @@
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
// file="EncodStream.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
using StgSharp.Data;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Text;
namespace StgSharpData
{
public class EncodeStream
{
byte[] buffer;
BitArray bufferArray;
byte count;
List<byte> data;
public EncodeStream()
{
count = 0;
buffer = new byte[1];
data = new List<byte>();
bufferArray = new BitArray(8, true);
}
public List<byte> EncodeData
=> data;
public int Length => data.Count;
public string Output => Encoding.UTF8.GetString(data.ToArray());
public void WriteBits<T>(HuffmanKey<T> key) where T : struct
{
BitArray bit = key.GetBitArray();
for (int i = 0; i < bit.Count; i++)
{
if (count == 8)
{
bufferArray.CopyTo(buffer, 0);
data.Add(buffer[0]);
count = 0;
}
bufferArray[count] = bit[i];
count++;
}
}
}
}
+55
View File
@@ -0,0 +1,55 @@
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
// file="Encoder.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
using StgSharpData;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Text;
namespace StgSharp.Data
{
public static class Encode
{
public static void Freeze<T>(
Dictionary<T, HuffmanKey<T>> indexDict
)
where T : struct
{
foreach (var key in indexDict)
{
key.Value.Freeze();
}
}
}
}
@@ -0,0 +1,237 @@
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
// file="HuffmanTree.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Xml;
using System.Xml.Linq;
namespace StgSharp.Data
{
public partial class HuffmanTree<T> where T : struct
{
HuffmanNodeList<T> leaves;
internal HuffmanTree(HuffmanNodeList<T> nodeList)
{
leaves = nodeList;
}
public HuffmanNodeList<T> LeafList
=> leaves;
public TreeNode<T> Root
=> leaves.Last;
public static HuffmanTree<T> BuildHuffmanTree(List<(T, ulong)> metaData)
{
if (metaData.Count <= 3)
{
throw new Exception("Data is too little, no need to compress");
}
HuffmanNodeList<T> leaves = new HuffmanNodeList<T>();
HuffmanNodeList<T> all = new HuffmanNodeList<T>();
foreach (var item in metaData)
{
leaves.Add(new TreeNode<T>(item));
}
while (leaves.Count > 2)
{
TreeNode<T> node, toMove;
if (leaves.Last._frequency < leaves.BeforeLast._frequency)
{
if (leaves.BeforeLast._frequency < leaves.OverLast._frequency)
{
//第三节点最大
node = new TreeNode<T>(leaves.BeforeLast, leaves.Last);
toMove = leaves.RemoveAndGetLast; all.Add(toMove);
toMove = leaves.RemoveAndGetLast; all.Add(toMove);
leaves.Add(node);
}
else
{
//第二节点最大
node = (leaves.Last._frequency < leaves.OverLast._frequency) ?
(new TreeNode<T>(leaves.OverLast, leaves.Last)) :
(new TreeNode<T>(leaves.Last, leaves.OverLast));
toMove = leaves.RemoveAndGetLast; all.Add(toMove);
toMove = leaves.RemoveAndGetBeforeLast; all.Add(toMove);
leaves.Add(node);
}
}
else
{
if (leaves.Last._frequency < leaves.OverLast._frequency)
{
//第三节点最大
node = new TreeNode<T>(leaves.Last, leaves.BeforeLast);
toMove = leaves.RemoveAndGetLast; all.Add(toMove);
toMove = leaves.RemoveAndGetLast; all.Add(toMove);
leaves.Add(node);
}
else
{
//第一节点最大
node = (leaves.BeforeLast._frequency < leaves.OverLast._frequency) ?
(new TreeNode<T>(leaves.OverLast, leaves.BeforeLast)) :
(new TreeNode<T>(leaves.BeforeLast, leaves.OverLast));
toMove = leaves.RemoveAndGetBeforeLast; all.Add(toMove);
toMove = leaves.RemoveAndGetBeforeLast; all.Add(toMove);
leaves.Add(node);
}
}
leaves = new HuffmanNodeList<T>(leaves.OrderByDescending(pair => pair.frequency)
.Select(pair => pair).ToList());
}
TreeNode<T> lastOne = leaves[0];
TreeNode<T> lastTwo = leaves[1];
TreeNode<T> root = (lastOne.frequency > lastTwo.frequency) ?
(new TreeNode<T>(lastOne, lastTwo)) :
(new TreeNode<T>(lastTwo, lastOne));
all.Add(lastOne);
all.Add(lastTwo);
all.Add(root);
leaves = all;
return new HuffmanTree<T>(all);
}
}
public class HuffmanNodeList<T> : List<TreeNode<T>> where T : struct
{
public HuffmanNodeList() : base()
{
}
public HuffmanNodeList(IEnumerable<TreeNode<T>> values) : base(values)
{ }
public TreeNode<T> BeforeLast
=> this[Count - 2];
public TreeNode<T> Last
=> this[Count - 1];
public TreeNode<T> OverLast
=> this[Count - 3];
public TreeNode<T> RemoveAndGetBeforeLast
{
get
{
TreeNode<T> ret = this[Count - 2];
RemoveAt(Count - 2);
return ret;
}
}
public TreeNode<T> RemoveAndGetLast
{
get
{
TreeNode<T> ret = this[Count - 1];
RemoveAt(Count - 1);
return ret;
}
}
public TreeNode<T> RemoveAndGetOverLast
{
get
{
TreeNode<T> ret = this[Count - 3];
RemoveAt(Count - 3);
return ret;
}
}
public void Add(T element, ulong f)
{
base.Add(new TreeNode<T>(element, f));
}
}
public class TreeNode<T> where T : struct
{
internal ulong _frequency;
internal TreeNode<T> _left;
internal TreeNode<T> _right;
internal T? _value;
internal TreeNode(TreeNode<T> left, TreeNode<T> right)
{
_left = left;
_right = right;
_value = null;
_frequency = left._frequency + right._frequency;
}
public TreeNode((T, ulong) data)
{
_value = data.Item1;
_frequency = data.Item2;
}
public TreeNode(T? value, ulong frequncy)
{
_value = value;
_frequency = frequncy;
}
public ulong frequency => _frequency;
public TreeNode<T> Left => _left;
public TreeNode<T> Right => _right;
public T? Value => _value;
}
}
@@ -0,0 +1,189 @@
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
// file="MappingTable.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
using System;
using System.Collections;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
namespace StgSharp.Data
{
public partial class HuffmanTree<T> where T : struct
{
public Dictionary<T, HuffmanKey<T>> BuildHuffmanDictionary()
{
Dictionary<T, HuffmanKey<T>> ret = new Dictionary<T, HuffmanKey<T>>();
TreeNode<T> root = Root;
HuffmanKey<T> key = new HuffmanKey<T>();
key = key.SetSide(KeySide.Right);
InternalBuildDictionary(ret, root, key);
return ret;
}
public void InternalBuildDictionary(
Dictionary<T, HuffmanKey<T>> dict,
TreeNode<T> node,
HuffmanKey<T> valueTemp
)
{
HuffmanKey<T> nativeNode = new HuffmanKey<T>(valueTemp.bitList);
nativeNode.AddIndexLevel();
//检查左节点
if (node._left.Value == null)
{
//左节点不是叶子节点,继续搜索
InternalBuildDictionary(
dict,
node._left,
nativeNode.SetSide(KeySide.Left));
}
else
{
//左节点是叶子节点,添加值
dict.Add(node._left.Value.Value, nativeNode.SetSide(KeySide.Left));
}
//检查右节点
if (node._right.Value == null)
{
//右节点不是叶子节点,继续搜索
InternalBuildDictionary(
dict,
node._right,
nativeNode.SetSide(KeySide.Right));
}
else
{
//右节点是叶子节点,添加值
dict.Add(node._right.Value.Value, nativeNode.SetSide(KeySide.Right));
}
//这里检查自己是不是叶子节点
//但是上面已经检查过了,不需要被检查
}
}
public class HuffmanKey<T> where T : struct
{
internal List<bool> bitList;
internal BitArray code;
public HuffmanKey()
{
bitList = new List<bool>();
bitList.Add(false);
}
public HuffmanKey(List<bool> former)
{
bitList = new List<bool>(former);
}
public bool IsSupported
{
get
{
return
(typeof(T) == typeof(byte)) ||
(typeof(T) == typeof(uint)) ||
(typeof(T) == typeof(ushort)) ||
(typeof(T) == typeof(ulong)) ||
#if NET7_0_OR_GREATER
typeof(T) == typeof(UInt128) ||
#endif
(typeof(T) == typeof(M128));
}
}
public unsafe void AddIndexLevel()
{
bitList.Add(true);
}
public void Freeze()
{
if (code != null)
{
return;
}
bitList.RemoveAt(0);
code = new BitArray(bitList.ToArray());
}
public BitArray GetBitArray()
{
if (code == null)
{
List<bool> list = new List<bool>(bitList);
list.RemoveAt(0);
return new BitArray(list.ToArray());
}
return code;
}
public unsafe byte[] GetBytes()
{
byte[] bytes = new byte[(bitList.Count / 8) + 1];
BitArray array = new BitArray(bitList.ToArray());
array.CopyTo(bytes, 0);
return bytes;
}
public unsafe HuffmanKey<T> SetSide(KeySide side)
{
HuffmanKey<T> newKey = new HuffmanKey<T>(bitList);
newKey.bitList[newKey.bitList.Count - 1] = side == KeySide.Left;
return newKey;
}
public override string ToString()
{
string str = string.Empty;
foreach (var bit in bitList)
{
str += $"{bit} ";
}
return str;
}
}
public enum KeySide
{
Left,
Right
}
}