daily update on 20240419 by nitload youmu, SSD merged into main stream

2024-04-20 02:36:36 +08:00
parent efeb4ba058
commit 7b67eb2f4d
81 changed files with 4568 additions and 33 deletions
@@ -0,0 +1,70 @@
+using StgSharp.Data;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Numerics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Text;
+
+namespace StgSharp.Data
+{
+    public static class CodeSeparator
+    {
+        public static unsafe T[] SeparateCode<T>(string filePath) where T : struct
+        {
+            byte[] allByte = File.ReadAllBytes(filePath);
+            List<T> rawList = new List<T>();
+
+            fixed (byte* allBytePtr = allByte)
+            {
+                for (int i = 0; i < allByte.Length; i += Marshal.SizeOf<T>()) 
+                {
+                    T t = new T();
+                    t= *(T*)(allBytePtr + i);
+                    rawList.Add(t);
+                }
+            }
+            return rawList.ToArray();
+        }
+
+        public static unsafe T[] SeparateCode<T>(byte[] stream) where T : struct
+        {
+            List<T> rawList = new List<T>();
+
+            fixed (byte* allBytePtr = stream)
+            {
+                for (int i = 0; i < stream.Length; i += Marshal.SizeOf<T>())
+                {
+                    T t = new T();
+                    t = *(T*)(allBytePtr + i);
+                    rawList.Add(t);
+                }
+            }
+            return rawList.ToArray();
+        }
+
+        public static List<(T, ulong)> CountCodeFrequency<T>(T[] rawArray) where T : struct
+        {
+            Dictionary<T, ulong> frequencyCollection = new Dictionary<T, ulong>();
+
+            foreach (var item in rawArray)
+            {
+                if (frequencyCollection.ContainsKey(item))
+                {
+                    frequencyCollection[item] += 1;
+                }
+                else 
+                {
+                    frequencyCollection.Add(item, 1);
+                }
+            }
+
+            return frequencyCollection.OrderByDescending(pair=>pair.Value)
+                .Select(pair=>(pair.Key, pair.Value)).ToList();
+        }
+
+    }
+}
@@ -0,0 +1,62 @@
+using StgSharp.Data;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Text;
+
+namespace StgSharpData
+{
+    public class EncodeStream
+    {
+        List<byte> data;
+        byte[] buffer;
+        byte count;
+        BitArray bufferArray;
+
+        public int Length
+        {
+            get { return data.Count; }
+        }
+
+        public List<byte> EncodeData
+        {
+            get=> data;
+        }
+
+        public EncodeStream()
+        {
+            count = 0;
+            buffer = new byte[1];
+            data = new List<byte>();
+            bufferArray = new BitArray(8,true);
+        }
+
+        public void WriteBits<T>(HuffmanKey<T> key) where T : struct
+        {
+            BitArray bit = key.GetBitArray();
+            for (int i = 0; i < bit.Count; i++)
+            {
+                if (count == 8)
+                {
+                    bufferArray.CopyTo(buffer, 0);
+                    data.Add(buffer[0]);
+                    count = 0;
+                }
+                bufferArray[count] = bit[i];
+                count++;
+            }
+        }
+
+        public string Output
+        {
+            get
+            {
+                return Encoding.UTF8.GetString(data.ToArray());
+            }
+        }
+
+    }
+
+
+}
@@ -0,0 +1,25 @@
+using StgSharpData;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Text;
+
+namespace StgSharp.Data
+{
+    public static class Encode
+    {
+
+        public static void Freeze<T>(
+            Dictionary<T,HuffmanKey<T>> indexDict
+            )
+            where T : struct
+        {
+            foreach ( var key in indexDict ) 
+            {
+                key.Value.Freeze();
+            }
+        }
+
+    }
+}
@@ -0,0 +1,214 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Xml;
+using System.Xml.Linq;
+
+namespace StgSharp.Data
+{
+    public partial class HuffmanTree<T> where T : struct
+    {
+        HuffmanNodeList<T> leaves;
+
+        public TreeNode<T> Root
+        {
+            get => leaves.Last;
+        }
+
+        public HuffmanNodeList<T> LeafList
+        {
+            get => leaves;
+        }
+
+        internal HuffmanTree(HuffmanNodeList<T> nodeList)
+        {
+            leaves = nodeList;
+        }
+
+        public static HuffmanTree<T> BuildHuffmanTree(List<(T, ulong)> metaData)
+        {
+            if (metaData.Count<=3)
+            {
+                throw new Exception("Data is too little, no need to compress");
+            }
+            HuffmanNodeList<T> leaves = new HuffmanNodeList<T>();
+            HuffmanNodeList<T> all = new HuffmanNodeList<T>();
+
+            foreach (var item in metaData)
+            {
+                leaves.Add(new TreeNode<T>(item));
+            }
+
+            int count = 0;
+
+            while (leaves.Count >2)
+            {
+                TreeNode<T> node, toMove;
+                if (leaves.Last._frequency < leaves.BeforeLast._frequency)
+                {
+                    if (leaves.BeforeLast._frequency < leaves.OverLast._frequency)
+                    {
+                        //第三节点最大
+                        node = new TreeNode<T>(leaves.BeforeLast, leaves.Last);
+                        toMove = leaves.RemoveAndGetLast; all.Add(toMove);
+                        toMove = leaves.RemoveAndGetLast; all.Add(toMove);
+                        leaves.Add(node);
+                    }
+                    else
+                    {
+                        //第二节点最大
+                        node = leaves.Last._frequency < leaves.OverLast._frequency ?
+                            new TreeNode<T>(leaves.OverLast, leaves.Last) :
+                            new TreeNode<T>(leaves.Last, leaves.OverLast);
+                        toMove = leaves.RemoveAndGetLast; all.Add(toMove);
+                        toMove = leaves.RemoveAndGetBeforeLast; all.Add(toMove);
+                        leaves.Add(node);
+                    }
+                }
+                else
+                {
+
+                    if (leaves.Last._frequency < leaves.OverLast._frequency)
+                    {
+                        //第三节点最大
+                        node = new TreeNode<T>(leaves.Last, leaves.BeforeLast);
+                        toMove = leaves.RemoveAndGetLast; all.Add(toMove);
+                        toMove = leaves.RemoveAndGetLast; all.Add(toMove);
+                        leaves.Add(node);
+                    }
+                    else
+                    {
+                        //第一节点最大
+                        node = leaves.BeforeLast._frequency < leaves.OverLast._frequency?
+                            new TreeNode<T>(leaves.OverLast, leaves.BeforeLast):
+                            new TreeNode<T>(leaves.BeforeLast, leaves.OverLast);
+                        toMove = leaves.RemoveAndGetBeforeLast; all.Add(toMove);
+                        toMove = leaves.RemoveAndGetBeforeLast; all.Add(toMove);
+                        leaves.Add(node);
+                    }
+                }
+
+                leaves = new HuffmanNodeList<T>( leaves.OrderByDescending(pair => pair.frequency)
+                    .Select(pair => pair).ToList());
+            }
+
+            TreeNode<T> lastOne = leaves[0];
+            TreeNode<T> lastTwo = leaves[1];
+            TreeNode<T> root = lastOne.frequency>lastTwo.frequency?
+                new TreeNode<T>(lastOne,lastTwo):
+                new TreeNode<T>(lastTwo, lastOne);
+
+            all.Add(lastOne);
+            all.Add(lastTwo);
+            all.Add(root);
+
+            leaves = all;
+            return new HuffmanTree<T>(all);
+        }
+
+
+    }
+
+    public class HuffmanNodeList<T> : List<TreeNode<T>> where T : struct
+    {
+        public HuffmanNodeList():base()
+        {
+            
+        }
+
+        public HuffmanNodeList(IEnumerable<TreeNode<T>> values):base(values)
+        {}
+
+        public void Add(T element, ulong f)
+        {
+            base.Add(new TreeNode<T>(element,f));
+        }
+
+        public TreeNode<T> Last
+        {
+            get => this[Count-1 ];
+        }
+
+        public TreeNode<T> RemoveAndGetLast
+        {
+            get 
+            {
+                TreeNode<T> ret = this[Count-1];
+                RemoveAt(Count-1);
+                return ret;
+            }
+        }
+
+        public TreeNode<T> BeforeLast
+        {
+            get => this[Count - 2];
+        }
+
+        public TreeNode<T> RemoveAndGetBeforeLast
+        {
+            get
+            {
+                TreeNode<T> ret = this[Count - 2];
+                RemoveAt(Count - 2);
+                return ret;
+            }
+        }
+
+        public TreeNode<T> OverLast
+        {
+            get => this[Count - 3];
+        }
+
+        public TreeNode<T> RemoveAndGetOverLast
+        {
+            get
+            {
+                TreeNode<T> ret = this[Count - 3];
+                RemoveAt(Count - 3);
+                return ret;
+            }
+        }
+
+    }
+
+    public class TreeNode<T> where T: struct
+    {
+        internal TreeNode<T> _left;
+        internal TreeNode<T> _right;
+        internal T? _value;
+        internal ulong _frequency;
+
+        public ulong frequency => _frequency;
+
+        public T? Value => _value;
+
+        public TreeNode<T> Left => _left;
+        public TreeNode<T> Right => _right;
+
+        public TreeNode( T? value, ulong frequncy)
+        {
+            _value = value;
+            _frequency = frequncy;
+
+        }
+
+        internal TreeNode(TreeNode<T> left, TreeNode<T> right)
+        {
+            _left = left;
+            _right = right;
+            _value = null;
+            _frequency = left._frequency + right._frequency;
+        }
+
+        public TreeNode((T, ulong) data)
+        {
+            _value = data.Item1;
+            _frequency = data.Item2;
+
+        }
+    }
+}
@@ -0,0 +1,156 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Text;
+
+namespace StgSharp.Data
+{
+    public partial class HuffmanTree<T> where T : struct
+    {
+        public Dictionary<T, HuffmanKey<T>> BuildHuffmanDictionary()
+        {
+            Dictionary<T, HuffmanKey<T>> ret = new Dictionary<T, HuffmanKey<T>>();
+            TreeNode<T> root = Root;
+            HuffmanKey<T> key = new HuffmanKey<T>();
+            key = key.SetSide(KeySide.Right);
+            InternalBuildDictionary(ret, root, key);
+            return ret;
+        }
+
+        public void InternalBuildDictionary(
+            Dictionary<T, HuffmanKey<T>> dict,
+            TreeNode<T> node,
+            HuffmanKey<T> valueTemp
+            )
+        {
+            HuffmanKey<T> nativeNode = new HuffmanKey<T>(valueTemp.bitList);
+            nativeNode.AddIndexLevel();
+            //检查左节点
+            if (node._left.Value == null)
+            {
+                //左节点不是叶子节点，继续搜索
+                InternalBuildDictionary(
+                    dict,
+                    node._left,
+                    nativeNode.SetSide(KeySide.Left));
+            }
+            else
+            {
+                //左节点是叶子节点，添加值
+                dict.Add(node._left.Value.Value, nativeNode.SetSide(KeySide.Left));
+            }
+            //检查右节点
+            if (node._right.Value == null)
+            {
+                //右节点不是叶子节点，继续搜索
+                InternalBuildDictionary(
+                    dict,
+                    node._right,
+                    nativeNode.SetSide(KeySide.Right));
+            }
+            else
+            {
+                //右节点是叶子节点，添加值
+                dict.Add(node._right.Value.Value, nativeNode.SetSide(KeySide.Right));
+            }
+            //这里检查自己是不是叶子节点
+            //但是上面已经检查过了，不需要被检查
+        }
+
+    }
+
+
+
+    public class HuffmanKey<T> where T : struct
+    {
+        internal List<bool> bitList;
+        internal BitArray code;
+
+        public HuffmanKey()
+        {
+            bitList = new List<bool>();
+            bitList.Add(false);
+        }
+
+        public HuffmanKey(List<bool> former)
+        {
+            bitList = new List<bool>(former);
+        }
+
+        public unsafe byte[] GetBytes()
+        {
+            byte[] bytes = new byte[bitList.Count / 8 +1];
+            BitArray array = new BitArray(bitList.ToArray());
+            array.CopyTo(bytes, 0);
+            return bytes;
+        }
+
+        public void Freeze()
+        {
+            if (code != null)
+            {
+                return;
+            }
+            bitList.RemoveAt(0); 
+            code = new BitArray(bitList.ToArray());
+        }
+
+        public BitArray GetBitArray()
+        {
+            if (code == null)
+            {
+                List<bool> list = new List<bool>(bitList);
+                list.RemoveAt(0);
+                return new BitArray(list.ToArray());
+            }
+            return code;
+        }
+
+        public unsafe HuffmanKey<T> SetSide(KeySide side)
+        {
+            HuffmanKey<T> newKey = new HuffmanKey<T>(bitList);
+            newKey.bitList[newKey.bitList.Count - 1] = (side == KeySide.Left);
+            return newKey;
+        }
+
+        public unsafe void AddIndexLevel()
+        {
+            bitList.Add(true);
+        }
+
+        public bool IsSupported
+        {
+            get
+            {
+                return
+                    typeof(T) == typeof(byte) ||
+                    typeof(T) == typeof(uint) ||
+                    typeof(T) == typeof(ushort) ||
+                    typeof(T) == typeof(ulong) ||
+#if NET7_0_OR_GREATER
+                    typeof(T) == typeof(UInt128) ||
+#endif
+                    typeof(T) == typeof(M128);
+            }
+        }
+
+        public override string ToString()
+        {
+            string str = "";
+            foreach (var bit in bitList) 
+            {
+                str += $"{bit}   ";
+            }
+            return str;
+        }
+    }
+
+    public enum KeySide
+    {
+        Left,
+        Right
+    }
+
+}