youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

after merge

Browse Source
This commit is contained in:
Nitload
2026-06-27 00:50:24 +08:00
parent 2a6fc7b633 359dd67287
commit 59f833178d
21 changed files with 1971 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/Collections/CapacityFixedList.cs
+66 -19
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="CapacityFixedList"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -35,7 +35,9 @@ namespace StgSharp.Collections
public static class CapacityFixedListBuilder
{
internal static CapacityFixedList<T> Create<T>(ReadOnlySpan<T> values)
internal static CapacityFixedList<T> Create<T>(
ReadOnlySpan<T> values
)
{
return new CapacityFixedList<T>(values);
}
@@ -50,28 +52,39 @@ namespace StgSharp.Collections
private readonly int _capacity;
private int _index;
public CapacityFixedList(int size)
public CapacityFixedList(
int size
)
{
_values = new T[size];
_capacity = size;
}
public CapacityFixedList(ReadOnlySpan<T> values)
public CapacityFixedList(
ReadOnlySpan<T> values
)
{
_values = values.ToArray();
}
public T this[int index]
public T this[
int index
]
{
get => index < _index ? _values[index] : throw new ArgumentOutOfRangeException(nameof(index));
set => _values[index] = index < _index ? value : throw new ArgumentOutOfRangeException(nameof(index));
get => index < _index ?
_values[index] :
throw new ArgumentOutOfRangeException(nameof(index));
set => _values[index] =
index < _index ? value : throw new ArgumentOutOfRangeException(nameof(index));
}
public bool IsReadOnly => false;
public int Count => _index + 1;
public void Add(T item)
public void Add(
T item
)
{
if (_index < _capacity - 1)
{
@@ -82,12 +95,19 @@ namespace StgSharp.Collections
public void Clear()
{
_values = new T[_capacity];
if (RuntimeHelpers.IsReferenceOrContainsReferences<T>()) {
Array.Fill(_values, default);
}
_index = 0;
}
public bool Contains(T item)
public bool Contains(
T item
)
{
if (item is null) {
return false;
}
for (int i = 0; i <= _index; i++)
{
if (item.Equals(_values[i])) {
@@ -97,7 +117,10 @@ namespace StgSharp.Collections
return false;
}
public void CopyTo(T[] array, int arrayIndex)
public void CopyTo(
T[] array,
int arrayIndex
)
{
_values.CopyTo(array, arrayIndex);
}
@@ -109,7 +132,9 @@ namespace StgSharp.Collections
}
}
public int IndexOf(T item)
public int IndexOf(
T item
)
{
T v;
for (int i = 0; i <= _index; i++)
@@ -121,7 +146,9 @@ namespace StgSharp.Collections
return -1;
}
public bool Remove(T item)
public bool Remove(
T item
)
{
T v;
for (int i = 0; i <= _index; i++)
@@ -136,8 +163,13 @@ namespace StgSharp.Collections
return false;
}
public void RemoveAll(Predicate<T> predicate)
public void RemoveAll(
Predicate<T> predicate
)
{
if (predicate is null) {
return;
}
for (int i = _index; i > 0; i--)
{
if (predicate(_values[i]))
@@ -148,26 +180,41 @@ namespace StgSharp.Collections
}
}
public void RemoveAt(int index)
public void RemoveAt(
int index
)
{
if (index <= _index)
{
_values[index] = _values[_index];
Array.Copy(_values, index + 1, _values, index, _index - index);
if (RuntimeHelpers.IsReferenceOrContainsReferences<T>()) {
_values[_index] = default!;
}
_index--;
}
}
public void Resize(int newSize)
public void Resize(
int newSize
)
{
Array.Resize(ref _values, newSize);
}
internal Span<T> ToSpan()
{
return _values;
}
IEnumerator IEnumerable.GetEnumerator()
{
yield return GetEnumerator();
}
void IList<T>.Insert(int index, T item)
void IList<T>.Insert(
int index,
T item
)
{
throw new NotSupportedException();
}
StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs
+9
View File
@@ -36,6 +36,15 @@ using System.Text;
namespace StgSharp.RegularAnalysis.Abstraction
{
public interface ITrinitySyntaxNode<TNode, TLabel> : ISyntaxNode<TNode, TLabel>
where TNode : ISyntaxNode<TNode, TLabel>
where TLabel : unmanaged
{
TNode Third { get; set; }
}
public interface ISyntaxNode<TNode, TLabel> where TNode : ISyntaxNode<TNode, TLabel>
where TLabel : unmanaged
{
StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs → StgSharp.RegularAnalysis/Text/RegexAstNode.cs
View File
StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs → StgSharp.RegularAnalysis/Text/RegexElementLabel.cs
View File
StgSharp.RegularAnalysis/TextRegex/RegexIR.cs → StgSharp.RegularAnalysis/Text/RegexIR.cs
+127 -54
View File
@@ -37,16 +37,57 @@ namespace StgSharp.RegularAnalysis.Text
internal enum RegexIRCommand
{
Search_Seq = 0,
Search_Set = 1,
SearchChar = 2,
ConfigFail = 3,
Condition = 4,
Try = 5,
Count = 6,
Count_Seq = 7,
Count_Set = 8,
Pop = 10
Match_Seq = 1 << 0,
Match_Set = 1 << 1,
Condition = 1 << 2,
Try = 1 << 3,
Count_Complex = 1 << 4,
Count_Seq = 1 << 5,
Count_Set = 1 << 6,
Pop = 1 << 7,
Pack = 1 << 8,
Find_Seq = 1 << 9,
Find_Set = 1 << 10,
Find_Complex = 1 << 11,
Match = Match_Seq | Match_Set,
Count = Count_Seq | Count_Set | Count_Complex,
}
internal interface IRegexCount
{
bool IsGreedy { get; }
bool IsStar { get; }
bool IsPlus { get; }
int Min { get; }
int Max { get; }
}
internal sealed class RegexFindComplexIR(
bool nearest,
int count
) : RegexIR
{
internal override RegexIRCommand Command => RegexIRCommand.Find_Complex;
private bool IsNearest { get; } = nearest;
private int CommandCount { get; } = count;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t FIND_COMPLEX {(IsNearest ? "NEAREST" : "FURTHEST")} COUNT: {CommandCount}";
}
}
@@ -61,6 +102,27 @@ namespace StgSharp.RegularAnalysis.Text
}
internal sealed class RegexFindSeqIR(
string pattern,
bool isFirst
) : RegexIR
{
public bool IsFirst { get; } = isFirst;
public string Pattern { get; } = pattern;
internal override RegexIRCommand Command => RegexIRCommand.Find_Seq;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t FIND_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "<EPSILON>" : Pattern)}\"";
}
}
/// <summary>
/// Searches for a certain string pattern or charset from current position.
/// </summary>
@@ -71,13 +133,13 @@ namespace StgSharp.RegularAnalysis.Text
public string Pattern { get; } = pattern;
internal override RegexIRCommand Command => RegexIRCommand.Search_Seq;
internal override RegexIRCommand Command => RegexIRCommand.Match_Seq;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t SEARCH_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "<EPSILON>" : Pattern)}\"";
return $"[{lineNum}]\t MATCH_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "<EPSILON>" : Pattern)}\"";
}
}
@@ -89,37 +151,13 @@ namespace StgSharp.RegularAnalysis.Text
public string Charset { get; } = charset;
internal override RegexIRCommand Command => RegexIRCommand.Search_Set;
internal override RegexIRCommand Command => RegexIRCommand.Match_Set;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t SEARCH_SET \"{Charset}\"";
}
}
/// <summary>
/// Configures the failure jump of current position. The failMod ranges 0-2
/// </summary>
/// <param name="target">
///
/// </param>
internal sealed class RegexConfigFailIR(
int failMod
) : RegexIR
{
public int FailMod { get; } = failMod & 0x3;
internal override RegexIRCommand Command => RegexIRCommand.ConfigFail;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t CONFIG_FAIL {FailMod}";
return $"[{lineNum}]\t MATCH_SET \"{Charset}\"";
}
}
@@ -145,29 +183,54 @@ namespace StgSharp.RegularAnalysis.Text
}
internal sealed class RegexCountIR(
int min,
int max,
bool isGreedy,
int operation
internal sealed class RegexPackIR(
bool condition,
string name = ""
) : RegexIR
{
public bool IsGreedy { get; } = isGreedy;
public bool Condition { get; } = condition;
public int Min { get; } = min;
public string GroupName { get; } = name;
public int Max { get; } = max;
public int OperationIndex { get; } = operation;
internal override RegexIRCommand Command { get; } = RegexIRCommand.Count;
internal override RegexIRCommand Command => RegexIRCommand.Pack;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {OperationIndex}";
return $"[{lineNum}]\t PACK {(Condition ? "SUCCESS" : "FAIL")} GROUP: \"{GroupName}\"";
}
}
internal sealed class RegexCountComplexIR(
int min,
int max,
bool isGreedy,
int operation
) : RegexIR, IRegexCount
{
public bool IsGreedy { get; } = isGreedy;
public bool IsStar => Min == 0 && Max < 0;
public bool IsPlus => Min == 1 && Max < 0;
public int Min { get; } = min;
public int Max { get; } = max;
public int OperationCount { get; } = operation;
internal override RegexIRCommand Command { get; } = RegexIRCommand.Count_Complex;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {OperationCount}";
}
}
@@ -177,11 +240,15 @@ namespace StgSharp.RegularAnalysis.Text
int max,
bool isGreedy,
string seq
) : RegexIR
) : RegexIR, IRegexCount
{
public bool IsGreedy { get; } = isGreedy;
public bool IsStar => Min == 0 && Max < 0;
public bool IsPlus => Min == 1 && Max < 0;
public int Min { get; } = min;
public int Max { get; } = max;
@@ -204,11 +271,17 @@ namespace StgSharp.RegularAnalysis.Text
int max,
bool isGreedy,
string set
) : RegexIR
) : RegexIR, IRegexCount
{
public bool IsGreedy { get; } = isGreedy;
public bool IsDotStar => CharSet == "." && Min == 0 && Max < 0;
public bool IsStar => Min == 0 && Max < 0;
public bool IsPlus => Min == 1 && Max < 0;
public int Min { get; } = min;
public int Max { get; } = max;
@@ -251,7 +324,7 @@ namespace StgSharp.RegularAnalysis.Text
public bool IsSuccess { get; } = isSuccess;
internal override RegexIRCommand Command => RegexIRCommand.Try;
internal override RegexIRCommand Command => RegexIRCommand.Pop;
internal override string AsCommand(
int lineNum
StgSharp.RegularAnalysis/TextRegex/RegexIRGenerator.cs → StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs
+11 -10
View File
@@ -56,22 +56,14 @@ namespace StgSharp.RegularAnalysis.Text
Count++;
}
public void EmitConfigFail(
int fail
)
{
_tail.Add(new RegexConfigFailIR(fail));
Count++;
}
public void EmitCount(
public void EmitCountComplex(
int min,
int max,
bool isGreedy,
int operation
)
{
_tail.Add(new RegexCountIR(min, max, isGreedy, operation));
_tail.Add(new RegexCountComplexIR(min, max, isGreedy, operation));
Count++;
}
@@ -125,6 +117,15 @@ namespace StgSharp.RegularAnalysis.Text
Count++;
}
public void EmitPack(
bool condition,
string name = ""
)
{
_tail.Add(new RegexPackIR(condition, name));
Count++;
}
public void EmitPop(
bool isSuccess
)
StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs → StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs
+2 -2
View File
@@ -1,6 +1,6 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexInterpreter.Optimize"
// file="RegexInterpreter.ASTOptimize"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
@@ -39,7 +39,7 @@ using System.Xml.Linq;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexInterpreter
public partial class RegexAnalyzer
{
private static RegexAstNode FlattenAlt(
StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.IREmit.cs → StgSharp.RegularAnalysis/Text/RegexInterpreter.IREmit.cs
+28 -23
View File
@@ -40,7 +40,7 @@ using System.Xml;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexInterpreter
public partial class RegexAnalyzer
{
internal static List<RegexIR> GenerateIR(AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree)
@@ -55,30 +55,34 @@ namespace StgSharp.RegularAnalysis.Text
int min,
int max,
bool isGreedy,
<<<<<<<< HEAD:StgSharp.RegularAnalysis/Text/RegexInterpreter.IREmit.cs
RegexAstNode right)
========
RegexAstNode body
)
>>>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs
{
bool isRightOperator = !RegexAstNode.IsNullOrEmpty(right) &&
(right.Source.Flag & RegexElementLabel.OPERATOR) != 0;
bool is_operator = !RegexAstNode.IsNullOrEmpty(body) &&
(body.Source.Flag & RegexElementLabel.OPERATOR) != 0;
if (isRightOperator) // seq to be count is complex
if (is_operator) // seq to be count is complex
{
ir.EmitCount(min, max, isGreedy, 1);
RegexIRGenerator rightIR = GenerateIRRecursion(right, curDepth + 1);
ir.EmitTry(rightIR.Count);
ir.EmitIRStream(rightIR);
RegexIRGenerator body_ir = GenerateIRRecursion(body, curDepth + 1);
ir.EmitCountComplex(min, max, isGreedy, body_ir.Count);
ir.EmitIRStream(body_ir);
} else
{
RegexElementLabel count_source_mask = right.Source.Flag;
RegexElementLabel count_source_mask = body.Source.Flag;
switch (count_source_mask)
{
case RegexElementLabel.UNIT:
ir.EmitCountSeq(min, max, isGreedy, right.Source.Value);
ir.EmitCountSeq(min, max, isGreedy, body.Source.Value);
break;
case RegexElementLabel.UNIT_SET:
ir.EmitCountSet(min, max, isGreedy, right.Source.Value);
ir.EmitCountSet(min, max, isGreedy, body.Source.Value);
break;
case RegexElementLabel.UNIT_SPAN:
ir.EmitCountSeq(min, max, isGreedy, right.Source.Value);
ir.EmitCountSeq(min, max, isGreedy, body.Source.Value);
break;
default:
throw new InvalidOperationException($"Unexpected regex element type for count operator: {count_source_mask}");
@@ -110,7 +114,7 @@ namespace StgSharp.RegularAnalysis.Text
bool is_group = group_source.Length == 1; // simple group: (pattern)
string group_name =
group_source.StartsWith(@"(?<", StringComparison.Ordinal) ?
group_source[3..^2] : /* named group (?<name>) */
group_source[3..^1] : /* named group (?<name>) */
string.Empty; /* unnamed group (?:) */
RegexIRGenerator groupIR = new();
@@ -118,7 +122,8 @@ namespace StgSharp.RegularAnalysis.Text
groupIR.EmitTry(rightIR.Count);
groupIR.EmitIRStream(rightIR);
groupIR.EmitCondition(2, 1);
/*generate pack ir here*/
groupIR.EmitPop(false);
groupIR.EmitPack(true, group_name);
return groupIR;
case RegexElementLabel.COUNT:
string count_source = token.Value;
@@ -137,25 +142,25 @@ namespace StgSharp.RegularAnalysis.Text
_ => throw new InvalidOperationException("Unexpected sequence mask for count operator"),
};
ReadOnlySpan<char> count_source_span = count_source;
RegexIRGenerator countIR = new();
RegexIRGenerator count_ir = new();
if (count_source_span[0] == '*') // infinite count
{
EmitCount(curDepth, countIR, 0, int.MaxValue, isGreedy, seq);
EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq);
} else if (count_source_span[0] == '+') // at least one
{
EmitCount(curDepth, countIR, 1, int.MaxValue, isGreedy, seq);
EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq);
} else if (count_source_span[0] == '?') // zero or one
{
EmitCount(curDepth, countIR, 0, 1, isGreedy, seq);
EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq);
} else if (count_source_span[0] == '{') // zero or one, non-greedy
{
int comma_pos = count_source_span.IndexOf(',');
if (comma_pos == -1)
{
if (int.TryParse(count_source[1..(isGreedy ? ^3 : ^2)], out int count))
if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count))
{
EmitCount(curDepth, countIR, count, count, isGreedy, seq);
EmitCount(curDepth, count_ir, count, count, isGreedy, seq);
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
@@ -163,9 +168,9 @@ namespace StgSharp.RegularAnalysis.Text
} else
{
if (int.TryParse(count_source[1..comma_pos], out int min) &&
int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^3 : ^2)], out int max))
int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max))
{
EmitCount(curDepth, countIR, min, max, isGreedy, seq);
EmitCount(curDepth, count_ir, min, max, isGreedy, seq);
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
@@ -176,7 +181,7 @@ namespace StgSharp.RegularAnalysis.Text
throw new InvalidOperationException();
} // unknown count format
return countIR;
return count_ir;
case RegexElementLabel.CONCAT:
RegexIRGenerator concat = new();
StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs
+280
View File
@@ -0,0 +1,280 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexInterpreter.IREmit"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.ComponentModel.Design;
using System.Data;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Reflection.Emit;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using System.Xml;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexAnalyzer
{
internal static List<RegexIR> GenerateIR(AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree)
{
return GenerateIRRecursion(tree.Root, 0).ToList();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void EmitCount(
int curDepth,
RegexIRGenerator ir,
int min,
int max,
bool isGreedy,
<<<<<<<< HEAD:StgSharp.RegularAnalysis/Text/RegexInterpreter.IREmit.cs
RegexAstNode right)
========
RegexAstNode body
)
>>>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs
{
bool is_operator = !RegexAstNode.IsNullOrEmpty(body) &&
(body.Source.Flag & RegexElementLabel.OPERATOR) != 0;
if (is_operator) // seq to be count is complex
{
RegexIRGenerator body_ir = GenerateIRRecursion(body, curDepth + 1);
ir.EmitCountComplex(min, max, isGreedy, body_ir.Count);
ir.EmitIRStream(body_ir);
} else
{
RegexElementLabel count_source_mask = body.Source.Flag;
switch (count_source_mask)
{
case RegexElementLabel.UNIT:
ir.EmitCountSeq(min, max, isGreedy, body.Source.Value);
break;
case RegexElementLabel.UNIT_SET:
ir.EmitCountSet(min, max, isGreedy, body.Source.Value);
break;
case RegexElementLabel.UNIT_SPAN:
ir.EmitCountSeq(min, max, isGreedy, body.Source.Value);
break;
default:
throw new InvalidOperationException($"Unexpected regex element type for count operator: {count_source_mask}");
}
}
}
private static RegexIRGenerator GenerateIRRecursion(RegexAstNode node, int depth)
{
#pragma warning disable IDE0010
if (depth > 1024)
{
throw new OverflowException("Regex parse exceeds maximum recursion depth");
}
int curDepth = depth;
RegexAstNode left = node.Left;
RegexAstNode right = node.Right;
RegexIRGenerator leftIR;
RegexIRGenerator rightIR;
Token<RegexElementLabel> token = node.Source;
switch (node.Source.Flag)
{
case RegexElementLabel.GROUP_BEGIN:
string group_source = token.Value;
// process group type
bool is_group = group_source.Length == 1; // simple group: (pattern)
string group_name =
group_source.StartsWith(@"(?<", StringComparison.Ordinal) ?
group_source[3..^1] : /* named group (?<name>) */
string.Empty; /* unnamed group (?:) */
RegexIRGenerator groupIR = new();
rightIR = GenerateIRRecursion(right, curDepth + 1);
groupIR.EmitTry(rightIR.Count);
groupIR.EmitIRStream(rightIR);
groupIR.EmitCondition(2, 1);
groupIR.EmitPop(false);
groupIR.EmitPack(true, group_name);
return groupIR;
case RegexElementLabel.COUNT:
string count_source = token.Value;
bool isGreedy = !(count_source.Length > 1 && count_source.EndsWith('?'));
int seq_mask = (RegexAstNode.IsNullOrEmpty(left) ? 0 : 2) + (RegexAstNode.IsNullOrEmpty(right) ?
0 :
1);
RegexAstNode seq = seq_mask switch
{
// no seq, invalid count operator
0 => throw new InvalidOperationException("Count operator must have a sequence to operate on"),
1 => right,// right seq
2 => left,// left seq
// both sides, invalid count operator
3 => throw new InvalidOperationException("Count operator cannot have two sequences to operate on"),
_ => throw new InvalidOperationException("Unexpected sequence mask for count operator"),
};
ReadOnlySpan<char> count_source_span = count_source;
RegexIRGenerator count_ir = new();
if (count_source_span[0] == '*') // infinite count
{
EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq);
} else if (count_source_span[0] == '+') // at least one
{
EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq);
} else if (count_source_span[0] == '?') // zero or one
{
EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq);
} else if (count_source_span[0] == '{') // zero or one, non-greedy
{
int comma_pos = count_source_span.IndexOf(',');
if (comma_pos == -1)
{
if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count))
{
EmitCount(curDepth, count_ir, count, count, isGreedy, seq);
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
}
} else
{
if (int.TryParse(count_source[1..comma_pos], out int min) &&
int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max))
{
EmitCount(curDepth, count_ir, min, max, isGreedy, seq);
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
}
}
} else
{
throw new InvalidOperationException();
} // unknown count format
return count_ir;
case RegexElementLabel.CONCAT:
RegexIRGenerator concat = new();
// process left node
if (RegexAstNode.IsNullOrEmpty(left))
{
// do nothing
} else if ((left.Source.Flag & RegexElementLabel.SEQUENCE) != 0)
{
switch (left.Source.Flag)
{
case RegexElementLabel.UNIT_SPAN:
goto case RegexElementLabel.UNIT;
case RegexElementLabel.UNIT:
concat.EmitMatchSeq(left.Source.Value);
break;
case RegexElementLabel.UNIT_SET:
concat.EmitMatchSet(left.Source.Value);
break;
default:
break;
}
} else if ((left.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{
leftIR = GenerateIRRecursion(left, curDepth + 1);
concat.EmitIRStream(leftIR);
}
// process right node
if (RegexAstNode.IsNullOrEmpty(right))
{
// do nothing
} else if ((right.Source.Flag & RegexElementLabel.SEQUENCE) != 0)
{
switch (right.Source.Flag)
{
case RegexElementLabel.UNIT_SPAN:
goto case RegexElementLabel.UNIT;
case RegexElementLabel.UNIT:
concat.EmitMatchSeq(right.Source.Value);
break;
case RegexElementLabel.UNIT_SET:
concat.EmitMatchSet(right.Source.Value);
break;
default:
break;
}
} else if ((right.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{
rightIR = GenerateIRRecursion(right, curDepth + 1);
concat.EmitIRStream(rightIR);
}
return concat;
case RegexElementLabel.ALT:
RegexIRGenerator alt = new();
List<RegexIRGenerator> casesIRList = [];
RegexAstNode cur_node = right;
int ir_count = 0;
while (!RegexAstNode.IsNullOrEmpty(cur_node))
{
RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1);
casesIRList.Add(_case);
ir_count += _case.Count;
cur_node = cur_node.Next;
}
int case_count = casesIRList.Count;
while (case_count != 0)
{
RegexIRGenerator cur_ir = casesIRList[^case_count];
ir_count -= cur_ir.Count;
case_count--;
alt.EmitTry(cur_ir.Count);
alt.EmitIRStream(cur_ir);
alt.EmitCondition(1/* need to calc here */, 2);
alt.EmitPop(true);
}
alt.EmitPop(false);
return alt;
case RegexElementLabel.UNIT_SPAN:
goto case RegexElementLabel.UNIT;
case RegexElementLabel.UNIT:
RegexIRGenerator unit_seq = new();
unit_seq.EmitMatchSeq(node.Source.Value);
return unit_seq;
case RegexElementLabel.UNIT_SET:
RegexIRGenerator unit_set = new();
unit_set.EmitMatchSet(node.Source.Value);
return unit_set;
default:
return null!;
}
#pragma warning restore IDE0010
}
}
}
StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs
+214
View File
@@ -0,0 +1,214 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexInterpreter.IROptimize"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexAnalyzer
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static RegexInfo ScanAndOptimizeIR(
List<RegexIR> baseIR,
out List<RegexIR> optIR
)
{
int index = 0;
Span<RegexIR> origin_span = CollectionsMarshal.AsSpan(baseIR);
List<RegexIR> opt_ir = new(baseIR.Count);
CapacityFixedList<RegexIR> opt_window = new(8);
Span<RegexIR> opt_win_sp = [.. opt_window];
RegexInfo info = new();
do
{
int process_count = 0;
int last = int.Min(origin_span.Length, index + 8);
origin_span[index..last].CopyTo(opt_win_sp);
do
{
// plus_expand:
int expand_result = TryExpandPlus(opt_win_sp, opt_ir);
if (expand_result == 0)
{
opt_win_sp[0] = opt_ir[^1];
process_count++;
// goto dot_count_count:
} else if (expand_result > 0)
{
// meets a count-complex
RegexCountComplexIR cnt_cmp_ir = Unsafe.As<RegexCountComplexIR>(opt_win_sp[0]);
if (TryExpandComplexPlus(cnt_cmp_ir.IsGreedy, origin_span[(index + 1)..(index + expand_result)], opt_ir))
{
process_count += expand_result + 1;
break;
}
}
// dot_count_count:
if (TryOptDotCountCount(opt_win_sp, opt_ir))
{
process_count += expand_result == 0 ? 1 : 2;
break;
}
if (TryFlattenCount(opt_win_sp, opt_ir)) { }
} while (false);
index += process_count;
} while (index < baseIR.Count);
optIR = opt_ir;
return info;
}
private static bool TryExpandComplexPlus(
bool is_greedy,
Span<RegexIR> complex,
List<RegexIR> opt
)
{
int count = complex.Length;
opt.AddRange(complex);
opt.Add(new RegexCountComplexIR(0, -1, is_greedy, count));
return true;
}
#region expand plus
private static int TryExpandPlus(
Span<RegexIR> window,
List<RegexIR> opt
)
{
/* a+
* convert to
* aa*
*/
RegexIR ir = window[0];
if (ir is not IRegexCount { IsPlus: true } count) {
return -1;
}
bool is_greedy = count.IsGreedy;
string pattern;
switch (ir.Command & RegexIRCommand.Count)
{
case RegexIRCommand.Count_Complex:
RegexCountComplexIR complex = Unsafe.As<RegexCountComplexIR>(ir);
return complex.OperationCount;
case RegexIRCommand.Count_Seq:
RegexCountSeqIR seq = Unsafe.As<RegexCountSeqIR>(ir);
pattern = seq.Sequence;
opt.Add(new RegexMatchSeqIR(pattern));
opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern));
return 0;
case RegexIRCommand.Count_Set:
RegexCountSeqIR set = Unsafe.As<RegexCountSeqIR>(ir);
pattern = set.Sequence;
opt.Add(new RegexMatchSeqIR(pattern));
opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern));
return 0;
default:
return -1;
}
}
#endregion
private static bool TryFlattenCount(
Span<RegexIR> window,
List<RegexIR> opt
)
{
RegexIR ir = window[0];
if (ir is not RegexCountSeqIR seq) {
return false;
}
return true;
}
#region dot count count
private static int TryOptDotCountCount(
Span<RegexIR> window,
List<RegexIR> opt
)
{
/* .* <any>*
* convert to
* .*
*/
RegexIR cur_ir = window[0];
if (cur_ir is not RegexCountSetIR { IsDotStar: true } dot) {
return -1;
}
if (window[1] is not IRegexCount { IsStar: true } after) {
return -1;
}
if (dot.IsGreedy)
{
opt.Add(window[0]);
return 0;
} else
{
/*
* .* ?a* is different
* it is something like
* FIND(a)?a* :.*
*/
RegexIRGenerator g = new();
g.EmitTry(1);
}
return -1;
}
#endregion
internal class RegexInfo
{
public int MinLength { get; set; }
public int TryDepth { get; set; }
public int GroupCount { get; set; }
}
}
}
StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.SourceEmit.cs → StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceEmit.cs
View File
StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs
+52
View File
@@ -0,0 +1,52 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexInterpreter.SourceGenerate"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexAnalyzer
{
internal static string GetSourceCode(
IEnumerable<RegexIR> irs
)
{
StringBuilder sb = new StringBuilder();
foreach (RegexIR ir in irs)
{
// ScanAndMarkIROptimization(ir);
}
return sb.ToString();
}
}
}
StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs → StgSharp.RegularAnalysis/Text/RegexInterpreter.cs
+9 -3
View File
@@ -37,7 +37,7 @@ using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexInterpreter
public partial class RegexAnalyzer
{
private AbstractSyntaxTree<RegexAstNode, RegexElementLabel> _tree = new();
@@ -45,7 +45,7 @@ namespace StgSharp.RegularAnalysis.Text
private Exception _interpreteException;
private string _source;
private RegexInterpreter() { }
private RegexAnalyzer() { }
internal AbstractSyntaxTree<RegexAstNode, RegexElementLabel> Tree
{
@@ -58,9 +58,15 @@ namespace StgSharp.RegularAnalysis.Text
}
}
<<<<<<< HEAD:StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs
public static RegexInterpreter Analyze(string regex)
=======
public static RegexAnalyzer Analyze(
string regex
)
>>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/RegexInterpreter.cs
{
RegexInterpreter interpreter = new();
RegexAnalyzer interpreter = new();
interpreter._source = regex;
bool isSuccess;
try
StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs → StgSharp.RegularAnalysis/Text/RegexTokenParser.cs
View File
StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs → StgSharp.RegularAnalysis/Text/RegexTokenReader.cs
View File
StgSharp.RegularAnalysis/TextRegex/RegexTokenUtil.cs → StgSharp.RegularAnalysis/Text/RegexTokenUtil.cs
View File
StgSharp.RegularAnalysis/Text/TextHelper.cs
+50
View File
@@ -0,0 +1,50 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="TextHelper"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public static class TextHelper
{
public static bool TryLastIndexOf(
this ReadOnlySpan<char> span,
ReadOnlySpan<char> value,
out int index
)
{
index = span.LastIndexOf(value);
return index != -1;
}
}
}
StgSharp.RegularAnalysis/TextRegex/TextRegex.cs → StgSharp.RegularAnalysis/Text/TextRegex.cs
+15
View File
@@ -28,6 +28,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
<<<<<<< HEAD:StgSharp.RegularAnalysis/TextRegex/TextRegex.cs
=======
using System.Runtime.CompilerServices;
>>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/TextRegex.cs
using System.Text;
using System.Threading.Tasks;
@@ -35,6 +39,7 @@ namespace StgSharp.RegularAnalysis.Text
{
public abstract class TextRegex
{
<<<<<<< HEAD:StgSharp.RegularAnalysis/TextRegex/TextRegex.cs
public ref struct Match
{
private ReadOnlySpan<char> _sequence;
@@ -42,4 +47,14 @@ namespace StgSharp.RegularAnalysis.Text
}
}
=======
public abstract bool IsMatch(
ReadOnlySpan<char> text
);
}
public ref struct MatchResult { }
>>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/TextRegex.cs
}
StgSharpDebug/TestRegexAnalyse.cs
+5 -4
View File
@@ -44,13 +44,14 @@ namespace StgSharp
@"abc[0-9]x|abc[0-9]y|abd[0-9]x|abd[0-9]y|xyz",
@"get_name|get_value|get_type|set_name|set_value|set_type",
@"helix|hello|help|hero|hers|her",
@"a*.[abc]*(abcd|a(bc)c)+(?<name>abc)"];
@"a*.[abc]*(abcd|a(bc)c)+(?<name>abc{11,13})"
];
public static void Run()
{
foreach (string regex in regexSource)
{
RegexInterpreter interpreter = RegexInterpreter.Analyze(regex);
RegexAnalyzer interpreter = RegexAnalyzer.Analyze(regex);
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree = interpreter.Tree;
Console.WriteLine($"result of \"{regex}\"");
TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(tree);
@@ -60,7 +61,7 @@ namespace StgSharp
Console.Write(new string('|', enumerator.Depth));
Console.WriteLine($"\t{node.Source.Value}\t{node.Source.Flag}");
}
List<RegexIR> ir = RegexInterpreter.GenerateIR(tree);
List<RegexIR> ir = RegexAnalyzer.GenerateIR(tree);
Console.WriteLine("-----IR-----");
for (int i = 0; i < ir.Count; i++) {
Console.WriteLine(ir[i].AsCommand(i));
@@ -71,7 +72,7 @@ namespace StgSharp
}
}
[GeneratedRegex(@"a*.[abc]*(abcd|a(bc)c)+(?<name>abc)", RegexOptions.ExplicitCapture)]
[GeneratedRegex(@".*a*b", RegexOptions.ExplicitCapture)]
internal static partial Regex GetRegex();
}
future/ngra-regex-ir-review.md
+1050
View File
File diff suppressed because it is too large Load Diff
future/ngra-regex-ir.md
+53 -9
View File
@@ -14,15 +14,59 @@
使用DEFINE指令定义的符号定义在符号区中，符号的类型通过第一次赋值确定，源生成模式下，可以通过第一个赋值指令推导其类型
## 算子
## HIR算子
1. 搜索算子（SEARCH %str; SEARCH %unicode），向后搜索确定长度序列或者满足规则的字符集，更新活动指针，不更新锚点指针，搜索结果保存在全局上下文中
2. 搜索算子（SEARCH %set_expression），向后匹配一个满足字符集要求的字符，更新活动指针，不更新锚点指针，搜索结果保存在全局上下文中
3. 失败配置指令（CINFIG_FAIL %fail），指定当前匹配栈后续如何处理一条指令的失败，可选为：FAST，立刻终止当前层级；CONTINUE，记录这个fail，但是继续操作；REVERSE，反过来，成功了才终止
4. 条件指令（CONDITION %cmd_true %cmd_fail），比较前一条SEARCH指令是否成功，并指定跳转到指定的位置，如果某种条件不需要跳转，可以写1。如果想要将fail向上级继续抛出，可以%cmd_fail写0.
5. 跳转执行指令（TRY %begin %end），尝试idx指向的IR块，并记住执行结果数据
6. 计数指令（COUNT %min %max %is_greedy %cmd），重复执行指令cmd，直到满足参数配置的数量为止
7. 快速序列计数指令（COUNT_SEQ %min %max %is_greedy %seq），快速搜索一个确定性序列，直到满足参数配置的数量为止
8. 快速字符集计数指令（COUNT_SET %min %max %is_greedy %set），快速搜索一个字符集，直到满足参数配置的数量为止
9. 尝试（TRY %count），对自身向后的一段内容在新的匹配栈层级中进行运行，被分层的指令数量由COUNT指令确定
10. 弹出（POP %is_success），提前终止一个try块，并用is_success表示弹出行为视为匹配成功还是失败
3. 条件指令（CONDITION %cmd_true %cmd_fail），比较前一条SEARCH指令是否成功，并指定跳转到指定的位置，如果某种条件不需要跳转，可以写1。如果想要将fail向上级继续抛出，可以%cmd_fail写0.
4. 计数指令（COUNT %min %max %is_greedy %cmd），重复执行指令cmd，直到满足参数配置的数量为止
5. 快速序列计数指令（COUNT_SEQ %min %max %is_greedy %seq），快速搜索一个确定性序列，直到满足参数配置的数量为止
6. 快速字符集计数指令（COUNT_SET %min %max %is_greedy %set），快速搜索一个字符集，直到满足参数配置的数量为止
7. 尝试（TRY %count），对自身向后的一段内容在新的匹配栈层级中进行运行，被分层的指令数量由COUNT指令确定
8. 弹出（POP %is_success），提前终止一个try块，并用is_success表示弹出行为视为匹配成功还是失败
9. 打包指令（PACK %is_success %name），检查上一条语句的结果，将匹配结果打包并记录为指定的名字。如果名字为空，则使用默认的序号进行索引
## 优化策略
应当注意的是，将HIT进行进一步优化或者展开的目的：
1. 尽可能降低源生成或者运行时执行中，对多指令上下文的依赖，尽可能保证每一个IR可以单独输出最佳的代码
2. 减少多余的结构，使得一行指令尽可能处理多的数据
### AST优化器
1. ALT合并
将多个ALT CASE合并到一个链表中
2. SPAN合并
将连续的CHAR CONCAT合并为一个char span
3. alt合并
将多个alt分支中的公共部分提取出来
### IR优化器
1. .*a特征，优化为
FIND_SEQ(a, last)
.*?a特征，优化为
FIND_SEQ(a, first)
2. a+特征，优化为
MATCH_SEQ(a)
COUNT_SEQ(a,0,+inf,greedy)
完成
3. a{1} -> a
4. .\*a\*特征，优化为.*
完成，最优先
5. a{m,n}特征，展开为
MATCH_SEQ(aa...m个a....aa)
COUNT_SEQ(a,0,n,greedy)
6. .*?a*$
7. 对于一般的.*?，应该将后续的所有东西全都放进一个巨大的try，code gen时使用最慢的一位一位offset直到直到第一个匹配rest的东西