youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Working on tree optimizer, char concat flatten ok, working on alt union

Browse Source
This commit is contained in:
Youmu Liu
2026-02-12 18:12:40 +08:00
parent 39dd32992d
commit 6c78373375
8 changed files with 285 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.RegularAnalysis/Abstraction/AbstractSyntaxTree.cs
+1 -1
View File
@@ -152,9 +152,9 @@ namespace StgSharp.RegularAnalysis.Abstraction
return true;
}
case 2:
_recycle.Push(_stack.Pop());
if (TNode.IsNullOrEmpty(cur.Node.Next))
{
_recycle.Push(_stack.Pop());
if (_stack.Count == 0)
{
_completed = true;
StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs
+8
View File
@@ -66,6 +66,14 @@ namespace StgSharp.RegularAnalysis.Abstraction
TNode nextToken
);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static virtual bool IsNullOrEmpty(
TNode node
)
{
return node is null || node.Equals(TNode.Empty);
}
void PrependNode(
TNode previousNode
);
StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs
+50 -27
View File
@@ -40,7 +40,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
internal class RegexAstNode : ISyntaxNode<RegexAstNode, RegexElementLabel>
{
private RegexAstNode _left, _right;
// private RegexAstNode _left, _right;
public RegexAstNode(
Token<RegexElementLabel> source
@@ -56,11 +56,11 @@ namespace StgSharp.RegularAnalysis.TextRegex
public static RegexAstNode Empty { get; }
= new EmptyRegexAstNode();
public RegexAstNode Previous { get; set; }
public virtual RegexAstNode Previous { get; set; }
public RegexAstNode Next { get; set; }
public virtual RegexAstNode Next { get; set; }
public RegexAstNode Parent { get; set; }
public virtual RegexAstNode Parent { get; set; }
public RegexAstNode Left
{
@@ -93,7 +93,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
public string CodeConvertTemplate { get; }
public Token<RegexElementLabel> Source { get; }
public Token<RegexElementLabel> Source { get; private set; }
public void AppendNode(
RegexAstNode nextToken
@@ -123,8 +123,9 @@ namespace StgSharp.RegularAnalysis.TextRegex
RegexAstNode node
)
{
return node is null || node.Equals(Empty);
return node is null || node is EmptyRegexAstNode;
}
/**/
public void PrependNode(
RegexAstNode previousNode
@@ -144,39 +145,61 @@ namespace StgSharp.RegularAnalysis.TextRegex
} else if (Right == node) {
Right = Empty;
}
node.Parent = Empty;
}
public static bool TryMergeUnit(
RegexAstNode prev,
RegexAstNode next,
out RegexAstNode result
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static RegexAstNode NewEmptyNode(
)
{
Token<RegexElementLabel> pSource = prev.Source;
Token<RegexElementLabel> nSource = next.Source;
if (pSource.IsAccurateChar() &&
nSource.IsAccurateChar() &&
pSource.Column + pSource.Value.Length == nSource.Column)
{
// both are unit types and adjacent
// and are all accurate unit
Token<RegexElementLabel> newToken = new Token<RegexElementLabel>($"{pSource.Value}{nSource.Value}",
pSource.Line,
pSource.Column,
RegexElementLabel.UNIT_SPAN);
result = new RegexAstNode(newToken);
return true;
return new EmptyRegexAstNode();
}
/**/
internal void ReplaceBy(
RegexAstNode node
)
{
Source = node.Source;
Left = node.Left;
Right = node.Right;
if (IsNullOrEmpty(Left)) {
Left.Parent = this;
}
if (IsNullOrEmpty(Right)) {
Right.Parent = this;
}
Previous = node.Previous;
Next = node.Next;
RegexAstNode prev = Previous;
RegexAstNode next = Next;
if (!IsNullOrEmpty(prev)) {
prev.Next = this;
}
if (!IsNullOrEmpty(next)) {
next.Previous = this;
}
result = Empty;
return false;
}
private class EmptyRegexAstNode : RegexAstNode
private sealed class EmptyRegexAstNode : RegexAstNode
{
public EmptyRegexAstNode() : base(Token<RegexElementLabel>.Empty) { }
public override RegexAstNode Next
{
get;
set { field = Empty; }
}
public override RegexAstNode Previous
{
get;
set { field = Empty; }
}
}
}
StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs
+1
View File
@@ -47,6 +47,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
CONCAT = 64,
ALT = 128,
SEQUENCE = UNIT | UNIT_SPAN | UNIT_SET,
SINGLE = UNIT | UNIT_SET,
OPERATOR = COUNT | ALT | CONCAT,
ATOM_BEGIN = SEQUENCE | GROUP_BEGIN ,
ATOM_END = SEQUENCE | GROUP_END | COUNT,
StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs
+205 -22
View File
@@ -30,7 +30,9 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Security;
using System.Runtime.CompilerServices;
using System.Security.Permissions;
using System.Text;
using System.Threading.Tasks;
@@ -39,8 +41,8 @@ namespace StgSharp.RegularAnalysis.TextRegex
public partial class RegexInterpreter
{
private static RegexAstNode FlattenAlt(
RegexAstNode root
private static List<RegexAstNode> FlattenAlt(
RegexAstNode root
)
{
TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(root);
@@ -50,44 +52,42 @@ namespace StgSharp.RegularAnalysis.TextRegex
while (enumerator.MoveNext())
{
current = enumerator.Current;
current.Parent.RemoveChild(current);
current.Parent = RegexAstNode.Empty;
if (current.Parent.EqualityTypeConvert == RegexElementLabel.ALT)
{
continue;
} else
if (current.EqualityTypeConvert != RegexElementLabel.ALT)
{
cases.Add(current);
enumerator.SkipNode(current);
}
current.Parent.RemoveChild(current);
}
for (int i = 0; i < cases.Count - 1; i++)
{
cases[i].AppendNode(cases[i + 1]);
cases[i].Parent = root;
}
return cases[0];
return cases;
}
private static void MergeConcat(
private static bool MergeConcat(
RegexAstNode root
)
{
if (root.EqualityTypeConvert != RegexElementLabel.CONCAT) {
return;
return false;
}
/*
* concat → concat
* ╱ ╲ /
* 'a' concat "abc"
* / \
* 'b' 'c'
*/
List<string> buffer = [];
int length = 0;
RegexAstNode right = root.Right;
RegexAstNode left = root.Left;
if (left.EqualityTypeConvert != RegexElementLabel.UNIT_SPAN &&
left.EqualityTypeConvert != RegexElementLabel.UNIT) {
return;
return false;
}
string str = left.Value;
buffer.Add(str);
length += str.Length;
while (RegexAstNode.IsNullOrEmpty(right) &&
while (!RegexAstNode.IsNullOrEmpty(right) &&
right.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
left = right.Left;
@@ -100,9 +100,16 @@ namespace StgSharp.RegularAnalysis.TextRegex
buffer.Add(str);
length += str.Length;
root.Right = right.Right;
// TODO: remove right from tree
right.Parent = RegexAstNode.Empty;
right = root.Right;
right.Parent = root;
}
if (right.EqualityTypeConvert == RegexElementLabel.UNIT_SPAN ||
right.EqualityTypeConvert == RegexElementLabel.UNIT)
{
root.Right = RegexAstNode.Empty;
buffer.Add(right.Value);
length += right.Value.Length;
}
str = string.Create(length, buffer, static(
span,
@@ -115,12 +122,12 @@ namespace StgSharp.RegularAnalysis.TextRegex
offset += buffer[i].Length;
}
});
left = root.Left;
root.Left = new RegexAstNode(new Token<RegexElementLabel>(str, left.Source.Line,
left.Source.Column,
str.Length == 1 ?
RegexElementLabel.UNIT :
RegexElementLabel.UNIT_SPAN));
return true;
}
private static void RotateConcat(
@@ -128,7 +135,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
)
{
RegexAstNode left = root.Left;
while (RegexAstNode.IsNullOrEmpty(left) &&
while (!RegexAstNode.IsNullOrEmpty(left) &&
left.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
/*
@@ -160,5 +167,181 @@ namespace StgSharp.RegularAnalysis.TextRegex
}
}
#region merge alt
/// <summary>
/// Get union beginning part of cases
/// </summary>
/// <param name="cases">
///
/// </param>
/// <returns>
///
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static RegexAstNode MergeAlt(
List<RegexAstNode> cases
)
{
UnionAlt union = new(0, cases);
return union.AsNormal();
}
private class UnionAlt : RegexAstNode
{
private const int MaxLevel = 8;
private UnionAlt(
int level,
RegexAstNode node
)
: base(new Token<RegexElementLabel>(string.Empty, 0, 0, 0))
{
Level = level;
Cases = new List<RegexAstNode>();
Cases.Add(node.Right ?? Empty);
UnionBegin = node;
UnionEnd = node;
}
public UnionAlt(
int level,
List<RegexAstNode> c
)
: base(new Token<RegexElementLabel>(string.Empty, 0, 0, 0))
{
Cases = c;
Level = level;
}
public List<RegexAstNode> Cases { get; set; }
public RegexAstNode UnionBegin { get; set; }
public RegexAstNode UnionEnd { get; set; }
private int Level { get; set; }
public RegexAstNode AsNormal()
{
if (Level >= MaxLevel)
{
goto build;
}
merge:
List<RegexAstNode> buffer = [];
for (int i = 0; i < Cases.Count; i++)
{
RegexAstNode current = Cases[i];
int j = 0;
for (; j < buffer.Count; j++)
{
if (CanMerge(current, buffer[j]))
{
// find a case in buffer that can merge with current case,
// then merge them and break
RegexAstNode b = buffer[j];
if (b is not UnionAlt alt)
{
alt = new UnionAlt(Level + 1, b);
alt.AddCase(current);
buffer[j] = alt;
} else
{
alt.AddCase(current);
}
break;
}
}
if (j >= buffer.Count) {
buffer.Add(current);
}
}
Cases = buffer;
// TODO problem1 没处理case链表
// TODO problem2 没有处理立刻返回策略
if (Cases.Count == 1)
{
// if there is only one case,
// then we suppose that its contained cases has a same beginning,
// so we attach it to UnionEnd and stay level unchanged
UnionAlt alt = (Cases[0] as UnionAlt)!;
if (IsNullOrEmpty(UnionBegin))
{
UnionBegin = alt.UnionBegin;
UnionEnd = alt.UnionEnd;
} else
{
alt.UnionBegin.Parent = UnionEnd;
UnionEnd.Right = alt.UnionBegin;
}
Cases = alt.Cases;
goto merge;
}
build:
// suppose that from UnionBegin to UnionEnd is a normal form without union
for (int i = 0; i < Cases.Count; i++)
{
if (Cases[i] is UnionAlt union) {
Cases[i] = union.AsNormal();
}
}
RegexAstNode branch = new RegexAstNode(new Token<RegexElementLabel>(string.Empty, 0,
UnionEnd.Source.Column,
RegexElementLabel.ALT));
UnionEnd.Right = branch;
branch.Parent = UnionEnd;
UnionEnd = branch;
for (int i = 0; i < Cases.Count - 1; i++)
{
Cases[i].AppendNode(Cases[i + 1]);
Cases[i].Parent = UnionEnd;
}
UnionEnd.Right = Cases[0];
return UnionBegin;
}
private void AddCase(
RegexAstNode _case
)
{
RegexAstNode right = _case.Right;
right ??= Empty;
Cases.Add(right);
}
private static bool CanMerge(
RegexAstNode origin,
RegexAstNode target
)
{
// suppose that origin is not a union,
if (target is UnionAlt alt)
{
// then we suppose that alt has only one are same
return origin .EqualityTypeConvert == RegexElementLabel.CONCAT &&
(origin.Left.EqualityTypeConvert & RegexElementLabel.SINGLE) != 0 &&
origin.Left.EqualityTypeConvert == alt.UnionEnd.Left.EqualityTypeConvert &&
origin.Left.Value == alt.UnionEnd.Left.Value;
} else
{
return origin.EqualityTypeConvert == RegexElementLabel.CONCAT &&
target.EqualityTypeConvert == RegexElementLabel.CONCAT &&
(origin.Left.EqualityTypeConvert & RegexElementLabel.SINGLE) != 0 &&
origin.Left.EqualityTypeConvert == target.Left.EqualityTypeConvert &&
origin.Left.Value == target.Left.Value;
}
}
}
#endregion
/**/
}
}
StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs
+15 -19
View File
@@ -89,18 +89,14 @@ namespace StgSharp.RegularAnalysis.TextRegex
if ((token.Flag & RegexElementLabel.SEQUENCE) != 0)
{
// char or charset
Console.WriteLine($"Unit {token.Value}");
_stack.PushOperand(new RegexAstNode(token));
} else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0)
{
Console.WriteLine($"Group begin {token.Value}");
_stack.PushOperator(new RegexAstNode(token));
_stack.IncreaseDepth(0);
Console.WriteLine(/**/_stack.OperatorInDepthCount/**/);
} else if ((token.Flag & RegexElementLabel.GROUP_END) != 0)
{
// close all nodes here
Console.WriteLine($"Group end {token.Value}");
RegexAstNode op;
while (_stack.TryPopOperator(out op)) {
ProcessOperator(op);
@@ -120,8 +116,6 @@ namespace StgSharp.RegularAnalysis.TextRegex
}
} else if ((token.Flag & RegexElementLabel.OPERATOR) != 0)
{
Console.Write($"Operator {token.Value} with flag {token.Flag}, ");
Console.WriteLine(/**/_stack.OperatorInDepthCount/**/);
if (_stack.OperatorInDepthCount == 0)
{
// the first operator in stack
@@ -150,7 +144,6 @@ namespace StgSharp.RegularAnalysis.TextRegex
}
}
}
Console.WriteLine(1);
if (_stack.Depth == 1)
{
if (_stack.OperandInDepthCount != 1 || _stack.OperatorInDepthCount != 0)
@@ -164,7 +157,6 @@ namespace StgSharp.RegularAnalysis.TextRegex
_tree.Root = _stack.PopOperand();
return;
}
throw new InvalidOperationException("Invalid regular expression syntax.");
}
throw new InvalidOperationException("Invalid regular expression syntax.");
}
@@ -178,33 +170,37 @@ namespace StgSharp.RegularAnalysis.TextRegex
RegexAstNode current;
// phase1: ALT flatten
// phase2: CONCAT normalize
// phase2: then ALT union
// phase3: CONCAT normalize
int i = 0;
while (enumerator.MoveNext())
{
current = enumerator.Current;
RegexAstNode cur = current.Next;
if (current.EqualityTypeConvert == RegexElementLabel.ALT)
{
RegexAstNode union = FlattenAlt(current);
current.Right = union;
current.Left = RegexAstNode.Empty;
List<RegexAstNode> list = FlattenAlt(current);
RegexAstNode union = MergeAlt(list);
current.ReplaceBy(union);
} else if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) {
RotateConcat(current);
}
}
// phase3: CONCAT merge
// phase4: CONCAT merge
enumerator.Reset();
while (enumerator.MoveNext())
{
// suppose all concat is now in normalized form,
// we can merge adjacent unit nodes into a single unit node with string value
current = enumerator.Current;
if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) {
MergeConcat(current);
if (current.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
if (MergeConcat(current)) {
enumerator.SkipOnce();
}
}
}
// phase4: ALT union
}
private void ProcessOperator(
@@ -244,8 +240,8 @@ namespace StgSharp.RegularAnalysis.TextRegex
if (_stack.TryPopOperand(out RegexAstNode? _1) &&
_stack.TryPopOperand(out RegexAstNode? _2))
{
op.Left = _1;
op.Right = _2;
op.Left = _2;
op.Right = _1;
_stack.PushOperand(op);
_tree.AddNode(op);
_tree.AddNode(_1);
StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs
+4
View File
@@ -79,6 +79,10 @@ namespace StgSharp.RegularAnalysis.TextRegex
}
}
throw new InvalidOperationException("Unclosed character set");
case '|':
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
RegexElementLabel.ALT);
case '(':
// very comlex here