From 6c7837337579a681e70dfffda70bf36cb2f48172 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Thu, 12 Feb 2026 18:12:40 +0800 Subject: [PATCH] Working on tree optimizer, char concat flatten ok, working on alt union --- .../Abstraction/AbstractSyntaxTree.cs | 2 +- .../Abstraction/ISyntaxNode.cs | 8 + .../TextRegex/RegexAstNode.cs | 77 +++--- .../TextRegex/RegexElementLabel.cs | 1 + .../TextRegex/RegexInterpreter.Optimize.cs | 227 ++++++++++++++++-- .../TextRegex/RegexInterpreter.cs | 34 ++- .../TextRegex/RegexTokenReader.cs | 4 + StgSharpDebug/TestRegexAnalyse.cs | 2 +- 8 files changed, 285 insertions(+), 70 deletions(-) diff --git a/StgSharp.RegularAnalysis/Abstraction/AbstractSyntaxTree.cs b/StgSharp.RegularAnalysis/Abstraction/AbstractSyntaxTree.cs index cde45b0f..af9dfc19 100644 --- a/StgSharp.RegularAnalysis/Abstraction/AbstractSyntaxTree.cs +++ b/StgSharp.RegularAnalysis/Abstraction/AbstractSyntaxTree.cs @@ -152,9 +152,9 @@ namespace StgSharp.RegularAnalysis.Abstraction return true; } case 2: + _recycle.Push(_stack.Pop()); if (TNode.IsNullOrEmpty(cur.Node.Next)) { - _recycle.Push(_stack.Pop()); if (_stack.Count == 0) { _completed = true; diff --git a/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs b/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs index 3911e7b4..0d816001 100644 --- a/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs +++ b/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs @@ -66,6 +66,14 @@ namespace StgSharp.RegularAnalysis.Abstraction TNode nextToken ); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static virtual bool IsNullOrEmpty( + TNode node + ) + { + return node is null || node.Equals(TNode.Empty); + } + void PrependNode( TNode previousNode ); diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs b/StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs index db926891..522bfa28 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs +++ b/StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs @@ -40,7 +40,7 @@ namespace StgSharp.RegularAnalysis.TextRegex internal class RegexAstNode : ISyntaxNode { - private RegexAstNode _left, _right; + // private RegexAstNode _left, _right; public RegexAstNode( Token source @@ -56,11 +56,11 @@ namespace StgSharp.RegularAnalysis.TextRegex public static RegexAstNode Empty { get; } = new EmptyRegexAstNode(); - public RegexAstNode Previous { get; set; } + public virtual RegexAstNode Previous { get; set; } - public RegexAstNode Next { get; set; } + public virtual RegexAstNode Next { get; set; } - public RegexAstNode Parent { get; set; } + public virtual RegexAstNode Parent { get; set; } public RegexAstNode Left { @@ -93,7 +93,7 @@ namespace StgSharp.RegularAnalysis.TextRegex public string CodeConvertTemplate { get; } - public Token Source { get; } + public Token Source { get; private set; } public void AppendNode( RegexAstNode nextToken @@ -123,8 +123,9 @@ namespace StgSharp.RegularAnalysis.TextRegex RegexAstNode node ) { - return node is null || node.Equals(Empty); + return node is null || node is EmptyRegexAstNode; } + /**/ public void PrependNode( RegexAstNode previousNode @@ -144,39 +145,61 @@ namespace StgSharp.RegularAnalysis.TextRegex } else if (Right == node) { Right = Empty; } + node.Parent = Empty; } - public static bool TryMergeUnit( - RegexAstNode prev, - RegexAstNode next, - out RegexAstNode result + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static RegexAstNode NewEmptyNode( ) { - Token pSource = prev.Source; - Token nSource = next.Source; - if (pSource.IsAccurateChar() && - nSource.IsAccurateChar() && - pSource.Column + pSource.Value.Length == nSource.Column) - { - // both are unit types and adjacent - // and are all accurate unit - Token newToken = new Token($"{pSource.Value}{nSource.Value}", - pSource.Line, - pSource.Column, - RegexElementLabel.UNIT_SPAN); - result = new RegexAstNode(newToken); - return true; + return new EmptyRegexAstNode(); + } + /**/ + + internal void ReplaceBy( + RegexAstNode node + ) + { + Source = node.Source; + Left = node.Left; + Right = node.Right; + if (IsNullOrEmpty(Left)) { + Left.Parent = this; + } + if (IsNullOrEmpty(Right)) { + Right.Parent = this; + } + Previous = node.Previous; + Next = node.Next; + + RegexAstNode prev = Previous; + RegexAstNode next = Next; + if (!IsNullOrEmpty(prev)) { + prev.Next = this; + } + if (!IsNullOrEmpty(next)) { + next.Previous = this; } - result = Empty; - return false; } - private class EmptyRegexAstNode : RegexAstNode + private sealed class EmptyRegexAstNode : RegexAstNode { public EmptyRegexAstNode() : base(Token.Empty) { } + public override RegexAstNode Next + { + get; + set { field = Empty; } + } + + public override RegexAstNode Previous + { + get; + set { field = Empty; } + } + } } diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs b/StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs index 0fc826d6..67ee3515 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs +++ b/StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs @@ -47,6 +47,7 @@ namespace StgSharp.RegularAnalysis.TextRegex CONCAT = 64, ALT = 128, SEQUENCE = UNIT | UNIT_SPAN | UNIT_SET, + SINGLE = UNIT | UNIT_SET, OPERATOR = COUNT | ALT | CONCAT, ATOM_BEGIN = SEQUENCE | GROUP_BEGIN , ATOM_END = SEQUENCE | GROUP_END | COUNT, diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs b/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs index dc98ee78..7da0f8f7 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs +++ b/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs @@ -30,7 +30,9 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net.Security; using System.Runtime.CompilerServices; +using System.Security.Permissions; using System.Text; using System.Threading.Tasks; @@ -39,8 +41,8 @@ namespace StgSharp.RegularAnalysis.TextRegex public partial class RegexInterpreter { - private static RegexAstNode FlattenAlt( - RegexAstNode root + private static List FlattenAlt( + RegexAstNode root ) { TreeEnumerator enumerator = new(root); @@ -50,44 +52,42 @@ namespace StgSharp.RegularAnalysis.TextRegex while (enumerator.MoveNext()) { current = enumerator.Current; - current.Parent.RemoveChild(current); - current.Parent = RegexAstNode.Empty; - if (current.Parent.EqualityTypeConvert == RegexElementLabel.ALT) - { - continue; - } else + if (current.EqualityTypeConvert != RegexElementLabel.ALT) { cases.Add(current); enumerator.SkipNode(current); } + current.Parent.RemoveChild(current); } - for (int i = 0; i < cases.Count - 1; i++) - { - cases[i].AppendNode(cases[i + 1]); - cases[i].Parent = root; - } - return cases[0]; + return cases; } - private static void MergeConcat( + private static bool MergeConcat( RegexAstNode root ) { if (root.EqualityTypeConvert != RegexElementLabel.CONCAT) { - return; + return false; } + /* + * concat → concat + * ╱ ╲ / + * 'a' concat "abc" + * / \ + * 'b' 'c' + */ List buffer = []; int length = 0; RegexAstNode right = root.Right; RegexAstNode left = root.Left; if (left.EqualityTypeConvert != RegexElementLabel.UNIT_SPAN && left.EqualityTypeConvert != RegexElementLabel.UNIT) { - return; + return false; } string str = left.Value; buffer.Add(str); length += str.Length; - while (RegexAstNode.IsNullOrEmpty(right) && + while (!RegexAstNode.IsNullOrEmpty(right) && right.EqualityTypeConvert == RegexElementLabel.CONCAT) { left = right.Left; @@ -100,9 +100,16 @@ namespace StgSharp.RegularAnalysis.TextRegex buffer.Add(str); length += str.Length; root.Right = right.Right; - - // TODO: remove right from tree + right.Parent = RegexAstNode.Empty; right = root.Right; + right.Parent = root; + } + if (right.EqualityTypeConvert == RegexElementLabel.UNIT_SPAN || + right.EqualityTypeConvert == RegexElementLabel.UNIT) + { + root.Right = RegexAstNode.Empty; + buffer.Add(right.Value); + length += right.Value.Length; } str = string.Create(length, buffer, static( span, @@ -115,12 +122,12 @@ namespace StgSharp.RegularAnalysis.TextRegex offset += buffer[i].Length; } }); - left = root.Left; root.Left = new RegexAstNode(new Token(str, left.Source.Line, left.Source.Column, str.Length == 1 ? RegexElementLabel.UNIT : RegexElementLabel.UNIT_SPAN)); + return true; } private static void RotateConcat( @@ -128,7 +135,7 @@ namespace StgSharp.RegularAnalysis.TextRegex ) { RegexAstNode left = root.Left; - while (RegexAstNode.IsNullOrEmpty(left) && + while (!RegexAstNode.IsNullOrEmpty(left) && left.EqualityTypeConvert == RegexElementLabel.CONCAT) { /* @@ -160,5 +167,181 @@ namespace StgSharp.RegularAnalysis.TextRegex } } + #region merge alt + + /// + /// Get union beginning part of cases + /// + /// + /// + /// + /// + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static RegexAstNode MergeAlt( + List cases + ) + { + UnionAlt union = new(0, cases); + return union.AsNormal(); + } + + private class UnionAlt : RegexAstNode + { + + private const int MaxLevel = 8; + + private UnionAlt( + int level, + RegexAstNode node + ) + : base(new Token(string.Empty, 0, 0, 0)) + { + Level = level; + Cases = new List(); + Cases.Add(node.Right ?? Empty); + UnionBegin = node; + UnionEnd = node; + } + + public UnionAlt( + int level, + List c + ) + : base(new Token(string.Empty, 0, 0, 0)) + { + Cases = c; + Level = level; + } + + public List Cases { get; set; } + + public RegexAstNode UnionBegin { get; set; } + + public RegexAstNode UnionEnd { get; set; } + + private int Level { get; set; } + + public RegexAstNode AsNormal() + { + if (Level >= MaxLevel) + { + goto build; + } + merge: + List buffer = []; + for (int i = 0; i < Cases.Count; i++) + { + RegexAstNode current = Cases[i]; + int j = 0; + for (; j < buffer.Count; j++) + { + if (CanMerge(current, buffer[j])) + { + // find a case in buffer that can merge with current case, + // then merge them and break + RegexAstNode b = buffer[j]; + if (b is not UnionAlt alt) + { + alt = new UnionAlt(Level + 1, b); + alt.AddCase(current); + buffer[j] = alt; + } else + { + alt.AddCase(current); + } + break; + } + } + if (j >= buffer.Count) { + buffer.Add(current); + } + } + Cases = buffer; + + // TODO problem1 没处理case链表 + // TODO problem2 没有处理立刻返回策略 + if (Cases.Count == 1) + { + // if there is only one case, + // then we suppose that its contained cases has a same beginning, + // so we attach it to UnionEnd and stay level unchanged + UnionAlt alt = (Cases[0] as UnionAlt)!; + if (IsNullOrEmpty(UnionBegin)) + { + UnionBegin = alt.UnionBegin; + UnionEnd = alt.UnionEnd; + } else + { + alt.UnionBegin.Parent = UnionEnd; + UnionEnd.Right = alt.UnionBegin; + } + + Cases = alt.Cases; + goto merge; + } + + build: + + // suppose that from UnionBegin to UnionEnd is a normal form without union + for (int i = 0; i < Cases.Count; i++) + { + if (Cases[i] is UnionAlt union) { + Cases[i] = union.AsNormal(); + } + } + RegexAstNode branch = new RegexAstNode(new Token(string.Empty, 0, + UnionEnd.Source.Column, + RegexElementLabel.ALT)); + UnionEnd.Right = branch; + branch.Parent = UnionEnd; + UnionEnd = branch; + for (int i = 0; i < Cases.Count - 1; i++) + { + Cases[i].AppendNode(Cases[i + 1]); + Cases[i].Parent = UnionEnd; + } + UnionEnd.Right = Cases[0]; + return UnionBegin; + } + + private void AddCase( + RegexAstNode _case + ) + { + RegexAstNode right = _case.Right; + right ??= Empty; + Cases.Add(right); + } + + private static bool CanMerge( + RegexAstNode origin, + RegexAstNode target + ) + { + // suppose that origin is not a union, + if (target is UnionAlt alt) + { + // then we suppose that alt has only one are same + return origin .EqualityTypeConvert == RegexElementLabel.CONCAT && + (origin.Left.EqualityTypeConvert & RegexElementLabel.SINGLE) != 0 && + origin.Left.EqualityTypeConvert == alt.UnionEnd.Left.EqualityTypeConvert && + origin.Left.Value == alt.UnionEnd.Left.Value; + } else + { + return origin.EqualityTypeConvert == RegexElementLabel.CONCAT && + target.EqualityTypeConvert == RegexElementLabel.CONCAT && + (origin.Left.EqualityTypeConvert & RegexElementLabel.SINGLE) != 0 && + origin.Left.EqualityTypeConvert == target.Left.EqualityTypeConvert && + origin.Left.Value == target.Left.Value; + } + } + + } + + #endregion + + /**/ } } diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs b/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs index 15ad6e3b..7794cd64 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs +++ b/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs @@ -89,18 +89,14 @@ namespace StgSharp.RegularAnalysis.TextRegex if ((token.Flag & RegexElementLabel.SEQUENCE) != 0) { // char or charset - Console.WriteLine($"Unit {token.Value}"); _stack.PushOperand(new RegexAstNode(token)); } else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0) { - Console.WriteLine($"Group begin {token.Value}"); _stack.PushOperator(new RegexAstNode(token)); _stack.IncreaseDepth(0); - Console.WriteLine(/**/_stack.OperatorInDepthCount/**/); } else if ((token.Flag & RegexElementLabel.GROUP_END) != 0) { // close all nodes here - Console.WriteLine($"Group end {token.Value}"); RegexAstNode op; while (_stack.TryPopOperator(out op)) { ProcessOperator(op); @@ -120,8 +116,6 @@ namespace StgSharp.RegularAnalysis.TextRegex } } else if ((token.Flag & RegexElementLabel.OPERATOR) != 0) { - Console.Write($"Operator {token.Value} with flag {token.Flag}, "); - Console.WriteLine(/**/_stack.OperatorInDepthCount/**/); if (_stack.OperatorInDepthCount == 0) { // the first operator in stack @@ -150,7 +144,6 @@ namespace StgSharp.RegularAnalysis.TextRegex } } } - Console.WriteLine(1); if (_stack.Depth == 1) { if (_stack.OperandInDepthCount != 1 || _stack.OperatorInDepthCount != 0) @@ -164,7 +157,6 @@ namespace StgSharp.RegularAnalysis.TextRegex _tree.Root = _stack.PopOperand(); return; } - throw new InvalidOperationException("Invalid regular expression syntax."); } throw new InvalidOperationException("Invalid regular expression syntax."); } @@ -178,33 +170,37 @@ namespace StgSharp.RegularAnalysis.TextRegex RegexAstNode current; // phase1: ALT flatten - // phase2: CONCAT normalize + // phase2: then ALT union + // phase3: CONCAT normalize + int i = 0; while (enumerator.MoveNext()) { current = enumerator.Current; + RegexAstNode cur = current.Next; if (current.EqualityTypeConvert == RegexElementLabel.ALT) { - RegexAstNode union = FlattenAlt(current); - current.Right = union; - current.Left = RegexAstNode.Empty; + List list = FlattenAlt(current); + RegexAstNode union = MergeAlt(list); + current.ReplaceBy(union); } else if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) { RotateConcat(current); } } - // phase3: CONCAT merge + // phase4: CONCAT merge enumerator.Reset(); while (enumerator.MoveNext()) { // suppose all concat is now in normalized form, // we can merge adjacent unit nodes into a single unit node with string value current = enumerator.Current; - if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) { - MergeConcat(current); + if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) + { + if (MergeConcat(current)) { + enumerator.SkipOnce(); + } } } - - // phase4: ALT union } private void ProcessOperator( @@ -244,8 +240,8 @@ namespace StgSharp.RegularAnalysis.TextRegex if (_stack.TryPopOperand(out RegexAstNode? _1) && _stack.TryPopOperand(out RegexAstNode? _2)) { - op.Left = _1; - op.Right = _2; + op.Left = _2; + op.Right = _1; _stack.PushOperand(op); _tree.AddNode(op); _tree.AddNode(_1); diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs b/StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs index 68922948..47aa4a24 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs +++ b/StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs @@ -79,6 +79,10 @@ namespace StgSharp.RegularAnalysis.TextRegex } } throw new InvalidOperationException("Unclosed character set"); + case '|': + position = pos + 1; + return new Token(_source[(pos)..(pos + 1)], 0, pos, + RegexElementLabel.ALT); case '(': // very comlex here diff --git a/StgSharpDebug/TestRegexAnalyse.cs b/StgSharpDebug/TestRegexAnalyse.cs index 96981606..c258a697 100644 --- a/StgSharpDebug/TestRegexAnalyse.cs +++ b/StgSharpDebug/TestRegexAnalyse.cs @@ -38,7 +38,7 @@ namespace StgSharp public class TestRegexAnalyse { - private const string regex = @"(ab[1-9])+(?abc)"; + private const string regex = @"abc|abd|abe|abf"; public static void Run() {