Working on tree optimizer, char concat flatten ok, working on alt union

StgSharp.RegularAnalysis/Abstraction/AbstractSyntaxTree.cs

@@ -152,9 +152,9 @@ namespace StgSharp.RegularAnalysis.Abstraction
                             return true;
                         }
                     case 2:
+                        _recycle.Push(_stack.Pop());
                         if (TNode.IsNullOrEmpty(cur.Node.Next))
                         {
-                            _recycle.Push(_stack.Pop());
                             if (_stack.Count == 0)
                             {
                                 _completed = true;

StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs

@@ -66,6 +66,14 @@ namespace StgSharp.RegularAnalysis.Abstraction
              TNode nextToken
         );
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static virtual bool IsNullOrEmpty(
+                                   TNode node
+        )
+        {
+            return node is null || node.Equals(TNode.Empty);
+        }
+
         void PrependNode(
              TNode previousNode
         );

StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs

@@ -40,7 +40,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
     internal class RegexAstNode : ISyntaxNode<RegexAstNode, RegexElementLabel>
     {
 
-        private RegexAstNode _left, _right;
+        // private RegexAstNode _left, _right;
 
         public RegexAstNode(
                Token<RegexElementLabel> source
@@ -56,11 +56,11 @@ namespace StgSharp.RegularAnalysis.TextRegex
         public static RegexAstNode Empty { get; }
         = new EmptyRegexAstNode();
 
-        public RegexAstNode Previous { get; set; }
+        public virtual RegexAstNode Previous { get; set; }
 
-        public RegexAstNode Next { get; set; }
+        public virtual RegexAstNode Next { get; set; }
 
-        public RegexAstNode Parent { get; set; }
+        public virtual RegexAstNode Parent { get; set; }
 
         public RegexAstNode Left
         {
@@ -93,7 +93,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
 
         public string CodeConvertTemplate { get; }
 
-        public Token<RegexElementLabel> Source { get; }
+        public Token<RegexElementLabel> Source { get; private set; }
 
         public void AppendNode(
                     RegexAstNode nextToken
@@ -123,8 +123,9 @@ namespace StgSharp.RegularAnalysis.TextRegex
                            RegexAstNode node
         )
         {
-            return node is null || node.Equals(Empty);
+            return node is null || node is EmptyRegexAstNode;
         }
+        /**/
 
         public void PrependNode(
                     RegexAstNode previousNode
@@ -144,39 +145,61 @@ namespace StgSharp.RegularAnalysis.TextRegex
             } else if (Right == node) {
                 Right = Empty;
             }
+            node.Parent = Empty;
         }
 
-        public static bool TryMergeUnit(
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-                           RegexAstNode prev,
+        internal static RegexAstNode NewEmptyNode(
-                           RegexAstNode next,
-                           out RegexAstNode result
         )
         {
-            Token<RegexElementLabel> pSource = prev.Source;
+            return new EmptyRegexAstNode();
-            Token<RegexElementLabel> nSource = next.Source;
+        }
-            if (pSource.IsAccurateChar() &&
+        /**/
-                nSource.IsAccurateChar() &&
+
-                pSource.Column + pSource.Value.Length == nSource.Column)
+        internal void ReplaceBy(
-            {
+                      RegexAstNode node
-                // both are unit types and adjacent
+        )
-                // and are all accurate unit
+        {
-                Token<RegexElementLabel> newToken = new Token<RegexElementLabel>($"{pSource.Value}{nSource.Value}",
+            Source = node.Source;
-                                                                                 pSource.Line,
+            Left = node.Left;
-                                                                                 pSource.Column,
+            Right = node.Right;
-                                                                                 RegexElementLabel.UNIT_SPAN);
+            if (IsNullOrEmpty(Left)) {
-                result = new RegexAstNode(newToken);
+                Left.Parent = this;
-                return true;
+            }
+            if (IsNullOrEmpty(Right)) {
+                Right.Parent = this;
+            }
+            Previous = node.Previous;
+            Next = node.Next;
+
+            RegexAstNode prev = Previous;
+            RegexAstNode next = Next;
+            if (!IsNullOrEmpty(prev)) {
+                prev.Next = this;
+            }
+            if (!IsNullOrEmpty(next)) {
+                next.Previous = this;
             }
-            result = Empty;
-            return false;
         }
 
-        private class EmptyRegexAstNode : RegexAstNode
+        private sealed class EmptyRegexAstNode : RegexAstNode
 
         {
 
             public EmptyRegexAstNode() : base(Token<RegexElementLabel>.Empty) { }
 
+            public override RegexAstNode Next
+            {
+                get;
+                set { field = Empty; }
+            }
+
+            public override RegexAstNode Previous
+            {
+                get;
+                set { field = Empty; }
+            }
+
         }
 
     }

StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs

@@ -47,6 +47,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
         CONCAT = 64,
         ALT = 128,
         SEQUENCE = UNIT | UNIT_SPAN | UNIT_SET,
+        SINGLE = UNIT | UNIT_SET,
         OPERATOR = COUNT | ALT | CONCAT,
         ATOM_BEGIN = SEQUENCE | GROUP_BEGIN ,
         ATOM_END = SEQUENCE | GROUP_END | COUNT,

StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs

@@ -30,7 +30,9 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Net.Security;
 using System.Runtime.CompilerServices;
+using System.Security.Permissions;
 using System.Text;
 using System.Threading.Tasks;
 
@@ -39,8 +41,8 @@ namespace StgSharp.RegularAnalysis.TextRegex
     public partial class RegexInterpreter
     {
 
-        private static RegexAstNode FlattenAlt(
+        private static List<RegexAstNode> FlattenAlt(
-                                    RegexAstNode root
+                                          RegexAstNode root
         )
         {
             TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(root);
@@ -50,44 +52,42 @@ namespace StgSharp.RegularAnalysis.TextRegex
             while (enumerator.MoveNext())
             {
                 current = enumerator.Current;
-                current.Parent.RemoveChild(current);
+                if (current.EqualityTypeConvert != RegexElementLabel.ALT)
-                current.Parent = RegexAstNode.Empty;
-                if (current.Parent.EqualityTypeConvert == RegexElementLabel.ALT)
-                {
-                    continue;
-                } else
                 {
                     cases.Add(current);
                     enumerator.SkipNode(current);
                 }
+                current.Parent.RemoveChild(current);
             }
-            for (int i = 0; i < cases.Count - 1; i++)
+            return cases;
-            {
-                cases[i].AppendNode(cases[i + 1]);
-                cases[i].Parent = root;
-            }
-            return cases[0];
         }
 
-        private static void MergeConcat(
+        private static bool MergeConcat(
                             RegexAstNode root
         )
         {
             if (root.EqualityTypeConvert != RegexElementLabel.CONCAT) {
-                return;
+                return false;
             }
+            /*
+             *    concat      →      concat
+             *    ╱    ╲             /
+             *  'a'   concat       "abc"
+             *        /    \
+             *       'b'    'c'
+            */
             List<string> buffer = [];
             int length = 0;
             RegexAstNode right = root.Right;
             RegexAstNode left = root.Left;
             if (left.EqualityTypeConvert != RegexElementLabel.UNIT_SPAN &&
                 left.EqualityTypeConvert != RegexElementLabel.UNIT) {
-                return;
+                return false;
             }
             string str = left.Value;
             buffer.Add(str);
             length += str.Length;
-            while (RegexAstNode.IsNullOrEmpty(right) &&
+            while (!RegexAstNode.IsNullOrEmpty(right) &&
                    right.EqualityTypeConvert == RegexElementLabel.CONCAT)
             {
                 left = right.Left;
@@ -100,9 +100,16 @@ namespace StgSharp.RegularAnalysis.TextRegex
                 buffer.Add(str);
                 length += str.Length;
                 root.Right = right.Right;
-
+                right.Parent = RegexAstNode.Empty;
-                // TODO: remove right from tree
                 right = root.Right;
+                right.Parent = root;
+            }
+            if (right.EqualityTypeConvert == RegexElementLabel.UNIT_SPAN ||
+                right.EqualityTypeConvert == RegexElementLabel.UNIT)
+            {
+                root.Right = RegexAstNode.Empty;
+                buffer.Add(right.Value);
+                length += right.Value.Length;
             }
             str = string.Create(length, buffer, static(
                                                 span,
@@ -115,12 +122,12 @@ namespace StgSharp.RegularAnalysis.TextRegex
                     offset += buffer[i].Length;
                 }
             });
-            left = root.Left;
             root.Left = new RegexAstNode(new Token<RegexElementLabel>(str, left.Source.Line,
                                                                       left.Source.Column,
                                                                       str.Length == 1 ?
                                                                       RegexElementLabel.UNIT :
                                                                       RegexElementLabel.UNIT_SPAN));
+            return true;
         }
 
         private static void RotateConcat(
@@ -128,7 +135,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
         )
         {
             RegexAstNode left = root.Left;
-            while (RegexAstNode.IsNullOrEmpty(left) &&
+            while (!RegexAstNode.IsNullOrEmpty(left) &&
                    left.EqualityTypeConvert == RegexElementLabel.CONCAT)
             {
                 /*
@@ -160,5 +167,181 @@ namespace StgSharp.RegularAnalysis.TextRegex
             }
         }
 
+        #region merge alt
+
+        /// <summary>
+        ///   Get union beginning part of cases
+        /// </summary>
+        /// <param name="cases">
+        ///
+        /// </param>
+        /// <returns>
+        ///
+        /// </returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static RegexAstNode MergeAlt(
+                                    List<RegexAstNode> cases
+        )
+        {
+            UnionAlt union = new(0, cases);
+            return union.AsNormal();
+        }
+
+        private class UnionAlt : RegexAstNode
+        {
+
+            private const int MaxLevel = 8;
+
+            private UnionAlt(
+                    int level,
+                    RegexAstNode node
+            )
+                : base(new Token<RegexElementLabel>(string.Empty, 0, 0, 0))
+            {
+                Level = level;
+                Cases = new List<RegexAstNode>();
+                Cases.Add(node.Right ?? Empty);
+                UnionBegin = node;
+                UnionEnd = node;
+            }
+
+            public UnionAlt(
+                   int level,
+                   List<RegexAstNode> c
+            )
+                : base(new Token<RegexElementLabel>(string.Empty, 0, 0, 0))
+            {
+                Cases = c;
+                Level = level;
+            }
+
+            public List<RegexAstNode> Cases { get; set; }
+
+            public RegexAstNode UnionBegin { get; set; }
+
+            public RegexAstNode UnionEnd { get; set; }
+
+            private int Level { get; set; }
+
+            public RegexAstNode AsNormal()
+            {
+                if (Level >= MaxLevel)
+                {
+                    goto build;
+                }
+                merge:
+                List<RegexAstNode> buffer = [];
+                for (int i = 0; i < Cases.Count; i++)
+                {
+                    RegexAstNode current = Cases[i];
+                    int j = 0;
+                    for (; j < buffer.Count; j++)
+                    {
+                        if (CanMerge(current, buffer[j]))
+                        {
+                            // find a case in buffer that can merge with current case,
+                            // then merge them and break
+                            RegexAstNode b = buffer[j];
+                            if (b is not UnionAlt alt)
+                            {
+                                alt = new UnionAlt(Level + 1, b);
+                                alt.AddCase(current);
+                                buffer[j] = alt;
+                            } else
+                            {
+                                alt.AddCase(current);
+                            }
+                            break;
+                        }
+                    }
+                    if (j >= buffer.Count) {
+                        buffer.Add(current);
+                    }
+                }
+                Cases = buffer;
+
+                // TODO problem1 没处理case链表
+                // TODO problem2 没有处理立刻返回策略
+                if (Cases.Count == 1)
+                {
+                    // if there is only one case,
+                    // then we suppose that its contained cases has a same beginning,
+                    // so we attach it to UnionEnd and stay level unchanged
+                    UnionAlt alt = (Cases[0] as UnionAlt)!;
+                    if (IsNullOrEmpty(UnionBegin))
+                    {
+                        UnionBegin = alt.UnionBegin;
+                        UnionEnd = alt.UnionEnd;
+                    } else
+                    {
+                        alt.UnionBegin.Parent = UnionEnd;
+                        UnionEnd.Right = alt.UnionBegin;
+                    }
+
+                    Cases = alt.Cases;
+                    goto merge;
+                }
+
+                build:
+
+                // suppose that from UnionBegin to UnionEnd is a normal form without union
+                for (int i = 0; i < Cases.Count; i++)
+                {
+                    if (Cases[i] is UnionAlt union) {
+                        Cases[i] = union.AsNormal();
+                    }
+                }
+                RegexAstNode branch = new RegexAstNode(new Token<RegexElementLabel>(string.Empty, 0,
+                                                                                    UnionEnd.Source.Column,
+                                                                                    RegexElementLabel.ALT));
+                UnionEnd.Right = branch;
+                branch.Parent = UnionEnd;
+                UnionEnd = branch;
+                for (int i = 0; i < Cases.Count - 1; i++)
+                {
+                    Cases[i].AppendNode(Cases[i + 1]);
+                    Cases[i].Parent = UnionEnd;
+                }
+                UnionEnd.Right = Cases[0];
+                return UnionBegin;
+            }
+
+            private void AddCase(
+                         RegexAstNode _case
+            )
+            {
+                RegexAstNode right = _case.Right;
+                right ??= Empty;
+                Cases.Add(right);
+            }
+
+            private static bool CanMerge(
+                                RegexAstNode origin,
+                                RegexAstNode target
+            )
+            {
+                // suppose that origin is not a union,
+                if (target is UnionAlt alt)
+                {
+                    // then we suppose that alt has only one are same
+                    return origin .EqualityTypeConvert == RegexElementLabel.CONCAT &&
+                           (origin.Left.EqualityTypeConvert & RegexElementLabel.SINGLE) != 0 &&
+                           origin.Left.EqualityTypeConvert == alt.UnionEnd.Left.EqualityTypeConvert &&
+                           origin.Left.Value == alt.UnionEnd.Left.Value;
+                } else
+                {
+                    return origin.EqualityTypeConvert == RegexElementLabel.CONCAT &&
+                           target.EqualityTypeConvert == RegexElementLabel.CONCAT &&
+                           (origin.Left.EqualityTypeConvert & RegexElementLabel.SINGLE) != 0 &&
+                           origin.Left.EqualityTypeConvert == target.Left.EqualityTypeConvert &&
+                           origin.Left.Value == target.Left.Value;
+                }
+            }
+
+        }
+
+        #endregion
+
+        /**/
     }
 }

StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs

@@ -89,18 +89,14 @@ namespace StgSharp.RegularAnalysis.TextRegex
                 if ((token.Flag & RegexElementLabel.SEQUENCE) != 0)
                 {
                     // char or charset
-                    Console.WriteLine($"Unit {token.Value}");
                     _stack.PushOperand(new RegexAstNode(token));
                 } else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0)
                 {
-                    Console.WriteLine($"Group begin {token.Value}");
                     _stack.PushOperator(new RegexAstNode(token));
                     _stack.IncreaseDepth(0);
-                    Console.WriteLine(/**/_stack.OperatorInDepthCount/**/);
                 } else if ((token.Flag & RegexElementLabel.GROUP_END) != 0)
                 {
                     // close all nodes here
-                    Console.WriteLine($"Group end {token.Value}");
                     RegexAstNode op;
                     while (_stack.TryPopOperator(out op)) {
                         ProcessOperator(op);
@@ -120,8 +116,6 @@ namespace StgSharp.RegularAnalysis.TextRegex
                     }
                 } else if ((token.Flag & RegexElementLabel.OPERATOR) != 0)
                 {
-                    Console.Write($"Operator {token.Value} with flag {token.Flag},  ");
-                    Console.WriteLine(/**/_stack.OperatorInDepthCount/**/);
                     if (_stack.OperatorInDepthCount == 0)
                     {
                         // the first operator in stack
@@ -150,7 +144,6 @@ namespace StgSharp.RegularAnalysis.TextRegex
                     }
                 }
             }
-            Console.WriteLine(1);
             if (_stack.Depth == 1)
             {
                 if (_stack.OperandInDepthCount != 1 || _stack.OperatorInDepthCount != 0)
@@ -164,7 +157,6 @@ namespace StgSharp.RegularAnalysis.TextRegex
                     _tree.Root = _stack.PopOperand();
                     return;
                 }
-                throw new InvalidOperationException("Invalid regular expression syntax.");
             }
             throw new InvalidOperationException("Invalid regular expression syntax.");
         }
@@ -178,33 +170,37 @@ namespace StgSharp.RegularAnalysis.TextRegex
             RegexAstNode current;
 
             // phase1: ALT flatten
-            // phase2: CONCAT normalize
+            // phase2: then ALT union
+            // phase3: CONCAT normalize
+            int i = 0;
             while (enumerator.MoveNext())
             {
                 current = enumerator.Current;
+                RegexAstNode cur = current.Next;
                 if (current.EqualityTypeConvert == RegexElementLabel.ALT)
                 {
-                    RegexAstNode union = FlattenAlt(current);
+                    List<RegexAstNode> list = FlattenAlt(current);
-                    current.Right = union;
+                    RegexAstNode union = MergeAlt(list);
-                    current.Left = RegexAstNode.Empty;
+                    current.ReplaceBy(union);
                 } else if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) {
                     RotateConcat(current);
                 }
             }
 
-            // phase3: CONCAT merge
+            // phase4: CONCAT merge
             enumerator.Reset();
             while (enumerator.MoveNext())
             {
                 // suppose all concat is now in normalized form,
                 // we can merge adjacent unit nodes into a single unit node with string value
                 current = enumerator.Current;
-                if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) {
+                if (current.EqualityTypeConvert == RegexElementLabel.CONCAT)
-                    MergeConcat(current);
+                {
+                    if (MergeConcat(current)) {
+                        enumerator.SkipOnce();
+                    }
                 }
             }
-
-            // phase4: ALT union
         }
 
         private void ProcessOperator(
@@ -244,8 +240,8 @@ namespace StgSharp.RegularAnalysis.TextRegex
                 if (_stack.TryPopOperand(out RegexAstNode? _1) &&
                     _stack.TryPopOperand(out RegexAstNode? _2))
                 {
-                    op.Left = _1;
+                    op.Left = _2;
-                    op.Right = _2;
+                    op.Right = _1;
                     _stack.PushOperand(op);
                     _tree.AddNode(op);
                     _tree.AddNode(_1);

StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs

@@ -79,6 +79,10 @@ namespace StgSharp.RegularAnalysis.TextRegex
                         }
                     }
                     throw new InvalidOperationException("Unclosed character set");
+                case '|':
+                    position = pos + 1;
+                    return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
+                                                        RegexElementLabel.ALT);
                 case '(':
 
                     // very comlex here

StgSharpDebug/TestRegexAnalyse.cs

@@ -38,7 +38,7 @@ namespace StgSharp
     public class TestRegexAnalyse
     {
 
-        private const string regex = @"(ab[1-9])+(?<name>abc)";
+        private const string regex = @"abc|abd|abe|abf";
 
         public static void Run()
         {