regex ir opt is almost done

StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs

@@ -0,0 +1,179 @@
+//-----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+// file="RegexAnalyzer.ASTGenerate"
+// Project: StgSharp
+// AuthorGroup: Nitload
+// Copyright (c) Nitload. All rights reserved.
+//     
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//     
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+using StgSharp.RegularAnalysis.Abstraction;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StgSharp.RegularAnalysis.Text
+{
+    public partial class RegexAnalyzer
+    {
+
+        private static AbstractSyntaxTree<RegexAstNode, RegexElementLabel> GenerateAST(
+                                                                           string _source
+        )
+        {
+            CompileStack<RegexAstNode, RegexElementLabel> _stack = new();
+            AbstractSyntaxTree<RegexAstNode, RegexElementLabel> _tree = new();
+            RegexTokenReader reader = new(_source);
+            TokenParser<RegexElementLabel, RegexElementLabel> lexer = reader.Pipe(() => new RegexTokenParser());
+
+            while (lexer.TryReadToken(out Token<RegexElementLabel> token))
+            {
+                // Console.WriteLine(token.Value);
+                if ((token.Flag & RegexElementLabel.SEQUENCE) != 0)
+                {
+                    // char or charset
+                    _stack.PushOperand(new RegexAstNode(token));
+                } else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0)
+                {
+                    _stack.PushOperator(new RegexAstNode(token));
+                    _stack.IncreaseDepth(0);
+                } else if ((token.Flag & RegexElementLabel.GROUP_END) != 0)
+                {
+                    // close all nodes here
+                    RegexAstNode op;
+                    while (_stack.TryPopOperator(out op)) {
+                        ProcessOperator(op);
+                    }
+                    if (_stack.OperandInDepthCount != 1) {
+                        throw new InvalidOperationException();
+                    }
+                    _stack.DecreaseDepth();
+                    op = _stack.PopOperator();
+                    if (op.Source.Flag != RegexElementLabel.GROUP_BEGIN) {
+                        throw new InvalidOperationException("Unmatched group begin and end symbol");
+                    }
+                    if (op.Source.Value.Length > 1)
+                    {
+                        op.Right = _stack.PopOperand();
+                        _stack.PushOperand(op);
+                    }
+                } else if ((token.Flag & RegexElementLabel.OPERATOR) != 0)
+                {
+                    if (_stack.OperatorInDepthCount == 0)
+                    {
+                        // the first operator in stack
+                        _stack.PushOperator(new RegexAstNode(token));
+                    } else
+                    {
+                        // process operator by precedence
+                        while (_stack.TryPeekOperator(out RegexAstNode topOp))
+                        {
+                            (bool Valid, int Value) cmp = RegexAstNode.ComparePrecedence(topOp.Source.Flag, token.Flag);
+
+                            if (!cmp.Valid) {
+                                throw new InvalidOperationException("Operators with same precedence are not supported.");
+                            }
+                            if (cmp.Value < 0)
+                            {
+                                _stack.PopOperator();
+                                ProcessOperator(topOp);
+                                continue;
+                            } else
+                            {
+                                break;
+                            }
+                        }
+                        _stack.PushOperator(new RegexAstNode(token));
+                    }
+                }
+            }
+            if (_stack.Depth == 1)
+            {
+                if (_stack.OperandInDepthCount != 1 || _stack.OperatorInDepthCount != 0)
+                {
+                    while (_stack.TryPopOperator(out RegexAstNode? op)) {
+                        ProcessOperator(op);
+                    }
+                }
+                if (_stack.OperandInDepthCount == 1 && _stack.OperatorInDepthCount == 0)
+                {
+                    _tree.Root = _stack.PopOperand();
+                    return _tree;
+                }
+            }
+            throw new InvalidOperationException("Invalid regular expression syntax.");
+
+
+            void ProcessOperator(
+                 RegexAstNode op
+            )
+            {
+                if (op.EqualityTypeConvert == RegexElementLabel.CONCAT)
+                {
+                    if (_stack.TryPopOperand(out RegexAstNode? _1) &&
+                        _stack.TryPopOperand(out RegexAstNode? _2))
+                    {
+                        op.Left = _2;
+                        op.Right = _1;
+                        _stack.PushOperand(op);
+                        _ = _tree.AddNode(op);
+                        _ = _tree.AddNode(_1);
+                        _ = _tree.AddNode(_2);
+                    } else
+                    {
+                        throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
+                    }
+                } else if (op.EqualityTypeConvert == RegexElementLabel.COUNT)
+                {
+                    if (_stack.TryPopOperand(out RegexAstNode? _1))
+                    {
+                        op.Left = _1;
+                        _stack.PushOperand(op);
+                        _ = _tree.AddNode(_1);
+                        _ = _tree.AddNode(op);
+                    } else
+                    {
+                        throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
+                    }
+                } else if (op.EqualityTypeConvert == RegexElementLabel.ALT)
+                {
+                    if (_stack.TryPopOperand(out RegexAstNode? _1) &&
+                        _stack.TryPopOperand(out RegexAstNode? _2))
+                    {
+                        op.Left = _2;
+                        op.Right = _1;
+                        _stack.PushOperand(op);
+                        _ = _tree.AddNode(op);
+                        _ = _tree.AddNode(_1);
+                        _ = _tree.AddNode(_2);
+                    } else
+                    {
+                        throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
+                    }
+                }
+            }
+        }
+
+    }
+}

StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs

@@ -1,6 +1,6 @@
 //-----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-// file="RegexInterpreter.ASTOptimize"
+// file="RegexAnalyzer.ASTOptimize"
 // Project: StgSharp
 // AuthorGroup: Nitload
 // Copyright (c) Nitload. All rights reserved.
@@ -156,6 +156,67 @@ namespace StgSharp.RegularAnalysis.Text
             return true;
         }
 
+        private static void OptimizeTree(
+                     AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree
+        )
+        {
+            // phase0: scan container and remove no parent nodes
+            _ = tree.AllNodes.RemoveWhere(node => node.Parent == null && node != tree.Root);
+            TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(tree.Root);
+            RegexAstNode current;
+
+            // phase1: ALT flatten
+            // phase2: CONCAT normalize
+            int i = 0;
+            List<RegexAstNode> altNodes = [];
+            while (enumerator.MoveNext())
+            {
+                current = enumerator.Current;
+                if (current.EqualityTypeConvert == RegexElementLabel.ALT)
+                {
+                    RegexAstNode union = FlattenAlt(current);
+                    altNodes.Add(current);
+                    current.Right = union;
+                    current.Left = RegexAstNode.Empty;
+                } else if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) {
+                    RotateConcat(current);
+                }
+            }
+
+            List<RegexAstNode> cases = [];
+
+            // phase3: then ALT union
+            foreach (RegexAstNode item in altNodes)
+            {
+                RegexAstNode _case = item.Right;
+                do
+                {
+                    cases.Add(_case);
+                    _case = _case.Next;
+                } while (!RegexAstNode.IsNullOrEmpty(_case));
+                RegexAstNode newAlt = MergeAlt(cases);
+                item.ReplaceBy(newAlt);
+            }
+
+            // return;
+
+            // phase4: CONCAT merge
+            enumerator.Reset();
+            while (enumerator.MoveNext())
+            {
+                // suppose all concat is now in normalized form,
+                // we can merge adjacent unit nodes into a single unit node with string value
+                current = enumerator.Current;
+                if (current.EqualityTypeConvert == RegexElementLabel.CONCAT)
+                {
+                    if (MergeConcat(current)) {
+                        enumerator.SkipOnce();
+                    }
+                }
+            }
+
+        }
+
         private static void RotateConcat(
                             RegexAstNode root
         )

StgSharp.RegularAnalysis/Text/RegexAnalyzer.IRGenerate.cs

@@ -1,30 +1,27 @@
 //-----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-// file="RegexInterpreter.IRGenerate.cs"
+// file="RegexInterpreter.IRGenerate"
 // Project: StgSharp
 // AuthorGroup: Nitload
 // Copyright (c) Nitload. All rights reserved.
 //     
-// Permission is hereby granted, free of charge, to any person 
+// Permission is hereby granted, free of charge, to any person obtaining a copy
-// obtaining a copy of this software and associated documentation 
+// of this software and associated documentation files (the "Software"), to deal
-// files (the “Software”), to deal in the Software without restriction, 
+// in the Software without restriction, including without limitation the rights
-// including without limitation the rights to use, copy, modify, merge,
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// publish, distribute, sublicense, and/or sell copies of the Software, 
+// copies of the Software, and to permit persons to whom the Software is
-// and to permit persons to whom the Software is furnished to do so, 
+// furnished to do so, subject to the following conditions:
-// subject to the following conditions:
 //
-// The above copyright notice and 
+// The above copyright notice and this permission notice shall be included in all
-// this permission notice shall be included in all copies 
+// copies or substantial portions of the Software.
-// or substantial portions of the Software.
 //
-// THE SOFTWARE IS PROVIDED “AS IS”, 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// ARISING FROM, OUT OF OR IN CONNECTION WITH 
+// SOFTWARE.
-// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 //     
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
@@ -46,7 +43,9 @@ namespace StgSharp.RegularAnalysis.Text
     public partial class RegexAnalyzer
     {
 
-        internal static List<RegexIR> GenerateIR(AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree)
+        internal static List<RegexIR> GenerateIR(
+                                      AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree
+        )
         {
             return GenerateIRRecursion(tree.Root, 0).ToList();
         }
@@ -58,7 +57,8 @@ namespace StgSharp.RegularAnalysis.Text
                             int min,
                             int max,
                             bool isGreedy,
-                            RegexAstNode right)
+                            RegexAstNode right
+        )
         {
             bool is_operator = !RegexAstNode.IsNullOrEmpty(right) &&
                                (right.Source.Flag & RegexElementLabel.OPERATOR) != 0;
@@ -88,7 +88,10 @@ namespace StgSharp.RegularAnalysis.Text
             }
         }
 
-        private static RegexIRGenerator GenerateIRRecursion(RegexAstNode node, int depth)
+        private static RegexIRGenerator GenerateIRRecursion(
+                                        RegexAstNode node,
+                                        int depth
+        )
         {
 #pragma warning disable IDE0010
             if (depth > 1024)
@@ -117,7 +120,7 @@ namespace StgSharp.RegularAnalysis.Text
 
                     RegexIRGenerator groupIR = new();
                     rightIR = GenerateIRRecursion(right, curDepth + 1);
-                    groupIR.EmitMatchComplex(rightIR.Count);
+                    groupIR.EmitTry(rightIR.Count);
                     groupIR.EmitIRStream(rightIR);
                     groupIR.EmitCondition(2, 1);
                     groupIR.EmitPop(false);
@@ -251,7 +254,7 @@ namespace StgSharp.RegularAnalysis.Text
                         RegexIRGenerator cur_ir = casesIRList[^case_count];
                         ir_count -= cur_ir.Count;
                         case_count--;
-                        alt.EmitMatchComplex(cur_ir.Count);
+                        alt.EmitTry(cur_ir.Count);
                         alt.EmitIRStream(cur_ir);
                         alt.EmitCondition(1/* need to calc here */, 2);
                         alt.EmitPop(true);

StgSharp.RegularAnalysis/Text/RegexAnalyzer.IROptimize.cs

@@ -1,6 +1,6 @@
 //-----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-// file="RegexInterpreter.IROptimize"
+// file="RegexAnalyzer.IROptimize"
 // Project: StgSharp
 // AuthorGroup: Nitload
 // Copyright (c) Nitload. All rights reserved.
@@ -40,80 +40,125 @@ namespace StgSharp.RegularAnalysis.Text
     public partial class RegexAnalyzer
     {
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static void ScanAndOptimizeIR(
-        internal static RegexInfo ScanAndOptimizeIR(
+                             List<RegexIR> baseIR,
-                                  List<RegexIR> baseIR,
+                             out List<RegexIR> optIR
-                                  out List<RegexIR> optIR
         )
         {
-            int index = 0;
+            int opt_count;
             List<RegexIR> origin_ir = baseIR;
-            Span<RegexIR> origin_span = CollectionsMarshal.AsSpan(baseIR);
+            List<RegexIR> opt_ir = new(origin_ir.Count);
-            List<RegexIR> opt_ir = new(baseIR.Count);
+            RegexIR[] opt_window = new RegexIR[8];
-            CapacityFixedList<RegexIR> opt_window = new(8);
+            Span<RegexIR> opt_win_sp = opt_window;
-            Span<RegexIR> opt_win_sp = [.. opt_window];
+            opt_cycle:
-            RegexInfo info = new();
+            opt_count = 0;
+            int index = 0;
+            Span<RegexIR> origin_span = CollectionsMarshal.AsSpan(origin_ir);
+
             do
             {
                 int process_count = 0;
                 int last = int.Min(origin_span.Length, index + 8);
                 origin_span[index..last].CopyTo(opt_win_sp);
 
-                do
+                int result;
+                if ((result = TryExpandPlus(opt_win_sp, opt_ir)) >= 0)
                 {
-                    int result = 0;
+                    process_count++;
-                    if ((result = TryExpandPlus(opt_win_sp, opt_ir)) >= 0)
+                    if (result > 0)
                     {
-                        process_count++;
+                        RegexCountComplexIR cc = Unsafe.As<RegexCountComplexIR>(opt_win_sp[0]);
-                        if (result > 0)
+                        Span<RegexIR> sp = origin_span.Slice(index + 1, result);
-                        {
+                        TryExpandComplexPlus(cc.IsGreedy, sp, opt_ir);
-                            RegexCountComplexIR cc = Unsafe.As<RegexCountComplexIR>(opt_win_sp[0]);
+                        process_count += result;
-                            Span<RegexIR> sp = origin_span.Slice(index + 1, result);
-                            TryExpandComplexPlus(cc.IsGreedy, sp, opt_ir);
-                            process_count += result;
-                        }
-                    } else if ((result = TryOptDotCountCount(opt_win_sp, opt_ir)) >= 0)
-                    {
-                        process_count++;
-                    } else if (TryFlattenCount(opt_win_sp, opt_ir)) {
-                        process_count++;
                     }
-                } while (false);
+                    opt_count++;
+                } else if (TryOptDotCountCount(opt_win_sp, opt_ir) >= 0)
+                {
+                    process_count += 2;
+                    opt_count++;
+                } else if (TryFlattenCount(opt_win_sp, opt_ir) >= 0)
+                {
+                    process_count++;
+                    opt_count++;
+                } else if ((result = TryExpandOnce(opt_win_sp, opt_ir)) >= 0)
+                {
+                    process_count++;
+                    if (result > 0)
+                    {
+                        RegexCountComplexIR cc = Unsafe.As<RegexCountComplexIR>(opt_win_sp[0]);
+                        Span<RegexIR> sp = origin_span.Slice(index + 1, result);
+                        TryExpandComplex(sp, opt_ir, true);
+                        process_count += result;
+                    }
+                    opt_count++;
+                } else if (opt_win_sp[0] is RegexTryIR{ IsFullyOptimized:false} try_ir)
+                {
+                    process_count++;
+                    int cmd_cnt = try_ir.CommandCount;
+                    Span<RegexIR> try_sp = origin_span.Slice(index + 1, cmd_cnt);
+                    ScanAndOptimizeIR(try_sp, out List<RegexIR> opt_try);
+                    RegexTryIR new_try = new(opt_try.Count)
+                    {
+                        IsFullyOptimized = true
+                    };
+                    opt_ir.Add(new_try);
+                    opt_ir.AddRange(opt_try);
+                    process_count += cmd_cnt;
+                    opt_count++;
+                } else
+                {
+                    opt_ir.Add(opt_win_sp[0]);
+                    process_count++;
+                }
 
                 index += process_count;
-            } while (index < baseIR.Count);
+            } while (index < origin_ir.Count);
+            if (opt_count > 0)
+            {
+                // need to swap the origin_ir and opt_ir, and clear the opt_ir for next round
+                if (ReferenceEquals(origin_ir, baseIR))
+                {
+                    origin_ir = new(opt_ir.Count);
+                }
+                (origin_ir, opt_ir) = (opt_ir, origin_ir);
+                opt_ir.Clear();
+                goto opt_cycle;
+            }
             optIR = opt_ir;
-            return info;
         }
 
-        private static RegexInfo ScanAndOptimizeIR(
+        private static void ScanAndOptimizeIR(
-                                 Span<RegexIR> baseIR,
+                            Span<RegexIR> baseIR,
-                                 List<RegexIR> opt_ir
+                            out List<RegexIR> opt_ir
         )
         {
             List<RegexIR> originList = new(baseIR.Length);
             CollectionsMarshal.SetCount(originList, baseIR.Length);
             baseIR.CopyTo(CollectionsMarshal.AsSpan(originList));
-            return ScanAndOptimizeIR(originList, out opt_ir);
+            ScanAndOptimizeIR(originList, out opt_ir);
         }
 
-        private static bool TryFlattenCount(
+        private static int TryFlattenCount(
-                            Span<RegexIR> window,
+                           Span<RegexIR> window,
-                            List<RegexIR> opt
+                           List<RegexIR> opt
         )
         {
             const int falttern_size = 32;
             RegexIR ir = window[0];
             if (ir is not RegexCountSeqIR seq) {
-                return false;
+                return -1;
+            }
+            int min = seq.Min;
+            int max = seq.Max;
+            if (min == 0) {
+                return -1;
             }
             string pattern = seq.Sequence;
             int pattern_size = pattern.Length;
             if (pattern_size > falttern_size) {
                 opt.Add(ir);
             }
-            int min = seq.Min;
-            int max = seq.Max;
             int min_expand = falttern_size / pattern_size;
             min_expand = int.Min(min_expand, min);
             string expand = string.Concat(Enumerable.Repeat(pattern, min_expand));
@@ -121,10 +166,10 @@ namespace StgSharp.RegularAnalysis.Text
             int new_rest = min % min_expand;
             if (new_min == 1)
             {
-                opt.Add(new RegexMatchSeqIR(pattern));
+                opt.Add(new RegexMatchSeqIR(expand));
             } else
             {
-                opt.Add(new RegexCountSeqIR(new_min, new_min, seq.IsGreedy, pattern));
+                opt.Add(new RegexCountSeqIR(new_min, new_min, seq.IsGreedy, expand));
             }
             if (new_rest > 0)
             {
@@ -134,7 +179,7 @@ namespace StgSharp.RegularAnalysis.Text
             int rest_count = max - min;
             rest_count = int.Max(rest_count, -1);
             opt.Add(new RegexCountSeqIR(0, rest_count, seq.IsGreedy, pattern));
-            return true;
+            return 0;
         }
 
         #region dot count count
@@ -176,17 +221,89 @@ namespace StgSharp.RegularAnalysis.Text
 
         }
 
+        #region expand once
+
+        private static void TryExpandComplex(
+                            Span<RegexIR> complex,
+                            List<RegexIR> opt,
+                            bool is_once
+        )
+        {
+            if (!is_once) {
+                opt.Add(new RegexTryIR(complex.Length));
+            }
+
+            opt.AddRange(complex);
+        }
+
+        private static int TryExpandOnce(
+                           Span<RegexIR> window,
+                           List<RegexIR> opt
+        )
+        {
+            RegexIR ir = window[0];
+            if (ir is IRegexCount { Min: 1, Max: 1 })
+            {
+                string pattern;
+                switch (ir.Command & RegexIRCommand.Count)
+                {
+                    case RegexIRCommand.Count_Complex:
+                        RegexCountComplexIR complex = Unsafe.As<RegexCountComplexIR>(ir);
+                        return complex.CommandCount;
+                    case RegexIRCommand.Count_Seq:
+                        RegexCountSeqIR seq = Unsafe.As<RegexCountSeqIR>(ir);
+                        pattern = seq.Sequence;
+                        opt.Add(new RegexMatchSeqIR(pattern));
+                        return 0;
+                    case RegexIRCommand.Count_Set:
+                        RegexCountSetIR set = Unsafe.As<RegexCountSetIR>(ir);
+                        pattern = set.CharSet;
+                        opt.Add(new RegexMatchSetIR(pattern));
+                        return 0;
+                    default:
+                        return -1;
+                }
+            } else if (ir is IRegexCount { Min: 0, Max: 1 })
+            {
+                string pattern;
+                switch (ir.Command & RegexIRCommand.Count)
+                {
+                    case RegexIRCommand.Count_Complex:
+                        RegexCountComplexIR complex = Unsafe.As<RegexCountComplexIR>(ir);
+                        return complex.CommandCount;
+                    case RegexIRCommand.Count_Seq:
+                        RegexCountSeqIR seq = Unsafe.As<RegexCountSeqIR>(ir);
+                        pattern = seq.Sequence;
+                        opt.Add(new RegexTryIR(1));
+                        opt.Add(new RegexMatchSeqIR(pattern));
+                        return 0;
+                    case RegexIRCommand.Count_Set:
+                        opt.Add(new RegexTryIR(1));
+                        RegexCountSetIR set = Unsafe.As<RegexCountSetIR>(ir);
+                        pattern = set.CharSet;
+                        opt.Add(new RegexMatchSetIR(pattern));
+                        return 0;
+                    default:
+                        return -1;
+                }
+            } else
+            {
+                return -1;
+            }
+        }
+
+            #endregion
+
+
         #region expand plus
 
-        private static bool TryExpandComplexPlus(
+        private static void TryExpandComplexPlus(
                             bool is_greedy,
                             Span<RegexIR> complex,
                             List<RegexIR> opt
         )
         {
-            int count = complex.Length;
+            ScanAndOptimizeIR(complex, out List<RegexIR>? part_opt);
-            List<RegexIR> part_opt = new(count);
-            RegexInfo info = ScanAndOptimizeIR(complex, part_opt);
             opt.AddRange(part_opt);
             RegexCountComplexIR c_ir = new(0, -1, is_greedy, part_opt.Count)
             {
@@ -194,7 +311,6 @@ namespace StgSharp.RegularAnalysis.Text
             };
             opt.Add(c_ir);
             opt.AddRange(part_opt);
-            return true;
         }
 
         /// <summary>

StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs

@@ -0,0 +1,94 @@
+//-----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+// file="RegexAnalyzer"
+// Project: StgSharp
+// AuthorGroup: Nitload
+// Copyright (c) Nitload. All rights reserved.
+//     
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//     
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+using StgSharp.RegularAnalysis.Abstraction;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection.Emit;
+using System.Runtime.CompilerServices;
+using System.Text;
+
+using System.Threading.Tasks;
+
+namespace StgSharp.RegularAnalysis.Text
+{
+    public record TextRegexSource
+    {
+
+        internal TextRegexSource(
+                 string pattern,
+                 AbstractSyntaxTree<RegexAstNode, RegexElementLabel> ast,
+                 List<RegexIR> base_ir,
+                 List<RegexIR> ir,
+                 Exception? analyseException = null
+        )
+        {
+            Pattern = pattern;
+            Ast = ast;
+            IRs = ir;
+            BaseIR = base_ir;
+            AnalyseException = analyseException;
+        }
+
+        public Exception? AnalyseException { get; set; }
+
+        public string Pattern { get; }
+
+        internal AbstractSyntaxTree<RegexAstNode, RegexElementLabel> Ast { get; }
+
+        internal List<RegexIR> BaseIR { get; }
+
+        internal List<RegexIR> IRs { get; }
+
+    }
+
+    public static partial class RegexAnalyzer
+    {
+
+        public static TextRegexSource Analyze(
+                                      string regex
+        )
+        {
+            // RegexAnalyzer interpreter = new();
+            // interpreter._source = regex;
+            try
+            {
+                AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree = GenerateAST(regex);
+                OptimizeTree(tree);
+                List<RegexIR> base_ir = GenerateIR(tree);
+                ScanAndOptimizeIR(base_ir, out List<RegexIR> opt_ir);
+                return new TextRegexSource(regex, tree, base_ir, opt_ir);
+            }
+            catch (Exception ex)
+            {
+                return new TextRegexSource(regex, null, null, null, ex);
+            }
+        }
+
+    }
+}

StgSharp.RegularAnalysis/Text/RegexIR.cs

@@ -312,16 +312,14 @@ namespace StgSharp.RegularAnalysis.Text
 
     }
 
-    internal sealed class RegexMatchComplexIR(
+    internal sealed class RegexTryIR(
                           int count
     ) : RegexIR, IRegexComplex
     {
 
         public bool IsFullyOptimized { get; set; }
 
-        public int CountOfCommand { get; } = count;
+        public int CommandCount { get; set; } = count;
-
-        public int CommandCount { get; set; }
 
         internal override RegexIRCommand Command => RegexIRCommand.Try;
 
@@ -329,7 +327,7 @@ namespace StgSharp.RegularAnalysis.Text
                                  int lineNum
         )
         {
-            return $"[{lineNum}]\t TRY {CountOfCommand}";
+            return $"[{lineNum}]\t TRY {CommandCount}";
         }
 
     }

StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs

@@ -134,11 +134,11 @@ namespace StgSharp.RegularAnalysis.Text
             Count++;
         }
 
-        public void EmitMatchComplex(
+        public void EmitTry(
                     int count
         )
         {
-            _tail.Add(new RegexMatchComplexIR(count));
+            _tail.Add(new RegexTryIR(count));
             Count++;
         }
 

StgSharp.RegularAnalysis/Text/RegexInterpreter.cs

@@ -1,275 +0,0 @@
-//-----------------------------------------------------------------------
-// -----------------------------------------------------------------------
-// file="RegexInterpreter"
-// Project: StgSharp
-// AuthorGroup: Nitload
-// Copyright (c) Nitload. All rights reserved.
-//     
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-//     
-// -----------------------------------------------------------------------
-// -----------------------------------------------------------------------
-using StgSharp.RegularAnalysis.Abstraction;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Reflection.Emit;
-using System.Runtime.CompilerServices;
-using System.Text;
-
-using System.Threading.Tasks;
-
-namespace StgSharp.RegularAnalysis.Text
-{
-    public partial class RegexAnalyzer
-    {
-
-        private AbstractSyntaxTree<RegexAstNode, RegexElementLabel> _tree = new();
-        private CompileStack<RegexAstNode, RegexElementLabel> _stack = new();
-        private Exception _interpreteException;
-        private string _source;
-
-        private RegexAnalyzer() { }
-
-        internal AbstractSyntaxTree<RegexAstNode, RegexElementLabel> Tree
-        {
-            get
-            {
-                if (_interpreteException != null) {
-                    throw new InvalidOperationException("The regex interpretation failed.", _interpreteException);
-                }
-                return _tree;
-            }
-        }
-
-<<<<<<< HEAD:StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs
-        public static RegexInterpreter Analyze(string regex)
-=======
-        public static RegexAnalyzer Analyze(
-                                       string regex
-        )
->>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/RegexInterpreter.cs
-        {
-            RegexAnalyzer interpreter = new();
-            interpreter._source = regex;
-            bool isSuccess;
-            try
-            {
-                interpreter.Analyze();
-                interpreter.OptimizeTree();
-                isSuccess = true;
-            }
-            catch (Exception ex)
-            {
-                interpreter._interpreteException = ex;
-                isSuccess = false;
-            }
-            return interpreter;
-        }
-
-        private void Analyze()
-        {
-            RegexTokenReader reader = new(_source);
-            TokenParser<RegexElementLabel, RegexElementLabel> lexer = reader.Pipe(() => new RegexTokenParser());
-
-            while (lexer.TryReadToken(out Token<RegexElementLabel> token))
-            {
-                // Console.WriteLine(token.Value);
-                if ((token.Flag & RegexElementLabel.SEQUENCE) != 0)
-                {
-                    // char or charset
-                    _stack.PushOperand(new RegexAstNode(token));
-                } else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0)
-                {
-                    _stack.PushOperator(new RegexAstNode(token));
-                    _stack.IncreaseDepth(0);
-                } else if ((token.Flag & RegexElementLabel.GROUP_END) != 0)
-                {
-                    // close all nodes here
-                    RegexAstNode op;
-                    while (_stack.TryPopOperator(out op)) {
-                        ProcessOperator(op);
-                    }
-                    if (_stack.OperandInDepthCount != 1) {
-                        throw new InvalidOperationException();
-                    }
-                    _stack.DecreaseDepth();
-                    op = _stack.PopOperator();
-                    if (op.Source.Flag != RegexElementLabel.GROUP_BEGIN) {
-                        throw new InvalidOperationException("Unmatched group begin and end symbol");
-                    }
-                    if (op.Source.Value.Length > 1)
-                    {
-                        op.Right = _stack.PopOperand();
-                        _stack.PushOperand(op);
-                    }
-                } else if ((token.Flag & RegexElementLabel.OPERATOR) != 0)
-                {
-                    if (_stack.OperatorInDepthCount == 0)
-                    {
-                        // the first operator in stack
-                        _stack.PushOperator(new RegexAstNode(token));
-                    } else
-                    {
-                        // process operator by precedence
-                        while (_stack.TryPeekOperator(out RegexAstNode topOp))
-                        {
-                            (bool Valid, int Value) cmp = RegexAstNode.ComparePrecedence(topOp.Source.Flag, token.Flag);
-
-                            if (!cmp.Valid) {
-                                throw new InvalidOperationException("Operators with same precedence are not supported.");
-                            }
-                            if (cmp.Value < 0)
-                            {
-                                _stack.PopOperator();
-                                ProcessOperator(topOp);
-                                continue;
-                            } else
-                            {
-                                break;
-                            }
-                        }
-                        _stack.PushOperator(new RegexAstNode(token));
-                    }
-                }
-            }
-            if (_stack.Depth == 1)
-            {
-                if (_stack.OperandInDepthCount != 1 || _stack.OperatorInDepthCount != 0)
-                {
-                    while (_stack.TryPopOperator(out RegexAstNode? op)) {
-                        ProcessOperator(op);
-                    }
-                }
-                if (_stack.OperandInDepthCount == 1 && _stack.OperatorInDepthCount == 0)
-                {
-                    _tree.Root = _stack.PopOperand();
-                    return;
-                }
-            }
-            throw new InvalidOperationException("Invalid regular expression syntax.");
-        }
-
-        private void OptimizeTree()
-        {
-            // phase0: scan container and remove no parent nodes
-            _ = _tree.AllNodes.RemoveWhere(node => node.Parent == null && node != _tree.Root);
-            TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(_tree.Root);
-            RegexAstNode current;
-
-            // phase1: ALT flatten
-            // phase2: CONCAT normalize
-            int i = 0;
-            List<RegexAstNode> altNodes = [];
-            while (enumerator.MoveNext())
-            {
-                current = enumerator.Current;
-                if (current.EqualityTypeConvert == RegexElementLabel.ALT)
-                {
-                    RegexAstNode union = FlattenAlt(current);
-                    altNodes.Add(current);
-                    current.Right = union;
-                    current.Left = RegexAstNode.Empty;
-                } else if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) {
-                    RotateConcat(current);
-                }
-            }
-
-            List<RegexAstNode> cases = [];
-
-            // phase3: then ALT union
-            foreach (RegexAstNode item in altNodes)
-            {
-                RegexAstNode _case = item.Right;
-                do
-                {
-                    cases.Add(_case);
-                    _case = _case.Next;
-                } while (!RegexAstNode.IsNullOrEmpty(_case));
-                RegexAstNode newAlt = MergeAlt(cases);
-                item.ReplaceBy(newAlt);
-            }
-
-            // return;
-
-            // phase4: CONCAT merge
-            enumerator.Reset();
-            while (enumerator.MoveNext())
-            {
-                // suppose all concat is now in normalized form,
-                // we can merge adjacent unit nodes into a single unit node with string value
-                current = enumerator.Current;
-                if (current.EqualityTypeConvert == RegexElementLabel.CONCAT)
-                {
-                    if (MergeConcat(current)) {
-                        enumerator.SkipOnce();
-                    }
-                }
-            }
-
-            // TODO enum AST again , do ALT tail merge and simple ALT front merge
-        }
-
-        private void ProcessOperator(RegexAstNode op)
-        {
-            if (op.EqualityTypeConvert == RegexElementLabel.CONCAT)
-            {
-                if (_stack.TryPopOperand(out RegexAstNode? _1) && _stack.TryPopOperand(out RegexAstNode? _2))
-                {
-                    op.Left = _2;
-                    op.Right = _1;
-                    _stack.PushOperand(op);
-                    _tree.AddNode(op);
-                    _tree.AddNode(_1);
-                    _tree.AddNode(_2);
-                } else
-                {
-                    throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
-                }
-            } else if (op.EqualityTypeConvert == RegexElementLabel.COUNT)
-            {
-                if (_stack.TryPopOperand(out RegexAstNode? _1))
-                {
-                    op.Left = _1;
-                    _stack.PushOperand(op);
-                    _tree.AddNode(_1);
-                    _tree.AddNode(op);
-                } else
-                {
-                    throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
-                }
-            } else if (op.EqualityTypeConvert == RegexElementLabel.ALT)
-            {
-                if (_stack.TryPopOperand(out RegexAstNode? _1) && _stack.TryPopOperand(out RegexAstNode? _2))
-                {
-                    op.Left = _2;
-                    op.Right = _1;
-                    _stack.PushOperand(op);
-                    _tree.AddNode(op);
-                    _tree.AddNode(_1);
-                    _tree.AddNode(_2);
-                } else
-                {
-                    throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
-                }
-            }
-        }
-
-    }
-}

StgSharp.RegularAnalysis/Text/TextRegex.cs

@@ -28,10 +28,6 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-<<<<<<< HEAD:StgSharp.RegularAnalysis/TextRegex/TextRegex.cs
-=======
-using System.Runtime.CompilerServices;
->>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/TextRegex.cs
 using System.Text;
 using System.Threading.Tasks;
 
@@ -39,22 +35,19 @@ namespace StgSharp.RegularAnalysis.Text
 {
     public abstract class TextRegex
     {
-<<<<<<< HEAD:StgSharp.RegularAnalysis/TextRegex/TextRegex.cs
-        public ref struct Match
-        {
-            private ReadOnlySpan<char> _sequence;
-
-
-        }
-    }
-=======
 
         public abstract bool IsMatch(
                              ReadOnlySpan<char> text
         );
 
+        public ref struct Match
+        {
+
+            private ReadOnlySpan<char> _sequence;
+
+        }
+
     }
 
     public ref struct MatchResult { }
->>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/TextRegex.cs
 }

StgSharpDebug/TestRegexAnalyse.cs

@@ -40,6 +40,7 @@ namespace StgSharp
     {
 
         private static string[] regexSource = [
+
             @"http|https|ftp|ftps",
             @"abc[0-9]x|abc[0-9]y|abd[0-9]x|abd[0-9]y|xyz",
             @"get_name|get_value|get_type|set_name|set_value|set_type",
@@ -51,8 +52,8 @@ namespace StgSharp
         {
             foreach (string regex in regexSource)
             {
-                RegexAnalyzer interpreter = RegexAnalyzer.Analyze(regex);
+                TextRegexSource interpreterSource = RegexAnalyzer.Analyze(regex);
-                AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree = interpreter.Tree;
+                AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree = interpreterSource.Ast;
                 Console.WriteLine($"result of \"{regex}\"");
                 TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(tree);
                 while (enumerator.MoveNext())
@@ -61,11 +62,16 @@ namespace StgSharp
                     Console.Write(new string('|', enumerator.Depth));
                     Console.WriteLine($"\t{node.Source.Value}\t{node.Source.Flag}");
                 }
-                List<RegexIR> ir = RegexAnalyzer.GenerateIR(tree);
                 Console.WriteLine("-----IR-----");
+                List<RegexIR> ir = interpreterSource.BaseIR;
                 for (int i = 0; i < ir.Count; i++) {
                     Console.WriteLine(ir[i].AsCommand(i));
                 }
+                Console.WriteLine("-----Optimized once-----");
+                List<RegexIR> opt_ir = interpreterSource.IRs;
+                for (int i = 0; i < opt_ir.Count; i++) {
+                    Console.WriteLine(opt_ir[i].AsCommand(i));
+                }
                 Console.WriteLine();
                 Console.WriteLine();
                 Console.WriteLine();

future/ngra-regex-ir.md

@@ -59,6 +59,8 @@ COUNT_SEQ(a,0,+inf,greedy)
 
 3. a{1} -> a
 
+完成
+
 4. .\*a\*特征，优化为.*
 
 完成，最优先
@@ -67,6 +69,4 @@ COUNT_SEQ(a,0,+inf,greedy)
 MATCH_SEQ(aa...m个a....aa)
 COUNT_SEQ(a,0,n,greedy)
 
-6. .*?a*$
+完成
-
-7. 对于一般的.*?，应该将后续的所有东西全都放进一个巨大的try，code gen时使用最慢的一位一位offset直到直到第一个匹配rest的东西