regex interpreter now can build partial correct AST (i forget groups), but in bad shape. building an optimizer.

StgSharp.RegularAnalysis/Abstraction/AbstractSyntaxTree.cs

@@ -29,6 +29,7 @@ using StgSharp.Collections;
 
 using System;
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using System.Text;
 using System.Xml.Schema;
 
@@ -38,6 +39,8 @@ namespace StgSharp.RegularAnalysis.Abstraction
         where TType : unmanaged
     {
 
+        private HashSet<TNode> _allToken = new();
+
         private TNode _root;
 
         public int Count
@@ -45,17 +48,31 @@ namespace StgSharp.RegularAnalysis.Abstraction
             get { return _allToken.Count; }
         }
 
-        public TNode Root => _root;
+        public TNode Root
+        {
+            get;
+            internal set
+            {
+                if (_allToken.Contains(value)) {
+                    field = value;
+                }
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool AddNode(
+                    TNode node
+        )
+        {
+            return _allToken.Add(node);
+        }
 
         public bool Contains(
                     TNode node
         )
         {
-            return _allToken<TLabel>.Contains(node);
+            return _allToken.Contains(node);
         }
 
-        private List<TNode> _allToken<TLabel> = 
-new();
-
     }
 }

StgSharp.RegularAnalysis/Abstraction/CompileStack.cs

@@ -47,7 +47,7 @@ namespace StgSharp.RegularAnalysis.Abstraction
 
         public CompileStack()
         {
-            _depthMarks.Push(new DepthMark(0, 0));
+            _depthMarks.Push(new DepthMark(0, 0, 0));
         }
 
         public int OperandInDepthCount
@@ -59,7 +59,7 @@ namespace StgSharp.RegularAnalysis.Abstraction
         public int OperatorInDepthCount
         {
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get => _operands.Count - _depthMarks.Peek().OperandPos;
+            get => _operators.Count - _depthMarks.Peek().OperandPos;
         }
 
         public int Depth
@@ -108,6 +108,20 @@ namespace StgSharp.RegularAnalysis.Abstraction
             return false;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool TryPeekOperator(
+                    out TNode op
+        )
+        {
+            if (OperatorInDepthCount > 0)
+            {
+                op = _operators.Peek();
+                return true;
+            }
+            op = TNode.Empty;
+            return false;
+        }
+
             #endregion
 
         #region operand
@@ -117,7 +131,7 @@ namespace StgSharp.RegularAnalysis.Abstraction
         )
         {
             if (OperatorInDepthCount > 0) {
-                return _operators.Pop();
+                return _operands.Pop();
             }
             return TNode.Empty;
         }
@@ -127,7 +141,7 @@ namespace StgSharp.RegularAnalysis.Abstraction
                     TNode operatorNode
         )
         {
-            _operators.Push(operatorNode);
+            _operands.Push(operatorNode);
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -144,6 +158,20 @@ namespace StgSharp.RegularAnalysis.Abstraction
             return false;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool TryPeekOperand(
+                    out TNode op
+        )
+        {
+            if (OperandInDepthCount > 0)
+            {
+                op = _operands.Peek();
+                return true;
+            }
+            op = TNode.Empty;
+            return false;
+        }
+
             #endregion
 
         #region depth

StgSharp.RegularAnalysis/Abstraction/Token.cs

@@ -45,6 +45,14 @@ namespace StgSharp.RegularAnalysis.Abstraction
         public readonly string Value;
         public readonly TLabel Flag;
 
+        public Token()
+        {
+            Column = 0;
+            Line = 0;
+            Value = string.Empty;
+            Flag = default;
+        }
+
         public Token(
                string chars,
                int lineNumber,

StgSharp.RegularAnalysis/StgSharp.RegularAnalysis.csproj

@@ -4,6 +4,7 @@
     <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+	<LangVersion>preview</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>

StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs

@@ -28,13 +28,15 @@
 using StgSharp.RegularAnalysis.Abstraction;
 using System;
 using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading.Tasks;
 
 namespace StgSharp.RegularAnalysis.TextRegex
 {
-    internal class RegexAstNode : ISyntaxNode<RegexAstNode, RegexElementLabel>
+    public class RegexAstNode : ISyntaxNode<RegexAstNode, RegexElementLabel>
     {
 
         private RegexAstNode _left, _right;
@@ -57,9 +59,27 @@ namespace StgSharp.RegularAnalysis.TextRegex
 
         public RegexAstNode Next { get; set; }
 
-        public RegexAstNode Left { get; set; }
+        public RegexAstNode Parent { get; set; }
 
-        public RegexAstNode Right { get; set; }
+        public RegexAstNode Left
+        {
+            get;
+            set
+            {
+                field = value;
+                value.Parent = this;
+            }
+        }
+
+        public RegexAstNode Right
+        {
+            get;
+            set
+            {
+                field = value;
+                value.Parent = this;
+            }
+        }
 
         public RegexElementLabel EqualityTypeConvert
         {
@@ -74,6 +94,21 @@ namespace StgSharp.RegularAnalysis.TextRegex
                     RegexAstNode nextToken
         ) { }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static (bool Valid,int Value) ComparePrecedence(
+                                             RegexElementLabel left,
+                                             RegexElementLabel right
+        )
+        {
+            left |= RegexElementLabel.OPERATOR;
+            right |= RegexElementLabel.OPERATOR;
+            int v_left = (int)left, v_right = (int)right;
+            if (v_left == 0 || v_right == 0) {
+                return (false,0);
+            }
+            return (true,v_left - v_right);
+        }
+
         public void PrependNode(
                     RegexAstNode previousNode
         ) { }

StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs

@@ -34,21 +34,22 @@ using System.Threading.Tasks;
 namespace StgSharp.RegularAnalysis.TextRegex
 {
     [Flags]
-    internal enum RegexElementLabel
+    public enum RegexElementLabel
     {
 
         NONE = 0,
         UNIT = 1,
-        GROUP_BEGIN = 2,
+        UNIT_SET = 2,
-        GROUP_END = 4,
+        UNIT_SPAN = 4,
-        COUNT = 8,
+        GROUP_BEGIN = 8,
-        ALT = 16,
+        GROUP_END = 16,
-        CONCAT = 32,
+        COUNT = 32,
-        UNIT_SPAN = 64,
+        CONCAT = 64,
-        SEQUENCE = UNIT | UNIT_SPAN,
+        ALT = 128,
+        SEQUENCE = UNIT | UNIT_SPAN | UNIT_SET,
         OPERATOR = COUNT | ALT | CONCAT,
-        ATOM_BEGIN = UNIT | GROUP_BEGIN | UNIT_SPAN,
+        ATOM_BEGIN = SEQUENCE | GROUP_BEGIN ,
-        ATOM_END = UNIT | GROUP_END | COUNT | UNIT_SPAN,
+        ATOM_END = SEQUENCE | GROUP_END,
 
     }
 }

StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs

@@ -53,13 +53,18 @@ namespace StgSharp.RegularAnalysis.TextRegex
             {
                 if ((token.Flag & RegexElementLabel.SEQUENCE) != 0)
                 {
+                    Console.WriteLine($"Unit {token.Value}");
+
                     // char or charset
                     _stack.PushOperand(new RegexAstNode(token));
                 } else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0)
                 {
+                    Console.WriteLine($"Group begin {token.Value}");
                     _stack.IncreaseDepth(0);
                 } else if ((token.Flag & RegexElementLabel.GROUP_END) != 0)
                 {
+                    Console.WriteLine($"Group end {token.Value}");
+
                     // TODO: close all nodes here
                     while (_stack.TryPopOperator(out RegexAstNode? op))
                     {
@@ -69,14 +74,61 @@ namespace StgSharp.RegularAnalysis.TextRegex
                         throw new InvalidOperationException();
                     }
                     _stack.DecreaseDepth();
-                } else if ((token.Flag & RegexElementLabel.OPERATOR) != 0) { }
+                } else if ((token.Flag & RegexElementLabel.OPERATOR) != 0)
+                {
+                    Console.WriteLine($"Operator {token.Value} with flag {token.Flag}");
+                    if (_stack.OperatorInDepthCount == 0)
+                    {
+                        // the first operator in stack
+                        _stack.PushOperator(new RegexAstNode(token));
+                    } else
+                    {
+                        // process operator by precedence
+                        while (_stack.TryPeekOperator(out RegexAstNode topOp))
+                        {
+                            (bool Valid, int Value) cmp = RegexAstNode.ComparePrecedence(token.Flag, topOp.Source.Flag);
+
+                            if (!cmp.Valid) {
+                                throw new InvalidOperationException("Operators with same precedence are not supported.");
+                            }
+                            if (cmp.Value < 0)
+                            {
+                                _stack.PopOperator();
+                                ProcessOperator(topOp);
+                                continue;
+                            } else
+                            {
+                                break;
+                            }
+                        }
+                        _stack.PushOperator(new RegexAstNode(token));
+                    }
+                }
             }
+            Console.WriteLine(1);
+            if (_stack.Depth == 1)
+            {
+                if (_stack.OperandInDepthCount != 1 || _stack.OperatorInDepthCount != 0)
+                {
+                    while (_stack.TryPopOperator(out RegexAstNode? op)) {
+                        ProcessOperator(op);
+                    }
+                }
+                if (_stack.OperandInDepthCount == 1 && _stack.OperatorInDepthCount == 0)
+                {
+                    _tree.Root = _stack.PopOperand();
+                    return _tree;
+                }
+                throw new InvalidOperationException("Invalid regular expression syntax.");
+            }
+            throw new InvalidOperationException("Invalid regular expression syntax.");
         }
 
         private void ProcessOperator(
                      RegexAstNode op
         )
         {
+            Console.WriteLine($"Processing a {op.Source.Flag} operator");
             if (op.EqualityTypeConvert == RegexElementLabel.CONCAT)
             {
                 if (_stack.TryPopOperand(out RegexAstNode? _1) &&
@@ -85,11 +137,15 @@ namespace StgSharp.RegularAnalysis.TextRegex
                     if (RegexAstNode.TryMergeUnit(_2, _1, out RegexAstNode? _3))
                     {
                         _stack.PushOperand(_3);
+                        _tree.AddNode(_3);
                     } else
                     {
                         op.Left = _2;
                         op.Right = _1;
                         _stack.PushOperand(op);
+                        _tree.AddNode(op);
+                        _tree.AddNode(_1);
+                        _tree.AddNode(_2);
                     }
                 } else
                 {
@@ -101,6 +157,8 @@ namespace StgSharp.RegularAnalysis.TextRegex
                 {
                     op.Left = _1;
                     _stack.PushOperand(op);
+                    _tree.AddNode(_1);
+                    _tree.AddNode(op);
                 } else
                 {
                     throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
@@ -113,6 +171,9 @@ namespace StgSharp.RegularAnalysis.TextRegex
                     op.Left = _1;
                     op.Right = _2;
                     _stack.PushOperand(op);
+                    _tree.AddNode(op);
+                    _tree.AddNode(_1);
+                    _tree.AddNode(_2);
                 } else
                 {
                     throw new InvalidOperationException("Insufficient operands for CONCAT operator.");

StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs

@@ -52,11 +52,12 @@ namespace StgSharp.RegularAnalysis.TextRegex
             }
             if (Source.TryReadToken(out Token<RegexElementLabel> orig))
             {
-                if (((int)(orig.Flag | RegexElementLabel.ATOM_BEGIN) != 0) && ((int)_prev != 0))
+                if (((int)(orig.Flag & RegexElementLabel.ATOM_BEGIN) != 0) && ((int)_prev != 0))
                 {
-                    Token<RegexElementLabel> ret = new Token<RegexElementLabel>(string.Empty, orig.Line,
+                    Token<RegexElementLabel> ret = new Token<RegexElementLabel>(string.Empty,
-                                                                            orig.Column,
+                                                                                orig.Line,
-                                                                            RegexElementLabel.OPERATOR);
+                                                                                orig.Column,
+                                                                                RegexElementLabel.CONCAT);
                     _buffer.Enqueue(orig);
                     _prev = orig.Flag & RegexElementLabel.ATOM_END;
                     return ret;

StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs

@@ -75,7 +75,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
                             position = i + 1;
                             return new Token<RegexElementLabel>(_source[(pos - 1)..(i - 1)], 0,
                                                                 position,
-                                                                RegexElementLabel.CHAR_SET);
+                                                                RegexElementLabel.UNIT_SET);
                         }
                     }
                     throw new InvalidOperationException("Unclosed character set");
@@ -85,7 +85,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
                     return ReadGroupBeginToken();
                 case ')':
                     position = pos + 1;
-                    return new Token<RegexElementLabel>(_source[(pos)..1], 0, pos,
+                    return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
                                                         RegexElementLabel.GROUP_END);
                 case '{':
                     for (int i = pos + 1; i < _source.Length; i++)

StgSharp.RegularAnalysis/TextRegex/RegexTokenUtil.cs

@@ -41,7 +41,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
                            this Token<RegexElementLabel> token
         )
         {
-            return ((token.Flag & RegexElementLabel.UNIT) != 0 && token.Value.Length == 0) ||
+            return ((token.Flag & RegexElementLabel.UNIT) != 0 && token.Value.Length == 1) ||
                    (token.Flag & RegexElementLabel.UNIT_SPAN) != 0;
         }
 

StgSharpDebug/Program.cs

@@ -50,7 +50,9 @@ namespace StgSharp
                      .UseModule<GraphicModule>()
                      .InitializeModule();
 
-                HlsfStressTest.Run();
+                TestRegexAnalyse.Run();
+
+                // HlsfStressTest.Run();
 
                 // TestMatrix();
 

StgSharpDebug/StgSharpDebug.csproj

@@ -77,6 +77,7 @@
       <ProjectReference Include="..\StgSharp.Common\StgSharp.Common.csproj" />
       <ProjectReference Include="..\StgSharp.Mathematics\StgSharp.Mathematics.csproj" />
       <ProjectReference Include="..\StgSharp.Model\StgSharp.Model.csproj" />
+      <ProjectReference Include="..\StgSharp.RegularAnalysis\StgSharp.RegularAnalysis.csproj" />
       <ProjectReference Include="..\StgSharp.Script\StgSharp.Script.csproj" />
     </ItemGroup>
 

StgSharpDebug/TestRegexAnalyse.cs

@@ -0,0 +1,50 @@
+//-----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+// file="TestRegexAnalyse"
+// Project: StgSharp
+// AuthorGroup: Nitload
+// Copyright (c) Nitload. All rights reserved.
+//     
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//     
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+using StgSharp.RegularAnalysis.Abstraction;
+using StgSharp.RegularAnalysis.TextRegex;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StgSharp
+{
+    public class TestRegexAnalyse
+    {
+
+        private const string regex = @"abc(?<name>abc)";
+
+        public static void Run()
+        {
+            RegexInterpreter interpreter = new RegexInterpreter();
+            AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree = interpreter.Analyze(regex);
+        }
+
+    }
+}