building regex parser

StgSharp.RegularAnalysis/Abstraction/CompileStack.cs

@@ -41,327 +41,129 @@ namespace StgSharp.RegularAnalysis.Abstraction
         where TLabel : unmanaged
     {
 
-        private Token<TLabel>? _lastToken;
+        private readonly RandomAccessibleStack<DepthMark> _depthMarks = [];
-
+        private readonly RandomAccessibleStack<TNode> _operands = [];
-        private TNode[] _nodeArray = new TNode[4];
+        private readonly RandomAccessibleStack<TNode> _operators = [];
-        private Token<TLabel>[] _operatorArray = new Token<TLabel>[4];
-        private Token<TLabel>[] _tokenArray = new Token<TLabel>[4];
-
-        private int _nodeCount,_tokenCount, _operatorCount;
-        private RandomAccessibleStack<CompileDepthMark> _depthStack = new RandomAccessibleStack<CompileDepthMark>(
-            );
-
-        private RandomAccessibleStack<OperandIndex> _isNode = new RandomAccessibleStack<OperandIndex>(
-            8);
-        private Stack<Stack<TNode>> _statementCache = new Stack<Stack<TNode>>();
-        private TNode _lastNode;
 
         public CompileStack()
         {
-            _depthStack.Push(new CompileDepthMark(0, 0, 0, null!));
+            _depthMarks.Push(new DepthMark(0, 0));
-            _statementCache.Push(new Stack<TNode>());
         }
 
-        public bool IsLastAddedOperator { get; private set; }
+        public int OperandInDepthCount
-
-        public CompileDepthMark CurrentDepthMark
         {
-            get => _depthStack.Peek();
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get => _operands.Count - _depthMarks.Peek().OperandPos;
         }
 
-        public int Depth => _depthStack.Count;
+        public int OperatorInDepthCount
-
-        public int OperandCount => _isNode.Count;
-
-        public int OperatorCount => _operatorCount;
-
-        public int OperatorAheadOfDepth
         {
-            get => OperatorCount - _depthStack.Peek().OperatorBegin;
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get => _operands.Count - _depthMarks.Peek().OperandPos;
         }
 
-        public int OperandAheadOfDepth
+        public int Depth
         {
-            get => OperandCount - _depthStack.Peek().OperandBegin;
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get => _depthMarks.Count;
         }
 
-        public Stack<TNode> StatementsInDepth
+        private record struct DepthMark(
-        {
+                              int OperandPos,
-            get => _statementCache.Peek();
+                              int OperatorPos,
-        }
+                              int Mark
-
-        public void DecreaseDepth()
-        {
-            CompileDepthMark mark = _depthStack.Pop();
-            Stack<TNode> statementsInDepth = _statementCache.Pop();
-        }
-
-        public bool GetOperandAt(
-                    Index i,
-                    out TNode operandNode,
-                    out Token<TLabel> operandToken
-        )
-        {
-            OperandIndex index = _isNode[i];
-            if (index.IsNode)
-            {
-                operandNode = _nodeArray[index.Index];
-                operandToken = Token<TLabel>.Empty;
-                return true;
-            } else
-            {
-                operandToken = _tokenArray[index.Index];
-                operandNode = default!;
-                return false;
-            }
-        }
-
-        public bool GetOperandAt(
-                    CompileDepthMark depth,
-                    Index i,
-                    out Token<TLabel> operandToken,
-                    out TNode operandNode
-        )
-        {
-            OperandIndex index = _isNode[i];
-
-            _depthStack.Contains(depth);
-            if (index.IsNode)
-            {
-                operandNode = _nodeArray[index.Index];
-                operandToken = Token<TLabel>.Empty;
-                return true;
-            } else
-            {
-                operandToken = _tokenArray[index.Index];
-                operandNode = default!;
-                return false;
-            }
-        }
-
-        public CompileDepthMark IncreaseDepth(
-                                int usage
-        )
-        {
-            CompileDepthMark mark =
-                new CompileDepthMark(OperatorCount, OperandCount, usage);
-
-            _depthStack.Push(mark);
-            _statementCache.Push(new Stack<TNode>());
-            return mark;
-        }
-
-        public TNode PackAllStatements()
-        {
-            if (StatementsInDepth.Count == 0) {
-                return TNode.Empty;
-            }
-            TNode end = StatementsInDepth.Peek();
-            TNode first = end;
-            while (StatementsInDepth.Count > 0)
-            {
-                first = StatementsInDepth.Pop();
-                end.PrependNode(first);
-                end = first;
-            }
-            return first;
-        }
-
-        public bool PeekOperand(
-                    out Token<TLabel> t,
-                    out TNode n
-        )
-        {
-            if (OperandAheadOfDepth <= 0)
-            {
-                t = Token<TLabel>.Empty;
-                n = TNode.Empty;
-                return false;
-            }
-            if (_isNode.Pop().IsNode)
-            {
-                t = Token<TLabel>.Empty;
-                n = _nodeArray[_nodeCount - 1];
-                return true;
-            } else
-            {
-                t = _tokenArray[_tokenCount - 1];
-                n = TNode.Empty;
-                return false;
-            }
-        }
-
-        public Token<TLabel> PeekOperator()
-        {
-            if (OperatorAheadOfDepth > 0) {
-                return _operatorArray[_operatorCount - 1];
-            }
-            return Token<TLabel>.Empty;
-        }
-
-        public bool PopOperand(
-                    out Token<TLabel> t,
-                    out TNode n
-        )
-        {
-            if (OperandAheadOfDepth <= 0)
-            {
-                t = Token<TLabel>.Empty;
-                n = TNode.Empty;
-                return false;
-            }
-            if (_isNode.Pop().IsNode)
-            {
-                t = Token<TLabel>.Empty;
-                n = _nodeArray[--_nodeCount];
-                return true;
-            } else
-            {
-                t = _tokenArray[--_tokenCount];
-                n = TNode.Empty;
-                return false;
-            }
-        }
-
-        public Token<TLabel> PopOperator()
-        {
-            if (OperatorAheadOfDepth > 0)
-            {
-                _operatorCount--;
-                return _operatorArray[_operatorCount];
-            }
-            return Token<TLabel>.Empty;
-        }
-
-        public void PushOperand(
-                    TNode node
-        )
-        {
-            if (_nodeArray.Length == _nodeCount) {
-                Array.Resize(ref _nodeArray, _nodeArray.Length * 2);
-            }
-            _nodeArray[_nodeCount] = node;
-            _isNode.Push(new(true, _nodeCount));
-            _nodeCount++;
-            IsLastAddedOperator = false;
-        }
-
-        public void PushOperand(
-                    Token<TLabel> token
-        )
-        {
-            if (_tokenArray.Length == _tokenCount) {
-                Array.Resize(ref _tokenArray, _tokenArray.Length * 2);
-            }
-            _tokenArray[_tokenCount] = token;
-            _isNode.Push(new(false, _tokenCount));
-            _tokenCount++;
-            IsLastAddedOperator = false;
-        }
-
-        public void PushOperator(
-                    Token<TLabel> token
-        )
-        {
-            if (_operatorArray.Length == _operatorCount) {
-                Array.Resize(ref _operatorArray, _operatorArray.Length * 2);
-            }
-            _operatorArray[_operatorCount] = token;
-            _operatorCount++;
-            IsLastAddedOperator = true;
-        }
-
-        public T StateOfCurrentDepth<T>() where T : class, ICompileDepthState
-        {
-            return (CurrentDepthMark.StateCache  as T)!;
-        }
-
-        public bool TryPeekOperand(
-                    out bool isNode,
-                    out Token<TLabel> t,
-                    out TNode n
-        )
-        {
-            if (OperandAheadOfDepth <= 0)
-            {
-                t = Token<TLabel>.Empty;
-                n = TNode.Empty;
-                isNode = false;
-                return false;
-            }
-            if (_isNode.Pop().IsNode)
-            {
-                t = Token<TLabel>.Empty;
-                n = _nodeArray[_nodeCount - 1];
-                isNode = true;
-                return true;
-            } else
-            {
-                t = _tokenArray[_tokenCount - 1];
-                n = TNode.Empty;
-                isNode = false;
-                return false;
-            }
-        }
-
-        public bool TryPeekOperator(
-                    out Token<TLabel> op
-        )
-        {
-            if (OperatorAheadOfDepth > 0)
-            {
-                op = _operatorArray[_operatorCount - 1];
-                return true;
-            }
-            op = Token<TLabel>.Empty;
-            return false;
-        }
-
-        public bool TryPopOperand(
-                    out bool isNode,
-                    out Token<TLabel> t,
-                    out TNode n
-        )
-        {
-            if (OperandAheadOfDepth <= 0)
-            {
-                t = Token<TLabel>.Empty;
-                n = TNode.Empty;
-                isNode = false;
-                return false;
-            }
-            if (_isNode.Pop().IsNode)
-            {
-                t = Token<TLabel>.Empty;
-                n = _nodeArray[--_nodeCount];
-                isNode = true;
-                return true;
-            } else
-            {
-                t = _tokenArray[--_tokenCount];
-                n = TNode.Empty;
-                isNode = false;
-                return true;
-            }
-        }
-
-        public bool TryPopOperator(
-                    out Token<TLabel> op
-        )
-        {
-            if (OperatorAheadOfDepth > 0)
-            {
-                op = _operatorArray[--_operatorCount];
-                return true;
-            }
-            op = Token<TLabel>.Empty;
-            return false;
-        }
-
-        private record struct OperandIndex(
-                              bool IsNode,
-                              int Index
         );
 
+        #region operator
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public TNode PopOperator(
+        )
+        {
+            if (OperatorInDepthCount > 0) {
+                return _operators.Pop();
+            }
+            return TNode.Empty;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void PushOperator(
+                    TNode operatorNode
+        )
+        {
+            _operators.Push(operatorNode);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool TryPopOperator(
+                    out TNode op
+        )
+        {
+            if (OperatorInDepthCount > 0)
+            {
+                op = _operators.Pop();
+                return true;
+            }
+            op = TNode.Empty;
+            return false;
+        }
+
+            #endregion
+
+        #region operand
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public TNode PopOperand(
+        )
+        {
+            if (OperatorInDepthCount > 0) {
+                return _operators.Pop();
+            }
+            return TNode.Empty;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void PushOperand(
+                    TNode operatorNode
+        )
+        {
+            _operators.Push(operatorNode);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool TryPopOperand(
+                    out TNode op
+        )
+        {
+            if (OperandInDepthCount > 0)
+            {
+                op = _operands.Pop();
+                return true;
+            }
+            op = TNode.Empty;
+            return false;
+        }
+
+            #endregion
+
+        #region depth
+
+        public void DecreaseDepth(
+        )
+        {
+            if (Depth > 1)
+            {
+                DepthMark mark = _depthMarks.Pop();
+            }
+        }
+
+        public void IncreaseDepth(
+                    int mark
+        )
+        {
+            _depthMarks.Push(new DepthMark(_operands.Count, _operators.Count, mark));
+        }
+
+        #endregion
     }
 }

StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs

@@ -49,9 +49,9 @@ namespace StgSharp.RegularAnalysis.Abstraction
 
         TNode Next { get; set; }
 
-        TNode Left { get; }
+        TNode Left { get; set; }
 
-        TNode Right { get; }
+        TNode Right { get; set; }
 
         static abstract TNode Empty { get; }
 

StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs

@@ -1,4 +1,32 @@
-using System;
+//-----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+// file="RegexAstNode"
+// Project: StgSharp
+// AuthorGroup: Nitload
+// Copyright (c) Nitload. All rights reserved.
+//     
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//     
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+using StgSharp.RegularAnalysis.Abstraction;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -6,7 +34,82 @@ using System.Threading.Tasks;
 
 namespace StgSharp.RegularAnalysis.TextRegex
 {
-    internal class RegexAstNode
+    internal class RegexAstNode : ISyntaxNode<RegexAstNode, RegexElementLabel>
     {
+
+        private RegexAstNode _left, _right;
+
+        public RegexAstNode(
+               Token<RegexElementLabel> source
+        )
+        {
+            Source = source;
+        }
+
+        public long NodeFlag => (long)Source.Flag;
+
+        public static RegexAstNode Empty
+        {
+            get => new EmptyRegexAstNode();
+        }
+
+        public RegexAstNode Previous { get; set; }
+
+        public RegexAstNode Next { get; set; }
+
+        public RegexAstNode Left { get; set; }
+
+        public RegexAstNode Right { get; set; }
+
+        public RegexElementLabel EqualityTypeConvert
+        {
+            get => Source.Flag;
+        }
+
+        public string CodeConvertTemplate { get; }
+
+        public Token<RegexElementLabel> Source { get; }
+
+        public void AppendNode(
+                    RegexAstNode nextToken
+        ) { }
+
+        public void PrependNode(
+                    RegexAstNode previousNode
+        ) { }
+
+        public static bool TryMergeUnit(
+                           RegexAstNode prev,
+                           RegexAstNode next,
+                           out RegexAstNode result
+        )
+        {
+            Token<RegexElementLabel> pSource = prev.Source;
+            Token<RegexElementLabel> nSource = next.Source;
+            if (pSource.IsAccurateChar() &&
+                nSource.IsAccurateChar() &&
+                pSource.Column + pSource.Value.Length == nSource.Column)
+            {
+                // both are unit types and adjacent
+                // and are all accurate unit
+                Token<RegexElementLabel> newToken = new Token<RegexElementLabel>($"{pSource.Value}{nSource.Value}",
+                                                                                 pSource.Line,
+                                                                                 pSource.Column,
+                                                                                 RegexElementLabel.UNIT_SPAN);
+                result = new RegexAstNode(newToken);
+                return true;
+            }
+            result = Empty;
+            return false;
+        }
+
+        private class EmptyRegexAstNode : RegexAstNode
+
+        {
+
+            public EmptyRegexAstNode() : base(Token<RegexElementLabel>.Empty) { }
+
+        }
+
     }
 }

StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs

@@ -44,9 +44,11 @@ namespace StgSharp.RegularAnalysis.TextRegex
         COUNT = 8,
         ALT = 16,
         CONCAT = 32,
+        UNIT_SPAN = 64,
+        SEQUENCE = UNIT | UNIT_SPAN,
         OPERATOR = COUNT | ALT | CONCAT,
-        ATOM_BEGIN = UNIT | GROUP_BEGIN,
+        ATOM_BEGIN = UNIT | GROUP_BEGIN | UNIT_SPAN,
-        ATOM_END = UNIT | GROUP_END | COUNT,
+        ATOM_END = UNIT | GROUP_END | COUNT | UNIT_SPAN,
 
     }
 }

StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs

@@ -38,17 +38,87 @@ namespace StgSharp.RegularAnalysis.TextRegex
     public class RegexInterpreter
     {
 
-        public static AbstractSyntaxTree<RegexAstNode, RegexElementLabel> Analyze(
+        private AbstractSyntaxTree<RegexAstNode, RegexElementLabel> _tree = new();
+
+        private CompileStack<RegexAstNode, RegexElementLabel> _stack = new();
+
+        public AbstractSyntaxTree<RegexAstNode, RegexElementLabel> Analyze(
                                                                    string regex
         )
         {
             RegexTokenReader reader = new RegexTokenReader(regex);
             TokenParser<RegexElementLabel, RegexElementLabel> lexer = reader.Pipe(() => new RegexTokenParser());
 
-            while (lexer.TryReadToken(out Token<RegexElementLabel> token)) { }
+            while (lexer.TryReadToken(out Token<RegexElementLabel> token))
+            {
+                if ((token.Flag & RegexElementLabel.SEQUENCE) != 0)
+                {
+                    // char or charset
+                    _stack.PushOperand(new RegexAstNode(token));
+                } else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0)
+                {
+                    _stack.IncreaseDepth(0);
+                } else if ((token.Flag & RegexElementLabel.GROUP_END) != 0)
+                {
+                    // TODO: close all nodes here
+                    while (_stack.TryPopOperator(out RegexAstNode? op))
+                    {
+                        ProcessOperator(op);
+                    }
+                    if (_stack.OperandInDepthCount != 1) {
+                        throw new InvalidOperationException();
+                    }
+                    _stack.DecreaseDepth();
+                } else if ((token.Flag & RegexElementLabel.OPERATOR) != 0) { }
+            }
         }
 
-        private static void
+        private void ProcessOperator(
+                     RegexAstNode op
+        )
+        {
+            if (op.EqualityTypeConvert == RegexElementLabel.CONCAT)
+            {
+                if (_stack.TryPopOperand(out RegexAstNode? _1) &&
+                    _stack.TryPopOperand(out RegexAstNode? _2))
+                {
+                    if (RegexAstNode.TryMergeUnit(_2, _1, out RegexAstNode? _3))
+                    {
+                        _stack.PushOperand(_3);
+                    } else
+                    {
+                        op.Left = _2;
+                        op.Right = _1;
+                        _stack.PushOperand(op);
+                    }
+                } else
+                {
+                    throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
+                }
+            } else if (op.EqualityTypeConvert == RegexElementLabel.COUNT)
+            {
+                if (_stack.TryPopOperand(out RegexAstNode? _1))
+                {
+                    op.Left = _1;
+                    _stack.PushOperand(op);
+                } else
+                {
+                    throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
+                }
+            } else if (op.EqualityTypeConvert == RegexElementLabel.ALT)
+            {
+                if (_stack.TryPopOperand(out RegexAstNode? _1) &&
+                    _stack.TryPopOperand(out RegexAstNode? _2))
+                {
+                    op.Left = _1;
+                    op.Right = _2;
+                    _stack.PushOperand(op);
+                } else
+                {
+                    throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
+                }
+            }
+        }
 
     }
 }

StgSharp.RegularAnalysis/TextRegex/RegexTokenUtil.cs

@@ -0,0 +1,49 @@
+//-----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+// file="RegexTokenUtil"
+// Project: StgSharp
+// AuthorGroup: Nitload
+// Copyright (c) Nitload. All rights reserved.
+//     
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//     
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+using StgSharp.RegularAnalysis.Abstraction;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StgSharp.RegularAnalysis.TextRegex
+{
+    internal static class RegexTokenUtil
+    {
+
+        public static bool IsAccurateChar(
+                           this Token<RegexElementLabel> token
+        )
+        {
+            return ((token.Flag & RegexElementLabel.UNIT) != 0 && token.Value.Length == 0) ||
+                   (token.Flag & RegexElementLabel.UNIT_SPAN) != 0;
+        }
+
+    }
+}