youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

building regex parser

Browse Source
This commit is contained in:
Youmu Liu
2026-02-05 18:35:42 +08:00
parent f22e90af55
commit 1bfb8b67c4
6 changed files with 342 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.RegularAnalysis/Abstraction/CompileStack.cs
+108 -306
View File
@@ -41,327 +41,129 @@ namespace StgSharp.RegularAnalysis.Abstraction
where TLabel : unmanaged
{
private Token<TLabel>? _lastToken;
private TNode[] _nodeArray = new TNode[4];
private Token<TLabel>[] _operatorArray = new Token<TLabel>[4];
private Token<TLabel>[] _tokenArray = new Token<TLabel>[4];
private int _nodeCount,_tokenCount, _operatorCount;
private RandomAccessibleStack<CompileDepthMark> _depthStack = new RandomAccessibleStack<CompileDepthMark>(
);
private RandomAccessibleStack<OperandIndex> _isNode = new RandomAccessibleStack<OperandIndex>(
8);
private Stack<Stack<TNode>> _statementCache = new Stack<Stack<TNode>>();
private TNode _lastNode;
private readonly RandomAccessibleStack<DepthMark> _depthMarks = [];
private readonly RandomAccessibleStack<TNode> _operands = [];
private readonly RandomAccessibleStack<TNode> _operators = [];
public CompileStack()
{
_depthStack.Push(new CompileDepthMark(0, 0, 0, null!));
_statementCache.Push(new Stack<TNode>());
_depthMarks.Push(new DepthMark(0, 0));
}
public bool IsLastAddedOperator { get; private set; }
public CompileDepthMark CurrentDepthMark
public int OperandInDepthCount
{
get => _depthStack.Peek();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => _operands.Count - _depthMarks.Peek().OperandPos;
}
public int Depth => _depthStack.Count;
public int OperandCount => _isNode.Count;
public int OperatorCount => _operatorCount;
public int OperatorAheadOfDepth
public int OperatorInDepthCount
{
get => OperatorCount - _depthStack.Peek().OperatorBegin;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => _operands.Count - _depthMarks.Peek().OperandPos;
}
public int OperandAheadOfDepth
public int Depth
{
get => OperandCount - _depthStack.Peek().OperandBegin;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => _depthMarks.Count;
}
public Stack<TNode> StatementsInDepth
{
get => _statementCache.Peek();
}
public void DecreaseDepth()
{
CompileDepthMark mark = _depthStack.Pop();
Stack<TNode> statementsInDepth = _statementCache.Pop();
}
public bool GetOperandAt(
Index i,
out TNode operandNode,
out Token<TLabel> operandToken
)
{
OperandIndex index = _isNode[i];
if (index.IsNode)
{
operandNode = _nodeArray[index.Index];
operandToken = Token<TLabel>.Empty;
return true;
} else
{
operandToken = _tokenArray[index.Index];
operandNode = default!;
return false;
}
}
public bool GetOperandAt(
CompileDepthMark depth,
Index i,
out Token<TLabel> operandToken,
out TNode operandNode
)
{
OperandIndex index = _isNode[i];
_depthStack.Contains(depth);
if (index.IsNode)
{
operandNode = _nodeArray[index.Index];
operandToken = Token<TLabel>.Empty;
return true;
} else
{
operandToken = _tokenArray[index.Index];
operandNode = default!;
return false;
}
}
public CompileDepthMark IncreaseDepth(
int usage
)
{
CompileDepthMark mark =
new CompileDepthMark(OperatorCount, OperandCount, usage);
_depthStack.Push(mark);
_statementCache.Push(new Stack<TNode>());
return mark;
}
public TNode PackAllStatements()
{
if (StatementsInDepth.Count == 0) {
return TNode.Empty;
}
TNode end = StatementsInDepth.Peek();
TNode first = end;
while (StatementsInDepth.Count > 0)
{
first = StatementsInDepth.Pop();
end.PrependNode(first);
end = first;
}
return first;
}
public bool PeekOperand(
out Token<TLabel> t,
out TNode n
)
{
if (OperandAheadOfDepth <= 0)
{
t = Token<TLabel>.Empty;
n = TNode.Empty;
return false;
}
if (_isNode.Pop().IsNode)
{
t = Token<TLabel>.Empty;
n = _nodeArray[_nodeCount - 1];
return true;
} else
{
t = _tokenArray[_tokenCount - 1];
n = TNode.Empty;
return false;
}
}
public Token<TLabel> PeekOperator()
{
if (OperatorAheadOfDepth > 0) {
return _operatorArray[_operatorCount - 1];
}
return Token<TLabel>.Empty;
}
public bool PopOperand(
out Token<TLabel> t,
out TNode n
)
{
if (OperandAheadOfDepth <= 0)
{
t = Token<TLabel>.Empty;
n = TNode.Empty;
return false;
}
if (_isNode.Pop().IsNode)
{
t = Token<TLabel>.Empty;
n = _nodeArray[--_nodeCount];
return true;
} else
{
t = _tokenArray[--_tokenCount];
n = TNode.Empty;
return false;
}
}
public Token<TLabel> PopOperator()
{
if (OperatorAheadOfDepth > 0)
{
_operatorCount--;
return _operatorArray[_operatorCount];
}
return Token<TLabel>.Empty;
}
public void PushOperand(
TNode node
)
{
if (_nodeArray.Length == _nodeCount) {
Array.Resize(ref _nodeArray, _nodeArray.Length * 2);
}
_nodeArray[_nodeCount] = node;
_isNode.Push(new(true, _nodeCount));
_nodeCount++;
IsLastAddedOperator = false;
}
public void PushOperand(
Token<TLabel> token
)
{
if (_tokenArray.Length == _tokenCount) {
Array.Resize(ref _tokenArray, _tokenArray.Length * 2);
}
_tokenArray[_tokenCount] = token;
_isNode.Push(new(false, _tokenCount));
_tokenCount++;
IsLastAddedOperator = false;
}
public void PushOperator(
Token<TLabel> token
)
{
if (_operatorArray.Length == _operatorCount) {
Array.Resize(ref _operatorArray, _operatorArray.Length * 2);
}
_operatorArray[_operatorCount] = token;
_operatorCount++;
IsLastAddedOperator = true;
}
public T StateOfCurrentDepth<T>() where T : class, ICompileDepthState
{
return (CurrentDepthMark.StateCache as T)!;
}
public bool TryPeekOperand(
out bool isNode,
out Token<TLabel> t,
out TNode n
)
{
if (OperandAheadOfDepth <= 0)
{
t = Token<TLabel>.Empty;
n = TNode.Empty;
isNode = false;
return false;
}
if (_isNode.Pop().IsNode)
{
t = Token<TLabel>.Empty;
n = _nodeArray[_nodeCount - 1];
isNode = true;
return true;
} else
{
t = _tokenArray[_tokenCount - 1];
n = TNode.Empty;
isNode = false;
return false;
}
}
public bool TryPeekOperator(
out Token<TLabel> op
)
{
if (OperatorAheadOfDepth > 0)
{
op = _operatorArray[_operatorCount - 1];
return true;
}
op = Token<TLabel>.Empty;
return false;
}
public bool TryPopOperand(
out bool isNode,
out Token<TLabel> t,
out TNode n
)
{
if (OperandAheadOfDepth <= 0)
{
t = Token<TLabel>.Empty;
n = TNode.Empty;
isNode = false;
return false;
}
if (_isNode.Pop().IsNode)
{
t = Token<TLabel>.Empty;
n = _nodeArray[--_nodeCount];
isNode = true;
return true;
} else
{
t = _tokenArray[--_tokenCount];
n = TNode.Empty;
isNode = false;
return true;
}
}
public bool TryPopOperator(
out Token<TLabel> op
)
{
if (OperatorAheadOfDepth > 0)
{
op = _operatorArray[--_operatorCount];
return true;
}
op = Token<TLabel>.Empty;
return false;
}
private record struct OperandIndex(
bool IsNode,
int Index
private record struct DepthMark(
int OperandPos,
int OperatorPos,
int Mark
);
#region operator
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public TNode PopOperator(
)
{
if (OperatorInDepthCount > 0) {
return _operators.Pop();
}
return TNode.Empty;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void PushOperator(
TNode operatorNode
)
{
_operators.Push(operatorNode);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPopOperator(
out TNode op
)
{
if (OperatorInDepthCount > 0)
{
op = _operators.Pop();
return true;
}
op = TNode.Empty;
return false;
}
#endregion
#region operand
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public TNode PopOperand(
)
{
if (OperatorInDepthCount > 0) {
return _operators.Pop();
}
return TNode.Empty;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void PushOperand(
TNode operatorNode
)
{
_operators.Push(operatorNode);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPopOperand(
out TNode op
)
{
if (OperandInDepthCount > 0)
{
op = _operands.Pop();
return true;
}
op = TNode.Empty;
return false;
}
#endregion
#region depth
public void DecreaseDepth(
)
{
if (Depth > 1)
{
DepthMark mark = _depthMarks.Pop();
}
}
public void IncreaseDepth(
int mark
)
{
_depthMarks.Push(new DepthMark(_operands.Count, _operators.Count, mark));
}
#endregion
}
}
StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs
+2 -2
View File
@@ -49,9 +49,9 @@ namespace StgSharp.RegularAnalysis.Abstraction
TNode Next { get; set; }
TNode Left { get; }
TNode Left { get; set; }
TNode Right { get; }
TNode Right { get; set; }
static abstract TNode Empty { get; }
StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs
+105 -2
View File
@@ -1,4 +1,32 @@
using System;
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexAstNode"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
@@ -6,7 +34,82 @@ using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.TextRegex
{
internal class RegexAstNode
internal class RegexAstNode : ISyntaxNode<RegexAstNode, RegexElementLabel>
{
private RegexAstNode _left, _right;
public RegexAstNode(
Token<RegexElementLabel> source
)
{
Source = source;
}
public long NodeFlag => (long)Source.Flag;
public static RegexAstNode Empty
{
get => new EmptyRegexAstNode();
}
public RegexAstNode Previous { get; set; }
public RegexAstNode Next { get; set; }
public RegexAstNode Left { get; set; }
public RegexAstNode Right { get; set; }
public RegexElementLabel EqualityTypeConvert
{
get => Source.Flag;
}
public string CodeConvertTemplate { get; }
public Token<RegexElementLabel> Source { get; }
public void AppendNode(
RegexAstNode nextToken
) { }
public void PrependNode(
RegexAstNode previousNode
) { }
public static bool TryMergeUnit(
RegexAstNode prev,
RegexAstNode next,
out RegexAstNode result
)
{
Token<RegexElementLabel> pSource = prev.Source;
Token<RegexElementLabel> nSource = next.Source;
if (pSource.IsAccurateChar() &&
nSource.IsAccurateChar() &&
pSource.Column + pSource.Value.Length == nSource.Column)
{
// both are unit types and adjacent
// and are all accurate unit
Token<RegexElementLabel> newToken = new Token<RegexElementLabel>($"{pSource.Value}{nSource.Value}",
pSource.Line,
pSource.Column,
RegexElementLabel.UNIT_SPAN);
result = new RegexAstNode(newToken);
return true;
}
result = Empty;
return false;
}
private class EmptyRegexAstNode : RegexAstNode
{
public EmptyRegexAstNode() : base(Token<RegexElementLabel>.Empty) { }
}
}
}
StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs
+4 -2
View File
@@ -44,9 +44,11 @@ namespace StgSharp.RegularAnalysis.TextRegex
COUNT = 8,
ALT = 16,
CONCAT = 32,
UNIT_SPAN = 64,
SEQUENCE = UNIT | UNIT_SPAN,
OPERATOR = COUNT | ALT | CONCAT,
ATOM_BEGIN = UNIT | GROUP_BEGIN,
ATOM_END = UNIT | GROUP_END | COUNT,
ATOM_BEGIN = UNIT | GROUP_BEGIN | UNIT_SPAN,
ATOM_END = UNIT | GROUP_END | COUNT | UNIT_SPAN,
}
}
StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs
+73 -3
View File
@@ -38,17 +38,87 @@ namespace StgSharp.RegularAnalysis.TextRegex
public class RegexInterpreter
{
public static AbstractSyntaxTree<RegexAstNode, RegexElementLabel> Analyze(
private AbstractSyntaxTree<RegexAstNode, RegexElementLabel> _tree = new();
private CompileStack<RegexAstNode, RegexElementLabel> _stack = new();
public AbstractSyntaxTree<RegexAstNode, RegexElementLabel> Analyze(
string regex
)
{
RegexTokenReader reader = new RegexTokenReader(regex);
TokenParser<RegexElementLabel, RegexElementLabel> lexer = reader.Pipe(() => new RegexTokenParser());
while (lexer.TryReadToken(out Token<RegexElementLabel> token)) { }
while (lexer.TryReadToken(out Token<RegexElementLabel> token))
{
if ((token.Flag & RegexElementLabel.SEQUENCE) != 0)
{
// char or charset
_stack.PushOperand(new RegexAstNode(token));
} else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0)
{
_stack.IncreaseDepth(0);
} else if ((token.Flag & RegexElementLabel.GROUP_END) != 0)
{
// TODO: close all nodes here
while (_stack.TryPopOperator(out RegexAstNode? op))
{
ProcessOperator(op);
}
if (_stack.OperandInDepthCount != 1) {
throw new InvalidOperationException();
}
_stack.DecreaseDepth();
} else if ((token.Flag & RegexElementLabel.OPERATOR) != 0) { }
}
}
private static void
private void ProcessOperator(
RegexAstNode op
)
{
if (op.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1) &&
_stack.TryPopOperand(out RegexAstNode? _2))
{
if (RegexAstNode.TryMergeUnit(_2, _1, out RegexAstNode? _3))
{
_stack.PushOperand(_3);
} else
{
op.Left = _2;
op.Right = _1;
_stack.PushOperand(op);
}
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
} else if (op.EqualityTypeConvert == RegexElementLabel.COUNT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1))
{
op.Left = _1;
_stack.PushOperand(op);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
} else if (op.EqualityTypeConvert == RegexElementLabel.ALT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1) &&
_stack.TryPopOperand(out RegexAstNode? _2))
{
op.Left = _1;
op.Right = _2;
_stack.PushOperand(op);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
}
}
}
}
StgSharp.RegularAnalysis/TextRegex/RegexTokenUtil.cs
+49
View File
@@ -0,0 +1,49 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexTokenUtil"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.TextRegex
{
internal static class RegexTokenUtil
{
public static bool IsAccurateChar(
this Token<RegexElementLabel> token
)
{
return ((token.Flag & RegexElementLabel.UNIT) != 0 && token.Value.Length == 0) ||
(token.Flag & RegexElementLabel.UNIT_SPAN) != 0;
}
}
}