youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

regex interpreter now can build partial correct AST (i forget groups), but in bad shape. building an optimizer.

Browse Source
This commit is contained in:
Youmu Liu
2026-02-06 17:07:03 +08:00
parent 1bfb8b67c4
commit 3a441b270c
13 changed files with 236 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.RegularAnalysis/Abstraction/AbstractSyntaxTree.cs
+22 -5
View File
@@ -29,6 +29,7 @@ using StgSharp.Collections;
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Text;
using System.Xml.Schema;
@@ -38,6 +39,8 @@ namespace StgSharp.RegularAnalysis.Abstraction
where TType : unmanaged
{
private HashSet<TNode> _allToken = new();
private TNode _root;
public int Count
@@ -45,17 +48,31 @@ namespace StgSharp.RegularAnalysis.Abstraction
get { return _allToken.Count; }
}
public TNode Root => _root;
public TNode Root
{
get;
internal set
{
if (_allToken.Contains(value)) {
field = value;
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool AddNode(
TNode node
)
{
return _allToken.Add(node);
}
public bool Contains(
TNode node
)
{
return _allToken<TLabel>.Contains(node);
return _allToken.Contains(node);
}
private List<TNode> _allToken<TLabel> =
new();
}
}
StgSharp.RegularAnalysis/Abstraction/CompileStack.cs
+32 -4
View File
@@ -47,7 +47,7 @@ namespace StgSharp.RegularAnalysis.Abstraction
public CompileStack()
{
_depthMarks.Push(new DepthMark(0, 0));
_depthMarks.Push(new DepthMark(0, 0, 0));
}
public int OperandInDepthCount
@@ -59,7 +59,7 @@ namespace StgSharp.RegularAnalysis.Abstraction
public int OperatorInDepthCount
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => _operands.Count - _depthMarks.Peek().OperandPos;
get => _operators.Count - _depthMarks.Peek().OperandPos;
}
public int Depth
@@ -108,6 +108,20 @@ namespace StgSharp.RegularAnalysis.Abstraction
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPeekOperator(
out TNode op
)
{
if (OperatorInDepthCount > 0)
{
op = _operators.Peek();
return true;
}
op = TNode.Empty;
return false;
}
#endregion
#region operand
@@ -117,7 +131,7 @@ namespace StgSharp.RegularAnalysis.Abstraction
)
{
if (OperatorInDepthCount > 0) {
return _operators.Pop();
return _operands.Pop();
}
return TNode.Empty;
}
@@ -127,7 +141,7 @@ namespace StgSharp.RegularAnalysis.Abstraction
TNode operatorNode
)
{
_operators.Push(operatorNode);
_operands.Push(operatorNode);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -144,6 +158,20 @@ namespace StgSharp.RegularAnalysis.Abstraction
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPeekOperand(
out TNode op
)
{
if (OperandInDepthCount > 0)
{
op = _operands.Peek();
return true;
}
op = TNode.Empty;
return false;
}
#endregion
#region depth
StgSharp.RegularAnalysis/Abstraction/Token.cs
+8
View File
@@ -45,6 +45,14 @@ namespace StgSharp.RegularAnalysis.Abstraction
public readonly string Value;
public readonly TLabel Flag;
public Token()
{
Column = 0;
Line = 0;
Value = string.Empty;
Flag = default;
}
public Token(
string chars,
int lineNumber,
StgSharp.RegularAnalysis/StgSharp.RegularAnalysis.csproj
+1
View File
@@ -4,6 +4,7 @@
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
</PropertyGroup>
<ItemGroup>
StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs
+38 -3
View File
@@ -28,13 +28,15 @@
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.TextRegex
{
internal class RegexAstNode : ISyntaxNode<RegexAstNode, RegexElementLabel>
public class RegexAstNode : ISyntaxNode<RegexAstNode, RegexElementLabel>
{
private RegexAstNode _left, _right;
@@ -57,9 +59,27 @@ namespace StgSharp.RegularAnalysis.TextRegex
public RegexAstNode Next { get; set; }
public RegexAstNode Left { get; set; }
public RegexAstNode Parent { get; set; }
public RegexAstNode Right { get; set; }
public RegexAstNode Left
{
get;
set
{
field = value;
value.Parent = this;
}
}
public RegexAstNode Right
{
get;
set
{
field = value;
value.Parent = this;
}
}
public RegexElementLabel EqualityTypeConvert
{
@@ -74,6 +94,21 @@ namespace StgSharp.RegularAnalysis.TextRegex
RegexAstNode nextToken
) { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static (bool Valid,int Value) ComparePrecedence(
RegexElementLabel left,
RegexElementLabel right
)
{
left |= RegexElementLabel.OPERATOR;
right |= RegexElementLabel.OPERATOR;
int v_left = (int)left, v_right = (int)right;
if (v_left == 0 || v_right == 0) {
return (false,0);
}
return (true,v_left - v_right);
}
public void PrependNode(
RegexAstNode previousNode
) { }
StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs
+11 -10
View File
@@ -34,21 +34,22 @@ using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.TextRegex
{
[Flags]
internal enum RegexElementLabel
public enum RegexElementLabel
{
NONE = 0,
UNIT = 1,
GROUP_BEGIN = 2,
GROUP_END = 4,
COUNT = 8,
ALT = 16,
CONCAT = 32,
UNIT_SPAN = 64,
SEQUENCE = UNIT | UNIT_SPAN,
UNIT_SET = 2,
UNIT_SPAN = 4,
GROUP_BEGIN = 8,
GROUP_END = 16,
COUNT = 32,
CONCAT = 64,
ALT = 128,
SEQUENCE = UNIT | UNIT_SPAN | UNIT_SET,
OPERATOR = COUNT | ALT | CONCAT,
ATOM_BEGIN = UNIT | GROUP_BEGIN | UNIT_SPAN,
ATOM_END = UNIT | GROUP_END | COUNT | UNIT_SPAN,
ATOM_BEGIN = SEQUENCE | GROUP_BEGIN ,
ATOM_END = SEQUENCE | GROUP_END,
}
}
StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs
+62 -1
View File
@@ -53,13 +53,18 @@ namespace StgSharp.RegularAnalysis.TextRegex
{
if ((token.Flag & RegexElementLabel.SEQUENCE) != 0)
{
Console.WriteLine($"Unit {token.Value}");
// char or charset
_stack.PushOperand(new RegexAstNode(token));
} else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0)
{
Console.WriteLine($"Group begin {token.Value}");
_stack.IncreaseDepth(0);
} else if ((token.Flag & RegexElementLabel.GROUP_END) != 0)
{
Console.WriteLine($"Group end {token.Value}");
// TODO: close all nodes here
while (_stack.TryPopOperator(out RegexAstNode? op))
{
@@ -69,14 +74,61 @@ namespace StgSharp.RegularAnalysis.TextRegex
throw new InvalidOperationException();
}
_stack.DecreaseDepth();
} else if ((token.Flag & RegexElementLabel.OPERATOR) != 0) { }
} else if ((token.Flag & RegexElementLabel.OPERATOR) != 0)
{
Console.WriteLine($"Operator {token.Value} with flag {token.Flag}");
if (_stack.OperatorInDepthCount == 0)
{
// the first operator in stack
_stack.PushOperator(new RegexAstNode(token));
} else
{
// process operator by precedence
while (_stack.TryPeekOperator(out RegexAstNode topOp))
{
(bool Valid, int Value) cmp = RegexAstNode.ComparePrecedence(token.Flag, topOp.Source.Flag);
if (!cmp.Valid) {
throw new InvalidOperationException("Operators with same precedence are not supported.");
}
if (cmp.Value < 0)
{
_stack.PopOperator();
ProcessOperator(topOp);
continue;
} else
{
break;
}
}
_stack.PushOperator(new RegexAstNode(token));
}
}
}
Console.WriteLine(1);
if (_stack.Depth == 1)
{
if (_stack.OperandInDepthCount != 1 || _stack.OperatorInDepthCount != 0)
{
while (_stack.TryPopOperator(out RegexAstNode? op)) {
ProcessOperator(op);
}
}
if (_stack.OperandInDepthCount == 1 && _stack.OperatorInDepthCount == 0)
{
_tree.Root = _stack.PopOperand();
return _tree;
}
throw new InvalidOperationException("Invalid regular expression syntax.");
}
throw new InvalidOperationException("Invalid regular expression syntax.");
}
private void ProcessOperator(
RegexAstNode op
)
{
Console.WriteLine($"Processing a {op.Source.Flag} operator");
if (op.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1) &&
@@ -85,11 +137,15 @@ namespace StgSharp.RegularAnalysis.TextRegex
if (RegexAstNode.TryMergeUnit(_2, _1, out RegexAstNode? _3))
{
_stack.PushOperand(_3);
_tree.AddNode(_3);
} else
{
op.Left = _2;
op.Right = _1;
_stack.PushOperand(op);
_tree.AddNode(op);
_tree.AddNode(_1);
_tree.AddNode(_2);
}
} else
{
@@ -101,6 +157,8 @@ namespace StgSharp.RegularAnalysis.TextRegex
{
op.Left = _1;
_stack.PushOperand(op);
_tree.AddNode(_1);
_tree.AddNode(op);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
@@ -113,6 +171,9 @@ namespace StgSharp.RegularAnalysis.TextRegex
op.Left = _1;
op.Right = _2;
_stack.PushOperand(op);
_tree.AddNode(op);
_tree.AddNode(_1);
_tree.AddNode(_2);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs
+5 -4
View File
@@ -52,11 +52,12 @@ namespace StgSharp.RegularAnalysis.TextRegex
}
if (Source.TryReadToken(out Token<RegexElementLabel> orig))
{
if (((int)(orig.Flag | RegexElementLabel.ATOM_BEGIN) != 0) && ((int)_prev != 0))
if (((int)(orig.Flag & RegexElementLabel.ATOM_BEGIN) != 0) && ((int)_prev != 0))
{
Token<RegexElementLabel> ret = new Token<RegexElementLabel>(string.Empty, orig.Line,
orig.Column,
RegexElementLabel.OPERATOR);
Token<RegexElementLabel> ret = new Token<RegexElementLabel>(string.Empty,
orig.Line,
orig.Column,
RegexElementLabel.CONCAT);
_buffer.Enqueue(orig);
_prev = orig.Flag & RegexElementLabel.ATOM_END;
return ret;
StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs
+2 -2
View File
@@ -75,7 +75,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
position = i + 1;
return new Token<RegexElementLabel>(_source[(pos - 1)..(i - 1)], 0,
position,
RegexElementLabel.CHAR_SET);
RegexElementLabel.UNIT_SET);
}
}
throw new InvalidOperationException("Unclosed character set");
@@ -85,7 +85,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
return ReadGroupBeginToken();
case ')':
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..1], 0, pos,
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
RegexElementLabel.GROUP_END);
case '{':
for (int i = pos + 1; i < _source.Length; i++)
StgSharp.RegularAnalysis/TextRegex/RegexTokenUtil.cs
+1 -1
View File
@@ -41,7 +41,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
this Token<RegexElementLabel> token
)
{
return ((token.Flag & RegexElementLabel.UNIT) != 0 && token.Value.Length == 0) ||
return ((token.Flag & RegexElementLabel.UNIT) != 0 && token.Value.Length == 1) ||
(token.Flag & RegexElementLabel.UNIT_SPAN) != 0;
}
StgSharpDebug/Program.cs
+3 -1
View File
@@ -50,7 +50,9 @@ namespace StgSharp
.UseModule<GraphicModule>()
.InitializeModule();
HlsfStressTest.Run();
TestRegexAnalyse.Run();
// HlsfStressTest.Run();
// TestMatrix();
StgSharpDebug/StgSharpDebug.csproj
+1
View File
@@ -77,6 +77,7 @@
<ProjectReference Include="..\StgSharp.Common\StgSharp.Common.csproj" />
<ProjectReference Include="..\StgSharp.Mathematics\StgSharp.Mathematics.csproj" />
<ProjectReference Include="..\StgSharp.Model\StgSharp.Model.csproj" />
<ProjectReference Include="..\StgSharp.RegularAnalysis\StgSharp.RegularAnalysis.csproj" />
<ProjectReference Include="..\StgSharp.Script\StgSharp.Script.csproj" />
</ItemGroup>
StgSharpDebug/TestRegexAnalyse.cs
+50
View File
@@ -0,0 +1,50 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="TestRegexAnalyse"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using StgSharp.RegularAnalysis.TextRegex;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp
{
public class TestRegexAnalyse
{
private const string regex = @"abc(?<name>abc)";
public static void Run()
{
RegexInterpreter interpreter = new RegexInterpreter();
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree = interpreter.Analyze(regex);
}
}
}