youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

regex ir opt is almost done

Browse Source
This commit is contained in:
Youmu Liu
2026-06-30 20:14:30 +08:00
parent e60eb0ada6
commit 1cd0257c93
12 changed files with 552 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs
+179
View File
@@ -0,0 +1,179 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexAnalyzer.ASTGenerate"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexAnalyzer
{
private static AbstractSyntaxTree<RegexAstNode, RegexElementLabel> GenerateAST(
string _source
)
{
CompileStack<RegexAstNode, RegexElementLabel> _stack = new();
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> _tree = new();
RegexTokenReader reader = new(_source);
TokenParser<RegexElementLabel, RegexElementLabel> lexer = reader.Pipe(() => new RegexTokenParser());
while (lexer.TryReadToken(out Token<RegexElementLabel> token))
{
// Console.WriteLine(token.Value);
if ((token.Flag & RegexElementLabel.SEQUENCE) != 0)
{
// char or charset
_stack.PushOperand(new RegexAstNode(token));
} else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0)
{
_stack.PushOperator(new RegexAstNode(token));
_stack.IncreaseDepth(0);
} else if ((token.Flag & RegexElementLabel.GROUP_END) != 0)
{
// close all nodes here
RegexAstNode op;
while (_stack.TryPopOperator(out op)) {
ProcessOperator(op);
}
if (_stack.OperandInDepthCount != 1) {
throw new InvalidOperationException();
}
_stack.DecreaseDepth();
op = _stack.PopOperator();
if (op.Source.Flag != RegexElementLabel.GROUP_BEGIN) {
throw new InvalidOperationException("Unmatched group begin and end symbol");
}
if (op.Source.Value.Length > 1)
{
op.Right = _stack.PopOperand();
_stack.PushOperand(op);
}
} else if ((token.Flag & RegexElementLabel.OPERATOR) != 0)
{
if (_stack.OperatorInDepthCount == 0)
{
// the first operator in stack
_stack.PushOperator(new RegexAstNode(token));
} else
{
// process operator by precedence
while (_stack.TryPeekOperator(out RegexAstNode topOp))
{
(bool Valid, int Value) cmp = RegexAstNode.ComparePrecedence(topOp.Source.Flag, token.Flag);
if (!cmp.Valid) {
throw new InvalidOperationException("Operators with same precedence are not supported.");
}
if (cmp.Value < 0)
{
_stack.PopOperator();
ProcessOperator(topOp);
continue;
} else
{
break;
}
}
_stack.PushOperator(new RegexAstNode(token));
}
}
}
if (_stack.Depth == 1)
{
if (_stack.OperandInDepthCount != 1 || _stack.OperatorInDepthCount != 0)
{
while (_stack.TryPopOperator(out RegexAstNode? op)) {
ProcessOperator(op);
}
}
if (_stack.OperandInDepthCount == 1 && _stack.OperatorInDepthCount == 0)
{
_tree.Root = _stack.PopOperand();
return _tree;
}
}
throw new InvalidOperationException("Invalid regular expression syntax.");
void ProcessOperator(
RegexAstNode op
)
{
if (op.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1) &&
_stack.TryPopOperand(out RegexAstNode? _2))
{
op.Left = _2;
op.Right = _1;
_stack.PushOperand(op);
_ = _tree.AddNode(op);
_ = _tree.AddNode(_1);
_ = _tree.AddNode(_2);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
} else if (op.EqualityTypeConvert == RegexElementLabel.COUNT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1))
{
op.Left = _1;
_stack.PushOperand(op);
_ = _tree.AddNode(_1);
_ = _tree.AddNode(op);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
} else if (op.EqualityTypeConvert == RegexElementLabel.ALT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1) &&
_stack.TryPopOperand(out RegexAstNode? _2))
{
op.Left = _2;
op.Right = _1;
_stack.PushOperand(op);
_ = _tree.AddNode(op);
_ = _tree.AddNode(_1);
_ = _tree.AddNode(_2);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
}
}
}
}
}
StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs → StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs
+62 -1
View File
@@ -1,6 +1,6 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexInterpreter.ASTOptimize"
// file="RegexAnalyzer.ASTOptimize"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
@@ -156,6 +156,67 @@ namespace StgSharp.RegularAnalysis.Text
return true;
}
private static void OptimizeTree(
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree
)
{
// phase0: scan container and remove no parent nodes
_ = tree.AllNodes.RemoveWhere(node => node.Parent == null && node != tree.Root);
TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(tree.Root);
RegexAstNode current;
// phase1: ALT flatten
// phase2: CONCAT normalize
int i = 0;
List<RegexAstNode> altNodes = [];
while (enumerator.MoveNext())
{
current = enumerator.Current;
if (current.EqualityTypeConvert == RegexElementLabel.ALT)
{
RegexAstNode union = FlattenAlt(current);
altNodes.Add(current);
current.Right = union;
current.Left = RegexAstNode.Empty;
} else if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) {
RotateConcat(current);
}
}
List<RegexAstNode> cases = [];
// phase3: then ALT union
foreach (RegexAstNode item in altNodes)
{
RegexAstNode _case = item.Right;
do
{
cases.Add(_case);
_case = _case.Next;
} while (!RegexAstNode.IsNullOrEmpty(_case));
RegexAstNode newAlt = MergeAlt(cases);
item.ReplaceBy(newAlt);
}
// return;
// phase4: CONCAT merge
enumerator.Reset();
while (enumerator.MoveNext())
{
// suppose all concat is now in normalized form,
// we can merge adjacent unit nodes into a single unit node with string value
current = enumerator.Current;
if (current.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
if (MergeConcat(current)) {
enumerator.SkipOnce();
}
}
}
}
private static void RotateConcat(
RegexAstNode root
)
StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs → StgSharp.RegularAnalysis/Text/RegexAnalyzer.IRGenerate.cs
+29 -26
View File
@@ -1,30 +1,27 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexInterpreter.IRGenerate.cs"
// file="RegexInterpreter.IRGenerate"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
@@ -46,7 +43,9 @@ namespace StgSharp.RegularAnalysis.Text
public partial class RegexAnalyzer
{
internal static List<RegexIR> GenerateIR(AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree)
internal static List<RegexIR> GenerateIR(
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree
)
{
return GenerateIRRecursion(tree.Root, 0).ToList();
}
@@ -58,7 +57,8 @@ namespace StgSharp.RegularAnalysis.Text
int min,
int max,
bool isGreedy,
RegexAstNode right)
RegexAstNode right
)
{
bool is_operator = !RegexAstNode.IsNullOrEmpty(right) &&
(right.Source.Flag & RegexElementLabel.OPERATOR) != 0;
@@ -88,7 +88,10 @@ namespace StgSharp.RegularAnalysis.Text
}
}
private static RegexIRGenerator GenerateIRRecursion(RegexAstNode node, int depth)
private static RegexIRGenerator GenerateIRRecursion(
RegexAstNode node,
int depth
)
{
#pragma warning disable IDE0010
if (depth > 1024)
@@ -117,7 +120,7 @@ namespace StgSharp.RegularAnalysis.Text
RegexIRGenerator groupIR = new();
rightIR = GenerateIRRecursion(right, curDepth + 1);
groupIR.EmitMatchComplex(rightIR.Count);
groupIR.EmitTry(rightIR.Count);
groupIR.EmitIRStream(rightIR);
groupIR.EmitCondition(2, 1);
groupIR.EmitPop(false);
@@ -251,7 +254,7 @@ namespace StgSharp.RegularAnalysis.Text
RegexIRGenerator cur_ir = casesIRList[^case_count];
ir_count -= cur_ir.Count;
case_count--;
alt.EmitMatchComplex(cur_ir.Count);
alt.EmitTry(cur_ir.Count);
alt.EmitIRStream(cur_ir);
alt.EmitCondition(1/* need to calc here */, 2);
alt.EmitPop(true);
StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs → StgSharp.RegularAnalysis/Text/RegexAnalyzer.IROptimize.cs
+164 -48
View File
@@ -1,6 +1,6 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexInterpreter.IROptimize"
// file="RegexAnalyzer.IROptimize"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
@@ -40,80 +40,125 @@ namespace StgSharp.RegularAnalysis.Text
public partial class RegexAnalyzer
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static RegexInfo ScanAndOptimizeIR(
List<RegexIR> baseIR,
out List<RegexIR> optIR
internal static void ScanAndOptimizeIR(
List<RegexIR> baseIR,
out List<RegexIR> optIR
)
{
int index = 0;
int opt_count;
List<RegexIR> origin_ir = baseIR;
Span<RegexIR> origin_span = CollectionsMarshal.AsSpan(baseIR);
List<RegexIR> opt_ir = new(baseIR.Count);
CapacityFixedList<RegexIR> opt_window = new(8);
Span<RegexIR> opt_win_sp = [.. opt_window];
RegexInfo info = new();
List<RegexIR> opt_ir = new(origin_ir.Count);
RegexIR[] opt_window = new RegexIR[8];
Span<RegexIR> opt_win_sp = opt_window;
opt_cycle:
opt_count = 0;
int index = 0;
Span<RegexIR> origin_span = CollectionsMarshal.AsSpan(origin_ir);
do
{
int process_count = 0;
int last = int.Min(origin_span.Length, index + 8);
origin_span[index..last].CopyTo(opt_win_sp);
do
int result;
if ((result = TryExpandPlus(opt_win_sp, opt_ir)) >= 0)
{
int result = 0;
if ((result = TryExpandPlus(opt_win_sp, opt_ir)) >= 0)
process_count++;
if (result > 0)
{
process_count++;
if (result > 0)
{
RegexCountComplexIR cc = Unsafe.As<RegexCountComplexIR>(opt_win_sp[0]);
Span<RegexIR> sp = origin_span.Slice(index + 1, result);
TryExpandComplexPlus(cc.IsGreedy, sp, opt_ir);
process_count += result;
}
} else if ((result = TryOptDotCountCount(opt_win_sp, opt_ir)) >= 0)
{
process_count++;
} else if (TryFlattenCount(opt_win_sp, opt_ir)) {
process_count++;
RegexCountComplexIR cc = Unsafe.As<RegexCountComplexIR>(opt_win_sp[0]);
Span<RegexIR> sp = origin_span.Slice(index + 1, result);
TryExpandComplexPlus(cc.IsGreedy, sp, opt_ir);
process_count += result;
}
} while (false);
opt_count++;
} else if (TryOptDotCountCount(opt_win_sp, opt_ir) >= 0)
{
process_count += 2;
opt_count++;
} else if (TryFlattenCount(opt_win_sp, opt_ir) >= 0)
{
process_count++;
opt_count++;
} else if ((result = TryExpandOnce(opt_win_sp, opt_ir)) >= 0)
{
process_count++;
if (result > 0)
{
RegexCountComplexIR cc = Unsafe.As<RegexCountComplexIR>(opt_win_sp[0]);
Span<RegexIR> sp = origin_span.Slice(index + 1, result);
TryExpandComplex(sp, opt_ir, true);
process_count += result;
}
opt_count++;
} else if (opt_win_sp[0] is RegexTryIR{ IsFullyOptimized:false} try_ir)
{
process_count++;
int cmd_cnt = try_ir.CommandCount;
Span<RegexIR> try_sp = origin_span.Slice(index + 1, cmd_cnt);
ScanAndOptimizeIR(try_sp, out List<RegexIR> opt_try);
RegexTryIR new_try = new(opt_try.Count)
{
IsFullyOptimized = true
};
opt_ir.Add(new_try);
opt_ir.AddRange(opt_try);
process_count += cmd_cnt;
opt_count++;
} else
{
opt_ir.Add(opt_win_sp[0]);
process_count++;
}
index += process_count;
} while (index < baseIR.Count);
} while (index < origin_ir.Count);
if (opt_count > 0)
{
// need to swap the origin_ir and opt_ir, and clear the opt_ir for next round
if (ReferenceEquals(origin_ir, baseIR))
{
origin_ir = new(opt_ir.Count);
}
(origin_ir, opt_ir) = (opt_ir, origin_ir);
opt_ir.Clear();
goto opt_cycle;
}
optIR = opt_ir;
return info;
}
private static RegexInfo ScanAndOptimizeIR(
Span<RegexIR> baseIR,
List<RegexIR> opt_ir
private static void ScanAndOptimizeIR(
Span<RegexIR> baseIR,
out List<RegexIR> opt_ir
)
{
List<RegexIR> originList = new(baseIR.Length);
CollectionsMarshal.SetCount(originList, baseIR.Length);
baseIR.CopyTo(CollectionsMarshal.AsSpan(originList));
return ScanAndOptimizeIR(originList, out opt_ir);
ScanAndOptimizeIR(originList, out opt_ir);
}
private static bool TryFlattenCount(
Span<RegexIR> window,
List<RegexIR> opt
private static int TryFlattenCount(
Span<RegexIR> window,
List<RegexIR> opt
)
{
const int falttern_size = 32;
RegexIR ir = window[0];
if (ir is not RegexCountSeqIR seq) {
return false;
return -1;
}
int min = seq.Min;
int max = seq.Max;
if (min == 0) {
return -1;
}
string pattern = seq.Sequence;
int pattern_size = pattern.Length;
if (pattern_size > falttern_size) {
opt.Add(ir);
}
int min = seq.Min;
int max = seq.Max;
int min_expand = falttern_size / pattern_size;
min_expand = int.Min(min_expand, min);
string expand = string.Concat(Enumerable.Repeat(pattern, min_expand));
@@ -121,10 +166,10 @@ namespace StgSharp.RegularAnalysis.Text
int new_rest = min % min_expand;
if (new_min == 1)
{
opt.Add(new RegexMatchSeqIR(pattern));
opt.Add(new RegexMatchSeqIR(expand));
} else
{
opt.Add(new RegexCountSeqIR(new_min, new_min, seq.IsGreedy, pattern));
opt.Add(new RegexCountSeqIR(new_min, new_min, seq.IsGreedy, expand));
}
if (new_rest > 0)
{
@@ -134,7 +179,7 @@ namespace StgSharp.RegularAnalysis.Text
int rest_count = max - min;
rest_count = int.Max(rest_count, -1);
opt.Add(new RegexCountSeqIR(0, rest_count, seq.IsGreedy, pattern));
return true;
return 0;
}
#region dot count count
@@ -176,17 +221,89 @@ namespace StgSharp.RegularAnalysis.Text
}
#region expand once
private static void TryExpandComplex(
Span<RegexIR> complex,
List<RegexIR> opt,
bool is_once
)
{
if (!is_once) {
opt.Add(new RegexTryIR(complex.Length));
}
opt.AddRange(complex);
}
private static int TryExpandOnce(
Span<RegexIR> window,
List<RegexIR> opt
)
{
RegexIR ir = window[0];
if (ir is IRegexCount { Min: 1, Max: 1 })
{
string pattern;
switch (ir.Command & RegexIRCommand.Count)
{
case RegexIRCommand.Count_Complex:
RegexCountComplexIR complex = Unsafe.As<RegexCountComplexIR>(ir);
return complex.CommandCount;
case RegexIRCommand.Count_Seq:
RegexCountSeqIR seq = Unsafe.As<RegexCountSeqIR>(ir);
pattern = seq.Sequence;
opt.Add(new RegexMatchSeqIR(pattern));
return 0;
case RegexIRCommand.Count_Set:
RegexCountSetIR set = Unsafe.As<RegexCountSetIR>(ir);
pattern = set.CharSet;
opt.Add(new RegexMatchSetIR(pattern));
return 0;
default:
return -1;
}
} else if (ir is IRegexCount { Min: 0, Max: 1 })
{
string pattern;
switch (ir.Command & RegexIRCommand.Count)
{
case RegexIRCommand.Count_Complex:
RegexCountComplexIR complex = Unsafe.As<RegexCountComplexIR>(ir);
return complex.CommandCount;
case RegexIRCommand.Count_Seq:
RegexCountSeqIR seq = Unsafe.As<RegexCountSeqIR>(ir);
pattern = seq.Sequence;
opt.Add(new RegexTryIR(1));
opt.Add(new RegexMatchSeqIR(pattern));
return 0;
case RegexIRCommand.Count_Set:
opt.Add(new RegexTryIR(1));
RegexCountSetIR set = Unsafe.As<RegexCountSetIR>(ir);
pattern = set.CharSet;
opt.Add(new RegexMatchSetIR(pattern));
return 0;
default:
return -1;
}
} else
{
return -1;
}
}
#endregion
#region expand plus
private static bool TryExpandComplexPlus(
private static void TryExpandComplexPlus(
bool is_greedy,
Span<RegexIR> complex,
List<RegexIR> opt
)
{
int count = complex.Length;
List<RegexIR> part_opt = new(count);
RegexInfo info = ScanAndOptimizeIR(complex, part_opt);
ScanAndOptimizeIR(complex, out List<RegexIR>? part_opt);
opt.AddRange(part_opt);
RegexCountComplexIR c_ir = new(0, -1, is_greedy, part_opt.Count)
{
@@ -194,7 +311,6 @@ namespace StgSharp.RegularAnalysis.Text
};
opt.Add(c_ir);
opt.AddRange(part_opt);
return true;
}
/// <summary>
StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs → StgSharp.RegularAnalysis/Text/RegexAnalyzer.SourceGenerate.cs
View File
StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs
+94
View File
@@ -0,0 +1,94 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexAnalyzer"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection.Emit;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public record TextRegexSource
{
internal TextRegexSource(
string pattern,
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> ast,
List<RegexIR> base_ir,
List<RegexIR> ir,
Exception? analyseException = null
)
{
Pattern = pattern;
Ast = ast;
IRs = ir;
BaseIR = base_ir;
AnalyseException = analyseException;
}
public Exception? AnalyseException { get; set; }
public string Pattern { get; }
internal AbstractSyntaxTree<RegexAstNode, RegexElementLabel> Ast { get; }
internal List<RegexIR> BaseIR { get; }
internal List<RegexIR> IRs { get; }
}
public static partial class RegexAnalyzer
{
public static TextRegexSource Analyze(
string regex
)
{
// RegexAnalyzer interpreter = new();
// interpreter._source = regex;
try
{
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree = GenerateAST(regex);
OptimizeTree(tree);
List<RegexIR> base_ir = GenerateIR(tree);
ScanAndOptimizeIR(base_ir, out List<RegexIR> opt_ir);
return new TextRegexSource(regex, tree, base_ir, opt_ir);
}
catch (Exception ex)
{
return new TextRegexSource(regex, null, null, null, ex);
}
}
}
}
StgSharp.RegularAnalysis/Text/RegexIR.cs
+3 -5
View File
@@ -312,16 +312,14 @@ namespace StgSharp.RegularAnalysis.Text
}
internal sealed class RegexMatchComplexIR(
internal sealed class RegexTryIR(
int count
) : RegexIR, IRegexComplex
{
public bool IsFullyOptimized { get; set; }
public int CountOfCommand { get; } = count;
public int CommandCount { get; set; }
public int CommandCount { get; set; } = count;
internal override RegexIRCommand Command => RegexIRCommand.Try;
@@ -329,7 +327,7 @@ namespace StgSharp.RegularAnalysis.Text
int lineNum
)
{
return $"[{lineNum}]\t TRY {CountOfCommand}";
return $"[{lineNum}]\t TRY {CommandCount}";
}
}
StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs
+2 -2
View File
@@ -134,11 +134,11 @@ namespace StgSharp.RegularAnalysis.Text
Count++;
}
public void EmitMatchComplex(
public void EmitTry(
int count
)
{
_tail.Add(new RegexMatchComplexIR(count));
_tail.Add(new RegexTryIR(count));
Count++;
}
StgSharp.RegularAnalysis/Text/RegexInterpreter.cs
-275
View File
@@ -1,275 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexInterpreter"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection.Emit;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexAnalyzer
{
private AbstractSyntaxTree<RegexAstNode, RegexElementLabel> _tree = new();
private CompileStack<RegexAstNode, RegexElementLabel> _stack = new();
private Exception _interpreteException;
private string _source;
private RegexAnalyzer() { }
internal AbstractSyntaxTree<RegexAstNode, RegexElementLabel> Tree
{
get
{
if (_interpreteException != null) {
throw new InvalidOperationException("The regex interpretation failed.", _interpreteException);
}
return _tree;
}
}
<<<<<<< HEAD:StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs
public static RegexInterpreter Analyze(string regex)
=======
public static RegexAnalyzer Analyze(
string regex
)
>>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/RegexInterpreter.cs
{
RegexAnalyzer interpreter = new();
interpreter._source = regex;
bool isSuccess;
try
{
interpreter.Analyze();
interpreter.OptimizeTree();
isSuccess = true;
}
catch (Exception ex)
{
interpreter._interpreteException = ex;
isSuccess = false;
}
return interpreter;
}
private void Analyze()
{
RegexTokenReader reader = new(_source);
TokenParser<RegexElementLabel, RegexElementLabel> lexer = reader.Pipe(() => new RegexTokenParser());
while (lexer.TryReadToken(out Token<RegexElementLabel> token))
{
// Console.WriteLine(token.Value);
if ((token.Flag & RegexElementLabel.SEQUENCE) != 0)
{
// char or charset
_stack.PushOperand(new RegexAstNode(token));
} else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0)
{
_stack.PushOperator(new RegexAstNode(token));
_stack.IncreaseDepth(0);
} else if ((token.Flag & RegexElementLabel.GROUP_END) != 0)
{
// close all nodes here
RegexAstNode op;
while (_stack.TryPopOperator(out op)) {
ProcessOperator(op);
}
if (_stack.OperandInDepthCount != 1) {
throw new InvalidOperationException();
}
_stack.DecreaseDepth();
op = _stack.PopOperator();
if (op.Source.Flag != RegexElementLabel.GROUP_BEGIN) {
throw new InvalidOperationException("Unmatched group begin and end symbol");
}
if (op.Source.Value.Length > 1)
{
op.Right = _stack.PopOperand();
_stack.PushOperand(op);
}
} else if ((token.Flag & RegexElementLabel.OPERATOR) != 0)
{
if (_stack.OperatorInDepthCount == 0)
{
// the first operator in stack
_stack.PushOperator(new RegexAstNode(token));
} else
{
// process operator by precedence
while (_stack.TryPeekOperator(out RegexAstNode topOp))
{
(bool Valid, int Value) cmp = RegexAstNode.ComparePrecedence(topOp.Source.Flag, token.Flag);
if (!cmp.Valid) {
throw new InvalidOperationException("Operators with same precedence are not supported.");
}
if (cmp.Value < 0)
{
_stack.PopOperator();
ProcessOperator(topOp);
continue;
} else
{
break;
}
}
_stack.PushOperator(new RegexAstNode(token));
}
}
}
if (_stack.Depth == 1)
{
if (_stack.OperandInDepthCount != 1 || _stack.OperatorInDepthCount != 0)
{
while (_stack.TryPopOperator(out RegexAstNode? op)) {
ProcessOperator(op);
}
}
if (_stack.OperandInDepthCount == 1 && _stack.OperatorInDepthCount == 0)
{
_tree.Root = _stack.PopOperand();
return;
}
}
throw new InvalidOperationException("Invalid regular expression syntax.");
}
private void OptimizeTree()
{
// phase0: scan container and remove no parent nodes
_ = _tree.AllNodes.RemoveWhere(node => node.Parent == null && node != _tree.Root);
TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(_tree.Root);
RegexAstNode current;
// phase1: ALT flatten
// phase2: CONCAT normalize
int i = 0;
List<RegexAstNode> altNodes = [];
while (enumerator.MoveNext())
{
current = enumerator.Current;
if (current.EqualityTypeConvert == RegexElementLabel.ALT)
{
RegexAstNode union = FlattenAlt(current);
altNodes.Add(current);
current.Right = union;
current.Left = RegexAstNode.Empty;
} else if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) {
RotateConcat(current);
}
}
List<RegexAstNode> cases = [];
// phase3: then ALT union
foreach (RegexAstNode item in altNodes)
{
RegexAstNode _case = item.Right;
do
{
cases.Add(_case);
_case = _case.Next;
} while (!RegexAstNode.IsNullOrEmpty(_case));
RegexAstNode newAlt = MergeAlt(cases);
item.ReplaceBy(newAlt);
}
// return;
// phase4: CONCAT merge
enumerator.Reset();
while (enumerator.MoveNext())
{
// suppose all concat is now in normalized form,
// we can merge adjacent unit nodes into a single unit node with string value
current = enumerator.Current;
if (current.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
if (MergeConcat(current)) {
enumerator.SkipOnce();
}
}
}
// TODO enum AST again , do ALT tail merge and simple ALT front merge
}
private void ProcessOperator(RegexAstNode op)
{
if (op.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1) && _stack.TryPopOperand(out RegexAstNode? _2))
{
op.Left = _2;
op.Right = _1;
_stack.PushOperand(op);
_tree.AddNode(op);
_tree.AddNode(_1);
_tree.AddNode(_2);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
} else if (op.EqualityTypeConvert == RegexElementLabel.COUNT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1))
{
op.Left = _1;
_stack.PushOperand(op);
_tree.AddNode(_1);
_tree.AddNode(op);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
} else if (op.EqualityTypeConvert == RegexElementLabel.ALT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1) && _stack.TryPopOperand(out RegexAstNode? _2))
{
op.Left = _2;
op.Right = _1;
_stack.PushOperand(op);
_tree.AddNode(op);
_tree.AddNode(_1);
_tree.AddNode(_2);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
}
}
}
}
StgSharp.RegularAnalysis/Text/TextRegex.cs
+7 -14
View File
@@ -28,10 +28,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
<<<<<<< HEAD:StgSharp.RegularAnalysis/TextRegex/TextRegex.cs
=======
using System.Runtime.CompilerServices;
>>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/TextRegex.cs
using System.Text;
using System.Threading.Tasks;
@@ -39,22 +35,19 @@ namespace StgSharp.RegularAnalysis.Text
{
public abstract class TextRegex
{
<<<<<<< HEAD:StgSharp.RegularAnalysis/TextRegex/TextRegex.cs
public ref struct Match
{
private ReadOnlySpan<char> _sequence;
}
}
=======
public abstract bool IsMatch(
ReadOnlySpan<char> text
);
public ref struct Match
{
private ReadOnlySpan<char> _sequence;
}
}
public ref struct MatchResult { }
>>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/TextRegex.cs
}