youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scanning min scan length

Browse Source
This commit is contained in:
Youmu Liu
2026-07-08 18:25:05 +08:00
parent 439128798e
commit 29b24191e1
8 changed files with 125 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs
+1 -1
View File
@@ -128,7 +128,7 @@ namespace StgSharp.RegularAnalysis.Text
{
if (_stack.TryPopOperand(out RegexAstNode? _1))
{
op.Left = _1;
op.Right = _1;
_stack.PushOperand(op);
_ = _tree.AddNode(_1);
_ = _tree.AddNode(op);
StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs
+37 -3
View File
@@ -137,7 +137,8 @@ namespace StgSharp.RegularAnalysis.Text
}
private static void OptimizeTree(
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree,
ref RegexInfo info
)
{
// phase0: scan container and remove no parent nodes
@@ -163,11 +164,10 @@ namespace StgSharp.RegularAnalysis.Text
}
}
List<RegexAstNode> cases = [];
// phase3: then ALT union
foreach (RegexAstNode item in altNodes)
{
List<RegexAstNode> cases = [];
RegexAstNode _case = item.Right;
do
{
@@ -194,6 +194,40 @@ namespace StgSharp.RegularAnalysis.Text
}
}
}
info.MinPredictLength = ComputeMinLength(tree.Root);
static int ComputeMinLength(
RegexAstNode node
)
{
if (node.EqualityTypeConvert is RegexElementLabel.UNIT_SPAN or
RegexElementLabel.UNIT)
{
return node.Value.Length;
} else if (node.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
return ComputeMinLength(node.Left) + ComputeMinLength(node.Right);
} else if (node.EqualityTypeConvert == RegexElementLabel.ALT)
{
RegexAstNode case_0 = node.Right;
int length = int.MaxValue;
do
{
length = int.Min(ComputeMinLength(case_0), length);
case_0 = case_0.Next;
} while (!RegexAstNode.IsNullOrEmpty(case_0));
return length;
} else if (node.EqualityTypeConvert == RegexElementLabel.COUNT)
{
string source = node.Source.Value;
int single_count = ComputeMinLength(node.Right);
return 0;
} else
{
return 0;
}
}
}
private static void RotateConcat(
StgSharp.RegularAnalysis/Text/RegexAnalyzer.IRGenerate.cs
+54 -27
View File
@@ -24,10 +24,11 @@ namespace StgSharp.RegularAnalysis.Text
{
internal static List<RegexIR> GenerateIR(
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree,
ref RegexInfo info
)
{
return GenerateIRRecursion(tree.Root, 0).ToList();
return GenerateIRRecursion(tree.Root, 0, ref info).ToList();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -37,7 +38,8 @@ namespace StgSharp.RegularAnalysis.Text
int min,
int max,
bool isGreedy,
RegexAstNode right
RegexAstNode right,
ref RegexInfo info
)
{
bool is_operator = !RegexAstNode.IsNullOrEmpty(right) &&
@@ -45,7 +47,7 @@ namespace StgSharp.RegularAnalysis.Text
if (is_operator) // seq to be count is complex
{
RegexIRGenerator right_ir = GenerateIRRecursion(right, curDepth + 1);
RegexIRGenerator right_ir = GenerateIRRecursion(right, curDepth + 1, ref info);
ir.EmitCountComplex(min, max, isGreedy, right_ir.Count);
ir.EmitIRStream(right_ir);
} else
@@ -70,7 +72,8 @@ namespace StgSharp.RegularAnalysis.Text
private static RegexIRGenerator GenerateIRRecursion(
RegexAstNode node,
int depth
int depth,
ref RegexInfo info
)
{
#pragma warning disable IDE0010
@@ -99,7 +102,7 @@ namespace StgSharp.RegularAnalysis.Text
string.Empty; /* unnamed group (?:) */
RegexIRGenerator groupIR = new();
rightIR = GenerateIRRecursion(right, curDepth + 1);
rightIR = GenerateIRRecursion(right, curDepth + 1, ref info);
groupIR.EmitTry(rightIR.Count);
groupIR.EmitIRStream(rightIR);
groupIR.EmitCondition(2, 1);
@@ -109,31 +112,19 @@ namespace StgSharp.RegularAnalysis.Text
case RegexElementLabel.COUNT:
string count_source = token.Value;
bool isGreedy = !(count_source.Length > 1 && count_source.EndsWith('?'));
int seq_mask = (RegexAstNode.IsNullOrEmpty(left) ? 0 : 2) + (RegexAstNode.IsNullOrEmpty(right) ?
0 :
1);
RegexAstNode seq = seq_mask switch
{
// no seq, invalid count operator
0 => throw new InvalidOperationException("Count operator must have a sequence to operate on"),
1 => right,// right seq
2 => left,// left seq
// both sides, invalid count operator
3 => throw new InvalidOperationException("Count operator cannot have two sequences to operate on"),
_ => throw new InvalidOperationException("Unexpected sequence mask for count operator"),
};
RegexAstNode seq = right;
ReadOnlySpan<char> count_source_span = count_source;
RegexIRGenerator count_ir = new();
if (count_source_span[0] == '*') // infinite count
{
EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq);
EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq, ref info);
} else if (count_source_span[0] == '+') // at least one
{
EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq);
EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq, ref info);
} else if (count_source_span[0] == '?') // zero or one
{
EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq);
EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq, ref info);
} else if (count_source_span[0] == '{') // zero or one, non-greedy
{
int comma_pos = count_source_span.IndexOf(',');
@@ -141,7 +132,8 @@ namespace StgSharp.RegularAnalysis.Text
{
if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count))
{
EmitCount(curDepth, count_ir, count, count, isGreedy, seq);
EmitCount(curDepth, count_ir, count, count, isGreedy, seq,
ref info);
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
@@ -151,7 +143,7 @@ namespace StgSharp.RegularAnalysis.Text
if (int.TryParse(count_source[1..comma_pos], out int min) &&
int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max))
{
EmitCount(curDepth, count_ir, min, max, isGreedy, seq);
EmitCount(curDepth, count_ir, min, max, isGreedy, seq, ref info);
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
@@ -187,7 +179,7 @@ namespace StgSharp.RegularAnalysis.Text
}
} else if ((left.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{
leftIR = GenerateIRRecursion(left, curDepth + 1);
leftIR = GenerateIRRecursion(left, curDepth + 1, ref info);
concat.EmitIRStream(leftIR);
}
@@ -212,7 +204,7 @@ namespace StgSharp.RegularAnalysis.Text
}
} else if ((right.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{
rightIR = GenerateIRRecursion(right, curDepth + 1);
rightIR = GenerateIRRecursion(right, curDepth + 1, ref info);
concat.EmitIRStream(rightIR);
}
return concat;
@@ -223,7 +215,7 @@ namespace StgSharp.RegularAnalysis.Text
int ir_count = 0;
while (!RegexAstNode.IsNullOrEmpty(cur_node))
{
RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1);
RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1, ref info);
casesIRList.Add(_case);
ir_count += _case.Count;
cur_node = cur_node.Next;
@@ -255,6 +247,41 @@ namespace StgSharp.RegularAnalysis.Text
return null!;
}
#pragma warning restore IDE0010
// info.MinPredictLength = ComputeMinLength(tree.Root);
static int ComputeMinLength(
RegexAstNode node
)
{
if (node.EqualityTypeConvert is RegexElementLabel.UNIT_SPAN or
RegexElementLabel.UNIT)
{
return node.Value.Length;
} else if (node.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
return ComputeMinLength(node.Left) + ComputeMinLength(node.Right);
} else if (node.EqualityTypeConvert == RegexElementLabel.ALT)
{
RegexAstNode case_0 = node.Right;
int length = int.MaxValue;
do
{
length = int.Min(ComputeMinLength(case_0), length);
case_0 = case_0.Next;
} while (!RegexAstNode.IsNullOrEmpty(case_0));
return length;
} else if (node.EqualityTypeConvert == RegexElementLabel.COUNT)
{
string source = node.Source.Value;
int single_count = ComputeMinLength(node.Right);
return 0;
} else
{
return 0;
}
}
}
}
StgSharp.RegularAnalysis/Text/RegexAnalyzer.IROptimize.cs
-11
View File
@@ -306,17 +306,6 @@ namespace StgSharp.RegularAnalysis.Text
}
}
internal class RegexInfo
{
public int MinLength { get; set; }
public int TryDepth { get; set; }
public int GroupCount { get; set; }
}
#region merge find and try-find
private static int TyrMergeFindRestTwice(
StgSharp.RegularAnalysis/Text/RegexAnalyzer.InfoScan.cs → StgSharp.RegularAnalysis/Text/RegexAnalyzer.ScanInfo.cs
+2 -13
View File
@@ -18,11 +18,11 @@ namespace StgSharp.RegularAnalysis.Text
public partial class RegexAnalyzer
{
internal static RegexInterpreterInfo ScanInfo(
internal static RegexInfo ScanInfo(
List<RegexIR> ir
)
{
RegexInterpreterInfo info = new();
RegexInfo info = new();
int process_count = 0;
ReadOnlySpan<RegexIR> irSpan = CollectionsMarshal.AsSpan(ir);
@@ -97,16 +97,5 @@ namespace StgSharp.RegularAnalysis.Text
}
}
internal struct RegexInterpreterInfo
{
public int RegionCount { get; set; }
public int MinPredictLength { get; set; }
public int SingleLineResultCount { get; set; }
}
}
}
StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs
+22 -5
View File
@@ -26,7 +26,9 @@ namespace StgSharp.RegularAnalysis.Text
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> ast,
List<RegexIR> base_ir,
List<RegexIR> ir,
RegexInfo info,
Exception? analyseException = null
)
{
Pattern = pattern;
@@ -34,6 +36,7 @@ namespace StgSharp.RegularAnalysis.Text
IRs = ir;
BaseIR = base_ir;
AnalyseException = analyseException;
Info = info;
}
public Exception? AnalyseException { get; set; }
@@ -46,6 +49,8 @@ namespace StgSharp.RegularAnalysis.Text
internal List<RegexIR> IRs { get; }
internal RegexInfo Info { get; }
}
public static partial class RegexAnalyzer
@@ -59,6 +64,7 @@ namespace StgSharp.RegularAnalysis.Text
// RegexAnalyzer interpreter = new();
// interpreter._source = regex;
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree;
RegexInfo info = new();
try
{
tree = GenerateAST(regex);
@@ -66,29 +72,40 @@ namespace StgSharp.RegularAnalysis.Text
}
catch (Exception ex)
{
return new TextRegexSource(regex, null, null, null, ex);
return new TextRegexSource(regex, null, null, null, info, ex);
}
List<RegexIR> base_ir;
try
{
base_ir = GenerateIR(tree);
base_ir = GenerateIR(tree, ref info);
}
catch (Exception ex)
{
return new TextRegexSource(regex, tree, null, null, ex);
return new TextRegexSource(regex, tree, null, null, info, ex);
}
try
{
ScanAndOptimizeIR(base_ir, out List<RegexIR> opt_ir);
List<RegexIR> prefix_ir = EmitPrefix(opt_ir);
return new TextRegexSource(regex, tree, base_ir, prefix_ir);
return new TextRegexSource(regex, tree, base_ir, prefix_ir, info);
}
catch (Exception ex)
{
return new TextRegexSource(regex, tree, base_ir, null, ex);
return new TextRegexSource(regex, tree, base_ir, null, info, ex);
}
#pragma warning restore CA1031
}
}
internal struct RegexInfo
{
public int RegionCount { get; set; }
public int MinPredictLength { get; set; }
public int SingleLineResultCount { get; set; }
}
}
StgSharp.RegularAnalysis/Text/TextHelper.cs
-30
View File
@@ -1,30 +0,0 @@
// -----------------------------------------------------------------------------
// file="TextHelper"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public static class TextHelper
{
public static bool TryLastIndexOf(
this ReadOnlySpan<char> span,
ReadOnlySpan<char> value,
out int index
)
{
index = span.LastIndexOf(value);
return index != -1;
}
}
}
StgSharpDebug/TestRegexAnalyse.cs
+8 -2
View File
@@ -10,6 +10,7 @@ using StgSharp.RegularAnalysis.Text;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.WebSockets;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
@@ -20,7 +21,7 @@ namespace StgSharp
{
private static string[] regexSource = [
@"(123(4|5)67){3,4}114514",
@"(a|b)(c|d)",
/**/
@"http|https|ftp|ftps",
@"abc[0-9]x|abc[0-9]y|abd[0-9]x|abd[0-9]y|xyz",
@@ -44,18 +45,23 @@ namespace StgSharp
Console.Write(new string('|', enumerator.Depth));
Console.WriteLine($"\t{node.Source.Value}\t{node.Source.Flag}");
}
/*
Console.WriteLine("-----IR-----");
List<RegexIR> ir = interpreterSource.BaseIR;
for (int i = 0; i < ir.Count; i++) {
Console.WriteLine(ir[i].AsCommand(i));
}
/**/
Console.WriteLine("-----Optimized once-----");
List<RegexIR> opt_ir = interpreterSource.IRs;
for (int i = 0; i < opt_ir.Count; i++) {
Console.WriteLine(opt_ir[i].AsCommand(i));
}
Console.WriteLine(interpreterSource.AnalyseException?.ToString() ?? "No error");
RegexInfo info = interpreterSource.Info;
Console.WriteLine("-----Regex Info-----");
Console.WriteLine($"Min length: {info.MinPredictLength}");
Console.WriteLine();
Console.WriteLine(interpreterSource.AnalyseException);
Console.WriteLine();
Console.WriteLine();
}