youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scanning min scan length

Browse Source
This commit is contained in:
Youmu Liu
2026-07-08 18:25:05 +08:00
parent 439128798e
commit 29b24191e1
8 changed files with 125 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs
+1 -1
View File
@@ -128,7 +128,7 @@ namespace StgSharp.RegularAnalysis.Text
{ {
if (_stack.TryPopOperand(out RegexAstNode? _1)) if (_stack.TryPopOperand(out RegexAstNode? _1))
{ {
op.Left = _1; op.Right = _1;
_stack.PushOperand(op); _stack.PushOperand(op);
_ = _tree.AddNode(_1); _ = _tree.AddNode(_1);
_ = _tree.AddNode(op); _ = _tree.AddNode(op);
StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs
+37 -3
View File
@@ -137,7 +137,8 @@ namespace StgSharp.RegularAnalysis.Text
} }
private static void OptimizeTree( private static void OptimizeTree(
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree,
ref RegexInfo info
) )
{ {
// phase0: scan container and remove no parent nodes // phase0: scan container and remove no parent nodes
@@ -163,11 +164,10 @@ namespace StgSharp.RegularAnalysis.Text
} }
} }
List<RegexAstNode> cases = [];
// phase3: then ALT union // phase3: then ALT union
foreach (RegexAstNode item in altNodes) foreach (RegexAstNode item in altNodes)
{ {
List<RegexAstNode> cases = [];
RegexAstNode _case = item.Right; RegexAstNode _case = item.Right;
do do
{ {
@@ -194,6 +194,40 @@ namespace StgSharp.RegularAnalysis.Text
} }
} }
} }
info.MinPredictLength = ComputeMinLength(tree.Root);
static int ComputeMinLength(
RegexAstNode node
)
{
if (node.EqualityTypeConvert is RegexElementLabel.UNIT_SPAN or
RegexElementLabel.UNIT)
{
return node.Value.Length;
} else if (node.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
return ComputeMinLength(node.Left) + ComputeMinLength(node.Right);
} else if (node.EqualityTypeConvert == RegexElementLabel.ALT)
{
RegexAstNode case_0 = node.Right;
int length = int.MaxValue;
do
{
length = int.Min(ComputeMinLength(case_0), length);
case_0 = case_0.Next;
} while (!RegexAstNode.IsNullOrEmpty(case_0));
return length;
} else if (node.EqualityTypeConvert == RegexElementLabel.COUNT)
{
string source = node.Source.Value;
int single_count = ComputeMinLength(node.Right);
return 0;
} else
{
return 0;
}
}
} }
private static void RotateConcat( private static void RotateConcat(
StgSharp.RegularAnalysis/Text/RegexAnalyzer.IRGenerate.cs
+54 -27
View File
@@ -24,10 +24,11 @@ namespace StgSharp.RegularAnalysis.Text
{ {
internal static List<RegexIR> GenerateIR( internal static List<RegexIR> GenerateIR(
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree,
ref RegexInfo info
) )
{ {
return GenerateIRRecursion(tree.Root, 0).ToList(); return GenerateIRRecursion(tree.Root, 0, ref info).ToList();
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -37,7 +38,8 @@ namespace StgSharp.RegularAnalysis.Text
int min, int min,
int max, int max,
bool isGreedy, bool isGreedy,
RegexAstNode right RegexAstNode right,
ref RegexInfo info
) )
{ {
bool is_operator = !RegexAstNode.IsNullOrEmpty(right) && bool is_operator = !RegexAstNode.IsNullOrEmpty(right) &&
@@ -45,7 +47,7 @@ namespace StgSharp.RegularAnalysis.Text
if (is_operator) // seq to be count is complex if (is_operator) // seq to be count is complex
{ {
RegexIRGenerator right_ir = GenerateIRRecursion(right, curDepth + 1); RegexIRGenerator right_ir = GenerateIRRecursion(right, curDepth + 1, ref info);
ir.EmitCountComplex(min, max, isGreedy, right_ir.Count); ir.EmitCountComplex(min, max, isGreedy, right_ir.Count);
ir.EmitIRStream(right_ir); ir.EmitIRStream(right_ir);
} else } else
@@ -70,7 +72,8 @@ namespace StgSharp.RegularAnalysis.Text
private static RegexIRGenerator GenerateIRRecursion( private static RegexIRGenerator GenerateIRRecursion(
RegexAstNode node, RegexAstNode node,
int depth int depth,
ref RegexInfo info
) )
{ {
#pragma warning disable IDE0010 #pragma warning disable IDE0010
@@ -99,7 +102,7 @@ namespace StgSharp.RegularAnalysis.Text
string.Empty; /* unnamed group (?:) */ string.Empty; /* unnamed group (?:) */
RegexIRGenerator groupIR = new(); RegexIRGenerator groupIR = new();
rightIR = GenerateIRRecursion(right, curDepth + 1); rightIR = GenerateIRRecursion(right, curDepth + 1, ref info);
groupIR.EmitTry(rightIR.Count); groupIR.EmitTry(rightIR.Count);
groupIR.EmitIRStream(rightIR); groupIR.EmitIRStream(rightIR);
groupIR.EmitCondition(2, 1); groupIR.EmitCondition(2, 1);
@@ -109,31 +112,19 @@ namespace StgSharp.RegularAnalysis.Text
case RegexElementLabel.COUNT: case RegexElementLabel.COUNT:
string count_source = token.Value; string count_source = token.Value;
bool isGreedy = !(count_source.Length > 1 && count_source.EndsWith('?')); bool isGreedy = !(count_source.Length > 1 && count_source.EndsWith('?'));
int seq_mask = (RegexAstNode.IsNullOrEmpty(left) ? 0 : 2) + (RegexAstNode.IsNullOrEmpty(right) ? RegexAstNode seq = right;
0 :
1);
RegexAstNode seq = seq_mask switch
{
// no seq, invalid count operator
0 => throw new InvalidOperationException("Count operator must have a sequence to operate on"),
1 => right,// right seq
2 => left,// left seq
// both sides, invalid count operator
3 => throw new InvalidOperationException("Count operator cannot have two sequences to operate on"),
_ => throw new InvalidOperationException("Unexpected sequence mask for count operator"),
};
ReadOnlySpan<char> count_source_span = count_source; ReadOnlySpan<char> count_source_span = count_source;
RegexIRGenerator count_ir = new(); RegexIRGenerator count_ir = new();
if (count_source_span[0] == '*') // infinite count if (count_source_span[0] == '*') // infinite count
{ {
EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq); EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq, ref info);
} else if (count_source_span[0] == '+') // at least one } else if (count_source_span[0] == '+') // at least one
{ {
EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq); EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq, ref info);
} else if (count_source_span[0] == '?') // zero or one } else if (count_source_span[0] == '?') // zero or one
{ {
EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq); EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq, ref info);
} else if (count_source_span[0] == '{') // zero or one, non-greedy } else if (count_source_span[0] == '{') // zero or one, non-greedy
{ {
int comma_pos = count_source_span.IndexOf(','); int comma_pos = count_source_span.IndexOf(',');
@@ -141,7 +132,8 @@ namespace StgSharp.RegularAnalysis.Text
{ {
if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count)) if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count))
{ {
EmitCount(curDepth, count_ir, count, count, isGreedy, seq); EmitCount(curDepth, count_ir, count, count, isGreedy, seq,
ref info);
} else } else
{ {
throw new InvalidCastException($"Invalid count format: {count_source}"); throw new InvalidCastException($"Invalid count format: {count_source}");
@@ -151,7 +143,7 @@ namespace StgSharp.RegularAnalysis.Text
if (int.TryParse(count_source[1..comma_pos], out int min) && if (int.TryParse(count_source[1..comma_pos], out int min) &&
int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max)) int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max))
{ {
EmitCount(curDepth, count_ir, min, max, isGreedy, seq); EmitCount(curDepth, count_ir, min, max, isGreedy, seq, ref info);
} else } else
{ {
throw new InvalidCastException($"Invalid count format: {count_source}"); throw new InvalidCastException($"Invalid count format: {count_source}");
@@ -187,7 +179,7 @@ namespace StgSharp.RegularAnalysis.Text
} }
} else if ((left.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0) } else if ((left.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{ {
leftIR = GenerateIRRecursion(left, curDepth + 1); leftIR = GenerateIRRecursion(left, curDepth + 1, ref info);
concat.EmitIRStream(leftIR); concat.EmitIRStream(leftIR);
} }
@@ -212,7 +204,7 @@ namespace StgSharp.RegularAnalysis.Text
} }
} else if ((right.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0) } else if ((right.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{ {
rightIR = GenerateIRRecursion(right, curDepth + 1); rightIR = GenerateIRRecursion(right, curDepth + 1, ref info);
concat.EmitIRStream(rightIR); concat.EmitIRStream(rightIR);
} }
return concat; return concat;
@@ -223,7 +215,7 @@ namespace StgSharp.RegularAnalysis.Text
int ir_count = 0; int ir_count = 0;
while (!RegexAstNode.IsNullOrEmpty(cur_node)) while (!RegexAstNode.IsNullOrEmpty(cur_node))
{ {
RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1); RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1, ref info);
casesIRList.Add(_case); casesIRList.Add(_case);
ir_count += _case.Count; ir_count += _case.Count;
cur_node = cur_node.Next; cur_node = cur_node.Next;
@@ -255,6 +247,41 @@ namespace StgSharp.RegularAnalysis.Text
return null!; return null!;
} }
#pragma warning restore IDE0010 #pragma warning restore IDE0010
// info.MinPredictLength = ComputeMinLength(tree.Root);
static int ComputeMinLength(
RegexAstNode node
)
{
if (node.EqualityTypeConvert is RegexElementLabel.UNIT_SPAN or
RegexElementLabel.UNIT)
{
return node.Value.Length;
} else if (node.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
return ComputeMinLength(node.Left) + ComputeMinLength(node.Right);
} else if (node.EqualityTypeConvert == RegexElementLabel.ALT)
{
RegexAstNode case_0 = node.Right;
int length = int.MaxValue;
do
{
length = int.Min(ComputeMinLength(case_0), length);
case_0 = case_0.Next;
} while (!RegexAstNode.IsNullOrEmpty(case_0));
return length;
} else if (node.EqualityTypeConvert == RegexElementLabel.COUNT)
{
string source = node.Source.Value;
int single_count = ComputeMinLength(node.Right);
return 0;
} else
{
return 0;
}
}
} }
} }
StgSharp.RegularAnalysis/Text/RegexAnalyzer.IROptimize.cs
-11
View File
@@ -306,17 +306,6 @@ namespace StgSharp.RegularAnalysis.Text
} }
} }
internal class RegexInfo
{
public int MinLength { get; set; }
public int TryDepth { get; set; }
public int GroupCount { get; set; }
}
#region merge find and try-find #region merge find and try-find
private static int TyrMergeFindRestTwice( private static int TyrMergeFindRestTwice(
StgSharp.RegularAnalysis/Text/RegexAnalyzer.InfoScan.cs → StgSharp.RegularAnalysis/Text/RegexAnalyzer.ScanInfo.cs
+2 -13
View File
@@ -18,11 +18,11 @@ namespace StgSharp.RegularAnalysis.Text
public partial class RegexAnalyzer public partial class RegexAnalyzer
{ {
internal static RegexInterpreterInfo ScanInfo( internal static RegexInfo ScanInfo(
List<RegexIR> ir List<RegexIR> ir
) )
{ {
RegexInterpreterInfo info = new(); RegexInfo info = new();
int process_count = 0; int process_count = 0;
ReadOnlySpan<RegexIR> irSpan = CollectionsMarshal.AsSpan(ir); ReadOnlySpan<RegexIR> irSpan = CollectionsMarshal.AsSpan(ir);
@@ -97,16 +97,5 @@ namespace StgSharp.RegularAnalysis.Text
} }
} }
internal struct RegexInterpreterInfo
{
public int RegionCount { get; set; }
public int MinPredictLength { get; set; }
public int SingleLineResultCount { get; set; }
}
} }
} }
StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs
+23 -6
View File
@@ -26,7 +26,9 @@ namespace StgSharp.RegularAnalysis.Text
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> ast, AbstractSyntaxTree<RegexAstNode, RegexElementLabel> ast,
List<RegexIR> base_ir, List<RegexIR> base_ir,
List<RegexIR> ir, List<RegexIR> ir,
RegexInfo info,
Exception? analyseException = null Exception? analyseException = null
) )
{ {
Pattern = pattern; Pattern = pattern;
@@ -34,6 +36,7 @@ namespace StgSharp.RegularAnalysis.Text
IRs = ir; IRs = ir;
BaseIR = base_ir; BaseIR = base_ir;
AnalyseException = analyseException; AnalyseException = analyseException;
Info = info;
} }
public Exception? AnalyseException { get; set; } public Exception? AnalyseException { get; set; }
@@ -46,6 +49,8 @@ namespace StgSharp.RegularAnalysis.Text
internal List<RegexIR> IRs { get; } internal List<RegexIR> IRs { get; }
internal RegexInfo Info { get; }
} }
public static partial class RegexAnalyzer public static partial class RegexAnalyzer
@@ -55,10 +60,11 @@ namespace StgSharp.RegularAnalysis.Text
string regex string regex
) )
{ {
#pragma warning disable CA1031 #pragma warning disable CA1031
// RegexAnalyzer interpreter = new(); // RegexAnalyzer interpreter = new();
// interpreter._source = regex; // interpreter._source = regex;
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree; AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree;
RegexInfo info = new();
try try
{ {
tree = GenerateAST(regex); tree = GenerateAST(regex);
@@ -66,29 +72,40 @@ namespace StgSharp.RegularAnalysis.Text
} }
catch (Exception ex) catch (Exception ex)
{ {
return new TextRegexSource(regex, null, null, null, ex); return new TextRegexSource(regex, null, null, null, info, ex);
} }
List<RegexIR> base_ir; List<RegexIR> base_ir;
try try
{ {
base_ir = GenerateIR(tree); base_ir = GenerateIR(tree, ref info);
} }
catch (Exception ex) catch (Exception ex)
{ {
return new TextRegexSource(regex, tree, null, null, ex); return new TextRegexSource(regex, tree, null, null, info, ex);
} }
try try
{ {
ScanAndOptimizeIR(base_ir, out List<RegexIR> opt_ir); ScanAndOptimizeIR(base_ir, out List<RegexIR> opt_ir);
List<RegexIR> prefix_ir = EmitPrefix(opt_ir); List<RegexIR> prefix_ir = EmitPrefix(opt_ir);
return new TextRegexSource(regex, tree, base_ir, prefix_ir); return new TextRegexSource(regex, tree, base_ir, prefix_ir, info);
} }
catch (Exception ex) catch (Exception ex)
{ {
return new TextRegexSource(regex, tree, base_ir, null, ex); return new TextRegexSource(regex, tree, base_ir, null, info, ex);
} }
#pragma warning restore CA1031 #pragma warning restore CA1031
} }
} }
internal struct RegexInfo
{
public int RegionCount { get; set; }
public int MinPredictLength { get; set; }
public int SingleLineResultCount { get; set; }
}
} }
StgSharp.RegularAnalysis/Text/TextHelper.cs
-30
View File
@@ -1,30 +0,0 @@
// -----------------------------------------------------------------------------
// file="TextHelper"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public static class TextHelper
{
public static bool TryLastIndexOf(
this ReadOnlySpan<char> span,
ReadOnlySpan<char> value,
out int index
)
{
index = span.LastIndexOf(value);
return index != -1;
}
}
}
StgSharpDebug/TestRegexAnalyse.cs
+8 -2
View File
@@ -10,6 +10,7 @@ using StgSharp.RegularAnalysis.Text;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Net.WebSockets;
using System.Text; using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Threading.Tasks; using System.Threading.Tasks;
@@ -20,7 +21,7 @@ namespace StgSharp
{ {
private static string[] regexSource = [ private static string[] regexSource = [
@"(123(4|5)67){3,4}114514", @"(a|b)(c|d)",
/**/ /**/
@"http|https|ftp|ftps", @"http|https|ftp|ftps",
@"abc[0-9]x|abc[0-9]y|abd[0-9]x|abd[0-9]y|xyz", @"abc[0-9]x|abc[0-9]y|abd[0-9]x|abd[0-9]y|xyz",
@@ -44,18 +45,23 @@ namespace StgSharp
Console.Write(new string('|', enumerator.Depth)); Console.Write(new string('|', enumerator.Depth));
Console.WriteLine($"\t{node.Source.Value}\t{node.Source.Flag}"); Console.WriteLine($"\t{node.Source.Value}\t{node.Source.Flag}");
} }
/*
Console.WriteLine("-----IR-----"); Console.WriteLine("-----IR-----");
List<RegexIR> ir = interpreterSource.BaseIR; List<RegexIR> ir = interpreterSource.BaseIR;
for (int i = 0; i < ir.Count; i++) { for (int i = 0; i < ir.Count; i++) {
Console.WriteLine(ir[i].AsCommand(i)); Console.WriteLine(ir[i].AsCommand(i));
} }
/**/
Console.WriteLine("-----Optimized once-----"); Console.WriteLine("-----Optimized once-----");
List<RegexIR> opt_ir = interpreterSource.IRs; List<RegexIR> opt_ir = interpreterSource.IRs;
for (int i = 0; i < opt_ir.Count; i++) { for (int i = 0; i < opt_ir.Count; i++) {
Console.WriteLine(opt_ir[i].AsCommand(i)); Console.WriteLine(opt_ir[i].AsCommand(i));
} }
Console.WriteLine(interpreterSource.AnalyseException?.ToString() ?? "No error");
RegexInfo info = interpreterSource.Info;
Console.WriteLine("-----Regex Info-----");
Console.WriteLine($"Min length: {info.MinPredictLength}");
Console.WriteLine(); Console.WriteLine();
Console.WriteLine(interpreterSource.AnalyseException);
Console.WriteLine(); Console.WriteLine();
Console.WriteLine(); Console.WriteLine();
} }