youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

regex ir emitter is working

Browse Source
This commit is contained in:
Youmu Liu
2026-06-18 19:10:30 +08:00
parent 5570c6f338
commit eeca5721fb
5 changed files with 164 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.RegularAnalysis/TextRegex/RegexIR.cs
+64 -44
View File
@@ -40,13 +40,13 @@ namespace StgSharp.RegularAnalysis.TextRegex
Search_Seq = 0,
Search_Set = 1,
SearchChar = 2,
Jump = 3,
ConfigFail = 3,
Condition = 4,
Try = 5,
Pop = 6,
Count = 7,
Count_Seq = 8,
Count_Set = 9
Count = 6,
Count_Seq = 7,
Count_Set = 8,
Pop = 10
}
@@ -55,30 +55,16 @@ namespace StgSharp.RegularAnalysis.TextRegex
internal abstract RegexIRCommand Command { get; }
internal abstract void Execute();
}
internal sealed class RegexPopIR(
bool isSuccess
) : RegexIR
{
public bool IsSuccess { get; } = isSuccess;
internal override RegexIRCommand Command => RegexIRCommand.Pop;
internal override void Execute()
{
throw new NotImplementedException();
}
internal abstract string AsCommand(
int lineNum
);
}
/// <summary>
/// Searches for a certain string pattern or charset from current position.
/// </summary>
internal sealed class RegexSearchSeqIR(
internal sealed class RegexMatchSeqIR(
string pattern
) : RegexIR
{
@@ -87,9 +73,11 @@ namespace StgSharp.RegularAnalysis.TextRegex
internal override RegexIRCommand Command => RegexIRCommand.Search_Seq;
internal override void Execute()
internal override string AsCommand(
int lineNum
)
{
throw new NotImplementedException();
return $"[{lineNum}]\t SEARCH_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "<EPSILON>" : Pattern)}\"";
}
}
@@ -103,31 +91,35 @@ namespace StgSharp.RegularAnalysis.TextRegex
internal override RegexIRCommand Command => RegexIRCommand.Search_Set;
internal override void Execute()
internal override string AsCommand(
int lineNum
)
{
throw new NotImplementedException();
return $"[{lineNum}]\t SEARCH_SET \"{Charset}\"";
}
}
/// <summary>
/// JUMP to a certain instruction index.
/// Configures the failure jump of current position. The failMod ranges 0-2
/// </summary>
/// <param name="target">
///
/// </param>
internal sealed class RegexJumpIR(
int target
internal sealed class RegexConfigFailIR(
int failMod
) : RegexIR
{
public int TargetIndex { get; } = target;
public int FailMod { get; } = failMod & 0x3;
internal override RegexIRCommand Command => RegexIRCommand.Jump;
internal override RegexIRCommand Command => RegexIRCommand.ConfigFail;
internal override void Execute()
internal override string AsCommand(
int lineNum
)
{
throw new NotImplementedException();
return $"[{lineNum}]\t CONFIG_FAIL {FailMod}";
}
}
@@ -144,9 +136,11 @@ namespace StgSharp.RegularAnalysis.TextRegex
internal override RegexIRCommand Command => RegexIRCommand.Condition;
internal override void Execute()
internal override string AsCommand(
int lineNum
)
{
throw new NotImplementedException();
return $"[{lineNum}]\t CONDITION SUCCESS: {SuccessCase} FAIL: {FailCase}";
}
}
@@ -169,9 +163,11 @@ namespace StgSharp.RegularAnalysis.TextRegex
internal override RegexIRCommand Command { get; } = RegexIRCommand.Count;
internal override void Execute()
internal override string AsCommand(
int lineNum
)
{
throw new NotImplementedException();
return $"[{lineNum}]\t COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {OperationIndex}";
}
}
@@ -194,9 +190,11 @@ namespace StgSharp.RegularAnalysis.TextRegex
internal override RegexIRCommand Command { get; } = RegexIRCommand.Count_Seq;
internal override void Execute()
internal override string AsCommand(
int lineNum
)
{
throw new NotImplementedException();
return $"[{lineNum}]\t COUNT_SEQ {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} SEQ: \"{Sequence}\"";
}
}
@@ -219,9 +217,11 @@ namespace StgSharp.RegularAnalysis.TextRegex
internal override RegexIRCommand Command => RegexIRCommand.Count_Set;
internal override void Execute()
internal override string AsCommand(
int lineNum
)
{
throw new NotImplementedException();
return $"[{lineNum}]\t COUNT_SET {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} SET: \"{CharSet}\"";
}
}
@@ -235,9 +235,29 @@ namespace StgSharp.RegularAnalysis.TextRegex
internal override RegexIRCommand Command => RegexIRCommand.Try;
internal override void Execute()
internal override string AsCommand(
int lineNum
)
{
throw new NotImplementedException();
return $"[{lineNum}]\t TRY {CountOfCommand}";
}
}
internal sealed class RegexPopIR(
bool isSuccess
) : RegexIR
{
public bool IsSuccess { get; } = isSuccess;
internal override RegexIRCommand Command => RegexIRCommand.Try;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t POP {(IsSuccess ? "SUCCESS" : "FAIL")}";
}
}
StgSharp.RegularAnalysis/TextRegex/RegexIRGenerator.cs
+17 -9
View File
@@ -56,6 +56,14 @@ namespace StgSharp.RegularAnalysis.TextRegex
Count++;
}
public void EmitConfigFail(
int fail
)
{
_tail.Add(new RegexConfigFailIR(fail));
Count++;
}
public void EmitCount(
int min,
int max,
@@ -101,14 +109,6 @@ namespace StgSharp.RegularAnalysis.TextRegex
Count += stream.Count;
}
public void EmitMatchSet(
string unit
)
{
_tail.Add(new RegexMatchSetIR(unit));
Count++;
}
public void EmitMatchSeq(
string unit
)
@@ -117,7 +117,15 @@ namespace StgSharp.RegularAnalysis.TextRegex
Count++;
}
public void EmitPopLevel(
public void EmitMatchSet(
string unit
)
{
_tail.Add(new RegexMatchSetIR(unit));
Count++;
}
public void EmitPop(
bool isSuccess
)
{
StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.IRGenerate.cs
+73 -32
View File
@@ -43,23 +43,21 @@ namespace StgSharp.RegularAnalysis.TextRegex
public partial class RegexInterpreter
{
private List<RegexIR> EmptyIR { get; } = [];
internal List<RegexIR> GenerateIR(
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree
internal static List<RegexIR> GenerateIR(
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree
)
{
return GenerateIRRecursion(tree.Root, 0).ToList();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void EmitCount(
int curDepth,
RegexIRGenerator ir,
int min,
int max,
bool isGreedy,
RegexAstNode right
private static void EmitCount(
int curDepth,
RegexIRGenerator ir,
int min,
int max,
bool isGreedy,
RegexAstNode right
)
{
bool isRightOperator = !RegexAstNode.IsNullOrEmpty(right) &&
@@ -83,7 +81,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
ir.EmitCountSet(min, max, isGreedy, right.Source.Value);
break;
case RegexElementLabel.UNIT_SPAN:
ir.EmitCountSet(min, max, isGreedy, right.Source.Value);
ir.EmitCountSeq(min, max, isGreedy, right.Source.Value);
break;
default:
throw new InvalidOperationException($"Unexpected regex element type for count operator: {count_source_mask}");
@@ -91,9 +89,9 @@ namespace StgSharp.RegularAnalysis.TextRegex
}
}
private RegexIRGenerator GenerateIRRecursion(
RegexAstNode node,
int depth
private static RegexIRGenerator GenerateIRRecursion(
RegexAstNode node,
int depth
)
{
#pragma warning disable IDE0010
@@ -107,11 +105,6 @@ namespace StgSharp.RegularAnalysis.TextRegex
RegexIRGenerator leftIR;
RegexIRGenerator rightIR;
bool isLeftOperator = !RegexAstNode.IsNullOrEmpty(left) &&
(left.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0;
bool isRightOperator = !RegexAstNode.IsNullOrEmpty(right) &&
(right.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0;
Token<RegexElementLabel> token = node.Source;
switch (node.Source.Flag)
@@ -131,26 +124,41 @@ namespace StgSharp.RegularAnalysis.TextRegex
groupIR.EmitTry(rightIR.Count);
groupIR.EmitIRStream(rightIR);
groupIR.EmitCondition(2, 1);
groupIR.EmitPopLevel(false);
// a grouping op needed here, before popping
groupIR.EmitPopLevel(true);
/*generate pack ir here*/
return groupIR;
case RegexElementLabel.COUNT:
string count_source = token.Value;
bool isGreedy = !(count_source.Length > 1 && !count_source.EndsWith('?'));
bool isGreedy = !(count_source.Length > 1 && count_source.EndsWith('?'));
int seq_mask = (RegexAstNode.IsNullOrEmpty(left) ? 0 : 2) + (RegexAstNode.IsNullOrEmpty(right) ?
0 :
1);
RegexAstNode seq = seq_mask switch
{
// no seq, invalid count operator
0 => throw new InvalidOperationException("Count operator must have a sequence to operate on"),
// right seq
1 => right,
// left seq
2 => left,
// both sides, invalid count operator
3 => throw new InvalidOperationException("Count operator cannot have two sequences to operate on"),
_ => throw new InvalidOperationException("Unexpected sequence mask for count operator"),
};
ReadOnlySpan<char> count_source_span = count_source;
RegexIRGenerator countIR = new();
if (count_source_span[0] == '*') // infinite count
{
EmitCount(curDepth, countIR, 0, int.MaxValue, isGreedy, right);
EmitCount(curDepth, countIR, 0, int.MaxValue, isGreedy, seq);
} else if (count_source_span[0] == '+') // at least one
{
EmitCount(curDepth, countIR, 1, int.MaxValue, isGreedy, right);
EmitCount(curDepth, countIR, 1, int.MaxValue, isGreedy, seq);
} else if (count_source_span[0] == '?') // zero or one
{
EmitCount(curDepth, countIR, 0, 1, isGreedy, right);
EmitCount(curDepth, countIR, 0, 1, isGreedy, seq);
} else if (count_source_span[0] == '{') // zero or one, non-greedy
{
int comma_pos = count_source_span.IndexOf(',');
@@ -158,7 +166,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
{
if (int.TryParse(count_source[1..(isGreedy ? ^3 : ^2)], out int count))
{
EmitCount(curDepth, countIR, count, count, isGreedy, right);
EmitCount(curDepth, countIR, count, count, isGreedy, seq);
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
@@ -168,7 +176,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
if (int.TryParse(count_source[1..comma_pos], out int min) &&
int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^3 : ^2)], out int max))
{
EmitCount(curDepth, countIR, min, max, isGreedy, right);
EmitCount(curDepth, countIR, min, max, isGreedy, seq);
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
@@ -192,7 +200,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
switch (left.Source.Flag)
{
case RegexElementLabel.UNIT_SPAN:
goto case RegexElementLabel.UNIT_SPAN;
goto case RegexElementLabel.UNIT;
case RegexElementLabel.UNIT:
concat.EmitMatchSeq(left.Source.Value);
break;
@@ -234,7 +242,40 @@ namespace StgSharp.RegularAnalysis.TextRegex
}
return concat;
case RegexElementLabel.ALT:
return null!;
RegexIRGenerator alt = new();
List<RegexIRGenerator> casesIRList = [];
RegexAstNode cur_node = right;
int ir_count = 0;
while (!RegexAstNode.IsNullOrEmpty(cur_node))
{
RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1);
casesIRList.Add(_case);
ir_count += _case.Count;
cur_node = cur_node.Next;
}
int case_count = casesIRList.Count;
while (case_count != 0)
{
RegexIRGenerator cur_ir = casesIRList[^case_count];
ir_count -= cur_ir.Count;
case_count--;
alt.EmitTry(cur_ir.Count);
alt.EmitIRStream(cur_ir);
alt.EmitCondition(1/* need to calc here */, 2);
alt.EmitPop(true);
}
alt.EmitPop(false);
return alt;
case RegexElementLabel.UNIT_SPAN:
goto case RegexElementLabel.UNIT;
case RegexElementLabel.UNIT:
RegexIRGenerator unit_seq = new();
unit_seq.EmitMatchSeq(node.Source.Value);
return unit_seq;
case RegexElementLabel.UNIT_SET:
RegexIRGenerator unit_set = new();
unit_set.EmitMatchSet(node.Source.Value);
return unit_set;
default:
return null!;
}
StgSharpDebug/TestRegexAnalyse.cs
+5
View File
@@ -61,6 +61,11 @@ namespace StgSharp
Console.Write(new string('|', enumerator.Depth));
Console.WriteLine($"\t{node.Source.Value}\t{node.Source.Flag}");
}
List<RegexIR> ir = RegexInterpreter.GenerateIR(tree);
Console.WriteLine("-----IR-----");
for (int i = 0; i < ir.Count; i++) {
Console.WriteLine(ir[i].AsCommand(i));
}
Console.WriteLine();
Console.WriteLine();
Console.WriteLine();
future/ngra-regex-ir.md
+4 -2
View File
@@ -18,9 +18,11 @@
1. 搜索算子（SEARCH %str; SEARCH %unicode），向后搜索确定长度序列或者满足规则的字符集，更新活动指针，不更新锚点指针，搜索结果保存在全局上下文中
2. 搜索算子（SEARCH %set_expression），向后匹配一个满足字符集要求的字符，更新活动指针，不更新锚点指针，搜索结果保存在全局上下文中
3. 跳转算子（JMP %cmd_idx），跳转到指定的IR指令之前，不变化任何指针，cmd_idx为相对当前位置的偏移量
4. 条件指令（CONDITION %cmd_true %cmd_fail），比较前一条SEARCH指令是否成功，并指定跳转到指定的位置，如果某种条件不需要跳转，请使用0进行填充
3. 失败配置指令（CINFIG_FAIL %fail），指定当前匹配栈后续如何处理一条指令的失败，可选为：FAST，立刻终止当前层级；CONTINUE，记录这个fail，但是继续操作；REVERSE，反过来，成功了才终止
4. 条件指令（CONDITION %cmd_true %cmd_fail），比较前一条SEARCH指令是否成功，并指定跳转到指定的位置，如果某种条件不需要跳转，可以写1。如果想要将fail向上级继续抛出，可以%cmd_fail写0.
5. 跳转执行指令（TRY %begin %end），尝试idx指向的IR块，并记住执行结果数据
6. 计数指令（COUNT %min %max %is_greedy %cmd），重复执行指令cmd，直到满足参数配置的数量为止
7. 快速序列计数指令（COUNT_SEQ %min %max %is_greedy %seq），快速搜索一个确定性序列，直到满足参数配置的数量为止
8. 快速字符集计数指令（COUNT_SET %min %max %is_greedy %set），快速搜索一个字符集，直到满足参数配置的数量为止
9. 尝试（TRY %count），对自身向后的一段内容在新的匹配栈层级中进行运行，被分层的指令数量由COUNT指令确定
10. 弹出（POP %is_success），提前终止一个try块，并用is_success表示弹出行为视为匹配成功还是失败