youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

source gen and ir gen now use the same seq generator

Browse Source
This commit is contained in:
Youmu Liu
2026-07-09 18:24:38 +08:00
parent 11ccd2568b
commit ef94bf1567
7 changed files with 327 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.RegularAnalysis/Abstraction/SequenceEmitter.cs
+177
View File
@@ -0,0 +1,177 @@
// -----------------------------------------------------------------------------
// file="SequenceEmitter"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.RegularAnalysis.Text;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Abstraction
{
public sealed class SequenceEmitter<T>
{
private bool _isRoot;
private readonly Node _head;
private Node _tail;
public SequenceEmitter()
{
_isRoot = true;
_head = new();
_tail = _head;
}
public int Count { get; private set; }
public SequenceEmitter<T> Append(
T unit
)
{
if (!_isRoot) {
throw new InvalidOperationException("This emitter has been appended to another emitter. It is now readonly.");
}
_tail.Add(unit);
Count++;
return this;
}
public SequenceEmitter<T> Append(
SequenceEmitter<T> emitter
)
{
if (emitter == null || emitter.Count == 0) {
return this;
}
if (ReferenceEquals(this, emitter)) {
throw new InvalidOperationException("Cannot append emitter to itself.");
}
if (!emitter._isRoot) {
throw new InvalidOperationException("The emitter has been appended to another emitter.");
}
_tail.Next = emitter._head;
_tail = emitter._tail;
Count += emitter.Count;
emitter._isRoot = false;
return this;
}
public Enumerator GetEnumerator()
{
return _isRoot ?
new Enumerator(this) :
throw new InvalidOperationException("This emitter has been appended to another emitter. You should not get its enumerator.");
}
public List<T> ToList()
{
List<T> result = new(Count);
Node current = _head;
while (current != null)
{
result.AddRange(current.IRList);
current = current.Next!;
}
return result;
}
#pragma warning disable CA1063
#pragma warning disable CA1816
public class Enumerator(
[NotNull] SequenceEmitter<T> emitter
) : IEnumerator<T>
{
private int _index = -1;
private Node _current = null!;
private readonly Node _root = emitter._head;
public T Current
{
get
{
if (_current is null || _index < 0 || _index >= _current.Count) {
throw new InvalidOperationException();
}
return _current[_index];
}
}
public void Dispose() { }
public bool MoveNext()
{
_current ??= _root;
int index = _index + 1;
if (index < _current.Count)
{
_index = index;
return true;
}
Node current = _current.Next!;
if (current is null || ReferenceEquals(current, _root))
{
_index = -1;
return false;
}
_current = current;
_index = -1;
return true;
}
public void Reset()
{
_current = _root;
_index = -1;
}
object IEnumerator.Current => Current!;
}
#pragma warning restore CA1063
#pragma warning restore CA1816
private sealed class Node
{
public T this[
int index
]
{
get => IRList[index];
set => IRList[index] = value;
}
public Node? Next { get; set; }
public int Count => IRList.Count;
public List<T> IRList { get; } = [];
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Add(
T ir
)
{
IRList.Add(ir);
}
}
}
}
StgSharp.RegularAnalysis/Text/RegexAnalyzer.IRGenerate.cs
+71 -40
View File
@@ -28,7 +28,9 @@ namespace StgSharp.RegularAnalysis.Text
ref RegexInfo info
)
{
return GenerateIRRecursion(tree.Root, 0, ref info).ToList();
List<RegexIR> list = GenerateIRRecursion(tree.Root, 0, out int length).ToList();
info.MinPredictLength = length;
return list;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -39,7 +41,7 @@ namespace StgSharp.RegularAnalysis.Text
int max,
bool isGreedy,
RegexAstNode right,
ref RegexInfo info
out int minLength
)
{
bool is_operator = !RegexAstNode.IsNullOrEmpty(right) &&
@@ -47,22 +49,25 @@ namespace StgSharp.RegularAnalysis.Text
if (is_operator) // seq to be count is complex
{
RegexIRGenerator right_ir = GenerateIRRecursion(right, curDepth + 1, ref info);
ir.EmitCountComplex(min, max, isGreedy, right_ir.Count);
ir.EmitIRStream(right_ir);
RegexIRGenerator right_ir = GenerateIRRecursion(right, curDepth + 1, out minLength);
_ = ir.EmitCountComplex(min, max, isGreedy, right_ir.Count);
_ = ir.EmitIRStream(right_ir);
} else
{
RegexElementLabel count_source_mask = right.Source.Flag;
switch (count_source_mask)
{
case RegexElementLabel.UNIT:
ir.EmitCountSeq(min, max, isGreedy, right.Source.Value);
minLength = 1;
_ = ir.EmitCountSeq(min, max, isGreedy, right.Value);
break;
case RegexElementLabel.UNIT_SET:
ir.EmitCountSet(min, max, isGreedy, right.Source.Value);
minLength = 1;
_ = ir.EmitCountSet(min, max, isGreedy, right.Value);
break;
case RegexElementLabel.UNIT_SPAN:
ir.EmitCountSeq(min, max, isGreedy, right.Source.Value);
minLength = 1;
_ = ir.EmitCountSeq(min, max, isGreedy, right.Value);
break;
default:
throw new InvalidOperationException($"Unexpected regex element type for count operator: {count_source_mask}");
@@ -73,7 +78,7 @@ namespace StgSharp.RegularAnalysis.Text
private static RegexIRGenerator GenerateIRRecursion(
RegexAstNode node,
int depth,
ref RegexInfo info
out int minLength
)
{
#pragma warning disable IDE0010
@@ -102,12 +107,12 @@ namespace StgSharp.RegularAnalysis.Text
string.Empty; /* unnamed group (?:) */
RegexIRGenerator groupIR = new();
rightIR = GenerateIRRecursion(right, curDepth + 1, ref info);
groupIR.EmitTry(rightIR.Count);
groupIR.EmitIRStream(rightIR);
groupIR.EmitCondition(2, 1);
groupIR.EmitPop(false);
groupIR.EmitPack(true, group_name);
rightIR = GenerateIRRecursion(right, curDepth + 1, out minLength);
_ = groupIR.EmitTry(rightIR.Count)
.EmitIRStream(rightIR)
.EmitCondition(2, 1)
.EmitPop(false)
.EmitPack(true, group_name);
return groupIR;
case RegexElementLabel.COUNT:
string count_source = token.Value;
@@ -115,16 +120,19 @@ namespace StgSharp.RegularAnalysis.Text
RegexAstNode seq = right;
ReadOnlySpan<char> count_source_span = count_source;
RegexIRGenerator count_ir = new();
int length_once;
if (count_source_span[0] == '*') // infinite count
{
EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq, ref info);
EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq, out _);
minLength = 0;
} else if (count_source_span[0] == '+') // at least one
{
EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq, ref info);
EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq, out length_once);
minLength = length_once;
} else if (count_source_span[0] == '?') // zero or one
{
EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq, ref info);
EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq, out _);
minLength = 0;
} else if (count_source_span[0] == '{') // zero or one, non-greedy
{
int comma_pos = count_source_span.IndexOf(',');
@@ -133,7 +141,8 @@ namespace StgSharp.RegularAnalysis.Text
if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count))
{
EmitCount(curDepth, count_ir, count, count, isGreedy, seq,
ref info);
out length_once);
minLength = length_once * count;
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
@@ -143,7 +152,9 @@ namespace StgSharp.RegularAnalysis.Text
if (int.TryParse(count_source[1..comma_pos], out int min) &&
int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max))
{
EmitCount(curDepth, count_ir, min, max, isGreedy, seq, ref info);
EmitCount(curDepth, count_ir, min, max, isGreedy, seq,
out length_once);
minLength = length_once * min;
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
@@ -159,6 +170,7 @@ namespace StgSharp.RegularAnalysis.Text
RegexIRGenerator concat = new();
// process left node
int left_length = 0;
if (RegexAstNode.IsNullOrEmpty(left))
{
// do nothing
@@ -167,23 +179,28 @@ namespace StgSharp.RegularAnalysis.Text
switch (left.Source.Flag)
{
case RegexElementLabel.UNIT_SPAN:
goto case RegexElementLabel.UNIT;
left_length = left.Value.Length;
_ = concat.EmitMatchSeq(left.Value);
break;
case RegexElementLabel.UNIT:
concat.EmitMatchSeq(left.Source.Value);
left_length = 1;
_ = concat.EmitMatchSeq(left.Value);
break;
case RegexElementLabel.UNIT_SET:
concat.EmitMatchSet(left.Source.Value);
left_length = 1;
_ = concat.EmitMatchSet(left.Value);
break;
default:
break;
}
} else if ((left.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{
leftIR = GenerateIRRecursion(left, curDepth + 1, ref info);
concat.EmitIRStream(leftIR);
leftIR = GenerateIRRecursion(left, curDepth + 1, out left_length);
_ = concat.EmitIRStream(leftIR);
}
// process right node
int right_length = 0;
if (RegexAstNode.IsNullOrEmpty(right))
{
// do nothing
@@ -192,30 +209,37 @@ namespace StgSharp.RegularAnalysis.Text
switch (right.Source.Flag)
{
case RegexElementLabel.UNIT_SPAN:
goto case RegexElementLabel.UNIT;
right_length = right.Value.Length;
_ = concat.EmitMatchSeq(right.Value);
break;
case RegexElementLabel.UNIT:
concat.EmitMatchSeq(right.Source.Value);
right_length = 1;
_ = concat.EmitMatchSeq(right.Value);
break;
case RegexElementLabel.UNIT_SET:
concat.EmitMatchSet(right.Source.Value);
right_length = 1;
_ = concat.EmitMatchSet(right.Value);
break;
default:
break;
}
} else if ((right.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{
rightIR = GenerateIRRecursion(right, curDepth + 1, ref info);
concat.EmitIRStream(rightIR);
rightIR = GenerateIRRecursion(right, curDepth + 1, out right_length);
_ = concat.EmitIRStream(rightIR);
}
minLength = left_length + right_length;
return concat;
case RegexElementLabel.ALT:
RegexIRGenerator alt = new();
List<RegexIRGenerator> casesIRList = [];
RegexAstNode cur_node = right;
int ir_count = 0;
int min_case_length = int.MaxValue;
while (!RegexAstNode.IsNullOrEmpty(cur_node))
{
RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1, ref info);
RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1, out int case_length);
min_case_length = int.Min(case_length, min_case_length);
casesIRList.Add(_case);
ir_count += _case.Count;
cur_node = cur_node.Next;
@@ -226,24 +250,31 @@ namespace StgSharp.RegularAnalysis.Text
RegexIRGenerator cur_ir = casesIRList[^case_count];
ir_count -= cur_ir.Count;
case_count--;
alt.EmitTry(cur_ir.Count);
alt.EmitIRStream(cur_ir);
alt.EmitCondition(1/* need to calc here */, 2);
alt.EmitPop(true);
_ = alt.EmitTry(cur_ir.Count)
.EmitIRStream(cur_ir)
.EmitCondition(1/* need to calc here */, 2)
.EmitPop(true);
}
alt.EmitPop(false);
_ = alt.EmitPop(false);
minLength = min_case_length;
return alt;
case RegexElementLabel.UNIT_SPAN:
goto case RegexElementLabel.UNIT;
RegexIRGenerator unit = new();
_ = unit.EmitMatchSeq(node.Value);
minLength = node.Value.Length;
return unit;
case RegexElementLabel.UNIT:
RegexIRGenerator unit_seq = new();
unit_seq.EmitMatchSeq(node.Source.Value);
_ = unit_seq.EmitMatchSeq(node.Value);
minLength = 1;
return unit_seq;
case RegexElementLabel.UNIT_SET:
RegexIRGenerator unit_set = new();
unit_set.EmitMatchSet(node.Source.Value);
_ = unit_set.EmitMatchSet(node.Value);
minLength = 1;
return unit_set;
default:
minLength = 0;
return null!;
}
#pragma warning restore IDE0010
StgSharp.RegularAnalysis/Text/RegexAnalyzer.PrefixEmit.cs
+17 -7
View File
@@ -18,7 +18,8 @@ namespace StgSharp.RegularAnalysis.Text
{
internal static List<RegexIR> EmitPrefix(
List<RegexIR> irList
List<RegexIR> irList,
ref RegexInfo info
)
{
List<RegexIR> origin = irList;
@@ -27,8 +28,9 @@ namespace StgSharp.RegularAnalysis.Text
int region_name = 1;
EmitPrefix(origin_span, result, 0, 0);
FixLineObservation(result);
int single_line_count = FixLineObservation(result);
info.RegionCount = region_name - 1;
info.SingleLineResultCount = single_line_count;
return result;
void EmitPrefix(
@@ -408,10 +410,14 @@ namespace StgSharp.RegularAnalysis.Text
}
}
static void FixLineObservation(
List<RegexIR> result
static int FixLineObservation(
List<RegexIR> result
)
{
// Observe_Line uses relative offsets while nested regions are emitted.
// Positive offsets point to larger line numbers; this pass resolves them
// into absolute IR line indexes after the final instruction order is known.
HashSet<int> single_required = [];
for (int i = 0; i < result.Count; i++)
{
RegexIRPrefix prefix = result[i].Prefix;
@@ -420,11 +426,15 @@ namespace StgSharp.RegularAnalysis.Text
continue;
}
result[i].Prefix = new RegexIRPrefix(
int observeSource = i + prefix.ObserveSource;
RegexIRPrefix prefix_new = new RegexIRPrefix(
prefix.WorkingRegion,
prefix.Mask,
i - 1);
observeSource);
result[i].Prefix = prefix_new;
_ = single_required.Add(prefix_new.ObserveSource);
}
return single_required.Count;
}
static bool IsFoldableSingleTryInstruction(
StgSharp.RegularAnalysis/Text/RegexAnalyzer.ScanInfo.cs
-101
View File
@@ -1,101 +0,0 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer.InfoScan"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexAnalyzer
{
internal static RegexInfo ScanInfo(
List<RegexIR> ir
)
{
RegexInfo info = new();
int process_count = 0;
ReadOnlySpan<RegexIR> irSpan = CollectionsMarshal.AsSpan(ir);
int max_region = 0;
int min_length = 0;
int single_line_count = 0;
ScanSpan(irSpan);
return info;
void ScanSpan(
ReadOnlySpan<RegexIR> ir_span
)
{
do
{
int count = int.Min(8, ir_span.Length - process_count);
ReadOnlySpan<RegexIR> window = ir_span.Slice(process_count, count);
RegexIRPrefix prefix;
if (window[0].CommandMask == RegexIRCommand.Match_Seq)
{
RegexMatchSeqIR match_seq = Unsafe.As<RegexMatchSeqIR>(window[0]);
min_length += match_seq.Pattern.Length;
prefix = match_seq.Prefix;
max_region = int.Max(max_region, prefix.WorkingRegion);
if ((prefix.Mask & RegexIRPrefixMask.ObserveMethod) == RegexIRPrefixMask.Observe_Line)
{
int line_idx = prefix.ObserveSource;
RegexIR source = ir[line_idx];
if (!source.SingleLineResultRequired)
{
source.SingleLineResultIndex = line_idx;
single_line_count++;
}
}
process_count++;
} else if (window[0] is RegexFindSeqIR find_seq)
{
min_length += find_seq.Pattern.Length;
prefix = find_seq.Prefix;
max_region = int.Max(max_region, prefix.WorkingRegion);
if ((prefix.Mask & RegexIRPrefixMask.ObserveMethod) == RegexIRPrefixMask.Observe_Line)
{
int line_idx = prefix.ObserveSource;
RegexIR source = ir[line_idx];
if (!source.SingleLineResultRequired)
{
source.SingleLineResultIndex = line_idx;
single_line_count++;
}
}
process_count++;
} else if (window[0] is RegexCountSeqIR count_seq)
{
min_length += count_seq.Pattern.Length * count_seq.Min;
prefix = count_seq.Prefix;
max_region = int.Max(max_region, prefix.WorkingRegion);
if ((prefix.Mask & RegexIRPrefixMask.ObserveMethod) == RegexIRPrefixMask.Observe_Line)
{
int line_idx = prefix.ObserveSource;
RegexIR source = ir[line_idx];
if (!source.SingleLineResultRequired)
{
source.SingleLineResultIndex = line_idx;
single_line_count++;
}
}
process_count++;
} else if (window[0] is RegexMatchSetIR) { }
} while (process_count < ir.Count);
}
}
}
}
StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs
+1 -1
View File
@@ -86,7 +86,7 @@ namespace StgSharp.RegularAnalysis.Text
try
{
ScanAndOptimizeIR(base_ir, out List<RegexIR> opt_ir);
List<RegexIR> prefix_ir = EmitPrefix(opt_ir);
List<RegexIR> prefix_ir = EmitPrefix(opt_ir, ref info);
return new TextRegexSource(regex, tree, base_ir, prefix_ir, info);
}
catch (Exception ex)
StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs
+59 -88
View File
@@ -5,6 +5,7 @@
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
@@ -14,143 +15,113 @@ using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal class RegexIRGenerator
internal sealed class RegexIRGenerator
{
private Node _head = new();
private Node _tail;
private readonly SequenceEmitter<RegexIR> _stream = new();
public RegexIRGenerator()
{
_tail = _head;
}
public int Count => _stream.Count;
public int Count { get; private set; }
public void EmitCondition(
int success,
int fail
public RegexIRGenerator EmitCondition(
int success,
int fail
)
{
_tail.Add(new RegexConditionIR(success, fail));
Count++;
return Emit(new RegexConditionIR(success, fail));
}
public void EmitCountComplex(
int min,
int max,
bool isGreedy,
int operation
public RegexIRGenerator EmitCountComplex(
int min,
int max,
bool isGreedy,
int operation
)
{
_tail.Add(new RegexCountComplexIR(min, max, isGreedy, operation));
Count++;
return Emit(new RegexCountComplexIR(min, max, isGreedy, operation));
}
public void EmitCountSeq(
int min,
int max,
bool isGreedy,
string seq
public RegexIRGenerator EmitCountSeq(
int min,
int max,
bool isGreedy,
string seq
)
{
_tail.Add(new RegexCountSeqIR(min, max, isGreedy, seq));
Count++;
return Emit(new RegexCountSeqIR(min, max, isGreedy, seq));
}
public void EmitCountSet(
int min,
int max,
bool isGreedy,
string set
public RegexIRGenerator EmitCountSet(
int min,
int max,
bool isGreedy,
string set
)
{
_tail.Add(new RegexCountSetIR(min, max, isGreedy, set));
Count++;
return Emit(new RegexCountSetIR(min, max, isGreedy, set));
}
public void EmitIRStream(
RegexIRGenerator stream
public RegexIRGenerator EmitIRStream(
RegexIRGenerator generator
)
{
if (stream is null || stream.Count == 0) {
return;
}
_tail.Next = stream._head;
_tail = stream._tail;
Count += stream.Count;
ArgumentNullException.ThrowIfNull(generator);
_ = _stream.Append(generator._stream);
return this;
}
public void EmitMatchSeq(
string unit
public RegexIRGenerator EmitMatchSeq(
string unit
)
{
_tail.Add(new RegexMatchSeqIR(unit));
Count++;
return Emit(new RegexMatchSeqIR(unit));
}
public void EmitMatchSet(
string unit
public RegexIRGenerator EmitMatchSet(
string unit
)
{
_tail.Add(new RegexMatchSetIR(unit));
Count++;
return Emit(new RegexMatchSetIR(unit));
}
public void EmitPack(
bool condition,
string name = ""
public RegexIRGenerator EmitPack(
bool condition,
string name = ""
)
{
_tail.Add(new RegexPackIR(condition, name));
Count++;
return Emit(new RegexPackIR(condition, name));
}
public void EmitPop(
bool isSuccess
public RegexIRGenerator EmitPop(
bool isSuccess
)
{
_tail.Add(new RegexPopIR(isSuccess));
Count++;
return Emit(new RegexPopIR(isSuccess));
}
public void EmitTry(
int count
public RegexIRGenerator EmitTry(
int count
)
{
_tail.Add(new RegexTryIR(count));
Count++;
return Emit(new RegexTryIR(count));
}
public SequenceEmitter<RegexIR>.Enumerator GetEnumerator()
{
return _stream.GetEnumerator();
}
public List<RegexIR> ToList()
{
List<RegexIR> result = new(Count);
Node current = _head;
while (current != null)
{
result.AddRange(current.IRList);
current = current.Next!;
}
return result;
return _stream.ToList();
}
private class Node
private RegexIRGenerator Emit(
RegexIR ir
)
{
public Node? Next { get; set; }
public int Count => IRList.Count;
public List<RegexIR> IRList { get; } = [];
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Add(
RegexIR ir
)
{
IRList.Add(ir);
}
_ = _stream.Append(ir);
return this;
}
}
StgSharpDebug/TestRegexAnalyse.cs
+2
View File
@@ -61,6 +61,8 @@ namespace StgSharp
RegexInfo info = interpreterSource.Info;
Console.WriteLine("-----Regex Info-----");
Console.WriteLine($"Min length: {info.MinPredictLength}");
Console.WriteLine($"Region count: {info.RegionCount}");
Console.WriteLine($"Single line dependency: {info.SingleLineResultCount}");
Console.WriteLine();
Console.WriteLine();
Console.WriteLine();