youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add single ir generator methods

Browse Source
This commit is contained in:
Youmu Liu
2026-08-03 20:03:16 +08:00
parent 85ca22d1aa
commit 029092b438
7 changed files with 280 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/StgSharp.RegularAnalysis/Global.cs
+7 -7
View File
@@ -5,11 +5,11 @@
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
global using System;
global using System.Collections.Generic;
global using System.Linq;
global using System.Runtime.CompilerServices;
global using System.Text;
global using System.Threading.Tasks;
[assembly: InternalsVisibleTo("StgSharpDebug")]
[assembly: InternalsVisibleTo("StgSharpDebug")]
src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs
+8 -4
View File
@@ -57,7 +57,9 @@ namespace StgSharp.RegularAnalysis.Text
{
public static TextRegexSource Analyze(
string regex
string regex,
bool require_ast = false,
bool require_origin_ir = false
)
{
#pragma warning disable CA1031
@@ -81,17 +83,19 @@ namespace StgSharp.RegularAnalysis.Text
}
catch (Exception ex)
{
return new TextRegexSource(regex, tree, null, null, info, ex);
return new TextRegexSource(regex, require_ast ? tree : null!, null, null, info, ex);
}
try
{
ScanAndOptimizeIR(base_ir, out List<RegexIR> opt_ir);
List<RegexIR> prefix_ir = EmitPrefix(opt_ir, ref info);
return new TextRegexSource(regex, tree, base_ir, prefix_ir, info);
return new TextRegexSource(regex, require_ast ? tree : null!,
require_origin_ir ? base_ir : null!, prefix_ir, info);
}
catch (Exception ex)
{
return new TextRegexSource(regex, tree, base_ir, null, info, ex);
return new TextRegexSource(regex, require_ast ? tree : null!,
require_origin_ir ? base_ir : null!, null!, info, ex);
}
#pragma warning restore CA1031
}
src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.MultiIR.cs
+1 -1
View File
@@ -227,7 +227,7 @@ namespace StgSharp.RegularAnalysis.Text
* [011] 0003, Sccs, DEF , LINE, 0010, PACK SUCCESS GROUP: "name"
* [012] 0000, DEF , DEF , NONE, 0000, MATCH_SEQ "234"
*/
if (ir_span[0] is not RegexCountSetIR { IsDotStar:true} ||
if (ir_span[0] is not RegexCountSetIR { IsDotStar: true } ||
ir_span[1] is not RegexPackIR pack)
{
return -1;
src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.SingleIR.cs
+126
View File
@@ -5,11 +5,44 @@
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System.Globalization;
using System.Text.RegularExpressions;
namespace StgSharp.RegularAnalysis.Text
{
internal partial class TextRegexSourceGen
{
private static int TryGenerateCountSeq(
int outer_level,
ReadOnlySpan<RegexIR> ir_span,
SourceGenContext sc,
in SequenceEmitter<string> code,
in List<SequenceEmitter<string>> local_func
)
{
if (ir_span[0] is not RegexCountSeqIR count_seq) {
return -1;
}
string seq = count_seq.Pattern;
string literal = FormatStringLiteral(seq);
bool is_greedy = count_seq.IsGreedy;
int min = count_seq.Min;
int max = count_seq.Max;
int leave = is_greedy ? min : max;
// TODO cannot check real count outside the cycle
_ = code.AppendLine(@$"for(int count = 0; count<{leave}; count++)")
.AppendLine("{")
.AppendLine(@$"if({sc.InputSpan}[{sc.Cursor}..].StartWith(""{literal}""))")
.AppendLine("{")
.AppendLine(@$"{sc.Cursor}+={seq.Length};")
.AppendLine("}")
.AppendLine("}");
return 1;
}
/*
* =======================================================================================
* L4, one instruction
@@ -70,5 +103,98 @@ namespace StgSharp.RegularAnalysis.Text
* - PACK decides nothing, so it should leave __ok untouched. Whether it runs at all is
* already expressed by its execution condition.
*/
private static int TryGenerateMatchSeq(
int outer_level,
ReadOnlySpan<RegexIR> ir_span,
SourceGenContext sc,
in SequenceEmitter<string> code,
in List<SequenceEmitter<string>> local_func
)
{
if (ir_span[0] is not RegexMatchSeqIR match_seq) {
return -1;
}
string seq = match_seq.Pattern;
string literal = FormatStringLiteral(seq);
_ = code.AppendLine(@$"if({sc.InputSpan}[{sc.Cursor}..].StartWith(""{literal}""))")
.AppendLine("{")
.AppendLine($"{sc.StepResult} = true;")
.AppendLine($@"{sc.Cursor} += {seq.Length};")
.AppendLine("}");
return 1;
}
private static int TryGenerateMatchSet(
int outer_level,
ReadOnlySpan<RegexIR> ir_span,
SourceGenContext sc,
in SequenceEmitter<string> code,
in List<SequenceEmitter<string>> local_func
)
{
if (ir_span[0] is not RegexMatchSetIR match_set) {
return -1;
}
string set_source = match_set.Charset;
RegexCharSet[] set_arr = FigureCharSetType(set_source, out bool accept);
_ = code.AppendLine(@$"if(");
for (int i = 1; i < set_arr.Length - 1; i++)
{
RegexCharSet set = set_arr[i];
switch (set.Type)
{
case CharSetType.Single:
string cases = set.Value;
if (cases.Length > 8)
{
// more than 8 different chars, using string match
string literal = FormatStringLiteral(cases);
_ = code.AppendLine($@"{literal}.Contains({sc.InputSpan}[{sc.Cursor}])");
} else
{
StringBuilder sb = new();
_ = sb.Append('(');
for (int j = 0; j < cases.Length - 1; j++) {
_ = sb.Append(CultureInfo.InvariantCulture, $@"{sc.InputSpan}[{sc.Cursor}] == {cases[j]} ||");
}
_ = sb.Append(CultureInfo.InvariantCulture, $@"{sc.InputSpan}[{sc.Cursor}] == {cases[^1]}")
.Append(")||");
_ = code.AppendLine(sb.ToString());
}
break;
case CharSetType.Range:
string rule = set.Value;
char lo = rule[0];
char hi = rule[1];
_ = code.AppendLine($@"({sc.InputSpan}[{sc.Cursor}] >= '{lo}' && {sc.InputSpan}[{sc.Cursor}] <= '{hi}') ||");
break;
case CharSetType.Set:
string accept_mark = set.Accept ? string.Empty : "!";
switch (set.Value)
{
case "s":
_ = code.AppendLine($@"{accept_mark}char.IsWhiteSpace({sc.InputSpan}[{sc.Cursor}]) ||");
break;
case "d":
_ = code.AppendLine($@"{accept_mark}char.IsDigit({sc.InputSpan}[{sc.Cursor}]) ||");
break;
case "w":
_ = code .AppendLine($@"{accept_mark}IsCharWord({sc.InputSpan}[{sc.Cursor}]) ||");
break;
default:
break;
}
break;
case CharSetType.None:
break;
default:
break;
}
}
return 1;
}
}
}
src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs
+56 -23
View File
@@ -72,9 +72,9 @@ namespace StgSharp.RegularAnalysis.Text
* lives in test/SingleFile/regex.cs.
*/
private const string current_ptr = "cur_ptr_global";
private const string offset = "offset";
private const string orig_text = "text";
// private const string current_ptr = "cur_ptr_global"; // global match cursor
// private const string offset = "offset"; //
// private const string orig_text = "text"; //
#region L0 method level
@@ -155,10 +155,11 @@ namespace StgSharp.RegularAnalysis.Text
.AppendLine($@"int region_{i}_end = -1;")
.AppendLine($@"bool region_{i}_success;");
}
/*
_ = se.AppendLine($@"int {current_ptr} = 0;")
.AppendLine($@"int {offset} = 0;")
.AppendLine();
/**/
// _ = sc.GenVarDefine(se);
@@ -177,6 +178,14 @@ namespace StgSharp.RegularAnalysis.Text
#endregion
private enum RegionExitMethod
{
Break,
Continue
}
#region shared helpers
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -224,14 +233,14 @@ namespace StgSharp.RegularAnalysis.Text
}
internal record struct RegexCharSet(
string Value,
CharSetType Type,
bool Accept
internal record class RegexCharSet(
string Value,
CharSetType Type,
bool Accept
)
{
public override readonly string ToString()
public override string ToString()
{
return $"{{value:{Value}, type:{Type}, accept:{Accept}}}";
}
@@ -244,34 +253,45 @@ namespace StgSharp.RegularAnalysis.Text
)
{
// range or single
HashSet<char> single_char = [];
if (source[0] == '[')
{
int idx;
accept = source[1] != '^';
idx = accept ? 1 : 2;
List<RegexCharSet> list = [];
ReadOnlySpan<char> seq = source.Slice(idx, source.Length - idx - 1);
if (seq.IndexOf('-') == -1)
{
return [new RegexCharSet(seq.ToString(), CharSetType.Single, true)];
SplitSet(single_char, seq, list);
StringBuilder sb = new();
foreach (char c in single_char) {
_ = sb.Append(c);
}
list.Add(new(sb.ToString(), CharSetType.Single, true));
} else
{
List<RegexCharSet> list = [];
for (; seq.Length > 0; )
{
int pos = seq.IndexOf('-');
if (pos == -1)
{
// has no range
SplitSet(seq, list);
SplitSet(single_char, seq, list);
break;
} else if (pos > 1)
{
// has other sequence before range
SplitSet(seq[0..(pos - 1)], list);
SplitSet(single_char, seq[0..(pos - 1)], list);
}
list.Add(new($"{seq[pos - 1]}{seq[pos + 1]}", CharSetType.Range, true));
seq = seq[(pos + 2)..];
}
StringBuilder sb = new();
foreach (char c in single_char) {
_ = sb.Append(c);
}
list.Add(new(sb.ToString(), CharSetType.Single, true));
return [.. list];
}
} else if (source[0] == '\\')
@@ -293,6 +313,7 @@ namespace StgSharp.RegularAnalysis.Text
}
static void SplitSet(
HashSet<char> single_char,
ReadOnlySpan<char> seq,
List<RegexCharSet> sets
)
@@ -310,19 +331,31 @@ namespace StgSharp.RegularAnalysis.Text
sets.Add(new RegexCharSet(singles.ToString(), CharSetType.Single, true));
}
bool accept = char.IsAsciiLetterUpper(remain[pos + 1]);
RegexCharSet set_set = char.ToLower(remain[pos + 1], CultureInfo.InvariantCulture)
switch
switch (char.ToLower(remain[pos + 1], CultureInfo.InvariantCulture))
{
's' => new("s", CharSetType.Set, accept),
'w' => new("w", CharSetType.Set, accept),
'd' => new("d", CharSetType.Set, accept),
_ => throw new NotSupportedException()
};
sets.Add(set_set);
case 's':
sets.Add(new("s", CharSetType.Set, accept));
break;
case 'w':
sets.Add(new("w", CharSetType.Set, accept));
break;
case 'd':
sets.Add(new("d", CharSetType.Set, accept));
break;
case '['or'('or')'or']'or'\\'or'\"':
_ = single_char.Add(remain[pos + 1]);
break;
default:
throw new InvalidCastException($"Unknown char or charset {remain[0..1]}");
}
remain = remain[(pos + 2)..];
}
if (remain.Length > 0) {
sets.Add(new RegexCharSet(remain.ToString(), CharSetType.Single, true));
if (remain.Length > 0)
{
foreach (char c in remain) {
_ = single_char.Add(c);
}
}
}
}
src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenContext.cs
+81 -82
View File
@@ -17,8 +17,6 @@ namespace StgSharp.RegularAnalysis.Text
internal partial class TextRegexSourceGen
{
#region var naming
/// <summary>
/// Tracks variable naming and declaration demand while generating one match method.
/// </summary>
@@ -34,35 +32,79 @@ namespace StgSharp.RegularAnalysis.Text
/// <summary>
/// Active pointer. Indexes the next unconsumed element of the input.
/// </summary>
public const string Cursor = "__cur";
public const string _cursor = "__cur";
/// <summary>
/// Name of the match method parameter holding the input sequence.
/// </summary>
public const string InputSpan = "text";
public const string _inputSpan = "text";
/// <summary>
/// Result of the instruction emitted immediately before the current one.
/// </summary>
public const string StepResult = "__ok";
public const string _stepResult = "__ok";
private readonly Dictionary<string, int> _groupIndex = [];
private readonly SortedSet<int> _lineSlots = [];
private readonly HashSet<int> _observedLines = [];
private readonly SortedSet<int> _regionSlots = [];
private int _anonymousGroup;
private int _scratchIndex;
private readonly SortedSet<int> _lineSlots = [];
private readonly SortedSet<int> _regionSlots = [];
public string Cursor { get; } = _cursor;
public string InputSpan { get; } = _inputSpan;
public string StepResult { get; } = _stepResult;
/// <summary>
/// Records that some prefix observes the result of the instruction at line
/// <paramref name="line" />.
/// Emits declarations for every slot requested during body generation.
/// </summary>
public void MarkLineObserved(
int line
/// <remarks>
/// Must be called after the method body has been generated, but the produced lines
/// have to be placed before it.
/// </remarks>
public SequenceEmitter<string> EmitDeclarationsTo(
SequenceEmitter<string> emitter
)
{
_ = _observedLines.Add(line);
if (emitter is null) {
return null!;
}
_ = emitter.AppendLine($@"int {_cursor} = 0;")
.AppendLine($@"bool {_stepResult} = true;");
foreach (int region in _regionSlots) {
_ = emitter.AppendLine($@"int {RegionAnchor(region)} = 0;")
.AppendLine($@"bool {RegionResult(region)} = false;");
}
foreach (int line in _lineSlots) {
_ = emitter.AppendLine($@"bool {LineResult(line)} = false;");
}
foreach (int group in _groupIndex.Values) {
_ = emitter.AppendLine($@"int __g{group}_begin = -1;")
.AppendLine($@"int __g{group}_end = -1;");
}
return emitter;
}
public string GroupBegin(
string name
)
{
return $"__g{ResolveGroup(name)}_begin";
}
public string GroupEnd(
string name
)
{
return $"__g{ResolveGroup(name)}_end";
}
/// <summary>
@@ -76,6 +118,30 @@ namespace StgSharp.RegularAnalysis.Text
return _observedLines.Contains(line);
}
/// <summary>
/// Name of the slot holding the match result of the instruction at line <paramref
/// name="line" />. Only lines actually referenced by an <see
/// cref="RegexIRPrefixMask.Observe_Line" /> prefix get a slot.
/// </summary>
public string LineResult(
int line
)
{
_ = _lineSlots.Add(line);
return $"__l{line}_ok";
}
/// <summary>
/// Records that some prefix observes the result of the instruction at line <paramref
/// name="line" />.
/// </summary>
public void MarkLineObserved(
int line
)
{
_ = _observedLines.Add(line);
}
/// <summary>
/// Allocates a fresh block scoped scratch variable name.
/// </summary>
@@ -91,8 +157,8 @@ namespace StgSharp.RegularAnalysis.Text
}
/// <summary>
/// Name of the slot holding the cursor value captured when region
/// <paramref name="region" /> was entered.
/// Name of the slot holding the cursor value captured when region <paramref
/// name="region" /> was entered.
/// </summary>
public string RegionAnchor(
int region
@@ -113,77 +179,11 @@ namespace StgSharp.RegularAnalysis.Text
return $"__r{region}_ok";
}
/// <summary>
/// Name of the slot holding the match result of the instruction at line
/// <paramref name="line" />. Only lines actually referenced by an
/// <see cref="RegexIRPrefixMask.Observe_Line" /> prefix get a slot.
/// </summary>
public string LineResult(
int line
)
{
_ = _lineSlots.Add(line);
return $"__l{line}_ok";
}
public string GroupBegin(
string name
)
{
return $"__g{ResolveGroup(name)}_begin";
}
public string GroupEnd(
string name
)
{
return $"__g{ResolveGroup(name)}_end";
}
/// <summary>
/// Emits declarations for every slot requested during body generation.
/// </summary>
/// <remarks>
/// Must be called after the method body has been generated, but the produced lines
/// have to be placed before it.
/// </remarks>
public SequenceEmitter<string> EmitDeclarations(
SequenceEmitter<string> emitter
)
{
if (emitter is null) {
return null!;
}
_ = emitter.AppendLine($@"int {Cursor} = 0;")
.AppendLine($@"bool {StepResult} = true;");
foreach (int region in _regionSlots)
{
_ = emitter.AppendLine($@"int {RegionAnchor(region)} = 0;")
.AppendLine($@"bool {RegionResult(region)} = false;");
}
foreach (int line in _lineSlots) {
_ = emitter.AppendLine($@"bool {LineResult(line)} = false;");
}
foreach (int group in _groupIndex.Values)
{
_ = emitter.AppendLine($@"int __g{group}_begin = -1;")
.AppendLine($@"int __g{group}_end = -1;");
}
return emitter;
}
private int ResolveGroup(
string name
)
{
string key = string.IsNullOrEmpty(name) ?
$"#{_anonymousGroup++}" :
name;
string key = string.IsNullOrEmpty(name) ? $"#{_anonymousGroup++}" : name;
if (_groupIndex.TryGetValue(key, out int index)) {
return index;
}
@@ -194,6 +194,5 @@ namespace StgSharp.RegularAnalysis.Text
}
#endregion
}
}
test/StgSharpDebug/TestRegexAnalyse.cs
+1 -1
View File
@@ -35,7 +35,7 @@ namespace StgSharp
{
foreach (string regex in regexSource)
{
TextRegexSource interpreterSource = RegexAnalyzer.Analyze(regex);
TextRegexSource interpreterSource = RegexAnalyzer.Analyze(regex, true, true);
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree = interpreterSource.Ast;
Console.WriteLine($"result of \"{regex}\"");
TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(tree);