youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

switch root namespace name to NITLOAD

Browse Source
This commit is contained in:
Youmu Liu
2026-09-02 18:15:03 +08:00
parent 4218baad48
commit 7c7101ee48
502 changed files with 710 additions and 721 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Nitload.RegularAnalysis/.discard/RegexAnalyzer.IRGenerate.cs.old
+331
View File
@@ -0,0 +1,331 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer.IRGenerate"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.ComponentModel.Design;
using System.Data;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Reflection.Emit;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using System.Xml;
namespace Nitload.RegularAnalysis.Text
{
public partial class RegexAnalyzer
{
internal static List<RegexIR> GenerateIR(
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree,
ref RegexInfo info
)
{
List<RegexIR> list = GenerateIRRecursion(tree.Root, 0, [], out int length).ToList();
info.MinPredictLength = length;
return list;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void EmitCount(
int curDepth,
RegexIRGenerator ir,
int min,
int max,
bool isGreedy,
RegexAstNode right,
HashSet<string> group_names,
out int minLength
)
{
bool is_operator = !RegexAstNode.IsNullOrEmpty(right) &&
(right.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0;
if (is_operator) // seq to be count is complex
{
RegexIRGenerator right_ir = GenerateIRRecursion(right, curDepth + 1, group_names,
out minLength);
_ = ir.EmitCountComplex(min, max, isGreedy, right_ir.Count);
_ = ir.EmitIRStream(right_ir);
} else
{
RegexElementLabel count_source_mask = right.Source.Flag;
switch (count_source_mask)
{
case RegexElementLabel.UNIT:
minLength = 1;
_ = ir.EmitCountSeq(min, max, isGreedy, right.Value);
break;
case RegexElementLabel.UNIT_SET:
minLength = 1;
_ = ir.EmitCountSet(min, max, isGreedy, right.Value);
break;
case RegexElementLabel.UNIT_SPAN:
minLength = 1;
_ = ir.EmitCountSeq(min, max, isGreedy, right.Value);
break;
default:
throw new InvalidOperationException($"Unexpected regex element type for count operator: {count_source_mask}");
}
}
}
private static RegexIRGenerator GenerateIRRecursion(
RegexAstNode node,
int depth,
HashSet<string> group_names,
out int minLength
)
{
#pragma warning disable IDE0010
if (depth > 1024)
{
throw new OverflowException("Regex parse exceeds maximum recursion depth");
}
int curDepth = depth;
RegexAstNode left = node.Left;
RegexAstNode right = node.Right;
RegexIRGenerator leftIR;
RegexIRGenerator rightIR;
Token<RegexElementLabel> token = node.Source;
HashSet<string> gn = group_names ?? [];
switch (node.Source.Flag)
{
case RegexElementLabel.GROUP_BEGIN:
string group_source = token.Value;
// process group type
bool has_name = group_source.StartsWith(@"(?<", StringComparison.Ordinal); // simple group: (pattern)
string group_name = has_name ?
group_source[3..^1] : /* named group (?<name>) */
string.Empty; /* unnamed group (?:) */
if (has_name)
{
if (!gn.Add(group_name)) {
throw new InvalidOperationException($"Duplicate group name: {group_name}");
}
}
RegexIRGenerator groupIR = new();
rightIR = GenerateIRRecursion(right, curDepth + 1, gn, out minLength);
_ = groupIR.EmitTry(rightIR.Count)
.EmitIRStream(rightIR)
.EmitCondition(2, 1)
.EmitPop(false)
.EmitPack(true, group_name);
return groupIR;
case RegexElementLabel.COUNT:
string count_source = token.Value;
bool isGreedy = !(count_source.Length > 1 && count_source.EndsWith('?'));
RegexAstNode seq = right;
ReadOnlySpan<char> count_source_span = count_source;
RegexIRGenerator count_ir = new();
int length_once;
if (count_source_span[0] == '*') // infinite count
{
EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq, gn, out _);
minLength = 0;
} else if (count_source_span[0] == '+') // at least one
{
EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq, gn, out length_once);
minLength = length_once;
} else if (count_source_span[0] == '?') // zero or one
{
EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq, gn, out _);
minLength = 0;
} else if (count_source_span[0] == '{') // zero or one, non-greedy
{
int comma_pos = count_source_span.IndexOf(',');
if (comma_pos == -1)
{
if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count))
{
EmitCount(curDepth, count_ir, count, count, isGreedy, seq, gn,
out length_once);
minLength = length_once * count;
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
}
} else
{
if (int.TryParse(count_source[1..comma_pos], out int min) &&
int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max))
{
EmitCount(curDepth, count_ir, min, max, isGreedy, seq, gn,
out length_once);
minLength = length_once * min;
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
}
}
} else
{
throw new InvalidOperationException();
} // unknown count format
return count_ir;
case RegexElementLabel.CONCAT:
RegexIRGenerator concat = new();
// process left node
int left_length = 0;
if (RegexAstNode.IsNullOrEmpty(left))
{
// do nothing
} else if ((left.Source.Flag & RegexElementLabel.SEQUENCE) != 0)
{
switch (left.Source.Flag)
{
case RegexElementLabel.UNIT_SPAN:
left_length = left.Value.Length;
_ = concat.EmitMatchSeq(left.Value);
break;
case RegexElementLabel.UNIT:
left_length = 1;
_ = concat.EmitMatchSeq(left.Value);
break;
case RegexElementLabel.UNIT_SET:
left_length = 1;
_ = concat.EmitMatchSet(left.Value);
break;
default:
break;
}
} else if ((left.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{
leftIR = GenerateIRRecursion(left, curDepth + 1, gn, out left_length);
_ = concat.EmitIRStream(leftIR);
}
// process right node
int right_length = 0;
if (RegexAstNode.IsNullOrEmpty(right))
{
// do nothing
} else if ((right.Source.Flag & RegexElementLabel.SEQUENCE) != 0)
{
switch (right.Source.Flag)
{
case RegexElementLabel.UNIT_SPAN:
right_length = right.Value.Length;
_ = concat.EmitMatchSeq(right.Value);
break;
case RegexElementLabel.UNIT:
right_length = 1;
_ = concat.EmitMatchSeq(right.Value);
break;
case RegexElementLabel.UNIT_SET:
right_length = 1;
_ = concat.EmitMatchSet(right.Value);
break;
default:
break;
}
} else if ((right.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{
rightIR = GenerateIRRecursion(right, curDepth + 1, gn, out right_length);
_ = concat.EmitIRStream(rightIR);
}
minLength = left_length + right_length;
return concat;
case RegexElementLabel.ALT:
RegexIRGenerator alt = new();
List<RegexIRGenerator> casesIRList = [];
RegexAstNode cur_node = right;
int ir_count = 0;
int min_case_length = int.MaxValue;
while (!RegexAstNode.IsNullOrEmpty(cur_node))
{
RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1, gn,
out int case_length);
min_case_length = int.Min(case_length, min_case_length);
casesIRList.Add(_case);
ir_count += _case.Count;
cur_node = cur_node.Next;
}
int case_count = casesIRList.Count;
while (case_count != 0)
{
RegexIRGenerator cur_ir = casesIRList[^case_count];
ir_count -= cur_ir.Count;
case_count--;
_ = alt.EmitTry(cur_ir.Count)
.EmitIRStream(cur_ir)
.EmitCondition(1/* need to calc here */, 2)
.EmitPop(true);
}
_ = alt.EmitPop(false);
minLength = min_case_length;
return alt;
case RegexElementLabel.UNIT_SPAN:
RegexIRGenerator unit = new();
_ = unit.EmitMatchSeq(node.Value);
minLength = node.Value.Length;
return unit;
case RegexElementLabel.UNIT:
RegexIRGenerator unit_seq = new();
_ = unit_seq.EmitMatchSeq(node.Value);
minLength = 1;
return unit_seq;
case RegexElementLabel.UNIT_SET:
RegexIRGenerator unit_set = new();
_ = unit_set.EmitMatchSet(node.Value);
minLength = 1;
return unit_set;
default:
minLength = 0;
return null!;
}
#pragma warning restore IDE0010
// info.MinPredictLength = ComputeMinLength(tree.Root);
static int ComputeMinLength(
RegexAstNode node
)
{
if (node.EqualityTypeConvert is RegexElementLabel.UNIT_SPAN or
RegexElementLabel.UNIT)
{
return node.Value.Length;
} else if (node.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
return ComputeMinLength(node.Left) + ComputeMinLength(node.Right);
} else if (node.EqualityTypeConvert == RegexElementLabel.ALT)
{
RegexAstNode case_0 = node.Right;
int length = int.MaxValue;
do
{
length = int.Min(ComputeMinLength(case_0), length);
case_0 = case_0.Next;
} while (!RegexAstNode.IsNullOrEmpty(case_0));
return length;
} else if (node.EqualityTypeConvert == RegexElementLabel.COUNT)
{
string source = node.Source.Value;
int single_count = ComputeMinLength(node.Right);
return 0;
} else
{
return 0;
}
}
}
}
}
src/Nitload.RegularAnalysis/.discard/RegexAnalyzer.IROptimize.cs.old
+493
View File
@@ -0,0 +1,493 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer.IROptimize"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
public partial class RegexAnalyzer
{
internal static void ScanAndOptimizeIR(
List<RegexIR> baseIR,
out List<RegexIR> optIR
)
{
int opt_count;
List<RegexIR> origin_ir = baseIR;
List<RegexIR> opt_ir = new(origin_ir.Count);
RegexIR[] opt_window = new RegexIR[8];
Span<RegexIR> opt_win_sp = opt_window;
int result;
// phase 0: common opt
phase_zero:
opt_count = 0;
int index = 0;
Span<RegexIR> origin_span = CollectionsMarshal.AsSpan(origin_ir);
do
{
int process_count = 0;
int last = int.Min(origin_span.Length, index + 8);
opt_win_sp.Clear();
origin_span[index..last].CopyTo(opt_win_sp);
if ((result = TryExpandPlus(opt_win_sp, opt_ir)) >= 0)
{
process_count++;
if (result > 0)
{
RegexCountComplexIR cc = Unsafe.As<RegexCountComplexIR>(opt_win_sp[0]);
Span<RegexIR> sp = origin_span.Slice(index + 1, result);
TryExpandComplexPlus(cc.IsGreedy, sp, opt_ir);
process_count += result;
}
opt_count++;
} else if (TryOptDotCountCount(opt_win_sp, opt_ir) >= 0)
{
process_count += 2;
opt_count++;
} else if (TryFlattenCount(opt_win_sp, opt_ir) >= 0)
{
process_count++;
opt_count++;
} else if ((result = TryExpandOnce(opt_win_sp, opt_ir)) >= 0)
{
process_count++;
if (result > 0)
{
RegexCountComplexIR cc = Unsafe.As<RegexCountComplexIR>(opt_win_sp[0]);
Span<RegexIR> sp = origin_span.Slice(index + 1, result);
TryExpandComplex(sp, opt_ir, true);
process_count += result;
}
opt_count++;
} else if (opt_win_sp[0] is IRegexComplex { IsFullyOptimized: true } skip)
{
int count = skip.CommandCount;
process_count += count + 1;
Span<RegexIR> opt_block = origin_span.Slice(index, count + 1);
opt_ir.AddRange(opt_block);
} else if (opt_win_sp[0] is RegexTryIR { IsFullyOptimized: false } try_ir)
{
process_count++;
int cmd_cnt = try_ir.CommandCount;
Span<RegexIR> try_sp = origin_span.Slice(index + 1, cmd_cnt);
ScanAndOptimizeIR(try_sp, out List<RegexIR> opt_try);
RegexTryIR new_try = new(opt_try.Count)
{
IsFullyOptimized = true
};
opt_ir.Add(new_try);
opt_ir.AddRange(opt_try);
process_count += cmd_cnt;
opt_count++;
} else
{
opt_ir.Add(opt_win_sp[0]);
process_count++;
}
index += process_count;
} while (index < origin_ir.Count);
if (opt_count > 0)
{
// need to swap the origin_ir and opt_ir, and clear the opt_ir for next round
if (ReferenceEquals(origin_ir, baseIR))
{
origin_ir = new(opt_ir.Count);
}
(origin_ir, opt_ir) = (opt_ir, origin_ir);
opt_ir.Clear();
goto phase_zero;
}
// phase 1: dot star replace
(origin_ir, opt_ir) = (opt_ir, origin_ir);
opt_ir.Clear();
index = 0;
opt_count = 0;
origin_span = CollectionsMarshal.AsSpan(origin_ir);
do
{
int process_count = 0;
int last = int.Min(origin_span.Length, index + 8);
opt_win_sp.Clear();
origin_span[index..last].CopyTo(opt_win_sp);
if (opt_win_sp[0] is IRegexComplex { IsFullyOptimized: true } skip)
{
int count = skip.CommandCount;
process_count += count + 1;
Span<RegexIR> opt_block = origin_span.Slice(index, count + 1);
opt_ir.AddRange(opt_block);
} else
if (TryReplaceDotStar(opt_win_sp, opt_ir) >= 0)
{
opt_count++;
process_count++;
} else
{
opt_ir.Add(opt_win_sp[0]);
process_count++;
}
index += process_count;
} while (index < origin_ir.Count);
if (opt_count == 0)
{
optIR = opt_ir;
return;
}
// phase 2: find-1 merge and try-find-1 rotate
do
{
(origin_ir, opt_ir) = (opt_ir, origin_ir);
opt_ir.Clear();
opt_count = 0;
index = 0;
origin_span = CollectionsMarshal.AsSpan(origin_ir);
do
{
int process_count = 0;
int last = int.Min(origin_span.Length, index + 8);
opt_win_sp.Clear();
origin_span[index..last].CopyTo(opt_win_sp);
if (opt_win_sp[0] is IRegexComplex { IsFullyOptimized: true, CommandCount: > 0 } skip)
{
int count = skip.CommandCount;
process_count += count + 1;
Span<RegexIR> opt_block = origin_span.Slice(index, count + 1);
opt_ir.AddRange(opt_block);
} else if (TyrMergeFindRestTwice(opt_win_sp, opt_ir) == 0)
{
process_count += 2;
opt_count++;
} else if ((result = TyrMergeFindRestTry(opt_win_sp, opt_ir)) >= 0)
{
Span<RegexIR> opt_block = origin_span.Slice(index + 2, result);
opt_ir.AddRange(opt_block);
process_count += 2;
process_count += result;
opt_count++;
} else
{
opt_ir.Add(opt_win_sp[0]);
process_count++;
}
index += process_count;
} while (index < origin_ir.Count);
} while (opt_count > 0);
optIR = opt_ir;
}
private static void ScanAndOptimizeIR(
Span<RegexIR> baseIR,
out List<RegexIR> opt_ir
)
{
List<RegexIR> originList = new(baseIR.Length);
CollectionsMarshal.SetCount(originList, baseIR.Length);
baseIR.CopyTo(CollectionsMarshal.AsSpan(originList));
ScanAndOptimizeIR(originList, out opt_ir);
}
private static int TryFlattenCount(
Span<RegexIR> window,
List<RegexIR> opt
)
{
const int flatten_size = 32;
RegexIR ir = window[0];
if (ir is not RegexCountSeqIR seq) {
return -1;
}
int min = seq.Min;
int max = seq.Max;
if (min == 0) {
return -1;
}
string pattern = seq.Pattern;
int pattern_size = pattern.Length;
if (pattern_size > flatten_size) {
opt.Add(ir);
}
int min_expand = flatten_size / pattern_size;
min_expand = int.Min(min_expand, min);
string expand = string.Concat(Enumerable.Repeat(pattern, min_expand));
int new_min = min / min_expand;
int new_rest = min % min_expand;
if (new_min == 1)
{
opt.Add(new RegexMatchSeqIR(expand));
} else
{
opt.Add(new RegexCountSeqIR(new_min, new_min, seq.IsGreedy, expand));
}
if (new_rest > 0)
{
string expand_tail = string.Concat(Enumerable.Repeat(pattern, new_rest));
opt.Add(new RegexMatchSeqIR(expand_tail));
}
int rest_count = max - min;
rest_count = int.Max(rest_count, -1);
opt.Add(new RegexCountSeqIR(0, rest_count, seq.IsGreedy, pattern));
return 0;
}
#region dot count count
private static int TryOptDotCountCount(
Span<RegexIR> window,
List<RegexIR> opt
)
{
/* .* <any>*
* convert to
* .*
*/
RegexIR cur_ir = window[0];
if (cur_ir is not RegexCountSetIR { IsDotStar: true } dot) {
return -1;
}
if (window[1] is not IRegexCount { IsStar: true } after) {
return -1;
}
if (dot.IsGreedy)
{
opt.Add(window[0]);
return 0;
}
return -1;
}
#endregion
private static int TryReplaceDotStar(
Span<RegexIR> window,
List<RegexIR> opt
)
{
if (window[0] is not RegexCountSetIR { IsDotStar: true } dot) {
return -1;
}
if (dot.IsGreedy)
{
RegexFindComplexIR find = new(false, -1)
{
IsFullyOptimized = true
};
opt.Add(find);
return 0;
} else
{
RegexFindComplexIR find = new(true, -1)
{
IsFullyOptimized = true
};
opt.Add(find);
return 0;
}
}
#region merge find and try-find
private static int TyrMergeFindRestTwice(
Span<RegexIR> window,
List<RegexIR> opt
)
{
if (window[0] is not RegexFindComplexIR { CommandCount: -1 } find_1 ||
window[1] is not RegexFindComplexIR find_2) {
return -1;
}
RegexFindComplexIR find_try = new(find_1.IsNearest && find_2.IsNearest, find_2.CommandCount)
{
IsFullyOptimized = true
};
opt.Add(find_try);
return 0;
}
private static int TyrMergeFindRestTry(
Span<RegexIR> window,
List<RegexIR> opt
)
{
if (window[0] is not RegexFindComplexIR { CommandCount: -1 } _find ||
window[1] is not RegexTryIR _try) {
return -1;
}
RegexFindComplexIR find_try = new(_find.IsNearest, _try.CommandCount)
{
IsFullyOptimized = true
};
opt.Add(find_try);
return find_try.CommandCount;
}
#endregion
#region expand once
private static void TryExpandComplex(
Span<RegexIR> complex,
List<RegexIR> opt,
bool is_once
)
{
if (!is_once) {
opt.Add(new RegexTryIR(complex.Length));
}
opt.AddRange(complex);
}
private static int TryExpandOnce(
Span<RegexIR> window,
List<RegexIR> opt
)
{
RegexIR ir = window[0];
if (ir is IRegexCount { Min: 1, Max: 1 })
{
string pattern;
switch (ir.CommandMask & RegexIRCommand.Count)
{
case RegexIRCommand.Count_Complex:
RegexCountComplexIR complex = Unsafe.As<RegexCountComplexIR>(ir);
return complex.CommandCount;
case RegexIRCommand.Count_Seq:
RegexCountSeqIR seq = Unsafe.As<RegexCountSeqIR>(ir);
pattern = seq.Pattern;
opt.Add(new RegexMatchSeqIR(pattern));
return 0;
case RegexIRCommand.Count_Set:
RegexCountSetIR set = Unsafe.As<RegexCountSetIR>(ir);
pattern = set.CharSet;
opt.Add(new RegexMatchSetIR(pattern));
return 0;
default:
return -1;
}
} else if (ir is IRegexCount { Min: 0, Max: 1 })
{
string pattern;
switch (ir.CommandMask & RegexIRCommand.Count)
{
case RegexIRCommand.Count_Complex:
RegexCountComplexIR complex = Unsafe.As<RegexCountComplexIR>(ir);
return complex.CommandCount;
case RegexIRCommand.Count_Seq:
RegexCountSeqIR seq = Unsafe.As<RegexCountSeqIR>(ir);
pattern = seq.Pattern;
opt.Add(new RegexTryIR(1));
opt.Add(new RegexMatchSeqIR(pattern));
return 0;
case RegexIRCommand.Count_Set:
opt.Add(new RegexTryIR(1));
RegexCountSetIR set = Unsafe.As<RegexCountSetIR>(ir);
pattern = set.CharSet;
opt.Add(new RegexMatchSetIR(pattern));
return 0;
default:
return -1;
}
} else
{
return -1;
}
}
#endregion
#region expand plus
private static void TryExpandComplexPlus(
bool is_greedy,
Span<RegexIR> complex,
List<RegexIR> opt
)
{
ScanAndOptimizeIR(complex, out List<RegexIR>? part_opt);
opt.AddRange(part_opt);
RegexCountComplexIR c_ir = new(0, -1, is_greedy, part_opt.Count)
{
IsFullyOptimized = true
};
opt.Add(c_ir);
opt.AddRange(part_opt);
}
/// <summary>
///
/// </summary>
/// <param name="window">
///
/// </param>
/// <param name="opt">
///
/// </param>
/// <returns>
/// Negative if cannot opt, 0 if match successfully, positive if meets an IRegexComplex
/// interface, need to call TryExpandComplexPlus
/// </returns>
private static int TryExpandPlus(
Span<RegexIR> window,
List<RegexIR> opt
)
{
/* a+
* convert to
* aa*
*/
RegexIR ir = window[0];
if (ir is not IRegexCount { IsPlus: true } count) {
return -1;
}
bool is_greedy = count.IsGreedy;
string pattern;
switch (ir.CommandMask & RegexIRCommand.Count)
{
case RegexIRCommand.Count_Complex:
RegexCountComplexIR complex = Unsafe.As<RegexCountComplexIR>(ir);
return complex.CommandCount;
case RegexIRCommand.Count_Seq:
RegexCountSeqIR seq = Unsafe.As<RegexCountSeqIR>(ir);
pattern = seq.Pattern;
opt.Add(new RegexMatchSeqIR(pattern));
opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern));
return 0;
case RegexIRCommand.Count_Set:
RegexCountSeqIR set = Unsafe.As<RegexCountSeqIR>(ir);
pattern = set.Pattern;
opt.Add(new RegexMatchSeqIR(pattern));
opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern));
return 0;
default:
return -1;
}
}
#endregion
}
}
src/Nitload.RegularAnalysis/.discard/RegexAnalyzer.PrefixEmit.cs.old
+531
View File
@@ -0,0 +1,531 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer.PrefixEmit"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
public static partial class RegexAnalyzer
{
internal static List<RegexIR> EmitPrefix(
List<RegexIR> irList,
ref RegexInfo info
)
{
List<RegexIR> origin = irList;
List<RegexIR> result = new(origin.Count);
Span<RegexIR> origin_span = CollectionsMarshal.AsSpan(origin);
int region_name = 1;
EmitPrefix(origin_span, result, 0, 0);
int single_line_count = FixLineObservation(result);
info.RegionCount = region_name - 1;
info.SingleLineResultCount = single_line_count;
return result;
void EmitPrefix(
ReadOnlySpan<RegexIR> region,
List<RegexIR> result,
int currentRegion,
int lineBase
)
{
ReadOnlySpan<RegexIR> emit_window;
int index = 0;
while (index < region.Length)
{
emit_window = region[index..];
int emitResult;
if ((emitResult =
TryEmitGroupFindPack(emit_window, result, currentRegion, lineBase)) > 0)
{
index += emitResult;
} else if ((emitResult =
TryEmitTryGroupTail(emit_window, result, currentRegion,
lineBase)) > 0)
{
index += emitResult;
} else if ((emitResult =
TryEmitAltChain(emit_window, result, currentRegion, lineBase)) > 0)
{
index += emitResult;
} else if ((emitResult =
TryEmitAltSuccessTail(emit_window, result, currentRegion,
lineBase)) > 0)
{
index += emitResult;
} else if ((emitResult =
TryEmitAltFailTail(emit_window, result, currentRegion,
lineBase)) > 0)
{
index += emitResult;
} else if ((emitResult =
TryEmitComplexIR(emit_window, result, currentRegion, lineBase)) > 0)
{
index += emitResult;
} else if ((emitResult =
TryEmitTryIR(emit_window, result, currentRegion, lineBase)) > 0)
{
index += emitResult;
} else
{
result.Add(CloneIR(emit_window[0], currentRegion));
index++;
}
}
}
int TryEmitTryGroupTail(
ReadOnlySpan<RegexIR> origin,
List<RegexIR> result,
int currentRegion,
int lineBase
)
{
if (origin.Length < 5 || origin[0] is not RegexTryIR _try) {
return -1;
}
int tailOffset = _try.CommandCount + 1;
if (origin.Length < tailOffset + 3 ||
origin[tailOffset] is not RegexConditionIR { SuccessCase: 2, FailCase: 1 } condition ||
origin[tailOffset + 1] is not RegexPopIR { IsSuccess: false } pop ||
origin[tailOffset + 2] is not RegexPackIR { Condition: true } pack) {
return -1;
}
int tryOutputIndex = result.Count;
int emitCount = TryEmitTryIR(origin, result, currentRegion, lineBase,
foldSingleInstructionTry:false);
if (emitCount <= 0 || result[tryOutputIndex] is not RegexTryIR emittedTry) {
return -1;
}
_ = condition;
int regionId = emittedTry.Prefix.WorkingRegion;
RegexIR prefixedPop = CloneIR(pop, regionId);
ApplyPrefix(prefixedPop, regionId,
RegexIRPrefixMask.Prev_Fail | RegexIRPrefixMask.Observe_Region,
regionId);
RegexIR prefixedPack = CloneIR(pack, regionId);
ApplyPrefix(prefixedPack, regionId,
RegexIRPrefixMask.Prev_Success | RegexIRPrefixMask.Observe_Region,
regionId);
result.Add(prefixedPop);
result.Add(prefixedPack);
return emitCount + 3;
}
int TryEmitAltChain(
ReadOnlySpan<RegexIR> origin,
List<RegexIR> result,
int currentRegion,
int lineBase
)
{
if (origin.Length < 3 ||
origin[0] is not RegexConditionIR { SuccessCase: 1, FailCase: 2 } ||
origin[1] is not RegexPopIR { IsSuccess: true } ||
origin[2] is not RegexTryIR) {
return -1;
}
int observeRegion = currentRegion;
RegexIRPrefixMask observeMask = RegexIRPrefixMask.Observe_Region;
int observeSource = currentRegion;
if (result.Count > 0)
{
RegexIR previous = result[^1];
observeRegion = GetObservedRegion(previous, currentRegion);
GetObservation(previous, result, lineBase, currentRegion, out observeMask,
out observeSource);
MarkPopResult(previous, RegexIRPrefixMask.Pop_Success);
}
RegexIRPrefixMask mask = RegexIRPrefixMask.Prev_Fail | observeMask;
int emit_count = TryEmitTryIR(origin[2..], result, currentRegion, lineBase, mask,
observeSource, observeRegion);
return emit_count < 0 ? -1 : emit_count + 2;
}
int TryEmitAltSuccessTail(
ReadOnlySpan<RegexIR> origin,
List<RegexIR> result,
int currentRegion,
int lineBase
)
{
if (origin.Length < 2 ||
origin[0] is not RegexConditionIR { SuccessCase: 1, FailCase: 2 } ||
origin[1] is not RegexPopIR { IsSuccess: true } ||
result.Count == 0) {
return -1;
}
RegexIR previous = result[^1];
MarkPopResult(previous, RegexIRPrefixMask.Pop_Success);
return 2;
}
int TryEmitAltFailTail(
ReadOnlySpan<RegexIR> origin,
List<RegexIR> result,
int currentRegion,
int lineBase
)
{
if (origin.Length < 1 ||
origin[0] is not RegexPopIR { IsSuccess: false } pop ||
result.Count == 0) {
return -1;
}
RegexIR previous = result[^1];
if ((previous.Prefix.Mask & RegexIRPrefixMask.Pop_Success) == 0) {
return -1;
}
GetObservation(previous, result, lineBase, currentRegion,
out RegexIRPrefixMask observeMask, out int observeSource);
RegexIR prefixedPop = CloneIR(pop, currentRegion);
ApplyPrefix(prefixedPop, currentRegion, RegexIRPrefixMask.Prev_Fail | observeMask,
observeSource);
result.Add(prefixedPop);
return 1;
}
int TryEmitComplexIR(
ReadOnlySpan<RegexIR> origin,
List<RegexIR> result,
int currentRegion,
int lineBase
)
{
if (origin.Length == 0) {
return -1;
}
if (origin[0] is RegexCountComplexIR count)
{
int commandCount = count.CommandCount;
if (origin.Length < commandCount + 1) {
return -1;
}
List<RegexIR> sub_ir = new(commandCount);
EmitPrefix(origin.Slice(1, commandCount), sub_ir, currentRegion,
lineBase + result.Count + 1);
RegexCountComplexIR newCount = new(count.Min, count.Max, count.IsGreedy,
sub_ir.Count)
{
IsFullyOptimized = count.IsFullyOptimized,
Prefix =
new RegexIRPrefix(
count.Prefix.WorkingRegion == 0 ?
currentRegion :
count.Prefix.WorkingRegion,
count.Prefix.Mask,
count.Prefix.ObserveSource)
};
result.Add(newCount);
result.AddRange(sub_ir);
return commandCount + 1;
}
if (origin[0] is RegexFindComplexIR { CommandCount: > 0 } find)
{
int commandCount = find.CommandCount;
if (origin.Length < commandCount + 1) {
return -1;
}
List<RegexIR> sub_ir = new(commandCount);
EmitPrefix(origin.Slice(1, commandCount), sub_ir, currentRegion,
lineBase + result.Count + 1);
RegexFindComplexIR newFind = new(find.IsNearest, sub_ir.Count)
{
IsFullyOptimized = find.IsFullyOptimized,
Prefix =
new RegexIRPrefix(
find.Prefix.WorkingRegion == 0 ?
currentRegion :
find.Prefix.WorkingRegion,
find.Prefix.Mask,
find.Prefix.ObserveSource)
};
result.Add(newFind);
result.AddRange(sub_ir);
return commandCount + 1;
}
return -1;
}
// convert TRY into region
int TryEmitTryIR(
ReadOnlySpan<RegexIR> origin,
List<RegexIR> result,
int currentRegion,
int lineBase,
RegexIRPrefixMask mask = 0,
int observeSource = 0,
int reuseRegion = 0,
bool foldSingleInstructionTry = true
)
{
if (origin.Length == 0 || origin[0] is not RegexTryIR _try) {
return -1;
}
int commandCount = _try.CommandCount;
if (origin.Length < commandCount + 1) {
return -1;
}
int regionId = reuseRegion == 0 ? region_name : reuseRegion;
if (reuseRegion == 0) {
region_name++;
}
if (foldSingleInstructionTry &&
commandCount == 1 &&
IsFoldableSingleTryInstruction(origin[1]))
{
RegexIR folded = CloneIR(origin[1], regionId);
int foldedObserveSource =
(mask & RegexIRPrefixMask.ObserveMethod) != 0 ?
observeSource :
_try.Prefix.ObserveSource;
folded.Prefix = new RegexIRPrefix(
regionId,
folded.Prefix.Mask | _try.Prefix.Mask | mask,
foldedObserveSource);
result.Add(folded);
return commandCount + 1;
}
List<RegexIR> sub_ir = new(commandCount);
ReadOnlySpan<RegexIR> try_region = origin.Slice(1, commandCount);
EmitPrefix(try_region, sub_ir, regionId, lineBase + result.Count + 1);
RegexTryIR newTry = new(sub_ir.Count)
{
IsFullyOptimized = _try.IsFullyOptimized,
Prefix = new RegexIRPrefix(regionId, _try.Prefix.Mask | mask, observeSource)
};
result.Add(newTry);
result.AddRange(sub_ir);
return commandCount + 1;
}
int TryEmitGroupFindPack(
ReadOnlySpan<RegexIR> origin,
List<RegexIR> result,
int currentRegion,
int lineBase
)
{
if (origin.Length < 5 ||
origin[0] is not RegexTryIR { CommandCount: 1 } _try ||
origin[1] is not RegexFindComplexIR find ||
origin[2] is not RegexConditionIR { SuccessCase: 2, FailCase: 1 } ||
origin[3] is not RegexPopIR { IsSuccess: false } ||
origin[4] is not RegexPackIR { Condition: true } pack) {
return -1;
}
int regionId = region_name;
region_name++;
RegexIRPrefixMask findMask = find.Prefix.Mask | _try.Prefix.Mask;
int findObserveSource =
(find.Prefix.Mask & RegexIRPrefixMask.ObserveMethod) != 0 ?
find.Prefix.ObserveSource :
_try.Prefix.ObserveSource;
RegexFindComplexIR newFind = new(find.IsNearest, find.CommandCount)
{
IsFullyOptimized = find.IsFullyOptimized,
Prefix = new RegexIRPrefix(regionId, findMask, findObserveSource)
};
RegexPackIR newPack = new(pack.Condition, pack.GroupName)
{
Prefix =
new RegexIRPrefix(
regionId,
pack.Prefix.Mask | RegexIRPrefixMask.Prev_Success | RegexIRPrefixMask.Observe_Line,
-1)
};
result.Add(newFind);
result.Add(newPack);
return 5;
}
static int GetObservedRegion(
RegexIR ir,
int currentRegion
)
{
return ir.Prefix.WorkingRegion == 0 ? currentRegion : ir.Prefix.WorkingRegion;
}
static void GetObservation(
RegexIR ir,
List<RegexIR> result,
int lineBase,
int currentRegion,
out RegexIRPrefixMask observeMask,
out int observeSource
)
{
if (IsLineObservable(ir))
{
observeMask = RegexIRPrefixMask.Observe_Line;
observeSource = -1;
} else
{
observeMask = RegexIRPrefixMask.Observe_Region;
observeSource = GetObservedRegion(ir, currentRegion);
}
}
static int FixLineObservation(
List<RegexIR> result
)
{
// Observe_Line uses relative offsets while nested regions are emitted.
// Positive offsets point to larger line numbers; this pass resolves them
// into absolute IR line indexes after the final instruction order is known.
HashSet<int> single_required = [];
for (int i = 0; i < result.Count; i++)
{
RegexIRPrefix prefix = result[i].Prefix;
if ((prefix.Mask & RegexIRPrefixMask.Observe_Line) == 0)
{
continue;
}
int observeSource = i + prefix.ObserveSource;
RegexIRPrefix prefix_new = new RegexIRPrefix(
prefix.WorkingRegion,
prefix.Mask,
observeSource);
result[i].Prefix = prefix_new;
_ = single_required.Add(prefix_new.ObserveSource);
}
return single_required.Count;
}
static bool IsFoldableSingleTryInstruction(
RegexIR ir
)
{
return IsLineObservable(ir) &&
(ir.Prefix.Mask & RegexIRPrefixMask.ExeCondition) == 0 &&
(ir.Prefix.Mask & RegexIRPrefixMask.ObserveMethod) == 0;
}
static bool IsLineObservable(
RegexIR ir
)
{
return ir is
RegexMatchSeqIR or
RegexMatchSetIR or
RegexCountSeqIR or
RegexCountSetIR or
RegexFindSeqIR or
//RegexFindSetIR or
RegexFindComplexIR;
}
static void ApplyPrefix(
RegexIR ir,
int workingRegion,
RegexIRPrefixMask mask,
int observeSource
)
{
ir.Prefix = new RegexIRPrefix(
workingRegion,
ir.Prefix.Mask | mask,
observeSource);
}
// Only annotate the pop result of an already emitted instruction.
// The working region and observe source belong to the execution guard
// and must stay untouched, otherwise a region id leaks into ObserveSource.
static void MarkPopResult(
RegexIR ir,
RegexIRPrefixMask popMask
)
{
ir.Prefix = new RegexIRPrefix(
ir.Prefix.WorkingRegion,
ir.Prefix.Mask | popMask,
ir.Prefix.ObserveSource);
}
static RegexIR CloneIR(
RegexIR ir,
int currentRegion
)
{
RegexIR clone = ir switch
{
RegexFindComplexIR find => new RegexFindComplexIR(find.IsNearest, find.CommandCount)
{
IsFullyOptimized = find.IsFullyOptimized
},
RegexFindSeqIR find => new RegexFindSeqIR(find.Pattern, find.IsFirst),
RegexMatchSeqIR match => new RegexMatchSeqIR(match.Pattern),
RegexMatchSetIR match => new RegexMatchSetIR(match.Charset),
RegexConditionIR condition => new RegexConditionIR(condition.SuccessCase, condition.FailCase),
RegexPackIR pack => new RegexPackIR(pack.Condition, pack.GroupName),
RegexCountComplexIR count => new RegexCountComplexIR(count.Min, count.Max,
count.IsGreedy,
count.CommandCount)
{
IsFullyOptimized = count.IsFullyOptimized
},
RegexCountSeqIR count => new RegexCountSeqIR(count.Min, count.Max,
count.IsGreedy, count.Pattern),
RegexCountSetIR count => new RegexCountSetIR(count.Min, count.Max,
count.IsGreedy, count.CharSet),
RegexTryIR tryIR => new RegexTryIR(tryIR.CommandCount)
{
IsFullyOptimized = tryIR.IsFullyOptimized
},
RegexPopIR pop => new RegexPopIR(pop.IsSuccess),
_ => throw new NotSupportedException($"Unsupported IR type: {ir.GetType().Name}")
};
clone.Prefix = new RegexIRPrefix(
ir.Prefix.WorkingRegion == 0 ? currentRegion : ir.Prefix.WorkingRegion,
ir.Prefix.Mask,
ir.Prefix.ObserveSource);
return clone;
}
}
}
}
src/Nitload.RegularAnalysis/.discard/RegexIR.Abstraction.cs.old
+71
View File
@@ -0,0 +1,71 @@
// -----------------------------------------------------------------------------
// file="RegexIR.Abstraction"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
internal interface IRegexComplex
{
bool IsFullyOptimized { get; set; }
int CommandCount { get; set; }
}
internal interface IRegexSequence
{
string Pattern { get; }
}
internal interface IRegexCount
{
bool IsGreedy { get; }
bool IsStar { get; }
bool IsPlus { get; }
int Min { get; }
int Max { get; }
}
internal abstract class RegexIR
{
protected RegexIR(RegexIRCommand command)
{
CommandMask = command;
}
public bool SingleLineResultRequired { get; set; }
public int SingleLineResultIndex { get; set; }
public RegexIRPrefix Prefix { get; set; }
internal RegexIRCommand CommandMask { get; init; }
internal string AsCommand(int lineNum)
{
return $"[{lineNum:D3}] {Prefix}, {AsCommand()}";
}
protected abstract string AsCommand();
}
}
src/Nitload.RegularAnalysis/.discard/RegexIR.Prefix.cs.old
+71
View File
@@ -0,0 +1,71 @@
// -----------------------------------------------------------------------------
// file="RegexIR.Prefix"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
internal enum RegexIRPrefixMask
{
Prev_Success = 1 << 0,
Prev_Fail = 1 << 1,
Always = 1 << 2,
ExeCondition = Prev_Fail | Prev_Success | Always,
Pop_Fail = 1 << 3,
Pop_Success = 1 << 4,
PopCondition = Pop_Success | Pop_Fail,
Observe_Line = 1 << 5,
Observe_Region = 1 << 6,
ObserveMethod = Observe_Line | Observe_Region
}
internal record struct RegexIRPrefix(int WorkingRegion, RegexIRPrefixMask Mask, int ObserveSource)
{
public override string ToString()
{
return $"{WorkingRegion:D4}, {DisplayMask(Mask , RegexIRPrefixMask.ExeCondition)}, {DisplayMask(Mask, RegexIRPrefixMask.PopCondition)}, {DisplayMask(Mask,RegexIRPrefixMask.ObserveMethod)}, {ObserveSource:D4}";
}
private static string DisplayMask(RegexIRPrefixMask mask, RegexIRPrefixMask range)
{
return range switch
{
RegexIRPrefixMask.ExeCondition => ((int)(mask & RegexIRPrefixMask.ExeCondition) >> 0) switch
{
0 => "DEF ",
(1 << 0) => "Sccs",
(1 << 1) => "Fail",
(1 << 2) => "Alws",
_ => "????"
},
RegexIRPrefixMask.PopCondition => ((int)(mask & RegexIRPrefixMask.PopCondition) >> 3) switch
{
0 => "DEF ",
(1 << 0) => "Fail",
(1 << 1) => "Sccs",
_ => "????"
},
RegexIRPrefixMask.ObserveMethod => ((int)(mask & RegexIRPrefixMask.ObserveMethod) >> 5) switch
{
0 => "NONE",
(1 << 0) => "LINE",
(1 << 1) => "REGI",
_ => "????"
},
_ => throw new ArgumentOutOfRangeException(nameof(range), range, null)
};
}
}
}
src/Nitload.RegularAnalysis/.discard/RegexIR.cs.old
+274
View File
@@ -0,0 +1,274 @@
// -----------------------------------------------------------------------------
// file="RegexIR"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Microsoft.CodeAnalysis.CSharp.Syntax;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
internal enum RegexIRCommand
{
Match_Seq = 1 << 0,
Match_Set = 1 << 1,
Condition = 1 << 2,
Try = 1 << 3,
Count_Complex = 1 << 4,
Count_Seq = 1 << 5,
Count_Set = 1 << 6,
Pop = 1 << 7,
Pack = 1 << 8,
Find_Seq = 1 << 9,
Find_Set = 1 << 10,
Find_Complex = 1 << 11,
Match = Match_Seq | Match_Set,
Count = Count_Seq | Count_Set | Count_Complex,
Find = Find_Seq | Find_Set | Find_Complex,
}
internal sealed class RegexFindComplexIR(
bool nearest,
int count
) : RegexIR(RegexIRCommand.Find_Complex), IRegexComplex
{
public bool IsFullyOptimized { get; set; }
public bool IsNearest { get; } = nearest;
public int CommandCount { get; set; } = count;
protected override string AsCommand()
{
return $"FIND_COMPLEX {(IsNearest ? "NEAREST" : "FURTHEST")} COUNT: {CommandCount}";
}
}
internal sealed class RegexFindSeqIR(
string pattern,
bool isFirst
) : RegexIR(RegexIRCommand.Find_Seq), IRegexSequence
{
public bool IsFirst { get; } = isFirst;
public string Pattern { get; } = pattern;
protected override string AsCommand()
{
return $"FIND_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "<EPSILON>" : Pattern)}\"";
}
}
internal interface IRegexMatch { }
/// <summary>
/// Searches for a certain string pattern or charset from current position.
/// </summary>
internal sealed class RegexMatchSeqIR(
string pattern
) : RegexIR(RegexIRCommand.Match_Seq), IRegexSequence, IRegexMatch
{
public string Pattern { get; } = pattern;
protected override string AsCommand()
{
return $"MATCH_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "<EPSILON>" : Pattern)}\"";
}
}
internal sealed class RegexMatchSetIR(
string charset
) : RegexIR(RegexIRCommand.Match_Set), IRegexMatch
{
public string Charset { get; } = charset;
protected override string AsCommand()
{
return $"MATCH_SET \"{Charset}\"";
}
}
internal sealed class RegexConditionIR(
int success,
int fail
) : RegexIR(RegexIRCommand.Condition)
{
public int SuccessCase { get; } = success;
public int FailCase { get; } = fail;
protected override string AsCommand()
{
return $"CONDITION SUCCESS: {SuccessCase} FAIL: {FailCase}";
}
}
internal sealed class RegexPackIR(
bool condition,
string name = ""
) : RegexIR(RegexIRCommand.Pack)
{
public bool Condition { get; } = condition;
public string GroupName { get; } = name;
protected override string AsCommand()
{
return $"PACK {(Condition ? "SUCCESS" : "FAIL")} GROUP: \"{GroupName}\"";
}
}
internal sealed class RegexCountComplexIR(
int min,
int max,
bool isGreedy,
int operation
) : RegexIR(RegexIRCommand.Count_Complex), IRegexCount, IRegexComplex
{
public bool IsGreedy { get; } = isGreedy;
public bool IsStar => Min == 0 && Max < 0;
public bool IsPlus => Min == 1 && Max < 0;
public bool IsFullyOptimized { get; set; }
public int Min { get; } = min;
public int Max { get; } = max;
public int CommandCount { get; set; } = operation;
protected override string AsCommand()
{
return $"COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {CommandCount}";
}
}
internal sealed class RegexCountSeqIR(
int min,
int max,
bool isGreedy,
string seq
) : RegexIR(RegexIRCommand.Count_Seq), IRegexCount, IRegexSequence
{
public bool IsGreedy { get; } = isGreedy;
public bool IsStar => Min == 0 && Max < 0;
public bool IsPlus => Min == 1 && Max < 0;
public int Min { get; } = min;
public int Max { get; } = max;
public string Pattern { get; } = seq.Length == 2 && seq[0] == '\\' ? seq[1..2] : seq;
protected override string AsCommand()
{
return $"COUNT_SEQ {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} SEQ: \"{Pattern}\"";
}
}
internal sealed class RegexCountSetIR(
int min,
int max,
bool isGreedy,
string set
) : RegexIR(RegexIRCommand.Count_Set), IRegexCount
{
public bool IsGreedy { get; } = isGreedy;
public bool IsDotStar => CharSet == "." && Min == 0 && Max < 0;
public bool IsStar => Min == 0 && Max < 0;
public bool IsPlus => Min == 1 && Max < 0;
public int Min { get; } = min;
public int Max { get; } = max;
public string CharSet { get; } = set;
protected override string AsCommand()
{
return $"COUNT_SET {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} SET: \"{CharSet}\"";
}
}
internal sealed class RegexTryIR(
int count
) : RegexIR(RegexIRCommand.Try), IRegexComplex, IRegexMatch
{
public bool IsFullyOptimized { get; set; }
public int CommandCount { get; set; } = count;
protected override string AsCommand()
{
return $"TRY {CommandCount}";
}
}
internal sealed class RegexFindSetIR(
string charset,
bool isFirst
) : RegexIR(RegexIRCommand.Find_Set)
{
public bool IsFirst { get; } = isFirst;
public string Charset { get; } = charset;
protected override string AsCommand()
{
return $"FIND_SET \"{Charset}\"";
}
}
internal sealed class RegexPopIR(
bool isSuccess
) : RegexIR(RegexIRCommand.Pop)
{
public bool IsSuccess { get; } = isSuccess;
protected override string AsCommand()
{
return $"POP {(IsSuccess ? "SUCCESS" : "FAIL")}";
}
}
}
src/Nitload.RegularAnalysis/.discard/RegexIRGenerator.cs.old
+128
View File
@@ -0,0 +1,128 @@
// -----------------------------------------------------------------------------
// file="RegexIRGenerator"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
internal sealed class RegexIRGenerator
{
private readonly SequenceEmitter<RegexIR> _stream = new();
public int Count => _stream.Count;
public RegexIRGenerator EmitCondition(
int success,
int fail
)
{
return Emit(new RegexConditionIR(success, fail));
}
public RegexIRGenerator EmitCountComplex(
int min,
int max,
bool isGreedy,
int operation
)
{
return Emit(new RegexCountComplexIR(min, max, isGreedy, operation));
}
public RegexIRGenerator EmitCountSeq(
int min,
int max,
bool isGreedy,
string seq
)
{
return Emit(new RegexCountSeqIR(min, max, isGreedy, seq));
}
public RegexIRGenerator EmitCountSet(
int min,
int max,
bool isGreedy,
string set
)
{
return Emit(new RegexCountSetIR(min, max, isGreedy, set));
}
public RegexIRGenerator EmitIRStream(
RegexIRGenerator generator
)
{
ArgumentNullException.ThrowIfNull(generator);
_ = _stream.Append(generator._stream);
return this;
}
public RegexIRGenerator EmitMatchSeq(
string unit
)
{
return Emit(new RegexMatchSeqIR(unit));
}
public RegexIRGenerator EmitMatchSet(
string unit
)
{
return Emit(new RegexMatchSetIR(unit));
}
public RegexIRGenerator EmitPack(
bool condition,
string name = ""
)
{
return Emit(new RegexPackIR(condition, name));
}
public RegexIRGenerator EmitPop(
bool isSuccess
)
{
return Emit(new RegexPopIR(isSuccess));
}
public RegexIRGenerator EmitTry(
int count
)
{
return Emit(new RegexTryIR(count));
}
public SequenceEmitter<RegexIR>.Enumerator GetEnumerator()
{
return _stream.GetEnumerator();
}
public List<RegexIR> ToList()
{
return _stream.ToList();
}
private RegexIRGenerator Emit(
RegexIR ir
)
{
_ = _stream.Append(ir);
return this;
}
}
}
src/Nitload.RegularAnalysis/.discard/TextRegexSourceGen.GenClass.MultiIR.cs.old
+298
View File
@@ -0,0 +1,298 @@
// -----------------------------------------------------------------------------
// file="TextRegexSourceGen.GenClass.MultiIR"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Runtime.Serialization.Formatters;
using System.Threading;
namespace Nitload.RegularAnalysis.Text
{
internal partial class TextRegexSourceGen
{
/*
* =======================================================================================
* L1, several regions plus a tail
* =======================================================================================
*
* Highest consumption in the dispatch chain, so these are offered the window first. Each
* shape here was already recognised once during prefix emission, so a generator should
* mirror the matching RegexAnalyzer.PrefixEmit recogniser rather than re-derive the shape.
*
* L1.2 ALT chain with a shared suffix
* window L1.1 followed by linear instructions in the parent region
* sample (ab|[cd]|.)x
* nature flattening, may return -1
*
* L1.3 group tail
* window region, POP FAIL with Prev_Fail, PACK SUCCESS with Prev_Success
* mirrors TryEmitTryGroupTail
* shape region block, capture written on the success branch
* sample (?<name>abc)
* nature flattening, may return -1
*
* L1.4 group tail wrapping an ALT chain
* sample (?<a>abcd|a(bc)c)
* nature flattening, may return -1
*
* L1.5 COUNT_COMPLEX plus body
* window RegexCountComplexIR, then CommandCount entries
* mirrors TryEmitComplexIR
* shape loop wrapping the generated body
* sample (ab){2,3}
* nature REQUIRED, no fallback exists
*
* L1.6 COUNT_COMPLEX whose body contains an ALT chain, plus a linear suffix
* sample (123(4|5)67){3,4}114514
* nature REQUIRED, no fallback exists
*
* L1.7 FIND_COMPLEX with CommandCount >= 0
* window RegexFindComplexIR, then CommandCount entries
* shape candidate retry loop wrapping the body
* sample .*(ab|cd)
* nature REQUIRED, no fallback exists
*
* L1.8 FIND_COMPLEX with CommandCount == -1
* window RegexFindComplexIR, then the rest of the region
* shape candidate retry loop wrapping the rest of the region
* sample (?<name>.*)234
* nature REQUIRED, no fallback exists
*
* L1.9 FIND_SEQ or FIND_SET
* window the instruction, then the rest of the region
* shape candidate retry loop wrapping the rest of the region
* sample .*abc
* nature REQUIRED, no fallback exists
*
* L1.5 to L1.9 are required because none of their bodies is a region of its own. L2
* cannot pick them up, so there is no degradation path if these are missing.
*
* CommandCount must be read from the prefixed IR. Prefix emission rebuilds these counts,
* so the value carried by the pre-prefix IR no longer applies.
*
* ---------------------------------------------------------------------------------------
* Backtracking semantics of the FIND family
* ---------------------------------------------------------------------------------------
*
* A FIND does not yield one position, it yields a priority ordered sequence of candidates.
* FURTHEST starts at the last one and steps backwards on failure, NEAREST starts at the
* first and steps forwards. The body is the verifier for a candidate: when it fails, the
* next candidate has to be tried, and only an exhausted candidate list is a real failure.
* A single LastIndexOf is wrong, because .*abcX has to fall back to the second to last abc
* when the last one is not followed by an X.
*
* CommandCount == -1 means the body is the rest of the region. TryReplaceDotStar emits
* FindComplex(-1) for a bare .*, then TyrMergeFindRestTwice and TyrMergeFindRestTry try to
* collapse it into a definite length. Whatever survives stays at -1. FIND_SEQ and FIND_SET
* carry no CommandCount at all and behave like -1. Only a FIND that happens to be the last
* instruction of its region degrades into a single search, consuming one entry.
*
* Target shape:
*
* int __bound = text.Length;
* __ok = false;
* while (true)
* {
* int __at = text[__cur..__bound].LastIndexOf("abc");
* if (__at < 0) { __ok = false; break; } // candidates exhausted, only failure
* int __save = __cur;
* __cur = __cur + __at + 3;
*
* // body, or the rest of the region, expands here
*
* if (__ok) { break; } // verified
* __cur = __save;
* __bound = __save + __at; // step to the previous candidate
* }
*
* POP conflicts with the retry loop. A POP lowers to break, and the retry while is nearer
* than the region do block, so the break would leave the wrong scope. The fix without goto
* is a region completion flag:
*
* do // region N
* {
* bool __rN_done = false;
* while (!__rN_done) // find retry
* {
* // candidate search
* do // isolation layer, POP breaks out here
* {
* // body, POP sets __rN_ok, sets __rN_done, breaks
* } while (false);
* if (__rN_done || __ok) { break; }
* // step to the next candidate
* }
* } while (false);
*
* Two further hazards:
* - nested FIND produces nested retry loops and the cost multiplies, which is the
* catastrophic backtracking shape, worth a guard or at least a marker
* - __ok is overwritten across retry iterations, so an Observe_Line prefix pointing into
* a retry body reads the value of the last iteration, a case the current prefix model
* does not cover
*
* Open questions to settle while implementing:
* - how the cursor is restored between ALT cases
* - which successful outcome a PACK uses as its capture span
* - whether a region body ever deserves its own generated method instead of inlining
*
* =======================================================================================
* L2, one region
* =======================================================================================
*
* Consumes CommandCount + 1, the whole block a TRY lowers to. Together with L4 and L5 this
* is the correctness core: once these three exist, every expression can be generated, just
* with deeper nesting than L1 and L3 would produce.
*
* L2.1 RegexTryIR plus its body sample (abc)
* L2.2 the same, body containing a nested region sample (a(bc))
* L2.3 a single instruction region folded by prefix emission sample (a)
*
* Target shape, names owned by SourceGenContext:
*
* __rN_begin = __cur;
* __rN_ok = false;
* do
* {
* // body, a POP breaks out here
* __rN_ok = __ok;
* } while (false);
* if (!__rN_ok) { __cur = __rN_begin; }
* __ok = __rN_ok;
*
* Points the implementation has to settle:
* - the failure path restores the cursor to the region entry
* - break only leaves the nearest enclosing loop, and a FIND retry while counts as one,
* so a region containing a FIND needs the __rN_done flag described above instead of a
* plain break
* - a nested region must not rely on break to exit an outer one
* - the whole match body is itself region zero, which is what makes a top level POP
* legal
*
* =======================================================================================
* L3, fixed multi instruction windows
* =======================================================================================
*
* Consumes two to eight entries. Pure code quality work, never required for correctness.
* Every one of these must return -1 cleanly when the window does not match, so the entry
* falls through to L4.
*
* L3.1 FIND_COMPLEX FURTHEST COUNT:-1, PACK, MATCH_SEQ 3 (?<name>.*)234
* L3.2 the same with a single element anchor 3 (?<name>.*)a
* L3.3 FIND_SEQ, MATCH_SEQ, merged into one search 2 .*abc
* L3.4 MATCH_SEQ, COUNT_SEQ, the expanded a+ shape 2 a{3,5}, a+
* L3.5 a run of MATCH_SEQ and MATCH_SET, merged into one condition 2 to 8 a[abc]d
* L3.6 trailing COUNT_SET over a dot, fast success 1 to 2 abc.*
*
* Two hazards to check for each of these:
* - merging must not break the line indices an Observe_Line prefix refers to
* - a merged window still has to leave __ok describing the whole window
*/
#region code
private static int TryGenerateTryBlock(
int outer_level,
ReadOnlySpan<RegexIR> ir_span,
SourceGenContext sc,
in SequenceEmitter<string> code,
in List<SequenceEmitter<string>> local_func
)
{
if (ir_span[0]is not RegexTryIR _try) {
return -1;
}
ReadOnlySpan<RegexIR> block = ir_span[1..(1 + _try.CommandCount)];
// TODO generate here
return _try.CommandCount;
}
private static int TryGenerateNamedCountAny(
int outer_level,
ReadOnlySpan<RegexIR> ir_span,
SourceGenContext sc,
in SequenceEmitter<string> code,
in List<SequenceEmitter<string>> local_func
)
{
/*
* (?<name>.*)234
* [010] 0003, DEF , DEF , NONE, 0000, FIND_COMPLEX FURTHEST COUNT: -1
* [011] 0003, Sccs, DEF , LINE, 0010, PACK SUCCESS GROUP: "name"
* [012] 0000, DEF , DEF , NONE, 0000, MATCH_SEQ "234"
*/
if (ir_span[0] is not RegexFindComplexIR { CommandCount: -1 } ||
ir_span[1] is not RegexPackIR pack)
{
return -1;
}
string group_name = pack.GroupName;
int i = 0;
for (; i < ir_span.Length; i++) { }
return -1;
}
private static int TryGenerateAltChain(
int outer_level,
ReadOnlySpan<RegexIR> ir_span,
SourceGenContext sc,
in SequenceEmitter<string> code,
in List<SequenceEmitter<string>> local_func
)
{
/*
* L1.1 ALT chain
* window region, then further regions carrying Prev_Fail
* mirrors TryEmitAltChain
* shape if / else if / else
* sample a|b, ab|cd|ef, a|bc|def
* nature flattening, may return -1
*/
/*
* [004] 0001, DEF , DEF , NONE, 0000, MATCH_SEQ "abc"
* [005] 0002, DEF , Sccs, NONE, 0000, MATCH_SEQ "d" <- an alt chain begins here
* [006] 0002, Fail, Sccs, LINE, 0005, MATCH_SEQ "c" <- an alt chain ends here
* [007] 0001, Fail, DEF , LINE, 0006, POP FAIL
*/
const RegexIRPrefixMask simple_alt = RegexIRPrefixMask.Prev_Fail | RegexIRPrefixMask.Pop_Success;
int cur_level = ir_span[0].Prefix.WorkingRegion;
if (cur_level == outer_level) {
return -1;
}
int i = 1;
RegexIR ir;
for (; i < ir_span.Length; i++)
{
ir = ir_span[i];
if ((ir.Prefix.Mask & simple_alt) == simple_alt)
{
if (TryGenerateSingleIR(outer_level, [ir], sc, code, local_func) > 0)
{
// TODO add additional code for the alt chain
}
// simple match
} else if (ir is RegexTryIR _try && (ir.Prefix.Mask & simple_alt) == simple_alt)
{
// match a try region
ReadOnlySpan<RegexIR> body_span = ir_span.Slice(i + 1, i + _try.CommandCount);
i += _try.CommandCount;
}
}
return i;
}
#endregion
}
}
src/Nitload.RegularAnalysis/.discard/TextRegexSourceGen.GenClass.Prefix.cs.old
+66
View File
@@ -0,0 +1,66 @@
// -----------------------------------------------------------------------------
// file="TextRegexSourceGen.GenClass.Prefix"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
namespace Nitload.RegularAnalysis.Text
{
internal partial class TextRegexSourceGen
{
/*
* =======================================================================================
* L5, prefix decoration
* =======================================================================================
*
* A prefix consumes no IR of its own. It wraps whatever an L1 to L4 body generator
* produced:
*
* [prefix open ] if (<observed result>) {
* [ body ] ... sets __ok, advances __cur
* [ line slot ] __lK_ok = __ok;
* [ pop ] if (__ok) { __rN_ok = true; break; }
* [prefix close] }
*
* Keeping this outside the body generators is what lets L2 and L4 stay small. They only
* describe what an instruction does, never when it runs or who reads the outcome. Without
* this layer the other layers cannot be assembled into correct control flow at all, so L5
* is part of the correctness core rather than an optimisation.
*
* Checklist, RegexIRPrefixMask on the left:
*
* L5.1 ExeCondition none run unconditionally
* L5.2 ExeCondition Prev_Success wrap in if (observed)
* L5.3 ExeCondition Prev_Fail wrap in if (!observed)
* L5.4 ExeCondition Always run unconditionally, deliberately detached from
* the surrounding condition chain
* L5.5 ObserveMethod none read the result of the preceding instruction
* L5.6 ObserveMethod Observe_Line read the slot named by ObserveSource
* L5.7 ObserveMethod Observe_Region read the slot named by ObserveSource
* L5.8 PopCondition Pop_Success write the region result true, then leave it
* L5.9 PopCondition Pop_Fail write the region result false, then leave it
* L5.10 WorkingRegion selects which region scope the code lands in
*
* Rules the implementation has to hold:
*
* - Observe_Line and Observe_Region must read the slot ObserveSource names. Neither may
* degrade into reading the previous instruction, which is what an absent observe
* method means and nothing else.
* - Line slots are allocated only for lines some prefix actually observes. That demand
* has to be collected in a pass before body generation, because an observer can be
* emitted before the line it reads, for instance when a nested region observes a line
* of its parent.
* - A pop lowers to break, which leaves the nearest enclosing do while(false) block.
* That block is the region the instruction works in, because every region is emitted
* as one and the whole match body is wrapped as region zero.
* - Open and close must be paired. A body generator that declines the window must be
* detected before the opening brace is emitted, so bodies are generated into a
* detached emitter first.
*
* The open call should report how many braces it opened, so the close call knows how many
* to emit rather than recomputing the condition.
*/
}
}
src/Nitload.RegularAnalysis/.discard/TextRegexSourceGen.GenClass.SingleIR.cs.old
+338
View File
@@ -0,0 +1,338 @@
// -----------------------------------------------------------------------------
// file="TextRegexSourceGen.GenClass.SingleIR"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Nitload.RegularAnalysis.Abstraction;
using System.Diagnostics.Metrics;
using System.Globalization;
using System.Text.RegularExpressions;
namespace Nitload.RegularAnalysis.Text
{
internal partial class TextRegexSourceGen
{
/*
* =======================================================================================
* L4, one instruction
* =======================================================================================
*
* The fallback of the dispatch chain. Every RegexIRCommand needs a generator here, because
* this is the only layer guaranteed to be offered every window. L1 and L3 may decline, L4
* may not.
*
* Each generator consumes exactly one entry and describes only what the instruction does.
* When it runs, which result it reads and whether it leaves the region early are all added
* by L5, so nothing here should inspect RegexIR.Prefix beyond WorkingRegion.
*
* Contract for a generated body, names owned by SourceGenContext:
*
* - assign the outcome to __ok
* - advance __cur only on success
* - never index __cur past the end of the input
* - declare scratch variables inside a block of their own, so a repeated shape does not
* collide with itself
*
* A body must be generated into a detached emitter and only spliced in once it is known to
* be claimed. Emitting a prefix for an instruction that turns out to have no generator
* would open a brace nobody closes.
*
* Checklist:
*
* L4.1 MATCH_SEQ abc, \., a\*b
* L4.2 MATCH_SET, discrete [abc], [123abc], [.]
* L4.3 MATCH_SET, range [1-9]
* L4.4 MATCH_SET, escape class \w
* L4.5 MATCH_SET, any element .
* L4.6 COUNT_SEQ a*, a?
* L4.7 COUNT_SET [abc]*, \w+, .{11,13}, [.]*
* L4.10 POP ALT tail
* L4.11 PACK (abc), ordinal group
*
* L4.8 and L4.9 were removed. FIND_SEQ and FIND_SET have no single instruction form,
* because they have to drive a candidate retry loop over the rest of the region. See L1.9
* in TextRegexSourceGen.GenClass.MultiIR.cs. Only a FIND that happens to be the last
* instruction of its region degrades into a single search, and that is a branch of L1.9
* rather than a generator of its own. The numbers are left vacant so existing references
* do not shift.
*
* Semantics worth pinning down before writing these:
*
* - [.] and . are different sets. The former is a literal dot, the latter is any
* element, and [.]* must not be folded into the dot star optimisation.
* - \w has to keep full .NET semantics. TextRegex.IsCharWord is inherited by the
* generated class and already carries them, so the generated code can call it
* directly instead of approximating with an ASCII test.
* - Repetition in this layer does not backtrack. A lazy repetition therefore consumes
* exactly its lower bound and leaves the rest to the following anchor. Backtracking
* does exist in this backend, but only inside the FIND retry loop at L1.7 to L1.9,
* never here.
* - An epsilon sequence and an empty repetition unit both reach these generators.
* Decide whether they are vacuous successes or an optimizer bug worth reporting.
* - PACK decides nothing, so it should leave __ok untouched. Whether it runs at all is
* already expressed by its execution condition.
*/
#region code
private static int TryGenerateSingleIR(
int outer_level,
ReadOnlySpan<RegexIR> ir_span,
SourceGenContext sc,
in SequenceEmitter<string> code,
in List<SequenceEmitter<string>> local_func
)
{
return TryGenerateCountSeq(outer_level, ir_span, sc, code,
local_func) > 0 ||
TryGenerateCountSet(outer_level, ir_span, sc, code,
local_func) > 0 ||
TryGenerateMatchSet(outer_level, ir_span, sc, code,
local_func) > 0 ||
TryGenerateMatchSeq(outer_level, ir_span, sc, code, local_func) > 0 ?
1 :
-1;
}
private static int TryGeneratePack(
int outer_level,
ReadOnlySpan<RegexIR> ir_span,
SourceGenContext sc,
in SequenceEmitter<string> code,
in List<SequenceEmitter<string>> local_func
)
{
if (ir_span[0] is not RegexPackIR pack) {
return -1;
}
string group_name = pack.GroupName;
string begin = sc.GetVarGroupBegin(group_name);
string end = sc.GetVarGroupEnd(group_name);
_ = code.AppendLine(begin).AppendLine(end);
return 1;
}
private static int TryGenerateCountSet(
int outer_level,
ReadOnlySpan<RegexIR> ir_span,
SourceGenContext sc,
in SequenceEmitter<string> code,
in List<SequenceEmitter<string>> local_func
)
{
if (ir_span[0] is not RegexCountSetIR count_set) {
return -1;
}
bool is_greedy = count_set.IsGreedy;
int min = count_set.Min;
int max = count_set.Max;
int leave = is_greedy ? max : min;
string counter = sc.RequireNewCounter();
string set_source = count_set.CharSet;
RegexCharSet[] set_arr = FigureCharSetType(set_source, out bool accept);
_ = code.AppendLine($@"for(; {counter}<{leave}; {counter}++)")
.AppendLine("{")
.AppendLine(accept ? @$"if(" : @$"if(!(");
string _case_code;
RegexCharSet set;
for (int i = 1; i < set_arr.Length - 1; i++)
{
set = set_arr[i];
_case_code = GenLine(set, sc, "||");
_ = code.AppendLine(_case_code);
}
set = set_arr[^1];
_case_code = GenLine(set, sc, ")");
_ = code.AppendLine(_case_code, //
"{", // in case match
"}", //
"else", //
"{", //
"break;", // in case miss match
"}", //
"}", // end of for
@$"if({counter}>{min})", // match here
"{", // in case count reaches min count request
"}", //
"else", //
"{", // in case count does not reach min count request
"}", //
string.Empty); //endof count
return 1;
}
private static int TryGenerateCountSeq(
int outer_level,
ReadOnlySpan<RegexIR> ir_span,
SourceGenContext sc,
in SequenceEmitter<string> code,
in List<SequenceEmitter<string>> local_func
)
{
if (ir_span[0] is not RegexCountSeqIR count_seq) {
return -1;
}
string seq = count_seq.Pattern;
string literal = FormatStringLiteral(seq);
bool is_greedy = count_seq.IsGreedy;
int min = count_seq.Min;
int max = count_seq.Max;
int leave = is_greedy ? max : min;
string counter = sc.RequireNewCounter();
// TODO cannot check real count outside the cycle
_ = code.AppendLine($@"int {counter} = 0;", //
@$"for(; {counter}<{leave}; {counter}++)", //
"{", //
@$"if(!{sc.RemainingSpan}.StartWith({literal}))", //
"{", //
$@"break;", //
"}", //
"}", //
@$"if({counter}>{min})", //
"{", // in case match
sc.MakeRemainOffset(counter), //
"}", //
$@"else", //
"{", // TODO fail here
"}"); //
return 1;
}
private static int TryGenerateMatchSeq(
int outer_level,
ReadOnlySpan<RegexIR> ir_span,
SourceGenContext sc,
in SequenceEmitter<string> code,
in List<SequenceEmitter<string>> local_func
)
{
if (ir_span[0] is not RegexMatchSeqIR match_seq) {
return -1;
}
string seq = match_seq.Pattern;
string literal = FormatStringLiteral(seq);
_ = code.AppendLine(@$"if({sc.RemainingSpan}.StartWith({literal}))")
.AppendLine("{")
.AppendLine($"{sc.StepResult} = true;")
.AppendLine(sc.MakeRemainOffset(seq.Length))
.AppendLine("}")
.AppendLine("else")
.AppendLine("{")//TODO fail here
.AppendLine("break;")
.AppendLine("}");
return 1;
}
private static int TryGenerateMatchSet(
int outer_level,
ReadOnlySpan<RegexIR> ir_span,
SourceGenContext sc,
in SequenceEmitter<string> code,
in List<SequenceEmitter<string>> local_func
)
{
if (ir_span[0] is not RegexMatchSetIR match_set) {
return -1;
}
string set_source = match_set.Charset;
RegexCharSet[] set_arr = FigureCharSetType(set_source, out bool accept);
_ = code.AppendLine(accept ? @$"if(" : @$"if(!(");
string _case_code;
RegexCharSet set;
/*
* in loop:
* for( is cas1 1 ||
* is case 2 ||
* .....
* is case last)
* {
* success here
* }
* else
* {
* fail case
* }
*/
for (int i = 1; i < set_arr.Length - 1; i++)
{
set = set_arr[i];
_case_code = GenLine(set, sc, "||");
_ = code.AppendLine(_case_code);
}
set = set_arr[^1];
_case_code = GenLine(set, sc, ")");
_ = code.AppendLine(_case_code)
.AppendLine("{")
.AppendLine(sc.MakeRemainOffset(1)) // TODO case success
.AppendLine("}")
.AppendLine("else")
.AppendLine("{")
.AppendLine("break;") // TODO case fail
.AppendLine("}");
return 1;
}
private static string GenLine(
RegexCharSet set,
SourceGenContext sc,
string end_of_line,
string symbol_of_offset = ""
)
{
string offset = string.IsNullOrEmpty(symbol_of_offset) ? "0" : $"{symbol_of_offset}";
switch (set.Type)
{
case CharSetType.Single:
string cases = set.Value;
if (cases.Length > 8)
{
// more than 8 different chars, using string match
string literal = FormatStringLiteral(cases);
return $@"{literal}.Contains({sc.RemainingSpan}[{offset}]) {end_of_line}";
} else
{
StringBuilder sb = new();
string literal;
_ = sb.Append('(');
for (int j = 0; j < cases.Length - 1; j++)
{
literal = FormatCharLiteral(cases[j]);
_ = sb.Append(CultureInfo.InvariantCulture, $@"{sc.RemainingSpan}[{offset}] == {literal} ||");
}
literal = FormatCharLiteral(cases[^1]);
_ = sb.Append(CultureInfo.InvariantCulture, $@"{sc.RemainingSpan}[{offset}] == {literal}")
.Append(CultureInfo.InvariantCulture, @$") {end_of_line}");
return sb.ToString();
}
case CharSetType.Range:
string rule = set.Value;
char lo = rule[0];
char hi = rule[1];
return $@"({sc.RemainingSpan}[{offset}] >= '{lo}' && {sc.RemainingSpan}[{offset}] <= '{hi}') {end_of_line}";
case CharSetType.Set:
string accept_mark = set.Accept ? string.Empty : "!";
return set.Value switch
{
"s" => $@"{accept_mark}char.IsWhiteSpace({sc.RemainingSpan}[{offset})] {end_of_line}",
"d" => $@"{accept_mark}char.IsDigit({sc.RemainingSpan}[{offset})] {end_of_line}",
"w" => $@"{accept_mark}IsCharWord({sc.RemainingSpan}[{offset})] {end_of_line}",
_ => string.Empty
};
case CharSetType.None:
return string.Empty;
default:
return string.Empty;
}
}
#endregion
}
}
src/Nitload.RegularAnalysis/Abstraction/AbstractSyntaxTree.cs
+238
View File
@@ -0,0 +1,238 @@
// -----------------------------------------------------------------------------
// file="AbstractSyntaxTree"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Xml.Linq;
using System.Xml.Schema;
namespace Nitload.RegularAnalysis.Abstraction
{
public sealed class AbstractSyntaxTree<TNode, TType> where TNode : ISyntaxNode<TNode, TType>
where TType : unmanaged
{
public HashSet<TNode> AllNodes => AllToken;
public int Count => AllToken.Count;
public TNode Root
{
get;
internal set
{
if (AllToken.Contains(value)) {
field = value;
}
}
}
private HashSet<TNode> AllToken { get; } = new();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool AddNode(
TNode node
)
{
return AllToken.Add(node);
}
public bool Contains(
TNode node
)
{
return AllToken.Contains(node);
}
public IEnumerator<TNode> GetEnumerator()
{
return new TreeEnumerator<TNode, TType>(this);
}
}
internal class TreeEnumerator<TNode, TType>(
TNode root
) : IEnumerator<TNode> where TNode : ISyntaxNode<TNode, TType> where TType : unmanaged
{
private bool _completed;
private int _state, _target; // 0: not started, 1: on right, 2: ended
private readonly Stack<Record> _recycle = [];
private readonly Stack<Record> _stack = [];
// private TNode _lRoot;
private TNode _root = root;
internal TreeEnumerator(
AbstractSyntaxTree<TNode, TType> tree
)
: this(tree.Root) { }
public int Depth => _stack.Count;
public TNode Current
{
get
{
if (_stack.TryPeek(out Record? r)) {
return r.Node;
}
return TNode.Empty;
}
}
public void Dispose()
{
return;
}
public bool MoveNext()
{
if (_completed) {
return false;
}
while (_stack.TryPeek(out Record? cur))
{
switch (cur.State)
{
case 0:
cur.State++;
if (TNode.IsNullOrEmpty(cur.Node.Left))
{
goto case 1;
} else
{
_stack.Push(GetNewRecord(cur.Node.Left));
return true;
}
case 1:
cur.State++;
if (TNode.IsNullOrEmpty(cur.Node.Right))
{
goto case 2;
} else
{
_stack.Push(GetNewRecord(cur.Node.Right));
return true;
}
case 2:
_recycle.Push(_stack.Pop());
if (TNode.IsNullOrEmpty(cur.Node.Next))
{
if (_stack.Count == 0)
{
_completed = true;
return false;
}
} else
{
_stack.Push(GetNewRecord(cur.Node.Next));
return true;
}
break;
default:
break;
}
}
_stack.Push(GetNewRecord(_root));
return true;
}
public void Reset()
{
_completed = false;
_stack.Clear();
_state = 0;
}
/// <summary>
/// Skips the node and its subtree if the current node is the given node. Otherwise, do
/// nothing.
/// </summary>
/// <param name="node">
/// The node to skip.
/// </param>
public void SkipNode(
TNode node
)
{
if (_stack.TryPeek(out Record? r) && ReferenceEquals(r.Node, node))
{
_ = _stack.Pop();
_recycle.Push(r);
}
}
public void SkipOnce()
{
if (_stack.TryPeek(out Record? cur))
{
switch (cur.State)
{
case 0:
cur.State++;
break;
case 1:
cur.State++;
break;
case 2:
if (TNode.IsNullOrEmpty(cur.Node.Next))
{
_recycle.Push(_stack.Pop());
if (_stack.Count == 0)
{
_completed = true;
return;
}
} else
{
_stack.Push(GetNewRecord(cur.Node.Next));
}
break;
default:
break;
}
}
}
private Record GetNewRecord(
TNode node
)
{
if (_recycle.TryPop(out Record? r))
{
r.State = 0;
r.Node = node;
return r;
}
return new Record(node, 0);
}
object IEnumerator.Current => Current;
private class Record(
TNode node,
int state
)
{
public int State { get; set; } = state;
public TNode Node { get; set; } = node;
}
}
}
src/Nitload.RegularAnalysis/Abstraction/CompileDepthState.cs
+62
View File
@@ -0,0 +1,62 @@
// -----------------------------------------------------------------------------
// file="CompileDepthState"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Abstraction
{
public interface ICompileDepthState
{
string CurrentState { get; set; }
}
public class CompileDepthMark
{
internal CompileDepthMark(
int operatorBegin,
int operandBegin,
int usage
)
{
OperatorBegin = operatorBegin;
OperandBegin = operandBegin;
Usage = usage;
}
internal CompileDepthMark(
int operatorBegin,
int operandBegin,
int usage,
ICompileDepthState stateCache
)
{
OperatorBegin = operatorBegin;
OperandBegin = operandBegin;
Usage = usage;
StateCache = stateCache;
}
public bool IsBeginOfBlock { get; internal set; }
public ICompileDepthState StateCache { get; internal set; }
public int OperandBegin { get; internal set; }
public int OperatorBegin { get; internal set; }
public int Usage { get; internal set; }
}
}
src/Nitload.RegularAnalysis/Abstraction/CompileStack.cs
+167
View File
@@ -0,0 +1,167 @@
// -----------------------------------------------------------------------------
// file="CompileStack"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Abstraction
{
public partial class CompileStack<TNode, TLabel> where TNode : ISyntaxNode<TNode, TLabel>
where TLabel : unmanaged
{
private readonly RandomAccessibleStack<DepthMark> _depthMarks = [];
private readonly RandomAccessibleStack<TNode> _operands = [];
private readonly RandomAccessibleStack<TNode> _operators = [];
public CompileStack()
{
_depthMarks.Push(new DepthMark(0, 0, 0));
}
public int OperandInDepthCount => _operands.Count - _depthMarks.Peek().OperandPos;
public int OperatorInDepthCount => _operators.Count - _depthMarks.Peek().OperatorPos;
public int Depth => _depthMarks.Count;
private record struct DepthMark(
int OperandPos,
int OperatorPos,
int Mark
);
#region operator
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public TNode PopOperator(
)
{
if (OperatorInDepthCount > 0) {
return _operators.Pop();
}
return TNode.Empty;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void PushOperator(
TNode operatorNode
)
{
_operators.Push(operatorNode);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPopOperator(
out TNode op
)
{
if (OperatorInDepthCount > 0)
{
op = _operators.Pop();
return true;
}
op = TNode.Empty;
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPeekOperator(
out TNode op
)
{
if (OperatorInDepthCount > 0)
{
op = _operators.Peek();
return true;
}
op = TNode.Empty;
return false;
}
#endregion
#region operand
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public TNode PopOperand(
)
{
if (OperandInDepthCount > 0) {
return _operands.Pop();
}
return TNode.Empty;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void PushOperand(
TNode operatorNode
)
{
_operands.Push(operatorNode);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPopOperand(
out TNode op
)
{
if (OperandInDepthCount > 0)
{
op = _operands.Pop();
return true;
}
op = TNode.Empty;
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPeekOperand(
out TNode op
)
{
if (OperandInDepthCount > 0)
{
op = _operands.Peek();
return true;
}
op = TNode.Empty;
return false;
}
#endregion
#region depth
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void DecreaseDepth(
)
{
if (Depth > 1)
{
DepthMark mark = _depthMarks.Pop();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void IncreaseDepth(
int mark
)
{
_depthMarks.Push(new DepthMark(_operands.Count, _operators.Count, mark));
}
#endregion
}
}
src/Nitload.RegularAnalysis/Abstraction/ISyntaxNode.cs
+69
View File
@@ -0,0 +1,69 @@
// -----------------------------------------------------------------------------
// file="ISyntaxNode"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Text;
namespace Nitload.RegularAnalysis.Abstraction
{
public interface ITrinitySyntaxNode<TNode, TLabel> : ISyntaxNode<TNode, TLabel>
where TNode : ISyntaxNode<TNode, TLabel>
where TLabel : unmanaged
{
TNode Third { get; set; }
}
public interface ISyntaxNode<TNode, TLabel> where TNode : ISyntaxNode<TNode, TLabel>
where TLabel : unmanaged
{
ISyntaxPayload<TLabel> Source { get; }
long NodeFlag { get; }
TLabel EqualityTypeConvert { get; }
TNode Previous { get; set; }
TNode Next { get; set; }
TNode Left { get; set; }
TNode Right { get; set; }
TNode Parent { get; set; }
static abstract TNode Empty { get; }
protected internal int EnumState { get; set; }
void AppendNode(
TNode nextToken
);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static virtual bool IsNullOrEmpty(
TNode node
)
{
return node is null || node.Equals(TNode.Empty);
}
void PrependNode(
TNode previousNode
);
}
}
src/Nitload.RegularAnalysis/Abstraction/ISyntaxPayload.cs
+28
View File
@@ -0,0 +1,28 @@
// -----------------------------------------------------------------------------
// file="ISyntaxPayload"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Abstraction
{
public interface ISyntaxPayload<TLabel> where TLabel : unmanaged
{
int Column { get; }
int Row { get; }
string Source { get; }
TLabel Flag { get; }
}
}
src/Nitload.RegularAnalysis/Abstraction/ITokenParser.cs
+31
View File
@@ -0,0 +1,31 @@
// -----------------------------------------------------------------------------
// file="ITokenParser"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Abstraction
{
public abstract class TokenParser<TIn, TOut> : ITokenReader<TOut> where TIn : unmanaged
where TOut : unmanaged
{
public abstract bool IsEmpty { get; }
protected internal ITokenReader<TIn> Source { get; internal set; }
public abstract Token<TOut> ReadToken();
public abstract bool TryReadToken(
out Token<TOut> t
);
}
}
src/Nitload.RegularAnalysis/Abstraction/ITokenReader.cs
+39
View File
@@ -0,0 +1,39 @@
// -----------------------------------------------------------------------------
// file="ITokenReader"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
namespace Nitload.RegularAnalysis.Abstraction
{
public static class TokenReader
{
public static TokenParser<TIn, TOut> Pipe<TIn, TOut>(
this ITokenReader<TIn> source,
Func<TokenParser<TIn, TOut>> builder
) where TIn : unmanaged where TOut : unmanaged
{
TokenParser<TIn, TOut> parser = builder();
parser.Source = source;
return parser;
}
}
public interface ITokenReader<TLabel> where TLabel : unmanaged
{
bool IsEmpty { get; }
Token<TLabel> ReadToken();
bool TryReadToken(
out Token<TLabel> t
);
}
}
src/Nitload.RegularAnalysis/Abstraction/SequenceEmitter.cs
+260
View File
@@ -0,0 +1,260 @@
// -----------------------------------------------------------------------------
// file="SequenceEmitter"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.RegularAnalysis.Text;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Abstraction
{
public static class SequenceEmitterExtensions
{
public static SequenceEmitter<string> AppendLine(
this SequenceEmitter<string> emitter
)
{
return emitter is null ? null! : emitter.Append(string.Empty);
}
public static SequenceEmitter<string> AppendLine(
this SequenceEmitter<string> emitter,
string line
)
{
return emitter is null ?
null! :
emitter.Append(string.IsNullOrEmpty(line) ? string.Empty : line);
}
public static SequenceEmitter<string> AppendLine(
this SequenceEmitter<string> emitter,
params string[] lines
)
{
if (lines is null || lines.Length == 0) {
return emitter;
}
foreach (string line in lines) {
emitter = emitter.AppendLine(line);
}
return emitter;
}
public static SequenceEmitter<string> AppendLine(
this SequenceEmitter<string> emitter,
string line,
bool condition
)
{
return emitter is null ? null! : condition ? emitter.Append(line) : emitter;
}
}
public sealed class SequenceEmitter<T>
{
private bool _isRoot;
private readonly Node _head;
private Node _tail;
private SequenceEmitter(
Node node
)
{
_isRoot = true;
_head = node;
_tail = _head;
}
public SequenceEmitter()
{
_isRoot = true;
_head = new();
_tail = _head;
}
public int Count { get; private set; }
public SequenceEmitter<T> Append(
T unit
)
{
if (!_isRoot) {
throw new InvalidOperationException("This emitter has been appended to another emitter. It is now readonly.");
}
_tail.Add(unit);
Count++;
return this;
}
public SequenceEmitter<T> Append(
SequenceEmitter<T> emitter
)
{
if (emitter == null || emitter.Count == 0) {
return this;
}
if (ReferenceEquals(this, emitter)) {
throw new InvalidOperationException("Cannot append emitter to itself.");
}
if (!emitter._isRoot) {
throw new InvalidOperationException("The emitter has been appended to another emitter.");
}
_tail.Next = emitter._head;
_tail = emitter._tail;
Count += emitter.Count;
emitter._isRoot = false;
return this;
}
public SequenceEmitter<T> Clone()
{
if (!_isRoot) {
throw new InvalidOperationException("This emitter has been appended to another emitter. It is now readonly.");
}
SequenceEmitter<T> clone = new();
Node current = _head;
return clone;
}
public Enumerator GetEnumerator()
{
return _isRoot ?
new Enumerator(this) :
throw new InvalidOperationException("This emitter has been appended to another emitter. You should not get its enumerator.");
}
public List<T> ToList()
{
List<T> result = new(Count);
Node current = _head;
while (current != null)
{
result.AddRange(current.IRList);
current = current.Next!;
}
return result;
}
#pragma warning disable CA1063
#pragma warning disable CA1816
public class Enumerator(
[NotNull] SequenceEmitter<T> emitter
) : IEnumerator<T>
{
private int _index = -1;
private Node _current = null!;
private readonly Node _root = emitter._head;
public T Current
{
get
{
if (_current is null || _index < 0 || _index >= _current.Count) {
throw new InvalidOperationException();
}
return _current[_index];
}
}
public void Dispose() { }
public bool MoveNext()
{
_current ??= _root;
int index = _index + 1;
if (index < _current.Count)
{
_index = index;
return true;
}
Node current = _current.Next!;
if (current is null || ReferenceEquals(current, _root))
{
_index = -1;
return false;
}
_current = current;
_index = -1;
return true;
}
public void Reset()
{
_current = _root;
_index = -1;
}
object IEnumerator.Current => Current!;
}
#pragma warning restore CA1063
#pragma warning restore CA1816
private sealed class Node
{
public Node()
{
IRList = [];
}
public Node(
int capacity
)
{
IRList = new(capacity);
}
public T this[
int index
]
{
get => IRList[index];
set => IRList[index] = value;
}
public Node? Next { get; set; }
public int Count => IRList.Count;
public List<T> IRList { get; }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Add(
T ir
)
{
IRList.Add(ir);
}
public void AddRange(
IEnumerable<T> irs
)
{
IRList.AddRange(irs);
}
}
}
}
src/Nitload.RegularAnalysis/Abstraction/Token.cs
+96
View File
@@ -0,0 +1,96 @@
// -----------------------------------------------------------------------------
// file="Token"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Abstraction
{
public readonly struct Token<TLabel> where TLabel : unmanaged
{
public readonly int Column;
public readonly int Line;
public readonly string Value;
public readonly TLabel Flag;
public Token()
{
Column = 0;
Line = 0;
Value = string.Empty;
Flag = default;
}
public Token(
string chars,
int lineNumber,
int columnNumber,
TLabel flag
)
{
Value = chars;
Line = lineNumber;
Column = columnNumber;
Flag = flag;
}
public static Token<TLabel> Empty { get; } = new Token<TLabel>();
public override bool Equals(
[NotNullWhen(true)] object? obj
)
{
return base.Equals(obj);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Token<TLabel> ReLocate(
int line,
int column
)
{
return new Token<TLabel>(Value, line, column, Flag);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Token<TNewLabel> Remark<TNewLabel>(
TNewLabel label
) where TNewLabel : unmanaged
{
return new Token<TNewLabel>(Value, Line, Column, label);
}
public static bool operator !=(
Token<TLabel> left,
Token<TLabel> right
)
{
return !left.Value.SequenceEqual(right.Value) ||
left.Line != right.Line ||
left.Column != right.Column ||
!left.Flag.Equals(right.Flag);
}
public static bool operator ==(
Token<TLabel> left,
Token<TLabel> right
)
{
return left.Value.SequenceEqual(right.Value) &&
left.Line == right.Line &&
left.Column == right.Column &&
left.Flag.Equals(right.Flag);
}
}
}
src/Nitload.RegularAnalysis/COUNT_UNIT_GENERATION.md
+204
View File
@@ -0,0 +1,204 @@
# NGRA `COUNT(UNIT)` 生成说明
本文记录 `TextRegexSourceGen.GenerateCount` 中单字符重复分支的当前设计，方便后续继续实现。
相关代码：[`Text/TextRegexSourceGen.GenClass.cs`](Text/TextRegexSourceGen.GenClass.cs)。
## 目标
对于 `a{min,max}`，不要把必需的 `min` 次匹配生成成逐字符循环。生成器先把字符重复 `min` 次，形成一个字符串字面量，并使用一次 `StartsWith` 验证下界：
```csharp
if (__remain.StartsWith("aaa"))
{
// 已经确认至少存在 min 个 a。
}
else
{
// 当前工作 region 失败。
}
```
匹配成功后一次性消费这个前缀，再只处理区间中可选的部分：
```text
必需前缀：min 次
可选尾部：max - min 次
```
`max < 0` 表示尾部没有上限。
## AST 数据归属
`COUNT` 配置属于操作符节点，重复的字符属于其子节点：
```csharp
RegexCountPayload payload = node.PayloadAs<RegexCountPayload>();
RegexAstNode value = RegexAstNode.IsNullOrEmpty(node.Left)
? node.Right
: node.Left;
char expected = value.Value[0];
```
不要从 `value` 读取 `RegexCountPayload`。`value` 携带的是 `RegexLiteralPayload` 或其他被重复元素的 payload。
字符应从 `value.Value` 读取，因为它是解码后的值。使用 `value.Source.Source[0]` 会把 `\t` 等转义序列错误地识别成 `'\\'`。
## 量词语义
| 量词 | `min` | `max` | 尾部处理 |
| --- | ---: | ---: | --- |
| `a{n}` | `n` | `n` | 无可选尾部 |
| `a{m,n}` | `m` | `n` | 最多再消费 `n - m` 次 |
| `a*` | `0` | `-1` | 无上限 |
| `a+` | `1` | `-1` | 验证一次后无上限 |
| `a?` | `0` | `1` | 最多消费一次 |
| `a{m,}` | `m` | `-1` | 验证 `m` 次后无上限 |
有限上限可以生成：
```csharp
__remain = __remain.Slice(min);
for (int i = 0;
i < max - min &&
!__remain.IsEmpty &&
__remain[0] == expected;
i++)
{
__remain = __remain.Slice(1);
}
```
无上限不能计算 `max - min`，否则结果为负数、循环不会执行。应单独生成：
```csharp
__remain = __remain.Slice(min);
while (!__remain.IsEmpty && __remain[0] == expected)
{
__remain = __remain.Slice(1);
}
```
`!__remain.IsEmpty` 必须位于 `__remain[0]` 之前。输入恰好在必需前缀后耗尽时，直接索引会抛出 `IndexOutOfRangeException`。
## 贪婪与惰性
贪婪量词可以先消费尽可能多的可选尾部，再尝试后续 continuation。如果后续失败，需要逐次回退消费数量并重试。
惰性量词只应先消费 `min`，立即尝试后续 continuation；后续失败时，才逐次多消费一个匹配字符。
因此 `is_greedy` 不能只改变扫描循环，它最终需要与后续节点的 continuation 和回溯状态一起实现。若 COUNT 位于匹配末尾，贪婪模式可直接扫描至上限，惰性模式保留在 `min` 即可。
## 失败工作 region
当前试点使用 `do { } while (false)`，使局部失败可以写成 `break`：
```csharp
bool regionSucceeded = false;
ReadOnlySpan<char> checkpoint = __remain;
do
{
if (!__remain.StartsWith(requiredPrefix))
{
break;
}
__remain = __remain.Slice(min);
// 匹配可选尾部和后续节点。
regionSucceeded = true;
}
while (false);
if (!regionSucceeded)
{
__remain = checkpoint;
// 向父 region 传播失败，或者在顶层返回失败结果。
}
```
只有 COUNT 自己被 `do/while(false)` 包裹时，`break` 结束后仍会执行下一个 CONCAT 节点，不能真正表达匹配失败。最终 region 应由以下之一持有：
- 包含后续 continuation 的 `CONCAT`；
- 一条完整的 ALT 候选路径；
- 顶层匹配方法。
父 region 负责保存 cursor 和 capture checkpoint。子节点失败后退出父 region，由父 region 统一恢复状态。ALT 和贪婪/惰性 COUNT 还需要保存多个可重试位置，不能只依赖一次 `break`。
## 摊平阈值
不要无条件执行：
```csharp
Span<char> required = stackalloc char[min];
```
量词来自用户输入，`a{1000000}` 会让源生成器尝试进行巨大的栈分配，并生成巨型字符串字面量。建议设置较小阈值，例如 32 或 64：
- `min <= threshold`：摊平为字面量并调用 `StartsWith`；
- `min > threshold`：循环比较固定长度的摊平块，最后比较不足一个块的余数；
- 非法或超过生成器限制的数量：报告 Roslyn diagnostic，而不是让生成器崩溃。
以阈值 64 为例，`a{150}` 被拆成两个 64 字符块和一个 22 字符余数块。运行时使用局部 span 逐块调用 `StartsWith`，全部成功后才一次性推进正式 cursor。这样失败时无需恢复已经提交的 `__remain`：
```csharp
if (__remain.Length < 150)
{
break;
}
ReadOnlySpan<char> __requiredRest_0 = __remain;
bool __requiredMatched_0 = true;
for (int __requiredChunkIndex_0 = 0; __requiredChunkIndex_0 < 2; __requiredChunkIndex_0++)
{
if (!__requiredRest_0.StartsWith("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
{
__requiredMatched_0 = false;
break;
}
__requiredRest_0 = __requiredRest_0.Slice(64);
}
if (__requiredMatched_0 && !__requiredRest_0.StartsWith("aaaaaaaaaaaaaaaaaaaaaa"))
{
__requiredMatched_0 = false;
}
if (!__requiredMatched_0)
{
break;
}
__remain = __remain.Slice(150);
```
## 后续实现顺序
1. 从 `node` 读取 `RegexCountPayload`，并用 Empty sentinel 安全选择子节点。
2. 改用 `value.Value[0]` 获取解码字符。
3. 为所有尾部循环加入 `!__remain.IsEmpty`。
4. 分开生成有限上限和 `max < 0` 两种循环。
5. 增加摊平阈值，避免巨大 `stackalloc` 和生成源膨胀。
6. 将工作 region 上移到 CONCAT/候选路径，完成失败传播和 checkpoint 恢复。
7. 接入 greedy/lazy continuation 回溯。
8. 最后处理 capture 状态的保存与恢复。
## 最小测试矩阵
至少覆盖以下模式和输入：
- `a{0}`：空输入、`a`；
- `a{3}`：`aa`、`aaa`、`aaaa`；
- `a{2,4}`：长度 1 到 5；
- `a*`、`a+`、`a?`、`a{2,}`；
- `\t{2}`：确认使用解码字符；
- `a{2,4}ab`：验证贪婪回退；
- `a{2,4}?ab`：验证惰性扩展；
- COUNT 位于 CONCAT、ALT 和捕获 GROUP 内的失败恢复；
- 输入恰好在 `min` 处结束，确认不会索引空 span；
- 超大 `min`，确认不会导致生成器栈溢出或生成巨型源码。
src/Nitload.RegularAnalysis/Global.cs
+15
View File
@@ -0,0 +1,15 @@
// -----------------------------------------------------------------------------
// file="Global"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
global using System;
global using System.Collections.Generic;
global using System.Linq;
global using System.Runtime.CompilerServices;
global using System.Text;
global using System.Threading.Tasks;
[assembly: InternalsVisibleTo("StgSharpDebug")]
src/Nitload.RegularAnalysis/GroupIndex.cs
+30
View File
@@ -0,0 +1,30 @@
// -----------------------------------------------------------------------------
// file="GroupIndex"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System.Runtime.InteropServices;
namespace Nitload.RegularAnalysis
{
public record struct GroupIndex
{
public GroupIndex(
int index,
string name
)
{
Index = index;
Name = name;
}
public int Index { get; internal set; }
public string Name { get; internal set; }
}
}
src/Nitload.RegularAnalysis/IR_RETIRED.md
+126
View File
@@ -0,0 +1,126 @@
# IR 层退役说明
2026-08-12。本项目的正则源生成路线从「AST → IR → 优化 → 前缀化 → 源码」
改为「AST → 树上规范化 → 直接递归生成源码」。
所有 `*.cs.old` 文件是退役的 IR 层实现，保留作参考，不参与编译。
## 为什么不要 IR 了
一句话：**前缀 IR 的整套机制都在补偿线性化亲手制造的问题，而源生成端最后又把树形还原了回来**。压平再还原，两头各付一遍成本。
逐项对照，前缀 IR 的每个机制在树上的对应物：
| 前缀机制 | 存在原因 | 树上的对应物 |
| --- | --- | --- |
| `Prev_Fail` / `Prev_Success` | 压平后失败链丢失 | ALT 节点的 case 链，天生就是 if/else |
| `Observe_Line` / `ObserveSource` | 平坦流里"读谁的结果"需显式编号 | 结构化控制流，结果就在作用域里 |
| `Pop_xxx` + region id | `break` 要知道跳出哪个作用域 | 树的作用域嵌套本身 |
| `RegionExtent` / 套山 | 切片找不到子表达式边界 | 子树边界白送 |
IR 窗口优化器（`ScanAndOptimizeIR`）为了在平坦序列上做局部重写，
搭了 8 条滑窗、`IsFullyOptimized` 防重扫、`goto phase_zero` 重启、
`CommandCount` 重算一整套脚手架——这些在树上是单节点替换。
历史讨论结论：.NET 官方 RegexGenerator 同样直接展开 `RegexNode` 树生成源码；
线性字节码的正当用途是**解释器**（`#run stepping` 模式），不是源生成。
IR 层代码因此归档而非删除：将来做 stepping 解释器时是字节码路线的起点。
## 需要在 AST 上补什么
### 节点重设计（先于一切）
`RegexAstNode` 不再挂载 `Token<RegexElementLabel>`：token 在 AST 构建时被**消费掉**，
转换为结构化载荷。节点只剩三样东西：`Label`（原 `Source.Flag`）、`IPayload`、拓扑。
这把 IR 层真正有价值的部分（类型化语义）请了回来，线性形态留在 `.old` 里。
| Label | Payload | 转换时机 |
| --- | --- | --- |
| `UNIT` / `UNIT_SPAN` | `LiteralPayload(Text)`，**转义已消解** | AST 构建 |
| `UNIT_SET` | `CharSetPayload(Set)` | AST 构建 |
| `COUNT` | `CountPayload(Min, Max, IsGreedy)`，即 `{m,n}` 的唯一解析点 | AST 构建 |
| `GROUP_BEGIN` | `GroupPayload(Name, Index)` | AST 构建 |
| `CONCAT` / `ALT` | 无（null） | — |
| `FIND`（新） | `FindPayload(Anchor, IsNearest)` | phase 6 融合 pass |
规则：
1. 载荷全部是不可变 record，深拷贝共享引用不复制
2. `ReplaceBy` / `RemoveAndAsChild` 搬运 `Label + Payload`，不再拷 Source
3. mask 决断载荷类型，取用走 `PayloadAs<T>()`，类型不符当场抛
4. pass 局部属性（`FirstAtom`/`LastAtom`）不住这个槽，放融合 pass 的 side table
改造点：`ASTGenerate` 构造处（token→payload 转换）、`ASTOptimize` 里读
`Value`/`Source` 的消费点、两个变形 API、`EmptyRegexAstNode`（Label=NONE）。
### 基础件（不存在，先补）
1. **子树深拷贝**——`(ab){2,3}` 展开需要克隆 body；配合破坏性移动可省一份；载荷共享不拷
2. **FIND 节点标签**——`RegexElementLabel` 新增 `FIND`，语义为"搜索连接"
（替代 IR 的 `FIND_COMPLEX`，见下）；合成节点直接 `new(label, payload)`，
无需伪造 token
### 树上规范化 pass（`OptimizeTree` 扩到六阶段）
| # | Pass | 状态 | 说明 |
| --- | --- | --- | --- |
| 0 | 清孤儿 | 已有 | 不变 |
| 1 | `FlattenAlt` + `RotateConcat` | 已有 | 不变 |
| 2 | **COUNT 展开族** | 待写 | `a+`→`CONCAT(a, a*)`、`a{1}`→`a`、`a{m,n}`→前缀展开+余数；产物是右脊 CONCAT，天然归一 |
| 3 | `MergeAlt` | 已有 | 不变 |
| 4 | `MergeConcat` | 已有 | 顺带吞掉 phase2 克隆出的相邻字面量 |
| 5 | **`FirstAtom`/`LastAtom` 综合** | 待写 | 一趟自底向上；GROUP 透传、COUNT 不透传、ALT 不一致则 null。必须在所有结构重写之后、字面量合并之后（锚要极大） |
| 6 | **FIND 融合** | 待写 | 后序遍历，`CONCAT(左.LastAtom==dot-star, 右.FirstAtom==字面量)` → 原地替换为 FIND 节点；覆盖有锚/穿 GROUP/无锚吞尾三种形态 |
FIND 节点语义契约（继承自 IR 版 FIND+PACK 协议）：
候选按优先级排序（FURTHEST 从远端往前退 / NEAREST 从近端往后进），
右子树是候选验证器，候选耗尽才是真失败；
GROUP 的捕获终点落在锚起点**之前**。
### 代码生成（按节点 label 一对一）
| 节点 | 形状 |
| --- | --- |
| `UNIT` / `UNIT_SPAN` | `StartsWith` + 推进 |
| `UNIT_SET` | 列表模式 char 测试 `__rest is [var c, ..] && ...` |
| `CONCAT` | 顺序发射，失败短路 |
| `ALT` | case 链 → `if / else if / else`；复杂 case 落回滚作用域 |
| `GROUP` | 进入存 span，成功写捕获槽（PACK 职责回归 GROUP） |
| `COUNT`（叶） | 计数循环 / `IndexOfAnyExcept` 向量化 |
| `COUNT`（复杂体） | 循环包递归 body |
| `FIND` | 候选重试循环，验证器 = 右子树 |
生成约定（沿用已定结论）：
- **破坏性遍历**：谁消费谁切，下游拿到无主子树根，物理上无法偷看上下文；
展开族用移动代替克隆省一份拷贝；生成后 `TextRegexSource.Ast` 是残骸，置空或立 flag
- **不变式契约**：进入时游标就位；离开时 `__ok` 必被赋值，
成功则游标已推进、失败则游标复位
- **剩余 span 写法**：游标是 `ReadOnlySpan<char> __rest`，绝对偏移按需反算
- `do{}while(false)` + `break` 只在真需要回滚的节点出现（ALT 复杂 case、
COUNT 复杂体、FIND 重试），不再每 region 强制
- 变量命名与槽位由 `SourceGenContext` 独有（该类保留，删掉 `RegionExtent` 即可）
## 当前编译错误（预期内，待重写）
| 文件 | 错误原因 | 处置 |
| --- | --- | --- |
| `RegexAnalyzer.cs` | `TextRegexSource` 记录携带 `List<RegexIR>` 字段，`Analyze` 调 `GenerateIR`/`ScanAndOptimizeIR`/`EmitPrefix` | 砍掉 IR 字段与三个调用，管线止于 `OptimizeTree` |
| `TextRegexSourceGen.GenClass.cs` | `GenerateSource` 读 `source.IRs`，`GenerateBlock`/`GenerateRegion`/`MeasureRegionExtents`/`FindRegionEnd` 全是 IR 机器 | 重写为读 `source.Ast` 的节点递归；保留 `FormatStringLiteral`/`FormatCharLiteral`；charset 解析迁入 `CharSetPayload` 转换 |
节点重设计落地时，`ASTGenerate`/`ASTOptimize` 会新增一批预期内错误（`Source`/`Value` 消费点），同批修复。
## `.old` 文件清单与参考价值
| 文件 | 参考价值 |
| --- | --- |
| `RegexIR.cs.old`、`RegexIR.Abstraction.cs.old`、`RegexIR.Prefix.cs.old`、`RegexIRGenerator.cs.old` | 字节码指令集定义，stepping 解释器的起点 |
| `RegexAnalyzer.IRGenerate.cs.old` | GROUP→TRY/POP/PACK 协议、COUNT 源文本解析（min/max/greedy 解析逻辑树上重写时照抄） |
| `RegexAnalyzer.IROptimize.cs.old` | 展开族与 FIND 合并的**语义定义**（树上 pass 的行为基准） |
| `RegexAnalyzer.PrefixEmit.cs.old` | region/观察/pop 语义的原始出处 |
| `TextRegexSourceGen.GenClass.SingleIR.cs.old` | L4 生成器 body（匹配逻辑可直接搬进节点生成器） |
| `TextRegexSourceGen.GenClass.MultiIR.cs.old`、`.Prefix.cs.old` | L1/L5 引导文档，FIND 重试循环与 POP 冲突的解法记录在此 |
配套文档：`future/ngra-regex-sourcegen-goldens.md` 的目标代码形状**全部继续有效**
（它记录的是生成产物的形状，与中间表示无关），
`test/SingleFile/regex.cs` 的手写样本同样有效。
`future/ngra-regex-ir.md` 的「优化策略」一节按旧认识写成，待按本文档修订。
src/Nitload.RegularAnalysis/NGRA_pattern_file.md
+1110
View File
File diff suppressed because it is too large Load Diff
src/Nitload.RegularAnalysis/Nitload.RegularAnalysis.csproj
+26
View File
@@ -0,0 +1,26 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<IsRoslynComponent>true</IsRoslynComponent>
<EnforceExtendedAnalyzerRules>true</EnforceExtendedAnalyzerRules>
<EmitCompilerGeneratedFiles>true</EmitCompilerGeneratedFiles>
<CompilerGeneratedFilesOutputPath>$(BaseIntermediateOutputPath)Generated</CompilerGeneratedFilesOutputPath>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Nitload.Common\Nitload.Common.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.CodeAnalysis.Analyzers" Version="3.11.0" PrivateAssets="all" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.13.0" PrivateAssets="all" />
</ItemGroup>
<ItemGroup>
<None Include="$(OutputPath)$(AssemblyName).dll" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false" />
<None Include="$(OutputPath)$(AssemblyName).pdb" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false" Condition="Exists('$(OutputPath)$(AssemblyName).pdb')" />
<None Include="$(OutputPath)Nitload.Common.dll" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false" Condition="Exists('$(OutputPath)Nitload.Common.dll')" />
<None Include="$(OutputPath)Nitload.Common.pdb" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false" Condition="Exists('$(OutputPath)Nitload.Common.pdb')" />
</ItemGroup>
</Project>
src/Nitload.RegularAnalysis/SourceGenerate.cs
+17
View File
@@ -0,0 +1,17 @@
// -----------------------------------------------------------------------------
// file="SourceGenerate"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis
{
internal class SourceGenerate { }
}
src/Nitload.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs
+189
View File
@@ -0,0 +1,189 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer.ASTGenerate"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.RegularAnalysis.Abstraction;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
public static partial class RegexAnalyzer
{
private static AbstractSyntaxTree<RegexAstNode, RegexElementLabel> GenerateAST(
string _source
)
{
CompileStack<RegexAstNode, RegexElementLabel> _stack = new();
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> _tree = new();
RegexTokenReader reader = new(_source);
TokenParser<RegexElementLabel, RegexElementLabel> lexer = reader.Pipe(() => new RegexTokenParser());
Stack<RegexElementLabel> outerLastTokens = [];
RegexElementLabel lastToken = RegexElementLabel.NONE;
while (lexer.TryReadToken(out Token<RegexElementLabel> token))
{
// Console.WriteLine(token.Value);
if ((token.Flag & RegexElementLabel.SEQUENCE) != 0)
{
// char or charset
_stack.PushOperand(new RegexAstNode(token));
} else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0)
{
_stack.PushOperator(new RegexAstNode(token));
_stack.IncreaseDepth(0);
outerLastTokens.Push(lastToken);
lastToken = RegexElementLabel.GROUP_BEGIN;
} else if ((token.Flag & RegexElementLabel.GROUP_END) != 0)
{
if (lastToken == RegexElementLabel.ALT) {
PushEpsilon(token.Line, token.Column);
}
// close all nodes here
RegexAstNode op;
while (_stack.TryPopOperator(out op)) {
ProcessOperator(op);
}
if (_stack.OperandInDepthCount != 1) {
throw new InvalidOperationException();
}
_stack.DecreaseDepth();
op = _stack.PopOperator();
if (op.Label != RegexElementLabel.GROUP_BEGIN) {
throw new InvalidOperationException("Unmatched group begin and end symbol");
}
if (op.Payload.Source.Length > 1)
{
op.Right = _stack.PopOperand();
_stack.PushOperand(op);
}
_ = outerLastTokens.Pop();
lastToken = RegexElementLabel.GROUP_END;
} else if ((token.Flag & RegexElementLabel.OPERATOR) != 0)
{
if (token.Flag == RegexElementLabel.ALT &&
lastToken is RegexElementLabel.NONE or
RegexElementLabel.GROUP_BEGIN or
RegexElementLabel.ALT)
{
PushEpsilon(token.Line, token.Column);
}
if (_stack.OperatorInDepthCount == 0)
{
// the first operator in stack
_stack.PushOperator(new RegexAstNode(token));
} else
{
// process operator by precedence
while (_stack.TryPeekOperator(out RegexAstNode topOp))
{
(bool Valid, int Value) cmp = RegexAstNode.ComparePrecedence(topOp.Label, token.Flag);
if (!cmp.Valid) {
throw new InvalidOperationException("Operators with same precedence are not supported.");
}
if (cmp.Value < 0)
{
_stack.PopOperator();
ProcessOperator(topOp);
continue;
} else
{
break;
}
}
_stack.PushOperator(new RegexAstNode(token));
}
}
if (token.Flag != RegexElementLabel.GROUP_BEGIN &&
token.Flag != RegexElementLabel.GROUP_END) {
lastToken = token.Flag;
}
}
if (lastToken == RegexElementLabel.ALT) {
PushEpsilon(0, _source.Length);
}
if (_stack.Depth == 1)
{
if (_stack.OperandInDepthCount != 1 || _stack.OperatorInDepthCount != 0)
{
while (_stack.TryPopOperator(out RegexAstNode? op)) {
ProcessOperator(op);
}
}
if (_stack.OperandInDepthCount == 1 && _stack.OperatorInDepthCount == 0)
{
_tree.Root = _stack.PopOperand();
return _tree;
}
}
throw new InvalidOperationException("Invalid regular expression syntax.");
void PushEpsilon(int row, int column)
{
RegexAstNode epsilon = new(new RegexEpsilonPayload(row, column));
_stack.PushOperand(epsilon);
_ = _tree.AddNode(epsilon);
}
void ProcessOperator(
RegexAstNode op
)
{
if (op.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1) &&
_stack.TryPopOperand(out RegexAstNode? _2))
{
op.Left = _2;
op.Right = _1;
_stack.PushOperand(op);
_ = _tree.AddNode(op);
_ = _tree.AddNode(_1);
_ = _tree.AddNode(_2);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
} else if (op.EqualityTypeConvert == RegexElementLabel.COUNT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1))
{
op.Right = _1;
_stack.PushOperand(op);
_ = _tree.AddNode(_1);
_ = _tree.AddNode(op);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
} else if (op.EqualityTypeConvert == RegexElementLabel.ALT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1) &&
_stack.TryPopOperand(out RegexAstNode? _2))
{
op.Left = _2;
op.Right = _1;
_stack.PushOperand(op);
_ = _tree.AddNode(op);
_ = _tree.AddNode(_1);
_ = _tree.AddNode(_2);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
}
}
}
}
}
src/Nitload.RegularAnalysis/Text/RegexAnalyzer.ASTNodeOptimize.cs
+25
View File
@@ -0,0 +1,25 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer.ASTNodeOptimize"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.RegularAnalysis.Abstraction;
using System;
namespace Nitload.RegularAnalysis.Text
{
public static partial class RegexAnalyzer
{
private static bool TryFlattenCount(
RegexAstNode node
)
{
// flatten count node if it matches raw string pattern
throw new NotImplementedException();
}
}
}
src/Nitload.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs
+470
View File
@@ -0,0 +1,470 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer.ASTOptimize"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Security;
using System.Runtime.CompilerServices;
using System.Security.Permissions;
using System.Text;
using System.Threading.Tasks;
using System.Xml.Linq;
namespace Nitload.RegularAnalysis.Text
{
public static partial class RegexAnalyzer
{
private static RegexAstNode FlattenAlt(
RegexAstNode root
)
{
TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(root);
List<RegexAstNode> cases = [];
RegexAstNode current;
_ = enumerator.MoveNext(); // skip root
while (enumerator.MoveNext())
{
current = enumerator.Current;
if (current.EqualityTypeConvert != RegexElementLabel.ALT)
{
cases.Add(current);
enumerator.SkipNode(current);
}
current.Parent.RemoveChild(current);
}
for (int i = 0; i < cases.Count - 1; i++)
{
cases[i].AppendNode(cases[i + 1]);
cases[i].Parent = root;
}
cases[^1].Parent = root;
return cases[0];
}
private static bool MergeConcat(
RegexAstNode root
)
{
if (root.EqualityTypeConvert != RegexElementLabel.CONCAT) {
return false;
}
/*
* concat → 'a'
* ╱
* 'a'
*/
if ((root.Left.EqualityTypeConvert & RegexElementLabel.SEQUENCE) != 0 &&
RegexAstNode.IsNullOrEmpty(root.Right))
{
root.RemoveAndAsChild(true);
return true;
}
if ((root.Right.EqualityTypeConvert & RegexElementLabel.SEQUENCE) != 0 &&
RegexAstNode.IsNullOrEmpty(root.Left))
{
root.RemoveAndAsChild(false);
return true;
}
/*
* concat → concat
* ╱ ╲ /
* 'a' concat "abc"
* / \
* 'b' 'c'
*/
List<string> buffer = [];
int length = 0;
RegexAstNode right = root.Right;
RegexAstNode left = root.Left;
if (left.EqualityTypeConvert is not RegexElementLabel.UNIT_SPAN and
not RegexElementLabel.UNIT) {
return false;
}
string str = left.Value;
buffer.Add(str);
length += str.Length;
while (!RegexAstNode.IsNullOrEmpty(right) &&
right.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
left = right.Left;
if (left.EqualityTypeConvert is not RegexElementLabel.UNIT_SPAN and
not RegexElementLabel.UNIT)
{
break;
}
str = left.Value;
buffer.Add(str);
length += str.Length;
root.Right = right.Right;
right.Parent = RegexAstNode.Empty;
right = root.Right;
right.Parent = root;
}
if (right.EqualityTypeConvert is RegexElementLabel.UNIT_SPAN or
RegexElementLabel.UNIT)
{
root.Right = RegexAstNode.Empty;
buffer.Add(right.Value);
length += right.Value.Length;
}
str = string.Create(length, buffer, static(
span,
buffer
) => {
int offset = 0;
for (int i = 0; i < buffer.Count; i++)
{
buffer[i].AsSpan().CopyTo(span[offset..]);
offset += buffer[i].Length;
}
});
root.Left = new RegexAstNode(new RegexLiteralPayload(str, str, left.Payload.Row,
left.Payload.Column,
str.Length == 1 ?
RegexElementLabel.UNIT :
RegexElementLabel.UNIT_SPAN));
return true;
}
private static void OptimizeTree(
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree
)
{
// phase0: scan container and remove no parent nodes
_ = tree.AllNodes.RemoveWhere(node => node.Parent == null && node != tree.Root);
TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(tree.Root);
RegexAstNode current;
// phase1: normalize associative binary operators, then flatten ALT
List<RegexAstNode> altNodes = [];
while (enumerator.MoveNext())
{
current = enumerator.Current;
if (current.EqualityTypeConvert is RegexElementLabel.CONCAT or
RegexElementLabel.ALT)
{
RotateAssociativeOperator(current, current.EqualityTypeConvert);
}
if (current.EqualityTypeConvert == RegexElementLabel.ALT)
{
RegexAstNode union = FlattenAlt(current);
altNodes.Add(current);
current.Right = union;
current.Left = RegexAstNode.Empty;
}
}
// phase3: then ALT union
foreach (RegexAstNode item in altNodes)
{
List<RegexAstNode> cases = [];
RegexAstNode _case = item.Right;
do
{
cases.Add(_case);
_case = _case.Next;
} while (!RegexAstNode.IsNullOrEmpty(_case));
RegexAstNode newAlt = MergeAlt(cases);
item.ReplaceBy(newAlt);
}
// return;
// phase4: CONCAT merge
enumerator.Reset();
while (enumerator.MoveNext())
{
// suppose all concat is now in normalized form,
// we can merge adjacent unit nodes into a single unit node with string value
current = enumerator.Current;
if (current.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
if (MergeConcat(current)) {
enumerator.SkipOnce();
}
}
}
}
/// <summary>
/// Converts a left-associated binary operator tree to a right spine while preserving
/// operand order. For example, <c>(a op b) op c</c> becomes
/// <c>a op (b op c)</c>.
/// </summary>
/// <remarks>
/// A child with another label is an operand and is not rotated across. Keeping the target
/// label explicit allows later associative operators to reuse the same tree operation.
/// </remarks>
private static void RotateAssociativeOperator(
RegexAstNode root,
RegexElementLabel operatorLabel
)
{
if (root.EqualityTypeConvert != operatorLabel) {
throw new ArgumentException("The root does not match the requested operator.",
nameof(operatorLabel));
}
RegexAstNode left = root.Left;
while (!RegexAstNode.IsNullOrEmpty(left) &&
left.EqualityTypeConvert == operatorLabel)
{
/*
* root → root
* ╱ ╲ ╱ ╲
* left c a left
* ╱ ╲ ╱ ╲
* a b b c
*/
RegexAstNode a = left.Left;
RegexAstNode b = left.Right;
RegexAstNode c = root.Right;
root.Left = a;
if (!RegexAstNode.IsNullOrEmpty(a)) {
a.Parent = root;
}
root.Right = left;
left.Left = b;
left.Right = c;
if (!RegexAstNode.IsNullOrEmpty(b)) {
b.Parent = left;
}
if (!RegexAstNode.IsNullOrEmpty(c)) {
c.Parent = left;
}
left = root.Left;
}
}
#region merge alt
/// <summary>
/// Get union beginning part of cases
/// </summary>
/// <param name="cases">
///
/// </param>
/// <returns>
///
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static RegexAstNode MergeAlt(
List<RegexAstNode> cases
)
{
UnionAlt union = new(0, cases);
RegexAstNode root = union.AsNormal();
return root;
}
private class UnionAlt : RegexAstNode
{
private const int MaxLevel = 8;
private UnionAlt(
int level,
RegexAstNode node
)
: base(node.Payload)
{
Level = level;
Cases = [];
RegexAstNode right = IsNullOrEmpty(node.Right) ?
new RegexAstNode(
new RegexLiteralPayload(node.Value, node.Value, node.Payload.Row,
node.Payload.Column, RegexElementLabel.UNIT)
) :
node.Right;
Cases.Add(right);
node.PrependNode(Empty);
node.AppendNode(Empty);
UnionBegin = node;
UnionEnd = node;
}
public UnionAlt(
int level,
List<RegexAstNode> c
)
: base(c[0].Payload)
{
Cases = c;
Level = level;
}
public List<RegexAstNode> Cases { get; set; }
public RegexAstNode UnionBegin { get; set; }
public RegexAstNode UnionEnd { get; set; }
private int Level { get; set; }
public RegexAstNode AsNormal()
{
if (Level >= MaxLevel)
{
goto build;
}
merge:
List<RegexAstNode> buffer = [];
for (int i = 0; i < Cases.Count; i++)
{
if ((Cases[i].EqualityTypeConvert & RegexElementLabel.SEQUENCE) != 0)
{
RegexAstNode concat = new(new Token<RegexElementLabel>(string.Empty, 0,
Cases[i].Payload.Column,
RegexElementLabel.CONCAT))
{
Left = Cases[i]
};
Cases[i].Parent = concat;
concat.Right = Empty;
Cases[i] = concat;
}
RegexAstNode current = Cases[i];
int j = 0;
for (; j < buffer.Count; j++)
{
if (CanMerge(current, buffer[j]))
{
// find a case in buffer that can merge with current case,
// then merge them and break
RegexAstNode b = buffer[j];
if (b is not UnionAlt alt)
{
alt = new UnionAlt(Level + 1, b);
alt.AddCase(current);
buffer[j] = alt;
} else
{
alt.AddCase(current);
}
break;
}
}
if (j >= buffer.Count) {
buffer.Add(current);
}
}
if (Cases.Count != buffer.Count) {
Cases = buffer;
}
if (buffer.Count == 1)
{
// if there is only one case,
// then we suppose that its contained cases has a same beginning,
// so we attach it to UnionEnd and stay level unchanged
UnionAlt alt = (Cases[0] as UnionAlt)!;
if (IsNullOrEmpty(UnionBegin))
{
UnionBegin = alt.UnionBegin;
UnionEnd = alt.UnionEnd;
} else
{
alt.UnionBegin.Parent = UnionEnd;
UnionEnd.Right = alt.UnionBegin;
UnionEnd = alt.UnionBegin;
}
Cases = alt.Cases;
goto merge;
}
build:
// suppose that from UnionBegin to UnionEnd is a normal form without union
for (int i = 0; i < Cases.Count; i++)
{
if (Cases[i] is UnionAlt union) {
Cases[i] = union.AsNormal();
}
}
RegexAstNode branch;
if (IsNullOrEmpty(UnionEnd))
{
branch = new RegexAstNode(new Token<RegexElementLabel>(string.Empty, 0,
Payload.Column,
RegexElementLabel.ALT));
UnionBegin = branch;
UnionEnd = branch;
} else
{
branch = new RegexAstNode(new Token<RegexElementLabel>(string.Empty, 0,
UnionEnd.Payload.Column,
RegexElementLabel.ALT));
UnionEnd.Right = branch;
branch.Parent = UnionEnd;
UnionEnd = branch;
}
for (int i = 0; i < Cases.Count - 1; i++)
{
Cases[i] ??= NewEmptyNode();
Cases[i].AppendNode(Cases[i + 1]);
Cases[i].Parent = UnionEnd;
}
UnionEnd.Right = Cases[0];
return UnionBegin;
}
private void AddCase(
RegexAstNode _case
)
{
RegexAstNode right = IsNullOrEmpty(_case.Right) ?
new RegexAstNode(
new RegexLiteralPayload(_case.Value, _case.Value, _case.Payload.Row,
_case.Payload.Column, RegexElementLabel.UNIT)
) :
_case.Right;
Cases.Add(right);
}
private static bool CanMerge(
RegexAstNode origin,
RegexAstNode target
)
{
// suppose that origin is not a union,
if (target is UnionAlt alt)
{
// then we suppose that alt has only one are same
return origin .EqualityTypeConvert == RegexElementLabel.CONCAT &&
(origin.Left.EqualityTypeConvert & RegexElementLabel.UNIT_SINGLE) != 0 &&
origin.Left.EqualityTypeConvert == alt.UnionEnd.Left.EqualityTypeConvert &&
origin.Left.Value == alt.UnionEnd.Left.Value;
} else
{
return origin.EqualityTypeConvert == RegexElementLabel.CONCAT &&
target.EqualityTypeConvert == RegexElementLabel.CONCAT &&
(origin.Left.EqualityTypeConvert & RegexElementLabel.UNIT_SINGLE) != 0 &&
origin.Left.EqualityTypeConvert == target.Left.EqualityTypeConvert &&
origin.Left.Value == target.Left.Value;
}
}
}
#endregion
/**/
}
}
src/Nitload.RegularAnalysis/Text/RegexAnalyzer.cs
+92
View File
@@ -0,0 +1,92 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection.Emit;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
public record TextRegexSource
{
internal TextRegexSource(
string pattern,
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> ast,
Exception? analyseException = null
)
{
Pattern = pattern;
Ast = ast;
AnalyseException = analyseException;
}
public Exception? AnalyseException { get; set; }
public string Pattern { get; }
internal AbstractSyntaxTree<RegexAstNode, RegexElementLabel> Ast { get; }
}
public static partial class RegexAnalyzer
{
public static TextRegexSource Analyze(
string regex
)
{
#pragma warning disable CA1031
// RegexAnalyzer interpreter = new();
// interpreter._source = regex;
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree;
try
{
tree = GenerateAST(regex);
OptimizeTree(tree);
return new TextRegexSource(regex, tree, null);
}
catch (Exception ex)
{
return new TextRegexSource(regex, null, ex);
}
/*
// List<RegexIR> base_ir;
try
{
// base_ir = GenerateIR(tree, ref info);
}
catch (Exception ex)
{
return new TextRegexSource(regex, require_ast ? tree : null!, null, null, info, ex);
}
try
{
// ScanAndOptimizeIR(base_ir, out List<RegexIR> opt_ir);
// List<RegexIR> prefix_ir = EmitPrefix(opt_ir, ref info);
return new TextRegexSource(regex, require_ast ? tree : null!,
require_origin_ir ? base_ir : null!, prefix_ir, info);
}
catch (Exception ex)
{
return new TextRegexSource(regex, require_ast ? tree : null!,
require_origin_ir ? base_ir : null!, null!, info, ex);
}
/**/
#pragma warning restore CA1031
}
}
}
src/Nitload.RegularAnalysis/Text/RegexAnanlyzer.ASTNodeOptimize.cs
View File
src/Nitload.RegularAnalysis/Text/RegexAstNode.AdditionalPayload.cs
+339
View File
@@ -0,0 +1,339 @@
// -----------------------------------------------------------------------------
// file="RegexAstNode.AdditionalPayload"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Text;
namespace Nitload.RegularAnalysis.Text
{
public abstract record RegexAstPayload(
string Source,
int Row,
int Column,
RegexElementLabel Flag
) : ISyntaxPayload<RegexElementLabel>;
public sealed record RegexEmptyPayload() : RegexAstPayload(string.Empty, 0, 0,
RegexElementLabel.NONE);
/// <summary>
/// A semantic empty expression. Unlike <see cref="RegexEmptyPayload"/>, this is a real AST
/// node and represents a successful match that consumes no input.
/// </summary>
public sealed record RegexEpsilonPayload(
int Row,
int Column
) : RegexAstPayload(string.Empty, Row, Column, RegexElementLabel.EPSILON);
/// <summary>
/// A deterministic literal sequence. Escapes represented as literals are consumed here.
/// </summary>
public sealed record RegexLiteralPayload(
string Text,
string Source,
int Row,
int Column,
RegexElementLabel Flag
) : RegexAstPayload(Source, Row, Column, Flag);
public enum RegexCharSetType
{
None,
Single,
Range,
Set,
Any
}
public sealed record RegexCharSet(
string Value,
RegexCharSetType Type,
bool Accept
);
/// <summary>
/// A character-set atom expanded into the rules formerly produced by FigureCharSetType.
/// </summary>
public sealed record RegexCharSetPayload(
IReadOnlyList<RegexCharSet> Set,
bool Accept,
string Source,
int Row,
int Column
) : RegexAstPayload(Source, Row, Column, RegexElementLabel.UNIT_SET)
{
public bool IsAny => Set.Count == 1 && Set[0].Type == RegexCharSetType.Any;
}
public sealed record RegexCountPayload(
int Min,
int Max,
bool IsGreedy,
string Source,
int Row,
int Column
) : RegexAstPayload(Source, Row, Column, RegexElementLabel.COUNT)
{
public bool IsStar => Min == 0 && Max < 0;
public bool IsPlus => Min == 1 && Max < 0;
}
public sealed record RegexGroupPayload(
string Name,
bool IsCapturing,
string Source,
int Row,
int Column
) : RegexAstPayload(Source, Row, Column, RegexElementLabel.GROUP_BEGIN);
public sealed record RegexCommonPayload(
string Source,
int Row,
int Column,
RegexElementLabel Flag
) : RegexAstPayload(Source, Row, Column, Flag)
{
public static RegexAstPayload FromToken(
Token<RegexElementLabel> token
)
{
return token.Flag switch
{
RegexElementLabel.UNIT or RegexElementLabel.UNIT_SPAN => new RegexLiteralPayload(DecodeLiteral(token.Value),
token.Value,
token.Line,
token.Column,
token.Flag),
RegexElementLabel.UNIT_SET => ParseCharSet(token),
RegexElementLabel.COUNT => ParseCount(token),
RegexElementLabel.GROUP_BEGIN => ParseGroup(token),
_ => new RegexCommonPayload(token.Value, token.Line, token.Column, token.Flag)
};
}
private static void AddEscape(
char escaped,
List<RegexCharSet> rules
)
{
bool ruleAccept = char.IsAsciiLetterLower(escaped);
char kind = char.ToLower(escaped, CultureInfo.InvariantCulture);
if (!IsNamedSet(escaped)) {
throw new InvalidCastException($"Unknown character set escape: \\{escaped}");
}
rules.Add(new(kind.ToString(), RegexCharSetType.Set, ruleAccept));
}
private static char DecodeEscapedChar(
char value
)
{
return value switch
{
'n' or 'r' => throw new NotSupportedException(
"Multi-line regular expressions are not supported yet."),
't' => '\t',
'0' => '\0',
_ => value
};
}
private static string DecodeLiteral(
string source
)
{
if (source.Length != 2 || source[0] != '\\') {
return source;
}
return source[1] switch
{
'n' => "\n",
'r' => "\r",
't' => "\t",
'0' => "\0",
_ => source[1].ToString()
};
}
private static bool IsNamedSet(
char value
)
{
return
char.ToLower(value, CultureInfo.InvariantCulture) is 's' or 'w' or 'd';
}
private static void ParseBracketContent(
ReadOnlySpan<char> content,
List<RegexCharSet> rules
)
{
HashSet<char> singles = [];
int index = 0;
while (index < content.Length)
{
if (content[index] is '\r' or '\n') {
throw new NotSupportedException(
"Multi-line regular expressions are not supported yet.");
}
if (content[index] == '\\')
{
if (++index >= content.Length) {
throw new InvalidCastException("Unclosed escape in character set.");
}
char escaped = content[index++];
if (IsNamedSet(escaped))
{
AddEscape(escaped, rules);
} else
{
_ = singles.Add(DecodeEscapedChar(escaped));
}
continue;
}
char first = content[index++];
if (index < content.Length - 0 &&
content[index] == '-' &&
index + 1 < content.Length)
{
index++;
char last;
if (content[index] == '\\')
{
if (++index >= content.Length) {
throw new InvalidCastException("Unclosed range in character set.");
}
last = DecodeEscapedChar(content[index++]);
} else
{
last = content[index++];
}
if (last < first) {
throw new InvalidCastException($"Invalid character range: {first}-{last}");
}
rules.Add(new($"{first}{last}", RegexCharSetType.Range, true));
} else
{
_ = singles.Add(first);
}
}
if (singles.Count > 0)
{
StringBuilder text = new(singles.Count);
foreach (char value in singles) {
_ = text.Append(value);
}
rules.Add(new(text.ToString(), RegexCharSetType.Single, true));
}
}
private static RegexCharSetPayload ParseCharSet(
Token<RegexElementLabel> token
)
{
string source = token.Value;
List<RegexCharSet> rules = [];
bool accept = true;
if (source == ".")
{
rules.Add(new(string.Empty, RegexCharSetType.Any, true));
} else if (source.Length >= 2 && source[0] == '[' && source[^1] == ']')
{
int start = source.Length > 2 && source[1] == '^' ? 2 : 1;
accept = start == 1;
ParseBracketContent(source.AsSpan(start, source.Length - start - 1), rules);
} else if (source.Length == 2 && source[0] == '\\')
{
AddEscape(source[1], rules);
} else
{
throw new InvalidCastException($"Unknown char set: {source}");
}
return new RegexCharSetPayload(rules.ToArray(), accept, source, token.Line,
token.Column);
}
private static RegexCountPayload ParseCount(
Token<RegexElementLabel> token
)
{
string source = token.Value;
bool greedy = source.Length == 1 || source[^1] != '?';
ReadOnlySpan<char> count = greedy ?
source.AsSpan() :
source.AsSpan(0, source.Length - 1);
return count[0] switch
{
'*' => new RegexCountPayload(0, -1, greedy, source, token.Line, token.Column),
'+' => new RegexCountPayload(1, -1, greedy, source, token.Line, token.Column),
'?' => new RegexCountPayload(0, 1, greedy, source, token.Line, token.Column),
'{' => ParseRange(count, greedy, token),
_ => throw new InvalidOperationException($"Invalid count format: {source}")
};
}
private static RegexGroupPayload ParseGroup(
Token<RegexElementLabel> token
)
{
string source = token.Value;
bool named = source.StartsWith("(?<", StringComparison.Ordinal);
string name = named ? source[3..^1] : string.Empty;
return new RegexGroupPayload(name, source != "(?:", source, token.Line, token.Column);
}
private static RegexCountPayload ParseRange(
ReadOnlySpan<char> count,
bool greedy,
Token<RegexElementLabel> token
)
{
if (count.Length < 3 || count[^1] != '}') {
throw new InvalidOperationException($"Invalid count format: {token.Value}");
}
ReadOnlySpan<char> body = count[1..^1];
int comma = body.IndexOf(',');
if (comma < 0)
{
if (!int.TryParse(body, out int exact)) {
throw new InvalidOperationException($"Invalid count format: {token.Value}");
}
return new RegexCountPayload(exact, exact, greedy, token.Value, token.Line,
token.Column);
}
if (!int.TryParse(body[..comma], out int min)) {
throw new InvalidOperationException($"Invalid count format: {token.Value}");
}
ReadOnlySpan<char> maxText = body[(comma + 1)..];
int max = maxText.IsEmpty ?
-1 :
int.TryParse(maxText, out int parsed) ?
parsed :
throw new InvalidOperationException($"Invalid count format: {token.Value}");
if (max >= 0 && max < min) {
throw new InvalidOperationException($"Invalid count range: {token.Value}");
}
return new RegexCountPayload(min, max, greedy, token.Value, token.Line, token.Column);
}
}
}
src/Nitload.RegularAnalysis/Text/RegexAstNode.cs
+196
View File
@@ -0,0 +1,196 @@
// -----------------------------------------------------------------------------
// file="RegexAstNode"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.RegularAnalysis.Abstraction;
using System.Runtime.CompilerServices;
namespace Nitload.RegularAnalysis.Text
{
internal class RegexAstNode : ISyntaxNode<RegexAstNode, RegexElementLabel>
{
// private RegexAstNode _left, _right;
public RegexAstNode(Token<RegexElementLabel> source) : this(RegexCommonPayload.FromToken(source)) { }
public RegexAstNode(RegexAstPayload payload)
{
Payload = payload;
}
public int EnumState { get; set; }
public ISyntaxPayload<RegexElementLabel> Source => Payload;
public long NodeFlag => (long)Label;
public static RegexAstNode Empty { get; }
= new EmptyRegexAstNode();
public RegexAstNode Previous { get; set; }
public RegexAstNode Next { get; set; }
public RegexAstNode Parent { get; set; }
public RegexAstNode Left
{
get;
set
{
_ = (field?.Parent = Empty);
field = value;
if (!IsNullOrEmpty(value)) {
value.Parent = this;
}
}
}
public RegexAstNode Right
{
get;
set
{
_ = (field?.Parent = Empty);
field = value;
if (!IsNullOrEmpty(value)) {
value.Parent = this;
}
}
}
public RegexAstPayload Payload { get; private protected set; }
public RegexElementLabel EqualityTypeConvert => Label;
public RegexElementLabel Label => Payload.Flag;
public string Value => Payload switch
{
RegexLiteralPayload literal => literal.Text,
RegexCharSetPayload set => set.Source,
_ => Payload.Source
};
public string SourceCode => Source.Source;
public void AppendNode(RegexAstNode nextToken)
{
Next = nextToken;
nextToken.Previous = this;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static (bool Valid, int Value) ComparePrecedence(RegexElementLabel left, RegexElementLabel right)
{
left &= RegexElementLabel.OPERATOR;
right &= RegexElementLabel.OPERATOR;
int v_left = (int)left, v_right = (int)right;
if (v_left == 0 || v_right == 0) {
return (false, 0);
}
return (true,v_left - v_right);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(RegexAstNode node)
{
return node is null || ReferenceEquals(node, Empty);
}
/**/
public void PrependNode(RegexAstNode previousNode)
{
Previous = previousNode;
previousNode.Next = this;
}
public void RemoveChild(RegexAstNode node)
{
if (Left == node)
{
Left = Empty;
} else if (Right == node) {
Right = Empty;
}
node.Parent = Empty;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static RegexAstNode NewEmptyNode(
)
{
return new EmptyRegexAstNode();
}
internal TPayload PayloadAs<TPayload>() where TPayload: RegexAstPayload
{
return
Payload as TPayload ??
throw new InvalidOperationException(
$"Node {Label} does not carry {typeof(TPayload).Name}.");
}
internal void RemoveAndAsChild(bool useLeft)
{
if (useLeft)
{
Payload = Left.Payload;
Left.Parent = Empty;
Left = Empty;
} else
{
Payload = Right.Payload;
Right.Parent = Empty;
Right = Empty;
}
}
/**/
/// <summary>
/// Replace the current node with another node, and update the parent and sibling
/// references accordingly. This is implemented by copying metadata and nearby
/// environment of the given node to the current node, and then update the nearby nodes to
/// point to the current node.
/// </summary>
/// <param name="node">
/// The node to replace the current node with.
/// </param>
internal void ReplaceBy(RegexAstNode node)
{
Payload = node.Payload;
Left = node.Left;
Right = node.Right;
if (!IsNullOrEmpty(Left)) {
Left.Parent = this;
}
if (!IsNullOrEmpty(Right)) {
Right.Parent = this;
}
Previous = node.Previous;
Next = node.Next;
RegexAstNode prev = Previous;
RegexAstNode next = Next;
if (!IsNullOrEmpty(prev)) {
prev.Next = this;
}
if (!IsNullOrEmpty(next)) {
next.Previous = this;
}
}
private sealed class EmptyRegexAstNode : RegexAstNode
{
public EmptyRegexAstNode() : base(new RegexEmptyPayload()) { }
}
}
}
src/Nitload.RegularAnalysis/Text/RegexElementLabel.cs
+39
View File
@@ -0,0 +1,39 @@
// -----------------------------------------------------------------------------
// file="RegexElementLabel"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
[Flags]
public enum RegexElementLabel
{
#pragma warning disable CA1707
NONE = 0,
UNIT = 1,
UNIT_SET = 2,
UNIT_SPAN = 4,
GROUP_BEGIN = 8,
GROUP_END = 16,
COUNT = 32,
CONCAT = 64,
ALT = 128,
EPSILON = 256,
SEQUENCE = UNIT | UNIT_SPAN | UNIT_SET,
UNIT_SINGLE = UNIT | UNIT_SET,
OPERATOR = COUNT | ALT | CONCAT,
VAST_OPERATOR = OPERATOR | GROUP_BEGIN,
ATOM_BEGIN = SEQUENCE | GROUP_BEGIN | EPSILON,
ATOM_END = SEQUENCE | GROUP_END | COUNT | EPSILON,
#pragma warning restore CA1707
}
}
src/Nitload.RegularAnalysis/Text/RegexTokenParser.cs
+67
View File
@@ -0,0 +1,67 @@
// -----------------------------------------------------------------------------
// file="RegexTokenParser"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Diagnostics.Tracing;
using System.IO.Pipes;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
internal class RegexTokenParser : TokenParser<RegexElementLabel, RegexElementLabel>
{
private readonly Queue<Token<RegexElementLabel>> _buffer = [];
private RegexElementLabel _prev;
public override bool IsEmpty => Source.IsEmpty && _buffer.Count == 0;
public override Token<RegexElementLabel> ReadToken()
{
if (_buffer.TryDequeue(out Token<RegexElementLabel> token)) {
return token;
}
if (Source.TryReadToken(out Token<RegexElementLabel> orig))
{
if (((orig.Flag & RegexElementLabel.ATOM_BEGIN) != 0) && (_prev != 0))
{
Token<RegexElementLabel> ret = new(string.Empty, orig.Line, orig.Column,
RegexElementLabel.CONCAT);
_buffer.Enqueue(orig);
_prev = orig.Flag & RegexElementLabel.ATOM_END;
return ret;
} else
{
_prev = orig.Flag & RegexElementLabel.ATOM_END;
return orig;
}
} else
{
return Token<RegexElementLabel>.Empty;
}
}
public override bool TryReadToken(
out Token<RegexElementLabel> t
)
{
if (IsEmpty)
{
t = Token<RegexElementLabel>.Empty;
return false;
}
t = ReadToken();
return t != Token<RegexElementLabel>.Empty;
}
}
}
src/Nitload.RegularAnalysis/Text/RegexTokenReader.cs
+313
View File
@@ -0,0 +1,313 @@
// -----------------------------------------------------------------------------
// file="RegexTokenReader"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Runtime.InteropServices;
using System.Security.Permissions;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
internal class RegexTokenReader : ITokenReader<RegexElementLabel>
{
private static string _source;
private int position = 0;
public RegexTokenReader() { }
public RegexTokenReader(string source)
{
_source = source;
}
public bool IsEmpty => string.IsNullOrEmpty(_source) || position >= _source.Length;
public Token<RegexElementLabel> ReadToken()
{
int pos = position;
switch (_source[pos])
{
case '\r' or '\n':
throw new NotSupportedException("Multi-line regular expressions are not supported yet.");
case '\\':
if (pos + 1 == _source.Length) {
throw new InvalidOperationException("Unclosed char");
}
if (_source[pos + 1] == 'x')
{
string value = _source[(pos + 1)..(pos + 3)];
pos += 4;
if (uint.TryParse(value, out uint code) && Rune.TryCreate(code, out Rune result))
{
return new Token<RegexElementLabel>(result.ToString(), 0, pos, RegexElementLabel.UNIT);
} else
{
throw new InvalidCastException($"Failed to convert {value} to a valid ascii code point.");
}
} else if (_source[pos + 1] == 'u')
{
const int unicodeEscapeLength = 6;
if (pos > _source.Length - unicodeEscapeLength)
{
throw new InvalidCastException(
"A Unicode escape must contain exactly four hexadecimal digits.");
}
ReadOnlySpan<char> value = _source.AsSpan(pos + 2, 4);
if (ushort.TryParse(value, NumberStyles.AllowHexSpecifier,
CultureInfo.InvariantCulture, out ushort code))
{
position = pos + unicodeEscapeLength;
return new Token<RegexElementLabel>(((char)code).ToString(), 0, pos,
RegexElementLabel.UNIT);
} else
{
throw new InvalidCastException(
$"Failed to convert {value.ToString()} to a UTF-16 code unit.");
}
} else if (_source[pos + 1] == 'p')
{
int i = pos + 2;
if (_source[i] != '{') {
throw new InvalidOperationException("Invalid Unicode property escape sequence.");
}
for (; i < _source.Length; i++)
{
if (_source[i] == '}')
{
position = i + 1;
return new Token<RegexElementLabel>(_source[(pos)..(i + 1)], 0, pos,
RegexElementLabel.UNIT_SET);
}
}
throw new InvalidOperationException("Unclosed Unicode property escape sequence.");
} else if (_source[pos + 1] == '.')
{
position = pos + 2;
return new Token<RegexElementLabel>(".", 0, pos, RegexElementLabel.UNIT);
} else
{
if (_source[pos + 1] is 'n' or 'r') {
throw new NotSupportedException("Multi-line regular expressions are not supported yet.");
}
position = pos + 2;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 2)], 0, pos,
RegexElementLabel.UNIT_SET);
}
case '[':
for (int i = pos + 1; i < _source.Length; i++)
{
if (_source[i] == ']' && _source[i - 1] != '\\')
{
int length = (i - pos - 1);
position = i + 1;
return new Token<RegexElementLabel>(_source[(pos)..(i + 1)], 0, position,
RegexElementLabel.UNIT_SET);
}
}
throw new InvalidOperationException("Unclosed character set");
case '|':
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.ALT);
case '(':
// very comlex here
return ReadGroupBeginToken();
case ')':
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.GROUP_END);
case '{':
for (int i = pos + 1; i < _source.Length; i++)
{
if (_source[i] == '}')
{
int end = i + 1;
if (end < _source.Length && _source[end] == '?') {
end++;
}
position = end;
return new Token<RegexElementLabel>(_source[pos..end], 0, pos,
RegexElementLabel.COUNT);
}
}
throw new InvalidOperationException("Unclosed count specifier");
case '*' or '+':
int count = pos + 1 >= _source.Length ? 1 : (_source[pos + 1] == '?' ? 2 : 1);
position = pos + count;
return new Token<RegexElementLabel>(_source[(pos)..(pos + count)], 0, pos, RegexElementLabel.COUNT);
case '?':
int optionalCount = pos + 1 < _source.Length && _source[pos + 1] == '?' ? 2 : 1;
position = pos + optionalCount;
return new Token<RegexElementLabel>(_source[pos..(pos + optionalCount)], 0, pos,
RegexElementLabel.COUNT);
case '.':
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.UNIT_SET);
default:
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.UNIT);
}
}
public bool TryReadToken(out Token<RegexElementLabel> t)
{
if (IsEmpty)
{
t = Token<RegexElementLabel>.Empty;
return false;
}
t = ReadToken();
return Token<RegexElementLabel>.Empty != t;
}
private Token<RegexElementLabel> ReadGroupBeginToken()
{
if (IsEmpty) {
throw new InvalidOperationException("Source is null.");
}
int start = position;
if (start >= _source.Length || _source[start] != '(') {
throw new InvalidOperationException("ReadGroupBeginToken called when current char is not '('.");
}
// Case 1: plain capturing group "("
if (start + 1 >= _source.Length || _source[start + 1] != '?')
{
position = start + 1;
return new Token<RegexElementLabel>("(", 0, start, RegexElementLabel.GROUP_BEGIN);
}
// From here we have "(?"
if (start + 2 >= _source.Length)
{
throw new InvalidOperationException("Unfinished group prefix '(?'.");
}
char c2 = _source[start + 2];
// Case 2: non-capturing group "(?:"
if (c2 == ':')
{
position = start + 3;
return new Token<RegexElementLabel>("(?:", 0, start, RegexElementLabel.GROUP_BEGIN);
}
// Assertions (write the first-level cases, but not supported yet)
if (c2 == '=')
{
// "(?="
position = start + 3;
throw new NotSupportedException("Positive lookahead '(?=...)' is not supported yet.");
}
if (c2 == '!')
{
// "(?!"
position = start + 3;
throw new NotSupportedException("Negative lookahead '(?!...)' is not supported yet.");
}
// Named capturing group or lookbehind starts with "(?<"
if (c2 == '<')
{
if (start + 3 >= _source.Length) {
throw new InvalidOperationException("Unfinished group prefix '(?<'.");
}
char c3 = _source[start + 3];
// Look behind prefixes: "(?<=" and "(?<!"
if (c3 == '=')
{
position = start + 4;
throw new NotSupportedException("Positive look behind '(?<=...)' is not supported yet.");
}
if (c3 == '!')
{
position = start + 4;
throw new NotSupportedException("Negative look behind '(?<!...)' is not supported yet.");
}
// Otherwise: "(?<name>"
int nameStart = start + 3;
// Read name until '>'
int i = nameStart;
// Name must start with [A-Za-z_]
if (i >= _source.Length)
{
throw new InvalidOperationException("Unfinished named group prefix.");
}
char first = _source[i];
if (!(first == '_' || char.IsLetter(first))) {
throw new InvalidOperationException($"Invalid named group name start '{first}' at position {i}.");
}
i++;
while (i < _source.Length)
{
char ch = _source[i];
if (ch == '>')
{
break;
}
if (!(ch == '_' || char.IsLetterOrDigit(ch))) {
throw new InvalidOperationException($"Invalid character '{ch}' in group name at position {i}.");
}
i++;
}
if (i >= _source.Length || _source[i] != '>') {
throw new InvalidOperationException("Unclosed named group prefix; missing '>'.");
}
// Consume "(?<name>"
int consumedLen = i - start + 1; // include '>'
string text = _source.Substring(start, consumedLen);
position = start + consumedLen;
return new Token<RegexElementLabel>(text, 0, start, RegexElementLabel.GROUP_BEGIN);
}
// Optional: comments "(?#...)" or inline options "(?i:...)" etc.
// First-level cases listed but not supported yet:
if (c2 == '#')
{
position = start + 3;
throw new NotSupportedException("Comment group '(?#...)' is not supported yet.");
}
// Inline options: (?i:...), (?m:...), (?-i:...), (?im), etc.
// Too many dialect differences; reserve the prefix but mark unsupported.
if (char.IsLetter(c2) || c2 == '-')
{
// We don't know how long the option spec is here; do NOT advance position too much.
// Keep position at '(' so caller can decide how to handle, or just throw.
position = start;
throw new NotSupportedException("Inline option groups like '(?i:...)' are not supported yet.");
}
// Unknown "(?X" prefix
position = start;
throw new InvalidOperationException($"Unknown or unsupported group prefix starting at position {start}: '{_source.Substring(start, Math.Min(4, _source.Length - start))}'");
}
}
}
src/Nitload.RegularAnalysis/Text/TextRegex.cs
+228
View File
@@ -0,0 +1,228 @@
// -----------------------------------------------------------------------------
// file="TextRegex"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Frozen;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
public abstract class TextRegex
{
private const int WordCategoriesMask =
(1 << (int)UnicodeCategory.UppercaseLetter) | (1 << (int)UnicodeCategory.LowercaseLetter) | (1 << (int)UnicodeCategory.TitlecaseLetter) | 1 << (int)UnicodeCategory.ModifierLetter | 1 << (int)UnicodeCategory.OtherLetter | 1 << (int)UnicodeCategory.NonSpacingMark | 1 << (int)UnicodeCategory.DecimalDigitNumber | 1 << (int)UnicodeCategory.ConnectorPunctuation;
public abstract MatchResult Match(
ReadOnlySpan<char> text
);
#region is char a word
/*
* Semantic reference:
*
* .NET regular-expression word character: \w
*
* A character is treated as a word character when its Unicode general
* category is one of:
*
* Lu UppercaseLetter
* Ll LowercaseLetter
* Lt TitlecaseLetter
* Lm ModifierLetter
* Lo OtherLetter
* Mn NonSpacingMark
* Nd DecimalDigitNumber
* Pc ConnectorPunctuation
*
* The behavior and the ASCII-fast-path design are based on:
*
* dotnet/runtime
* System.Text.RegularExpressions.RegexCharClass.IsWordChar
*
* Source:
* https://github.com/dotnet/runtime/blob/main/src/libraries/
* System.Text.RegularExpressions/src/System/Text/RegularExpressions/
* RegexCharClass.cs
*
* dotnet/runtime is licensed under the MIT License.
*
* This implementation is written for NGRA rather than copied verbatim,
* but the source is recorded here because its intended semantics are
* explicitly compatible with .NET Regex.
*/
/// <summary>
/// Bit mask containing every Unicode general category accepted by the default .NET
/// regular-expression <c> \w </c> character class.
/// </summary>
/// <remarks>
/// <see cref="UnicodeCategory" /> values are small integer identifiers, so each category
/// can be represented by one bit.
/// </remarks>
private const uint WordCategoryMask =
(1u << (int)UnicodeCategory.UppercaseLetter) | (1u << (int)UnicodeCategory.LowercaseLetter) | (1u << (int)UnicodeCategory.TitlecaseLetter) | (1u << (int)UnicodeCategory.ModifierLetter) | (1u << (int)UnicodeCategory.OtherLetter) | (1u << (int)UnicodeCategory.NonSpacingMark) | (1u << (int)UnicodeCategory.DecimalDigitNumber) | (1u << (int)UnicodeCategory.ConnectorPunctuation);
/// <summary>
/// Determines whether one UTF-16 code unit belongs to the default .NET regular-expression
/// <c> \w </c> character class.
/// </summary>
/// <param name="value">
/// The UTF-16 code unit to classify.
/// </param>
/// <returns>
/// <see langword="true" /> when <paramref name="value" /> is a word character; otherwise,
/// <see langword="false" />.
/// </returns>
/// <remarks>
/// <para>/// This method operates on a single <see cref="char" />, and therefore /// on a
/// single UTF-16 code unit rather than on a complete Unicode scalar /// value. ///
/// </para> <para>/// Characters in the Unicode Basic Multilingual Plane are classified
/// /// normally. A supplementary-plane character is represented by a /// surrogate pair
/// and is therefore presented to this method as two /// separate surrogate code units.
/// Neither surrogate is independently /// treated as a word character. This follows the
/// char-based behavior of /// the corresponding .NET Regex helper. /// </para> <para>///
/// For ASCII input, the Unicode-category lookup is avoided. In ASCII, /// the accepted
/// set is exactly: /// <c> [A-Za-z0-9_] </c>. /// </para> <para>/// This method
/// implements ordinary .NET <c> \w </c> semantics. It does /// not implement the ASCII-
/// oriented behavior enabled by /// <c> RegexOptions.ECMAScript </c>. /// </para>
/// </remarks>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected internal static bool IsCharWord(
char value
)
{
/*
* ASCII fast path.
*
* Unsigned subtraction combines the lower-bound and upper-bound
* checks into one comparison:
*
* value in ['A', 'Z']
* => value - 'A' is in [0, 25]
*
* Values below 'A' underflow after conversion to uint and therefore
* also fail the comparison.
*/
if (value <= '\x7F')
{
return
(uint)(value - 'A') <= 'Z' - 'A' ||
(uint)(value - 'a') <= 'z' - 'a' ||
(uint)(value - '0') <= '9' - '0' ||
value == '_';
}
/*
* Non-ASCII path.
*
* CharUnicodeInfo classifies the current UTF-16 code unit using
* Unicode general categories. The resulting category selects one
* bit from WordCategoryMask.
*/
UnicodeCategory category =
CharUnicodeInfo.GetUnicodeCategory(value);
return
(WordCategoryMask & (1u << (int)category)) != 0;
}
#endregion
}
internal readonly struct CaptureRange
{
private readonly uint _encodedStart;
public CaptureRange(
int start,
int length
)
{
ArgumentOutOfRangeException.ThrowIfNegative(start);
ArgumentOutOfRangeException.ThrowIfNegative(length);
_encodedStart = (uint)start + 1u;
Length = length;
}
public bool Success => _encodedStart != 0;
public int Length { get; }
public int Start => unchecked((int)(_encodedStart - 1u));
}
public readonly ref struct MatchResult
{
private readonly CaptureRange[]? _groupRanges;
private readonly FrozenDictionary<string, int>? _namedGroups;
private readonly ReadOnlySpan<char> _text;
internal MatchResult(
ReadOnlySpan<char> text,
FrozenDictionary<string, int> namedGroups,
CaptureRange[] groupRanges
)
{
ArgumentNullException.ThrowIfNull(namedGroups);
ArgumentNullException.ThrowIfNull(groupRanges);
_text = text;
_namedGroups = namedGroups;
_groupRanges = groupRanges;
}
public ReadOnlySpan<char> this[
string name
]
{
get
{
ArgumentNullException.ThrowIfNull(name);
if (_namedGroups is null || !_namedGroups.TryGetValue(name, out int index)) {
throw new KeyNotFoundException($"The match result does not define a group named '{name}'.");
}
CaptureRange range = _groupRanges![index];
return range.Success ? _text.Slice(range.Start, range.Length) : default;
}
}
public bool TryGetGroup(
string name,
out ReadOnlySpan<char> value
)
{
ArgumentNullException.ThrowIfNull(name);
if (_namedGroups is not null && _namedGroups.TryGetValue(name, out int index))
{
CaptureRange range = _groupRanges![index];
if (range.Success)
{
value = _text.Slice(range.Start, range.Length);
return true;
}
}
value = default;
return false;
}
}
}
src/Nitload.RegularAnalysis/Text/TextRegexAttribute.cs
+25
View File
@@ -0,0 +1,25 @@
// -----------------------------------------------------------------------------
// file="TextRegexAttribute"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public sealed class TextRegexAttribute(
string pattern
) : Attribute
{
public string Pattern { get; } = pattern;
}
}
src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.Diagnostic.cs
+63
View File
@@ -0,0 +1,63 @@
// -----------------------------------------------------------------------------
// file="TextRegexSourceGen.Diagnostic"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Microsoft.CodeAnalysis;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
internal partial class TextRegexSourceGen
{
private static readonly DiagnosticDescriptor CanNotBeRecordDiag = new(id:"NGRA004",
title:$"Containing type cannot be record",
messageFormat:$"Containing type: {{0}} cannot be record",
defaultSeverity:DiagnosticSeverity.Error,
category:"NGRA.TextRegex",
isEnabledByDefault:true);
private static readonly DiagnosticDescriptor CompileFailDiag = new(id:"NGRA006",
title:"Regular expression compilation failed",
messageFormat:"Failed to compile expression \"{0}\": {1}",
defaultSeverity:DiagnosticSeverity.Error,
category:"NGRA.TextRegex",
isEnabledByDefault:true);
private static readonly DiagnosticDescriptor IncorrectContainingTypeDiag = new(id:"NGRA005",
title:$"Containing type can only be struct or class",
messageFormat:$"Containing type: {{0}} is a {{1}}, but not class or struct",
defaultSeverity:DiagnosticSeverity.Error,
category:"NGRA.TextRegex",
isEnabledByDefault:true);
private static readonly DiagnosticDescriptor IncorrectReturnTypeDiag = new(id:"NGRA003",
title:$"Returning type must be {typeof(TextRegex).FullName}",
messageFormat:$"Method {{0}} must return {typeof(TextRegex).FullName}",
defaultSeverity:DiagnosticSeverity.Error,
category:"NGRA.TextRegex",
isEnabledByDefault:true);
private static readonly DiagnosticDescriptor NotParameterLessDiag = new(id:"NGRA002",
title:"Text regex binding must not have any parameter",
messageFormat:"Method {0} must not have any parameter",
defaultSeverity:DiagnosticSeverity.Error,
category:"NGRA.TextRegex",
isEnabledByDefault:true);
private static readonly DiagnosticDescriptor NotPartialDiag = new(id:"NGRA001",
title:"Text regex binding must be partial",
messageFormat:"Method {0} must be partial to receive generated code",
defaultSeverity:DiagnosticSeverity.Error,
category:"NGRA.TextRegex",
isEnabledByDefault:true);
}
}
src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs
+698
View File
@@ -0,0 +1,698 @@
// -----------------------------------------------------------------------------
// file="TextRegexSourceGen.GenClass"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Microsoft.CodeAnalysis.CSharp;
using Nitload.RegularAnalysis.Abstraction;
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Reflection.Metadata.Ecma335;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace Nitload.RegularAnalysis.Text
{
internal partial class TextRegexSourceGen
{
internal static void GenerateMethodSource(
RegexAstNode ast,
in SequenceEmitter<string> sc,
in List<SequenceEmitter<string>> func_list,
in SourceGenContext context,
bool is_find = false
)
{
RegexAstNode root = ast;
if (root.Label == RegexElementLabel.EPSILON) {
return;
}
RegexElementLabel op = root.Label & (RegexElementLabel.VAST_OPERATOR);
if (op != 0)
{
switch (op)
{
case RegexElementLabel.CONCAT:
GenerateConcat(root, sc, func_list, context);
break;
case RegexElementLabel.COUNT:
GenerateCount(root, sc, func_list, context);
break;
case RegexElementLabel.GROUP_BEGIN:
GenerateGroup(root, sc, func_list, context);
break;
case RegexElementLabel.ALT:
GenerateAlt(root, sc, func_list, context);
break;
default:
// TODO(NGRA): Report an unsupported operator instead of silently emitting
// an incomplete matcher.
break;
}
} else
{
op = root.Label & (RegexElementLabel.SEQUENCE);
switch (op)
{
case RegexElementLabel.UNIT:
// TODO(NGRA): Emit the remaining-span length guard, literal comparison,
// cursor advance, and failure transfer for one character.
break;
case RegexElementLabel.UNIT_SET:
// TODO(NGRA): Emit the remaining-span length guard, charset predicate,
// cursor advance, and failure transfer.
break;
case RegexElementLabel.UNIT_SPAN:
// TODO(NGRA): Emit StartsWith for the complete literal span and advance by
// the literal length; integrate is_find candidate retry semantics.
break;
default:
// TODO(NGRA): Diagnose malformed leaf nodes instead of generating no code.
break;
}
}
}
#region supporting method
private static string FormatStringLiteral(
string value
)
{
return SyntaxFactory.Literal(value).Text;
}
private static string FormatStringLiteral(
ReadOnlySpan<char> value
)
{
return SyntaxFactory.Literal(value.ToString()).Text;
}
private static string FormatCharLiteral(
char value
)
{
return SyntaxFactory.Literal(value).Text;
}
private static string GenerateCharSetExpression(
RegexCharSet set
)
{
string remain_span = SourceGenContext._remain_span;
string accept = set.Accept ? string.Empty : "!";
switch (set.Type)
{
case RegexCharSetType.Single:
return $"{accept}({FormatStringLiteral(set.Value)}.{nameof(MemoryExtensions.Contains)}({remain_span}[0]))";
case RegexCharSetType.Range:
return $"{accept}({remain_span}[0] >= {FormatCharLiteral(set.Value[0])} && {remain_span}[0] <= {FormatCharLiteral(set.Value[1])})";
case RegexCharSetType.Set:
switch (set.Value[0])
{
case 's':
return $"{accept}char.IsWhiteSpace({remain_span}[0])";
case 'w':
return $"{accept}{nameof(TextRegex.IsCharWord)}({remain_span}[0])";
case 'd':
return $"{accept}char.IsDigit({remain_span}[0])";
default:
break;
}
goto default;
default:
// TODO(NGRA): Unsupported/empty charset rules must become an analyzer
// diagnostic; returning an empty expression produces invalid generated C#.
return string.Empty;
}
}
private static bool IsNamedFind(
RegexAstNode node
)
{
// (?<name>.*)abc
if (node is null)
{
return false;
}
if ((node.Label & RegexElementLabel.CONCAT) != 0) {
return false;
}
RegexAstNode group = node.Left;
// TODO(NGRA): Validate the normalized CONCAT/GROUP/COUNT shape with Empty sentinels
// before dereferencing children; this helper currently assumes every edge exists.
if ((group.Label & RegexElementLabel.GROUP_BEGIN) == 0)
{
return false;
}
RegexAstNode count = group.Right;
// .*
return (count.Label & RegexElementLabel.COUNT) != 0 &&
(count.Right.Label & RegexElementLabel.UNIT_SINGLE) != 0 &&
count.Right.Source.Source == ".";
}
#endregion
#region basic verb generate
private static void GenerateAlt(
RegexAstNode node,
[NotNull] in SequenceEmitter<string> sc,
in List<SequenceEmitter<string>> func_list,
in SourceGenContext context
)
{
RegexAstNode current = node.Right;
string exit_label = context.RequestExitLabel();
// TODO(NGRA): Walk every normalized ALT candidate exactly once. The final traversal
// shape and failure continuation are not wired yet.
while (RegexAstNode.IsNullOrEmpty(current))
{
RegexElementLabel op = current.Label & RegexElementLabel.SEQUENCE;
if (op != 0)
{
string code = current.SourceCode;
switch (op)
{
case RegexElementLabel.UNIT:
// TODO(NGRA): Guard against an empty remaining span before indexing it.
_ = sc.AppendLine(@$"if({context.RemainSpan}[0] == {FormatCharLiteral(code[0])})")
.AppendLine("{")
.AppendLine($@"{context.RemainSpan} = {context.RemainSpan}.Slice(1);")
.AppendLine($@"goto {exit_label};")
.AppendLine("}");
break;
case RegexElementLabel.UNIT_SET:
// TODO(NGRA): Guard the remaining span, handle empty/Any sets, and
// validate negated-set composition before emitting this predicate.
RegexCharSetPayload payload = current.PayloadAs<RegexCharSetPayload>();
IReadOnlyList<RegexCharSet> split = payload.Set;
bool accept = payload.Accept;
RegexCharSet _set = split[0];
_ = sc.AppendLine($"if({(accept ? string.Empty : "!(")} {GenerateCharSetExpression(_set)} ||");
for (int i = 1; i < split.Count - 1; i++)
{
_set = split[i];
_ = sc.AppendLine($" {GenerateCharSetExpression(_set)} ||");
}
_set = split[^1];
_ = sc.AppendLine($" {GenerateCharSetExpression(_set)} ){(accept ? string.Empty : ')')}")
.AppendLine("{")
.AppendLine($@"{context.RemainSpan} = {context.RemainSpan}.Slice(1);")
.AppendLine($@"goto {exit_label};")
.AppendLine("}");
break;
case RegexElementLabel.UNIT_SPAN:
// TODO(NGRA): Advance by code.Length rather than one character and
// preserve the ALT rollback cursor on failure.
_ = sc.AppendLine(@$"if({context.RemainSpan}.{MemoryExtensions.StartsWith}({FormatStringLiteral(code)}))")
.AppendLine("{")
.AppendLine($@"{context.RemainSpan} = {context.RemainSpan}.Slice(1);")
.AppendLine($@"goto {exit_label};")
.AppendLine("}");
break;
default:
// TODO(NGRA): Route unsupported ALT candidates through the recursive
// matcher or report a source-generation diagnostic.
break;
}
// TODO(NGRA): Advance to the next ALT candidate after a scalar candidate
// misses; the current sequence branch does not update current.
} else
{
GenerateMethodSource(current, sc, func_list, context);
current = current.Next;
}
}
// TODO(NGRA): Emit the all-alternatives-failed continuation before the shared success
// label, including restoration of the entry cursor and capture state.
_ = sc.AppendLine($@"{exit_label}: {{ }}");
}
private static void GenerateGroup(
RegexAstNode node,
[NotNull] in SequenceEmitter<string> sc,
in List<SequenceEmitter<string>> func_list,
in SourceGenContext context
)
{
RegexAstNode left = node.Left;
RegexAstNode right = node.Right;
RegexGroupPayload config = node.PayloadAs<RegexGroupPayload>();
// TODO(NGRA): Select the owned group body using Empty-sentinel checks, snapshot its
// start cursor, and restore it when the child matcher fails.
if (left is not null)
{
GenerateMethodSource(left, sc, func_list, context);
} else
if (right is not null) {
GenerateMethodSource(right, sc, func_list, context);
}
string group_name = config.Name;
// TODO(NGRA): Allocate numeric/named capture slots and emit the successful capture
// span. Non-capturing groups still need a rollback scope but no stored capture.
throw new NotImplementedException();
}
private static void GenerateConcat(
RegexAstNode node,
[NotNull] in SequenceEmitter<string> sc,
in List<SequenceEmitter<string>> func_list,
in SourceGenContext context
)
{
RegexAstNode left = node.Left;
RegexAstNode right = node.Right;
if (left is not null)
{
// find
if ((left.Label & RegexElementLabel.COUNT) == RegexElementLabel.COUNT &&
(left.Right.Label & RegexElementLabel.UNIT_SINGLE) == RegexElementLabel.UNIT_SINGLE &&
left.Right.Source.Source == "." &&
right is not null)
{
// TODO(NGRA): Emit the unnamed FIND candidate loop and retry the right subtree
// until it succeeds or no anchor candidate remains.
GenerateMethodSource(right, sc, func_list, context, true);
} else if (IsNamedFind(node))
{
if (right is not null)
{
// TODO(NGRA): Emit named FIND retry and capture the text preceding the
// successful anchor candidate.
GenerateMethodSource(right, sc, func_list, context, true);
} else
{
// TODO(NGRA): Capture the remaining input for a terminal named dot-star.
}
} else
{
GenerateMethodSource(left, sc, func_list, context);
if (right is not null) {
GenerateMethodSource(right, sc, func_list, context);
}
}
}
// TODO(NGRA): Normalize ownership so the right subtree is emitted exactly once. It is
// currently reachable both inside the ordinary CONCAT branch and here.
if (right is not null)
{
GenerateMethodSource(right, sc, func_list, context);
}
}
private const int flattened_unit_count_threshold = 32;
private static void GenerateUnitCount(
RegexAstNode value,
int min,
int max,
bool is_greedy,
[NotNull] in SequenceEmitter<string> sc,
in SourceGenContext context
)
{
if (min < 0 || (max >= 0 && max < min)) {
throw new InvalidOperationException($"Invalid COUNT range: {min}..{max}.");
}
string literal = value.Value;
if (literal.Length != 1) {
throw new InvalidOperationException(
$"COUNT(UNIT) requires one decoded character, but received {literal.Length}.");
}
char expected = literal[0];
string expected_literal = FormatCharLiteral(expected);
string remain_span = context.RemainSpan;
int scratch_scope_index = context.RentScratchScopeIndex();
string required_rest = $"__required_rest_{scratch_scope_index}";
string required_matched = $"__required_matched_{scratch_scope_index}";
string required_chunk_index = $"__required_chunk_index_{scratch_scope_index}";
string additional_count = $"__additional_count_{scratch_scope_index}";
// TODO(NGRA): Move ownership of this work region to the enclosing CONCAT or match
// method. Until then, break demonstrates the local failure path but cannot prevent a
// later sibling node from being emitted and executed.
_ = sc.AppendLine("do").AppendLine("{");
if (min <= flattened_unit_count_threshold)
{
if (min > 0)
{
string required_prefix = new(expected, min);
_ = sc.AppendLine($"if (!{remain_span}.{nameof(MemoryExtensions.StartsWith)}({FormatStringLiteral(required_prefix)}))")
.AppendLine("{")
.AppendLine("break;")
.AppendLine("}")
.AppendLine($"{remain_span} = {remain_span}.Slice({min});");
}
} else
{
// Keep large lower bounds vectorizable without emitting a giant literal. Validate
// repeated threshold-sized chunks against a local span and commit the cursor only
// after every chunk has matched.
string required_chunk = new(expected, flattened_unit_count_threshold);
int required_chunk_count = min / flattened_unit_count_threshold;
int required_remainder_length = min % flattened_unit_count_threshold;
_ = sc.AppendLine($"if ({remain_span}.Length < {min})")
.AppendLine("{")
.AppendLine("break;")
.AppendLine("}")
.AppendLine($"{ROS_char} {required_rest} = {remain_span};")
.AppendLine($"bool {required_matched} = true;")
.AppendLine($"for (int {required_chunk_index} = 0; {required_chunk_index} < {required_chunk_count}; {required_chunk_index}++)")
.AppendLine("{")
.AppendLine($"if (!{required_rest}.{nameof(MemoryExtensions.StartsWith)}({FormatStringLiteral(required_chunk)}))")
.AppendLine("{")
.AppendLine($"{required_matched} = false;")
.AppendLine("break;")
.AppendLine("}")
.AppendLine($"{required_rest} = {required_rest}.Slice({flattened_unit_count_threshold});")
.AppendLine("}");
if (required_remainder_length > 0)
{
string required_remainder = new(expected, required_remainder_length);
_ = sc.AppendLine($"if ({required_matched} && !{required_rest}.{nameof(MemoryExtensions.StartsWith)}({FormatStringLiteral(required_remainder)}))")
.AppendLine("{")
.AppendLine($"{required_matched} = false;")
.AppendLine("}");
}
_ = sc.AppendLine($"if (!{required_matched})")
.AppendLine("{")
.AppendLine("break;")
.AppendLine("}")
.AppendLine($"{remain_span} = {remain_span}.Slice({min});");
}
int additional_limit = max < 0 ? -1 : max - min;
if (is_greedy && additional_limit != 0)
{
if (additional_limit < 0)
{
_ = sc.AppendLine($"while (!{remain_span}.IsEmpty && {remain_span}[0] == {expected_literal})")
.AppendLine("{")
.AppendLine($"{remain_span} = {remain_span}.Slice(1);")
.AppendLine("}");
} else
{
_ = sc.AppendLine($"for (int {additional_count} = 0; {additional_count} < {additional_limit} && !{remain_span}.IsEmpty && {remain_span}[0] == {expected_literal}; {additional_count}++)")
.AppendLine("{")
.AppendLine($"{remain_span} = {remain_span}.Slice(1);")
.AppendLine("}");
}
} else if (!is_greedy && additional_limit != 0)
{
// Lazy COUNT deliberately stops at Min. Its optional tail is consumed only when
// a future continuation attempt fails and requests another repetition.
_ = sc.AppendLine("// TODO(NGRA): Retry the lazy COUNT tail from its continuation.");
}
_ = sc.AppendLine("} while (false);");
context.ReturnScratchScopeIndex(scratch_scope_index);
}
private static void GenerateUnitSetCount(
RegexAstNode value,
int min,
int max,
bool is_greedy,
[NotNull] in SequenceEmitter<string> sc,
in SourceGenContext context
)
{
if (min < 0 || (max >= 0 && max < min)) {
throw new InvalidOperationException($"Invalid COUNT range: {min}..{max}.");
}
string literal = value.Value;
if (literal.Length != 1) {
throw new InvalidOperationException(
$"COUNT(UNIT) requires one decoded character, but received {literal.Length}.");
}
RegexCharSetPayload payload = value.PayloadAs<RegexCharSetPayload>();
IReadOnlyList<RegexCharSet> charset = payload.Set;
string remain_span = context.RemainSpan;
int scratch_scope_index = context.RentScratchScopeIndex();
string required_rest = $"__required_rest_{scratch_scope_index}";
string required_matched = $"__required_matched_{scratch_scope_index}";
string required_chunk_index = $"__required_chunk_index_{scratch_scope_index}";
string additional_count = $"__additional_count_{scratch_scope_index}";
// TODO(NGRA): Move ownership of this work region to the enclosing CONCAT or match
// method. Until then, break demonstrates the local failure path but cannot prevent a
// later sibling node from being emitted and executed.
bool accept = payload.Accept;
RegexCharSet _set = charset[0];
_ = sc.AppendLine($"{ROS_char} {required_matched} = {remain_span}")
.AppendLine($"int {additional_count} = 0;")
.AppendLine()
.AppendLine() //TODO loop head
.AppendLine($"if({(accept ? string.Empty : "!(")} {GenerateCharSetExpression(_set)} ||");
for (int i = 1; i < charset.Count - 1; i++)
{
_set = charset[i];
_ = sc.AppendLine($" {GenerateCharSetExpression(_set)} ||");
}
_set = charset[^1];
_ = sc.AppendLine($" {GenerateCharSetExpression(_set)} ){(accept ? string.Empty : ')')}")
.AppendLine("{")
.AppendLine($@"{context.RemainSpan} = {context.RemainSpan}.Slice(1);")
.AppendLine($@"break;")
.AppendLine("}");
}
private static void GenerateUnitSpanCount(
RegexAstNode value,
int min,
int max,
bool is_greedy,
[NotNull] in SequenceEmitter<string> sc,
in SourceGenContext context
)
{
if (min < 0 || (max >= 0 && max < min)) {
throw new InvalidOperationException($"Invalid COUNT range: {min}..{max}.");
}
string literal = value.Value;
if (literal.Length == 0) {
throw new InvalidOperationException(
"COUNT(UNIT_SPAN) requires a non-empty literal.");
}
string expected_literal = FormatStringLiteral(literal);
string remain_span = context.RemainSpan;
int scratch_scope_index = context.RentScratchScopeIndex();
string required_rest = $"__required_rest_{scratch_scope_index}";
string required_matched = $"__required_matched_{scratch_scope_index}";
string required_chunk_index = $"__required_chunk_index_{scratch_scope_index}";
string required_count = $"__required_count_{scratch_scope_index}";
string additional_count = $"__additional_count_{scratch_scope_index}";
string starts_with = $"{nameof(MemoryExtensions.StartsWith)}";
// TODO(NGRA): Move ownership of this work region to the enclosing CONCAT or match
// method. Until then, break demonstrates the local failure path but cannot prevent a
// later sibling node from being emitted and executed.
_ = sc.AppendLine("do").AppendLine("{");
if (literal.Length > flattened_unit_count_threshold)
{
// One repetition is already larger than the flattening threshold. Match the
// original literal directly and keep all cursor mutations local until Min has
// succeeded; the common tail below remains responsible for Min..Max.
_ = sc.AppendLine($"int {required_count} = 0;")
.AppendLine($"{ROS_char} {required_rest} = {remain_span};")
.AppendLine($"for (; {required_count} < {min} && {required_rest}.{starts_with}({expected_literal}); {required_count}++)")
.AppendLine("{")
.AppendLine($"{required_rest} = {required_rest}.Slice({literal.Length});")
.AppendLine("}")
.AppendLine($"if ({required_count} != {min})")
.AppendLine("{")
.AppendLine("break;")
.AppendLine("}")
.AppendLine($"{remain_span} = {required_rest};");
} else if (min <= flattened_unit_count_threshold / literal.Length)
{
if (min > 0)
{
Span<char> required_prefix_span = stackalloc char[literal.Length * min];
for (int i = 0; i < min; i++) {
literal.AsSpan().CopyTo(required_prefix_span[(i * literal.Length)..]);
}
string required_prefix = required_prefix_span.ToString();
_ = sc.AppendLine($"if (!{remain_span}.{starts_with}({FormatStringLiteral(required_prefix)}))")
.AppendLine("{")
.AppendLine("break;")
.AppendLine("}")
.AppendLine($"{remain_span} = {remain_span}.Slice({min * literal.Length});");
}
} else
{
// Keep large lower bounds vectorizable without emitting a giant literal. Validate
// repeated threshold-sized chunks against a local span and commit the cursor only
// after every chunk has matched.
int count = flattened_unit_count_threshold / literal.Length;
Span<char> required_prefix_span = stackalloc char[literal.Length * count];
for (int i = 0; i < count; i++) {
literal.AsSpan().CopyTo(required_prefix_span[(i * literal.Length)..]);
}
string required_chunk = required_prefix_span.ToString();
int required_chunk_count = min / count;
int required_remainder_length = min % count;
_ = sc.AppendLine($"{ROS_char} {required_rest} = {remain_span};")
.AppendLine($"bool {required_matched} = true;")
.AppendLine($"for (int {required_chunk_index} = 0; {required_chunk_index} < {required_chunk_count}; {required_chunk_index}++)")
.AppendLine("{")
.AppendLine($"if (!{required_rest}.{starts_with}({FormatStringLiteral(required_chunk)}))")
.AppendLine("{")
.AppendLine($"{required_matched} = false;")
.AppendLine("break;")
.AppendLine("}")
.AppendLine($"{required_rest} = {required_rest}.Slice({count * literal.Length});")
.AppendLine("}");
if (required_remainder_length > 0)
{
Span<char> required_remainder_span = stackalloc char[literal.Length * required_remainder_length];
for (int i = 0; i < required_remainder_length; i++) {
literal.AsSpan().CopyTo(required_remainder_span[(i * literal.Length)..]);
}
string required_remainder = required_remainder_span.ToString();
_ = sc.AppendLine($"if ({required_matched} && !{required_rest}.{starts_with}({FormatStringLiteral(required_remainder)}))")
.AppendLine("{")
.AppendLine($"{required_matched} = false;")
.AppendLine("}")
.AppendLine($"if ({required_matched})")
.AppendLine("{")
.AppendLine($"{required_rest} = {required_rest}.Slice({literal.Length * required_remainder_length});")
.AppendLine("}");
}
_ = sc.AppendLine($"if (!{required_matched})")
.AppendLine("{")
.AppendLine("break;")
.AppendLine("}")
.AppendLine($"{remain_span} = {required_rest};");
}
int additional_limit = max < 0 ? -1 : max - min;
if (is_greedy && additional_limit != 0)
{
if (additional_limit < 0)
{
_ = sc.AppendLine($"while ({remain_span}.{starts_with}({expected_literal}))")
.AppendLine("{")
.AppendLine($"{remain_span} = {remain_span}.Slice({literal.Length});")
.AppendLine("}");
} else
{
_ = sc.AppendLine($"for (int {additional_count} = 0; {additional_count} < {additional_limit} && {remain_span}.{starts_with}({expected_literal}); {additional_count}++)")
.AppendLine("{")
.AppendLine($"{remain_span} = {remain_span}.Slice({literal.Length});")
.AppendLine("}");
}
} else if (!is_greedy && additional_limit != 0)
{
// Lazy COUNT deliberately stops at Min. Its optional tail is consumed only when
// a future continuation attempt fails and requests another repetition.
_ = sc.AppendLine("// TODO(NGRA): Retry the lazy COUNT tail from its continuation.");
}
_ = sc.AppendLine("} while (false);");
context.ReturnScratchScopeIndex(scratch_scope_index);
}
private static void GenerateCount(
RegexAstNode node,
[NotNull] in SequenceEmitter<string> sc,
in List<SequenceEmitter<string>> func_list,
in SourceGenContext context
)
{
RegexAstNode value = RegexAstNode.IsNullOrEmpty(node.Left) ? node.Right : node.Left;
if (RegexAstNode.IsNullOrEmpty(value)) {
throw new InvalidOperationException("COUNT requires an operand.");
}
RegexCountPayload payload = node.PayloadAs<RegexCountPayload>();
int min = payload.Min;
int max = payload.Max;
bool is_greedy = payload.IsGreedy;
string exit_label = context.RequestExitLabel();
if ((value.Label & RegexElementLabel.SEQUENCE) == 0)
{
// TODO(NGRA): Emit min/max repetition, zero-width progress protection, greedy vs.
// lazy ordering, and per-attempt rollback for a complex COUNT body.
GenerateMethodSource(value, sc, func_list, context);
} else
{
switch (value.Label & RegexElementLabel.SEQUENCE)
{
case RegexElementLabel.UNIT:
GenerateUnitCount(value, min, max, is_greedy, sc, context);
break;
case RegexElementLabel.UNIT_SET:
// TODO(NGRA): Emit a bounded/unbounded charset counting loop, preferably
// using IndexOfAnyExcept where the charset shape permits it.
break;
case RegexElementLabel.UNIT_SPAN:
GenerateUnitSpanCount(value, min, max, is_greedy, sc, context);
break;
default:
// TODO(NGRA): Diagnose malformed COUNT children instead of emitting an
// empty branch.
break;
}
}
// TODO(NGRA): Connect COUNT success/failure continuations to the surrounding rollback
// scope; the label is currently only a placeholder.
_ = sc.AppendLine($@"{exit_label}: {{ }}");
}
#endregion
}
}
src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenContext.cs
+93
View File
@@ -0,0 +1,93 @@
// -----------------------------------------------------------------------------
// file="TextRegexSourceGen.GenContext"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Microsoft.CodeAnalysis.Text;
using Nitload.Common.PipeLine;
using Nitload.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
internal partial class TextRegexSourceGen
{
/// <summary>
/// Tracks variable naming and declaration demand while generating one match method.
/// </summary>
/// <remarks>
/// Method level state is limited to the active cursor and the result of the most recent
/// instruction. Everything else is either a region slot, a line slot, a capture slot, or
/// a block scoped scratch variable. Scratch indices remain unique while rented and are
/// recycled after their generated lexical scope has closed.
/// </remarks>
internal sealed class SourceGenContext
{
public const string __input_span = "input";
public const string _remain_span = "__remain";
private int _exit_alt_count;
private int _scratch_scope_count;
private readonly Stack<int> _free_scratch_scope_indices = [];
public string RemainSpan { get; } = _remain_span;
public string FirstOfRemain { get; } = $@"{_remain_span}[0]";
public string InputString { get; } = __input_span;
public SequenceEmitter<string> DeclareGlobalFields()
{
SequenceEmitter<string> sc = new();
return sc.AppendLine($@"{ROS_char} {RemainSpan};");
}
public string RequestExitLabel()
{
string code = $@"_exit_alt_{_exit_alt_count}";
_exit_alt_count++;
return code;
}
public int RentScratchScopeIndex()
{
if (_free_scratch_scope_indices.TryPop(out int recycled_index)) {
return recycled_index;
}
int index = _scratch_scope_count;
_scratch_scope_count++;
return index;
}
public void ReturnScratchScopeIndex(
int index
)
{
if ((uint)index >= (uint)_scratch_scope_count) {
throw new ArgumentOutOfRangeException(nameof(index));
}
if (_free_scratch_scope_indices.Contains(index)) {
throw new InvalidOperationException($"Scratch scope {index} was returned twice.");
}
_free_scratch_scope_indices.Push(index);
}
}
}
}
src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenFile.cs
+129
View File
@@ -0,0 +1,129 @@
// -----------------------------------------------------------------------------
// file="TextRegexSourceGen.GenFile"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Microsoft.CodeAnalysis;
using Nitload.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
internal partial class TextRegexSourceGen
{
public const string ROS_char = "global::System.ReadOnlySpan<char>";
public const string Text_Regex = "global::Nitload.RegularAnalysis.Text.TextRegex";
private static readonly SymbolDisplayFormat TypeSourceFormat =
SymbolDisplayFormat.FullyQualifiedFormat
.AddMiscellaneousOptions(
SymbolDisplayMiscellaneousOptions.IncludeNullableReferenceTypeModifier);
private static string AsSource<T>()
{
Type t = typeof(T);
return t.IsGenericType ? throw new InvalidOperationException() : $"global::{t.FullName}";
}
private static void GenerateRegexSourceFile(
SourceProductionContext context,
TextRegexBinding binding
)
{
StringBuilder file_name_sb = new();
SequenceEmitter<string> ce = new();
SourceRecord gen_source = binding.Source;
TypeSource type = binding.Source.ContainingType;
SequenceEmitter<string> method_body = binding.GeneratedSource;
SourceGenContext regex_gen_context = binding.SourceContext;
_ = file_name_sb.Append(CultureInfo.InvariantCulture, $"{gen_source.NamespaceName}.");
_ = ce.AppendLine($@"// <auto-generated/>")
.AppendLine($@"#nullable enable")
.AppendLine($@"using Nitload.RegularAnalysis.Text;")
.AppendLine($@"using Nitload.RegularAnalysis.Abstraction;")
.AppendLine($@"using System;")
.AppendLine($@"using System.Text;")
.AppendLine($@"using System.Threading.Tasks;")
.AppendLine()
.AppendLine($@"namespace {gen_source.NamespaceName};");
string static_modifier;
int brace_count = 0;
do
{
_ = file_name_sb.Append(CultureInfo.InvariantCulture, $@"{type.TypeName}.");
string declare_type = type.DeclType switch
{
TypeKind.Class => "class",
TypeKind.Struct => "struct",
_ => throw new InvalidOperationException($"Unexpected type kind: {type.DeclType}"),
};
static_modifier = type.IsStatic ? "static" : string.Empty;
_ = ce.AppendLine($@"{ToKeyword(type.Accessibility)} {static_modifier} partial {declare_type} {type.TypeName}")
.AppendLine("{");
brace_count++;
type = type.InnerType!;
} while (type is not null);
static_modifier = gen_source.IsMethodStatic ? "static" : string.Empty;
_ = ce.AppendLine(@$"{ToKeyword(gen_source.MethodAccessibility)} {static_modifier} {Text_Regex} {gen_source.MethodName}({ROS_char} {regex_gen_context.InputString})")
.AppendLine("{")
.Append(method_body)
.AppendLine("}");
for (int i = 0; i < brace_count; i++) {
_ = ce.AppendLine("}");
}
StringBuilder code_source = new();
brace_count = 0;
foreach (string line in ce)
{
bool is_right_brace = (line == "}") ||
line.StartsWith("} while", StringComparison.InvariantCulture);
brace_count -= is_right_brace ? 1 : 0;
for (int i = 0; i < brace_count; i++) {
_ = code_source.Append(" ");
}
_ = code_source.AppendLine(line);
brace_count += is_right_brace ? 0 : (line == "{" ? 1 : 0);
}
_ = file_name_sb.Append(CultureInfo.InvariantCulture, $@"{binding.Source.MethodName}.g.cs");
string file_name = file_name_sb.ToString();
context.AddSource(file_name, code_source.ToString());
}
private static string ToKeyword(
Accessibility accessibility
)
{
return accessibility switch
{
Accessibility.Public => "public",
Accessibility.Private => "private",
Accessibility.Internal => "internal",
Accessibility.Protected => "protected",
Accessibility.ProtectedAndInternal => "private protected",
Accessibility.ProtectedOrInternal => "protected internal",
_ => throw new InvalidOperationException(
$"Unexpected accessibility: {accessibility}"),
};
}
}
}
src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.Source.cs
+149
View File
@@ -0,0 +1,149 @@
// -----------------------------------------------------------------------------
// file="TextRegexSourceGen.Source"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Nitload.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.RegularAnalysis.Text
{
internal partial class TextRegexSourceGen
{
private readonly record struct SourceRecord(
bool IsClassStatic,
bool IsMethodStatic,
string Pattern,
string MethodName,
string NamespaceName,
Accessibility MethodAccessibility,
TypeSource ContainingType,
Location PatternDefineLocation
);
private sealed record TypeSource(
bool IsStatic,
string TypeDisplayName,
string TypeName,
TypeKind DeclType,
Accessibility Accessibility,
TypeSource? InnerType
);
private struct Source
{
public Accessibility MethodAccessibility { get; set; }
public bool IsClassStatic { get; set; }
public bool IsMethodStatic { get; set; }
public Location PatternDefineLocation { get; set; }
public string NameSpaceName { get; set; }
public string Pattern { get; set; }
public string MethodName { get; set; }
public TypeSource ContainingType { get; set; }
public readonly SourceRecord AsRecord()
{
return new SourceRecord(IsClassStatic, IsMethodStatic, Pattern, MethodName,
NameSpaceName, MethodAccessibility, ContainingType,
PatternDefineLocation);
}
}
private class TextRegexBinding
{
private readonly List<Diagnostic> _diagnostics = [];
private TextRegexBinding(
List<Diagnostic> diagnostics,
SequenceEmitter<string> generatedSource,
SourceRecord source,
SourceGenContext sourceContext,
TextRegexSource? analyzedSource
)
{
_diagnostics = diagnostics ?? [];
GeneratedSource = generatedSource ?? new();
Source = source;
AnalyzedSource = analyzedSource;
IsValid = !_diagnostics.Exists(static diagnostic => diagnostic.Severity == DiagnosticSeverity.Error);
SourceContext = sourceContext;
}
public TextRegexBinding()
{
_diagnostics = [];
}
public TextRegexSource? AnalyzedSource { get; set; }
public bool IsValid { get; private set; } = true;
public IReadOnlyList<Diagnostic> RegexDiagnostic => _diagnostics;
public SequenceEmitter<string> GeneratedSource { get; set; } = new();
public SourceGenContext SourceContext { get; set; } = null!;
public SourceRecord Source { get; set; }
public void AddDiagnostic(
Diagnostic diag
)
{
_diagnostics.Add(diag);
IsValid = IsValid && (diag.DefaultSeverity != DiagnosticSeverity.Error);
}
public void AddDiagnostic(
DiagnosticDescriptor descriptor,
Location? location,
params object[] messageArgs
)
{
Diagnostic diag = Diagnostic.Create(descriptor, location, messageArgs);
_diagnostics.Add(diag);
IsValid = IsValid && (diag.Severity != DiagnosticSeverity.Error);
}
public TextRegexBinding Clone()
{
SequenceEmitter<string> code = GeneratedSource.Clone();
List<Diagnostic> diag = new(_diagnostics.Count);
diag.AddRange(this._diagnostics);
TextRegexBinding binding = new(diag, code, Source with { }, SourceContext,
AnalyzedSource);
return binding;
}
public static TextRegexBinding Invalid()
{
return new TextRegexBinding
{
IsValid = false
};
}
}
}
}
src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.cs
+208
View File
@@ -0,0 +1,208 @@
// -----------------------------------------------------------------------------
// file="TextRegexSourceGen"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Nitload.RegularAnalysis.Abstraction;
using System.Collections.Generic;
using System.Threading;
namespace Nitload.RegularAnalysis.Text
{
internal partial class TextRegexSourceGen : IIncrementalGenerator
{
private static readonly string TextRegexTypeName = typeof(TextRegexAttribute).FullName ??
string.Empty;
public void Initialize(
IncrementalGeneratorInitializationContext context
)
{
IncrementalValuesProvider<TextRegexBinding> candidates =
//
context.SyntaxProvider
.ForAttributeWithMetadataName<TextRegexBinding>(
TextRegexTypeName,
static(
_,
_
) => true, BindTextRegexCandidate);
IncrementalValuesProvider<TextRegexBinding> validCandidates =
RegisterDiagnosticBypass(context, candidates);
IncrementalValuesProvider<TextRegexBinding> analyzed =
validCandidates.Select(AnalyzeRegexMethod);
IncrementalValuesProvider<TextRegexBinding> validAnalyzed =
RegisterDiagnosticBypass(context, analyzed);
IncrementalValuesProvider<TextRegexBinding> generated =
validAnalyzed.Select(CompileRegexMethodSource);
IncrementalValuesProvider<TextRegexBinding> validGenerated =
RegisterDiagnosticBypass(context, generated);
context.RegisterSourceOutput(validGenerated, GenerateRegexSourceFile);
}
private static TextRegexBinding AnalyzeRegexMethod(
TextRegexBinding binding,
CancellationToken cancellationToken
)
{
cancellationToken.ThrowIfCancellationRequested();
if (!binding.IsValid) {
return binding;
}
TextRegexBinding result = binding.Clone();
string pattern = result.Source.Pattern;
TextRegexSource analyzedSource = RegexAnalyzer.Analyze(pattern);
result.AnalyzedSource = analyzedSource;
if (analyzedSource.AnalyseException is not null) {
result.AddDiagnostic(CompileFailDiag, result.Source.PatternDefineLocation, pattern,
analyzedSource.AnalyseException.Message);
}
return result;
}
private static TextRegexBinding BindTextRegexCandidate(
GeneratorAttributeSyntaxContext context,
CancellationToken ct
)
{
ct.ThrowIfCancellationRequested();
if (context.TargetNode is not MethodDeclarationSyntax method_decl ||
context.TargetSymbol is not IMethodSymbol method_symbol) {
return TextRegexBinding.Invalid();
}
/*
*return node is MethodDeclarationSyntax method
* && method_decl.Modifiers.Any(SyntaxKind.PartialKeyword)
* && method_decl.Modifiers.Any(SyntaxKind.StaticKeyword)
* && method_decl.ParameterList.Parameters.Count == 0
* && method_decl.Body is null
* && method_decl.ExpressionBody is null
* && method_decl.SemicolonToken.IsKind(SyntaxKind.SemicolonToken);
*/
TextRegexBinding bind = new();
if (!method_decl.Modifiers.Any(SyntaxKind.PartialKeyword)) {
bind.AddDiagnostic(NotPartialDiag, method_decl.GetLocation(), method_symbol.Name);
}
if (method_decl.ParameterList.Parameters.Count != 0) {
bind.AddDiagnostic(NotParameterLessDiag, method_decl.GetLocation(), method_decl.GetText());
}
INamedTypeSymbol? expected_return_type =
context.SemanticModel.Compilation
.GetTypeByMetadataName(TextRegexTypeName
);
ITypeSymbol ret_value = method_symbol.ReturnType;
if (!SymbolEqualityComparer.Default.Equals(expected_return_type, ret_value)) {
bind.AddDiagnostic(IncorrectReturnTypeDiag, method_decl.ReturnType.GetLocation(), method_symbol.Name);
}
AttributeData attribute = context.Attributes[0];
INamedTypeSymbol contain = method_symbol.ContainingType;
INamespaceSymbol namespace_symbol = contain.ContainingNamespace;
TypeSource type_source = null!;
while (contain is not null)
{
if (contain.IsRecord) {
bind.AddDiagnostic(CanNotBeRecordDiag, method_decl.GetLocation(), contain.Name);
}
if (contain.TypeKind is not (TypeKind.Class or TypeKind.Struct)) {
bind.AddDiagnostic(IncorrectContainingTypeDiag, method_decl.GetLocation(),
method_symbol.Name, contain.TypeKind);
}
if (!bind.IsValid) {
return bind;
}
string display = contain.ToDisplayString(TypeSourceFormat);
TypeSource t = new(contain.IsStatic, display, contain.Name, contain.TypeKind,
contain.DeclaredAccessibility, type_source);
type_source = t;
contain = contain.ContainingType!;
}
Source s = new()
{
MethodName = method_symbol.Name,
NameSpaceName = namespace_symbol.ToDisplayString(),
PatternDefineLocation = method_decl.GetLocation(),
IsMethodStatic = method_decl.Modifiers.Any(SyntaxKind.StaticKeyword),
ContainingType = type_source,
MethodAccessibility = method_symbol.DeclaredAccessibility,
};
foreach (KeyValuePair<string, TypedConstant> item in attribute.NamedArguments)
{
if (item.Key == nameof(TextRegexAttribute.Pattern)) {
s.Pattern = item.Value.Value as string ?? string.Empty;
}
}
bind.Source = s.AsRecord();
return bind;
}
private static TextRegexBinding CompileRegexMethodSource(
TextRegexBinding binding,
CancellationToken cancellationToken
)
{
cancellationToken.ThrowIfCancellationRequested();
TextRegexBinding result = binding.Clone();
RegexAstNode root = result.AnalyzedSource!.Ast.Root;
SourceGenContext generationContext = new();
SequenceEmitter<string> methodSource = new();
List<SequenceEmitter<string>> localFunctions = [];
GenerateMethodSource(root, methodSource, localFunctions, generationContext);
foreach (SequenceEmitter<string> localFunction in localFunctions) {
_ = methodSource.AppendLine()
.AppendLine()
.AppendLine("// local method for matching")
.Append(localFunction);
}
result.SourceContext = generationContext;
result.GeneratedSource = methodSource;
return result;
}
private static IncrementalValuesProvider<TextRegexBinding> RegisterDiagnosticBypass(
IncrementalGeneratorInitializationContext context,
IncrementalValuesProvider<TextRegexBinding> bindings
)
{
IncrementalValuesProvider<Diagnostic> errorPath =
bindings.Where(static binding => !binding.IsValid)
.SelectMany(static(
binding,
_
) => binding.RegexDiagnostic);
context.RegisterSourceOutput(errorPath, static(
productionContext,
diagnostic
) => productionContext.ReportDiagnostic(diagnostic));
return bindings.Where(static binding => binding.IsValid);
}
}
}
src/Nitload.RegularAnalysis/regex_node_transaction.md
+24
View File
@@ -0,0 +1,24 @@
### basic
char/charset->char/charset
### count
char/charset-match->count increment
count increment-loop back->char/charset
char/charset-miss match->count end
count end-exit->count increment-judge->next node
### match group/lookaround
left boarder-> match group start //push stack, add group index
-> match logic // recored index
-> match group end //pop stack, set group end, build sub sequence