youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

do some source gen work

Browse Source
This commit is contained in:
Youmu Liu
2026-07-15 18:05:38 +08:00
parent 4d2a4eeb18
commit ee7cbd869c
10 changed files with 276 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/StgSharp.Native/CMakeLists.txt
+1 -1
View File
@@ -52,7 +52,7 @@ else()
set(STGSHARP_ARCH_TAG ${SN_RESOLVED_ARCH_NAME})
endif()
set(STGSHARP_BUILD_OUTPUT_DIR ${STGSHARP_SOLUTION_ROOT}/bin/${STGSHARP_OS_TAG}_${STGSHARP_ARCH_TAG})
set(STGSHARP_BUILD_OUTPUT_DIR ${STGSHARP_SOLUTION_ROOT}/bin/${STGSHARP_OS_TAG}-${STGSHARP_ARCH_TAG})
# Compile-time intermediates (native PDB / .obj side files) live inside the CMake
# build tree (cmake_build/<preset>) so they get cleaned up with it.
set(STGSHARP_OBJECT_ROOT_DIR ${CMAKE_CURRENT_BINARY_DIR}/obj/${PROJECT_NAME})
src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.IRGenerate.cs
+29 -17
View File
@@ -28,7 +28,7 @@ namespace StgSharp.RegularAnalysis.Text
ref RegexInfo info
)
{
List<RegexIR> list = GenerateIRRecursion(tree.Root, 0, out int length).ToList();
List<RegexIR> list = GenerateIRRecursion(tree.Root, 0, [], out int length).ToList();
info.MinPredictLength = length;
return list;
}
@@ -41,15 +41,17 @@ namespace StgSharp.RegularAnalysis.Text
int max,
bool isGreedy,
RegexAstNode right,
HashSet<string> group_names,
out int minLength
)
{
bool is_operator = !RegexAstNode.IsNullOrEmpty(right) &&
(right.Source.Flag & RegexElementLabel.OPERATOR) != 0;
(right.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0;
if (is_operator) // seq to be count is complex
{
RegexIRGenerator right_ir = GenerateIRRecursion(right, curDepth + 1, out minLength);
RegexIRGenerator right_ir = GenerateIRRecursion(right, curDepth + 1, group_names,
out minLength);
_ = ir.EmitCountComplex(min, max, isGreedy, right_ir.Count);
_ = ir.EmitIRStream(right_ir);
} else
@@ -78,6 +80,7 @@ namespace StgSharp.RegularAnalysis.Text
private static RegexIRGenerator GenerateIRRecursion(
RegexAstNode node,
int depth,
HashSet<string> group_names,
out int minLength
)
{
@@ -94,20 +97,28 @@ namespace StgSharp.RegularAnalysis.Text
Token<RegexElementLabel> token = node.Source;
HashSet<string> gn = group_names ?? [];
switch (node.Source.Flag)
{
case RegexElementLabel.GROUP_BEGIN:
string group_source = token.Value;
// process group type
bool is_group = group_source.Length == 1; // simple group: (pattern)
string group_name =
group_source.StartsWith(@"(?<", StringComparison.Ordinal) ?
group_source[3..^1] : /* named group (?<name>) */
string.Empty; /* unnamed group (?:) */
bool has_name = group_source.StartsWith(@"(?<", StringComparison.Ordinal); // simple group: (pattern)
string group_name = has_name ?
group_source[3..^1] : /* named group (?<name>) */
string.Empty; /* unnamed group (?:) */
if (has_name)
{
if (!gn.Add(group_name)) {
throw new InvalidOperationException($"Duplicate group name: {group_name}");
}
}
RegexIRGenerator groupIR = new();
rightIR = GenerateIRRecursion(right, curDepth + 1, out minLength);
rightIR = GenerateIRRecursion(right, curDepth + 1, gn, out minLength);
_ = groupIR.EmitTry(rightIR.Count)
.EmitIRStream(rightIR)
.EmitCondition(2, 1)
@@ -123,15 +134,15 @@ namespace StgSharp.RegularAnalysis.Text
int length_once;
if (count_source_span[0] == '*') // infinite count
{
EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq, out _);
EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq, gn, out _);
minLength = 0;
} else if (count_source_span[0] == '+') // at least one
{
EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq, out length_once);
EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq, gn, out length_once);
minLength = length_once;
} else if (count_source_span[0] == '?') // zero or one
{
EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq, out _);
EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq, gn, out _);
minLength = 0;
} else if (count_source_span[0] == '{') // zero or one, non-greedy
{
@@ -140,7 +151,7 @@ namespace StgSharp.RegularAnalysis.Text
{
if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count))
{
EmitCount(curDepth, count_ir, count, count, isGreedy, seq,
EmitCount(curDepth, count_ir, count, count, isGreedy, seq, gn,
out length_once);
minLength = length_once * count;
} else
@@ -152,7 +163,7 @@ namespace StgSharp.RegularAnalysis.Text
if (int.TryParse(count_source[1..comma_pos], out int min) &&
int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max))
{
EmitCount(curDepth, count_ir, min, max, isGreedy, seq,
EmitCount(curDepth, count_ir, min, max, isGreedy, seq, gn,
out length_once);
minLength = length_once * min;
} else
@@ -195,7 +206,7 @@ namespace StgSharp.RegularAnalysis.Text
}
} else if ((left.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{
leftIR = GenerateIRRecursion(left, curDepth + 1, out left_length);
leftIR = GenerateIRRecursion(left, curDepth + 1, gn, out left_length);
_ = concat.EmitIRStream(leftIR);
}
@@ -225,7 +236,7 @@ namespace StgSharp.RegularAnalysis.Text
}
} else if ((right.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{
rightIR = GenerateIRRecursion(right, curDepth + 1, out right_length);
rightIR = GenerateIRRecursion(right, curDepth + 1, gn, out right_length);
_ = concat.EmitIRStream(rightIR);
}
minLength = left_length + right_length;
@@ -238,7 +249,8 @@ namespace StgSharp.RegularAnalysis.Text
int min_case_length = int.MaxValue;
while (!RegexAstNode.IsNullOrEmpty(cur_node))
{
RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1, out int case_length);
RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1, gn,
out int case_length);
min_case_length = int.Min(case_length, min_case_length);
casesIRList.Add(_case);
ir_count += _case.Count;
src/StgSharp.RegularAnalysis/Text/RegexElementLabel.cs
+2 -1
View File
@@ -16,7 +16,7 @@ namespace StgSharp.RegularAnalysis.Text
[Flags]
public enum RegexElementLabel
{
#pragma warning disable CA1707
NONE = 0,
UNIT = 1,
UNIT_SET = 2,
@@ -33,5 +33,6 @@ namespace StgSharp.RegularAnalysis.Text
ATOM_BEGIN = SEQUENCE | GROUP_BEGIN ,
ATOM_END = SEQUENCE | GROUP_END | COUNT,
#pragma warning restore CA1707
}
}
src/StgSharp.RegularAnalysis/Text/RegexTokenParser.cs
+3 -5
View File
@@ -32,12 +32,10 @@ namespace StgSharp.RegularAnalysis.Text
}
if (Source.TryReadToken(out Token<RegexElementLabel> orig))
{
if (((int)(orig.Flag & RegexElementLabel.ATOM_BEGIN) != 0) && ((int)_prev != 0))
if (((orig.Flag & RegexElementLabel.ATOM_BEGIN) != 0) && (_prev != 0))
{
Token<RegexElementLabel> ret = new Token<RegexElementLabel>(string.Empty,
orig.Line,
orig.Column,
RegexElementLabel.CONCAT);
Token<RegexElementLabel> ret = new(string.Empty, orig.Line, orig.Column,
RegexElementLabel.CONCAT);
_buffer.Enqueue(orig);
_prev = orig.Flag & RegexElementLabel.ATOM_END;
return ret;
src/StgSharp.RegularAnalysis/Text/RegexTokenUtil.cs
-29
View File
@@ -1,29 +0,0 @@
// -----------------------------------------------------------------------------
// file="RegexTokenUtil"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal static class RegexTokenUtil
{
public static bool IsAccurateChar(
this Token<RegexElementLabel> token
)
{
return ((token.Flag & RegexElementLabel.UNIT) != 0 && token.Value.Length == 1) ||
(token.Flag & RegexElementLabel.UNIT_SPAN) != 0;
}
}
}
src/StgSharp.RegularAnalysis/Text/TextRegexAttribute.cs
+4 -9
View File
@@ -14,17 +14,12 @@ using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public sealed class TextRegexAttribute : Attribute
public sealed class TextRegexAttribute(
string pattern
) : Attribute
{
public TextRegexAttribute(
string pattern
)
{
Pattern = pattern;
}
public string Pattern { get; }
public string Pattern { get; } = pattern;
}
}
src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs
+211 -23
View File
@@ -8,6 +8,7 @@
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
@@ -18,12 +19,16 @@ namespace StgSharp.RegularAnalysis.Text
internal partial class TextRegexSourceGen
{
private const string current_ptr = "current_ptr";
private const string orig_text = "text";
internal static SequenceEmitter<string> GenerateSource(
TextRegexSource source
)
{
SequenceEmitter<string> se = new();
List<SequenceEmitter<string>> sub_match_list = [];
VarNameGenerator ng = new();
RegexInfo info = source.Info;
List<RegexIR> ir = source.IRs;
@@ -31,8 +36,6 @@ namespace StgSharp.RegularAnalysis.Text
RegexIR[] window = new RegexIR[8];
Span<RegexIR> scan_win = window;
const string current_ptr = "current_ptr";
_ = se.AppendLine($@"public override MatchResult Match({ROS_char} text)")//
.AppendLine(@"{");
@@ -49,9 +52,18 @@ namespace StgSharp.RegularAnalysis.Text
int index = 0;
do
{
int result;
int count = int.Min(8, ir_sp.Length - index);
ir_sp.Slice(index, count).CopyTo(scan_win);
if (TryGenerateMatch(scan_win, se) > 0) { }
if (TryGenerateMatchSeq(scan_win, se, ng) > 0)
{
count++;
} else if (TryGenerateMatchSet(scan_win, se, ng) > 0)
{
count++;
} else { }
index += count;
} while (index < ir.Count);
foreach (SequenceEmitter<string> _sub in sub_match_list)
@@ -63,27 +75,203 @@ namespace StgSharp.RegularAnalysis.Text
_ = se.Append(@"}");
return se;
/**/
int TryGenerateMatch(
Span<RegexIR> scan_win,
SequenceEmitter<string> sb
)
{
if (scan_win[0] is not RegexMatchSeqIR seq) {
return -1;
}
string pattern = seq.Pattern;
int length = pattern.Length;
_ = sb.AppendLine()
.AppendLine($@"if(text[{current_ptr}..({current_ptr} + {length})] == {pattern})")
.AppendLine($@"{{")
.AppendLine($@"{current_ptr} += {length};")
.AppendLine($@"}}");
return 1;
}
/**/
}
#region var naming
private class VarNameGenerator
{
private int char_counter;
public string GenerateCharName()
{
string name = $"__char_{char_counter}";
char_counter++;
return name;
}
}
#endregion
#region sub generator
private static int TryGenerateMatchSeq(
Span<RegexIR> scan_win,
SequenceEmitter<string> se,
VarNameGenerator ng
)
{
if (scan_win[0] is not RegexMatchSeqIR seq) {
return -1;
}
string pattern = seq.Pattern;
int length = pattern.Length;
_ = se.AppendLine()
.AppendLine($@"if(text[{current_ptr}..({current_ptr} + {length})] == {pattern})")
.AppendLine($@"{{")
.AppendLine($@"{current_ptr} += {length};")
.AppendLine($@"}}");
return 1;
}
private static int TryGenerateMatchSet(
Span<RegexIR> scan_win,
SequenceEmitter<string> se,
VarNameGenerator ng
)
{
if (scan_win[0] is not RegexMatchSeqIR seq) {
return -1;
}
string pattern = seq.Pattern;
RegexCharSet[] set_pattern = FigureCharSetType(pattern);
string var_char = ng.GenerateCharName();
_ = se.AppendLine($@"// match charset of {pattern}")
.AppendLine($@"char {var_char} = {orig_text}[{current_ptr}];")
.AppendLine();
bool accept = set_pattern[0].Accept;
_ = se.AppendLine("if(");
RegexCharSet p = set_pattern[0];
string _case = GenerateCase(p);
string _case_begin = accept ? string.Empty : "!(";
_ = se.AppendLine($@"if ({_case_begin}{_case}||");
for (int i = 1; i < set_pattern.Length - 1; i++)
{
p = set_pattern[i];
_case = GenerateCase(p);
if (string.IsNullOrEmpty(_case))
{
continue;
}
_ = se.AppendLine($@" {_case} ||");
}
p = set_pattern[^1];
_case = GenerateCase(p);
string _case_end = accept ? string.Empty : ")";
_ = se.AppendLine($@"{_case}{_case_end})")
.AppendLine("{")
.AppendLine($@"{current_ptr} += 1;")
.AppendLine("}");
return 1;
string GenerateCase(
RegexCharSet c
)
{
return c.Type switch
{
// Handle single character
CharSetType.Single => $@"{var_char} == '{c.Value}'",
// Handle range of characters
CharSetType.Range => $@"({var_char} >= '{c.Value[0]}' && {var_char} <= '{c.Value[1]}')",
// Handle set of characters
CharSetType.Set => c.Value switch
{
"s" => $@"char.IsWhiteSpace({var_char})",
"w" => $@"char.IsLetterOrDigit({var_char}) || {var_char} == '_'",
"d" => $@"char.IsDigit({var_char})",
_ => throw new InvalidCastException(),
},
CharSetType.None => string.Empty,
_ => string.Empty,
};
}
}
#endregion
#region charset config
internal enum CharSetType
{
None,
Single,
Range,
Set
}
internal record struct RegexCharSet(
string Value,
CharSetType Type,
bool Accept
)
{
public override readonly string ToString()
{
return $"{{value:{Value}, type:{Type}, accept:{Accept}}}";
}
}
public static RegexCharSet[] FigureCharSetType(
ReadOnlySpan<char> source
)
{
// range or single
bool accept;
if (source[0] == '[')
{
int idx;
accept = source[1] != '^';
idx = accept ? 1 : 2;
ReadOnlySpan<char> seq = source.Slice(idx, source.Length - idx - 1);
if (seq.IndexOf('-') == -1)
{
return [new RegexCharSet(seq.ToString(), CharSetType.Single, accept)];
} else
{
List<RegexCharSet> list = [];
for (; seq.Length > 0; )
{
int pos = seq.IndexOf('-');
if (pos == -1)
{
RegexCharSet single = new(seq.ToString(), CharSetType.Single, accept);
list.Add(single);
break;
} else
if (pos > 1) {
list.Add(new(seq[0..(pos - 1)].ToString(), CharSetType.Single, accept));
}
list.Add(new($"{seq[pos - 1]}{seq[pos + 1]}", CharSetType.Range, accept));
seq = seq[(pos + 2)..];
}
return [.. list];
}
} else if (source[0] == '\\')
{
// set
accept = char.IsAsciiLetterUpper(source[0]);
return char.ToLower(source[0], CultureInfo.InvariantCulture)
switch
{
's' => [new("s", CharSetType.Set, accept)],
'w' => [new("w", CharSetType.Set, accept)],
'd' => [new("d", CharSetType.Set, accept)],
_ => throw new NotSupportedException()
};
} else
{
// unknown, throw
throw new InvalidCastException($"Unknown char set: {source}");
}
}
#endregion
}
}