diff --git a/src/StgSharp.RegularAnalysis/Global.cs b/src/StgSharp.RegularAnalysis/Global.cs index fb0bc749..e0c76008 100644 --- a/src/StgSharp.RegularAnalysis/Global.cs +++ b/src/StgSharp.RegularAnalysis/Global.cs @@ -5,11 +5,11 @@ // SPDX-License-Identifier: MIT // ----------------------------------------------------------------------------- -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading.Tasks; +global using System; +global using System.Collections.Generic; +global using System.Linq; +global using System.Runtime.CompilerServices; +global using System.Text; +global using System.Threading.Tasks; -[assembly: InternalsVisibleTo("StgSharpDebug")] \ No newline at end of file +[assembly: InternalsVisibleTo("StgSharpDebug")] diff --git a/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs b/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs index b1f8033d..826e791e 100644 --- a/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs +++ b/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs @@ -57,7 +57,9 @@ namespace StgSharp.RegularAnalysis.Text { public static TextRegexSource Analyze( - string regex + string regex, + bool require_ast = false, + bool require_origin_ir = false ) { #pragma warning disable CA1031 @@ -81,17 +83,19 @@ namespace StgSharp.RegularAnalysis.Text } catch (Exception ex) { - return new TextRegexSource(regex, tree, null, null, info, ex); + return new TextRegexSource(regex, require_ast ? tree : null!, null, null, info, ex); } try { ScanAndOptimizeIR(base_ir, out List opt_ir); List prefix_ir = EmitPrefix(opt_ir, ref info); - return new TextRegexSource(regex, tree, base_ir, prefix_ir, info); + return new TextRegexSource(regex, require_ast ? tree : null!, + require_origin_ir ? base_ir : null!, prefix_ir, info); } catch (Exception ex) { - return new TextRegexSource(regex, tree, base_ir, null, info, ex); + return new TextRegexSource(regex, require_ast ? tree : null!, + require_origin_ir ? base_ir : null!, null!, info, ex); } #pragma warning restore CA1031 } diff --git a/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.MultiIR.cs b/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.MultiIR.cs index 782235bb..027fc1b6 100644 --- a/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.MultiIR.cs +++ b/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.MultiIR.cs @@ -227,7 +227,7 @@ namespace StgSharp.RegularAnalysis.Text * [011] 0003, Sccs, DEF , LINE, 0010, PACK SUCCESS GROUP: "name" * [012] 0000, DEF , DEF , NONE, 0000, MATCH_SEQ "234" */ - if (ir_span[0] is not RegexCountSetIR { IsDotStar:true} || + if (ir_span[0] is not RegexCountSetIR { IsDotStar: true } || ir_span[1] is not RegexPackIR pack) { return -1; diff --git a/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.SingleIR.cs b/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.SingleIR.cs index 55a120ad..c15af0d2 100644 --- a/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.SingleIR.cs +++ b/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.SingleIR.cs @@ -5,11 +5,44 @@ // SPDX-License-Identifier: MIT // ----------------------------------------------------------------------------- +using StgSharp.RegularAnalysis.Abstraction; +using System.Globalization; +using System.Text.RegularExpressions; + namespace StgSharp.RegularAnalysis.Text { internal partial class TextRegexSourceGen { + private static int TryGenerateCountSeq( + int outer_level, + ReadOnlySpan ir_span, + SourceGenContext sc, + in SequenceEmitter code, + in List> local_func + ) + { + if (ir_span[0] is not RegexCountSeqIR count_seq) { + return -1; + } + string seq = count_seq.Pattern; + string literal = FormatStringLiteral(seq); + bool is_greedy = count_seq.IsGreedy; + int min = count_seq.Min; + int max = count_seq.Max; + int leave = is_greedy ? min : max; + + // TODO cannot check real count outside the cycle + _ = code.AppendLine(@$"for(int count = 0; count<{leave}; count++)") + .AppendLine("{") + .AppendLine(@$"if({sc.InputSpan}[{sc.Cursor}..].StartWith(""{literal}""))") + .AppendLine("{") + .AppendLine(@$"{sc.Cursor}+={seq.Length};") + .AppendLine("}") + .AppendLine("}"); + return 1; + } + /* * ======================================================================================= * L4, one instruction @@ -70,5 +103,98 @@ namespace StgSharp.RegularAnalysis.Text * - PACK decides nothing, so it should leave __ok untouched. Whether it runs at all is * already expressed by its execution condition. */ + + private static int TryGenerateMatchSeq( + int outer_level, + ReadOnlySpan ir_span, + SourceGenContext sc, + in SequenceEmitter code, + in List> local_func + ) + { + if (ir_span[0] is not RegexMatchSeqIR match_seq) { + return -1; + } + string seq = match_seq.Pattern; + string literal = FormatStringLiteral(seq); + _ = code.AppendLine(@$"if({sc.InputSpan}[{sc.Cursor}..].StartWith(""{literal}""))") + .AppendLine("{") + .AppendLine($"{sc.StepResult} = true;") + .AppendLine($@"{sc.Cursor} += {seq.Length};") + .AppendLine("}"); + return 1; + } + + private static int TryGenerateMatchSet( + int outer_level, + ReadOnlySpan ir_span, + SourceGenContext sc, + in SequenceEmitter code, + in List> local_func + ) + { + if (ir_span[0] is not RegexMatchSetIR match_set) { + return -1; + } + string set_source = match_set.Charset; + RegexCharSet[] set_arr = FigureCharSetType(set_source, out bool accept); + + _ = code.AppendLine(@$"if("); + for (int i = 1; i < set_arr.Length - 1; i++) + { + RegexCharSet set = set_arr[i]; + switch (set.Type) + { + case CharSetType.Single: + string cases = set.Value; + if (cases.Length > 8) + { + // more than 8 different chars, using string match + string literal = FormatStringLiteral(cases); + _ = code.AppendLine($@"{literal}.Contains({sc.InputSpan}[{sc.Cursor}])"); + } else + { + StringBuilder sb = new(); + _ = sb.Append('('); + for (int j = 0; j < cases.Length - 1; j++) { + _ = sb.Append(CultureInfo.InvariantCulture, $@"{sc.InputSpan}[{sc.Cursor}] == {cases[j]} ||"); + } + _ = sb.Append(CultureInfo.InvariantCulture, $@"{sc.InputSpan}[{sc.Cursor}] == {cases[^1]}") + .Append(")||"); + _ = code.AppendLine(sb.ToString()); + } + break; + case CharSetType.Range: + string rule = set.Value; + char lo = rule[0]; + char hi = rule[1]; + _ = code.AppendLine($@"({sc.InputSpan}[{sc.Cursor}] >= '{lo}' && {sc.InputSpan}[{sc.Cursor}] <= '{hi}') ||"); + break; + case CharSetType.Set: + string accept_mark = set.Accept ? string.Empty : "!"; + switch (set.Value) + { + case "s": + _ = code.AppendLine($@"{accept_mark}char.IsWhiteSpace({sc.InputSpan}[{sc.Cursor}]) ||"); + break; + case "d": + _ = code.AppendLine($@"{accept_mark}char.IsDigit({sc.InputSpan}[{sc.Cursor}]) ||"); + break; + case "w": + _ = code .AppendLine($@"{accept_mark}IsCharWord({sc.InputSpan}[{sc.Cursor}]) ||"); + break; + default: + break; + } + break; + case CharSetType.None: + break; + default: + break; + } + } + return 1; + } + } } diff --git a/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs b/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs index ebcbc23a..cf550287 100644 --- a/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs +++ b/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs @@ -72,9 +72,9 @@ namespace StgSharp.RegularAnalysis.Text * lives in test/SingleFile/regex.cs. */ - private const string current_ptr = "cur_ptr_global"; - private const string offset = "offset"; - private const string orig_text = "text"; + // private const string current_ptr = "cur_ptr_global"; // global match cursor + // private const string offset = "offset"; // + // private const string orig_text = "text"; // #region L0 method level @@ -155,10 +155,11 @@ namespace StgSharp.RegularAnalysis.Text .AppendLine($@"int region_{i}_end = -1;") .AppendLine($@"bool region_{i}_success;"); } - + /* _ = se.AppendLine($@"int {current_ptr} = 0;") .AppendLine($@"int {offset} = 0;") .AppendLine(); + /**/ // _ = sc.GenVarDefine(se); @@ -177,6 +178,14 @@ namespace StgSharp.RegularAnalysis.Text #endregion + private enum RegionExitMethod + { + + Break, + Continue + + } + #region shared helpers [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -224,14 +233,14 @@ namespace StgSharp.RegularAnalysis.Text } - internal record struct RegexCharSet( - string Value, - CharSetType Type, - bool Accept + internal record class RegexCharSet( + string Value, + CharSetType Type, + bool Accept ) { - public override readonly string ToString() + public override string ToString() { return $"{{value:{Value}, type:{Type}, accept:{Accept}}}"; } @@ -244,34 +253,45 @@ namespace StgSharp.RegularAnalysis.Text ) { // range or single + HashSet single_char = []; if (source[0] == '[') { int idx; accept = source[1] != '^'; idx = accept ? 1 : 2; + List list = []; ReadOnlySpan seq = source.Slice(idx, source.Length - idx - 1); if (seq.IndexOf('-') == -1) { - return [new RegexCharSet(seq.ToString(), CharSetType.Single, true)]; + SplitSet(single_char, seq, list); + StringBuilder sb = new(); + foreach (char c in single_char) { + _ = sb.Append(c); + } + list.Add(new(sb.ToString(), CharSetType.Single, true)); } else { - List list = []; for (; seq.Length > 0; ) { int pos = seq.IndexOf('-'); if (pos == -1) { // has no range - SplitSet(seq, list); + SplitSet(single_char, seq, list); break; } else if (pos > 1) { // has other sequence before range - SplitSet(seq[0..(pos - 1)], list); + SplitSet(single_char, seq[0..(pos - 1)], list); } list.Add(new($"{seq[pos - 1]}{seq[pos + 1]}", CharSetType.Range, true)); seq = seq[(pos + 2)..]; } + StringBuilder sb = new(); + foreach (char c in single_char) { + _ = sb.Append(c); + } + list.Add(new(sb.ToString(), CharSetType.Single, true)); return [.. list]; } } else if (source[0] == '\\') @@ -293,6 +313,7 @@ namespace StgSharp.RegularAnalysis.Text } static void SplitSet( + HashSet single_char, ReadOnlySpan seq, List sets ) @@ -310,19 +331,31 @@ namespace StgSharp.RegularAnalysis.Text sets.Add(new RegexCharSet(singles.ToString(), CharSetType.Single, true)); } bool accept = char.IsAsciiLetterUpper(remain[pos + 1]); - RegexCharSet set_set = char.ToLower(remain[pos + 1], CultureInfo.InvariantCulture) - switch + + switch (char.ToLower(remain[pos + 1], CultureInfo.InvariantCulture)) { - 's' => new("s", CharSetType.Set, accept), - 'w' => new("w", CharSetType.Set, accept), - 'd' => new("d", CharSetType.Set, accept), - _ => throw new NotSupportedException() - }; - sets.Add(set_set); + case 's': + sets.Add(new("s", CharSetType.Set, accept)); + break; + case 'w': + sets.Add(new("w", CharSetType.Set, accept)); + break; + case 'd': + sets.Add(new("d", CharSetType.Set, accept)); + break; + case '['or'('or')'or']'or'\\'or'\"': + _ = single_char.Add(remain[pos + 1]); + break; + default: + throw new InvalidCastException($"Unknown char or charset {remain[0..1]}"); + } remain = remain[(pos + 2)..]; } - if (remain.Length > 0) { - sets.Add(new RegexCharSet(remain.ToString(), CharSetType.Single, true)); + if (remain.Length > 0) + { + foreach (char c in remain) { + _ = single_char.Add(c); + } } } } diff --git a/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenContext.cs b/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenContext.cs index 41a7db7a..dc604f8b 100644 --- a/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenContext.cs +++ b/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenContext.cs @@ -17,8 +17,6 @@ namespace StgSharp.RegularAnalysis.Text internal partial class TextRegexSourceGen { - #region var naming - /// /// Tracks variable naming and declaration demand while generating one match method. /// @@ -34,35 +32,79 @@ namespace StgSharp.RegularAnalysis.Text /// /// Active pointer. Indexes the next unconsumed element of the input. /// - public const string Cursor = "__cur"; + public const string _cursor = "__cur"; /// /// Name of the match method parameter holding the input sequence. /// - public const string InputSpan = "text"; + public const string _inputSpan = "text"; /// /// Result of the instruction emitted immediately before the current one. /// - public const string StepResult = "__ok"; + public const string _stepResult = "__ok"; private readonly Dictionary _groupIndex = []; - private readonly SortedSet _lineSlots = []; private readonly HashSet _observedLines = []; - private readonly SortedSet _regionSlots = []; private int _anonymousGroup; private int _scratchIndex; + private readonly SortedSet _lineSlots = []; + private readonly SortedSet _regionSlots = []; + + public string Cursor { get; } = _cursor; + + public string InputSpan { get; } = _inputSpan; + + public string StepResult { get; } = _stepResult; /// - /// Records that some prefix observes the result of the instruction at line - /// . + /// Emits declarations for every slot requested during body generation. /// - public void MarkLineObserved( - int line + /// + /// Must be called after the method body has been generated, but the produced lines + /// have to be placed before it. + /// + public SequenceEmitter EmitDeclarationsTo( + SequenceEmitter emitter ) { - _ = _observedLines.Add(line); + if (emitter is null) { + return null!; + } + + _ = emitter.AppendLine($@"int {_cursor} = 0;") + .AppendLine($@"bool {_stepResult} = true;"); + + foreach (int region in _regionSlots) { + _ = emitter.AppendLine($@"int {RegionAnchor(region)} = 0;") + .AppendLine($@"bool {RegionResult(region)} = false;"); + } + + foreach (int line in _lineSlots) { + _ = emitter.AppendLine($@"bool {LineResult(line)} = false;"); + } + + foreach (int group in _groupIndex.Values) { + _ = emitter.AppendLine($@"int __g{group}_begin = -1;") + .AppendLine($@"int __g{group}_end = -1;"); + } + + return emitter; + } + + public string GroupBegin( + string name + ) + { + return $"__g{ResolveGroup(name)}_begin"; + } + + public string GroupEnd( + string name + ) + { + return $"__g{ResolveGroup(name)}_end"; } /// @@ -76,6 +118,30 @@ namespace StgSharp.RegularAnalysis.Text return _observedLines.Contains(line); } + /// + /// Name of the slot holding the match result of the instruction at line . Only lines actually referenced by an prefix get a slot. + /// + public string LineResult( + int line + ) + { + _ = _lineSlots.Add(line); + return $"__l{line}_ok"; + } + + /// + /// Records that some prefix observes the result of the instruction at line . + /// + public void MarkLineObserved( + int line + ) + { + _ = _observedLines.Add(line); + } + /// /// Allocates a fresh block scoped scratch variable name. /// @@ -91,8 +157,8 @@ namespace StgSharp.RegularAnalysis.Text } /// - /// Name of the slot holding the cursor value captured when region - /// was entered. + /// Name of the slot holding the cursor value captured when region was entered. /// public string RegionAnchor( int region @@ -113,77 +179,11 @@ namespace StgSharp.RegularAnalysis.Text return $"__r{region}_ok"; } - /// - /// Name of the slot holding the match result of the instruction at line - /// . Only lines actually referenced by an - /// prefix get a slot. - /// - public string LineResult( - int line - ) - { - _ = _lineSlots.Add(line); - return $"__l{line}_ok"; - } - - public string GroupBegin( - string name - ) - { - return $"__g{ResolveGroup(name)}_begin"; - } - - public string GroupEnd( - string name - ) - { - return $"__g{ResolveGroup(name)}_end"; - } - - /// - /// Emits declarations for every slot requested during body generation. - /// - /// - /// Must be called after the method body has been generated, but the produced lines - /// have to be placed before it. - /// - public SequenceEmitter EmitDeclarations( - SequenceEmitter emitter - ) - { - if (emitter is null) { - return null!; - } - - _ = emitter.AppendLine($@"int {Cursor} = 0;") - .AppendLine($@"bool {StepResult} = true;"); - - foreach (int region in _regionSlots) - { - _ = emitter.AppendLine($@"int {RegionAnchor(region)} = 0;") - .AppendLine($@"bool {RegionResult(region)} = false;"); - } - - foreach (int line in _lineSlots) { - _ = emitter.AppendLine($@"bool {LineResult(line)} = false;"); - } - - foreach (int group in _groupIndex.Values) - { - _ = emitter.AppendLine($@"int __g{group}_begin = -1;") - .AppendLine($@"int __g{group}_end = -1;"); - } - - return emitter; - } - private int ResolveGroup( string name ) { - string key = string.IsNullOrEmpty(name) ? - $"#{_anonymousGroup++}" : - name; + string key = string.IsNullOrEmpty(name) ? $"#{_anonymousGroup++}" : name; if (_groupIndex.TryGetValue(key, out int index)) { return index; } @@ -194,6 +194,5 @@ namespace StgSharp.RegularAnalysis.Text } - #endregion } } diff --git a/test/StgSharpDebug/TestRegexAnalyse.cs b/test/StgSharpDebug/TestRegexAnalyse.cs index 927b5cfc..4bcaff95 100644 --- a/test/StgSharpDebug/TestRegexAnalyse.cs +++ b/test/StgSharpDebug/TestRegexAnalyse.cs @@ -35,7 +35,7 @@ namespace StgSharp { foreach (string regex in regexSource) { - TextRegexSource interpreterSource = RegexAnalyzer.Analyze(regex); + TextRegexSource interpreterSource = RegexAnalyzer.Analyze(regex, true, true); AbstractSyntaxTree tree = interpreterSource.Ast; Console.WriteLine($"result of \"{regex}\""); TreeEnumerator enumerator = new(tree);