diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs b/StgSharp.RegularAnalysis/Text/RegexAstNode.cs similarity index 99% rename from StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs rename to StgSharp.RegularAnalysis/Text/RegexAstNode.cs index 9b6de712..b5238042 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs +++ b/StgSharp.RegularAnalysis/Text/RegexAstNode.cs @@ -28,7 +28,7 @@ using StgSharp.RegularAnalysis.Abstraction; using System.Runtime.CompilerServices; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { internal class RegexAstNode : ISyntaxNode { diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs b/StgSharp.RegularAnalysis/Text/RegexElementLabel.cs similarity index 97% rename from StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs rename to StgSharp.RegularAnalysis/Text/RegexElementLabel.cs index dec57691..96e3b61b 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs +++ b/StgSharp.RegularAnalysis/Text/RegexElementLabel.cs @@ -31,7 +31,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { [Flags] public enum RegexElementLabel diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexIR.cs b/StgSharp.RegularAnalysis/Text/RegexHIR.cs similarity index 81% rename from StgSharp.RegularAnalysis/TextRegex/RegexIR.cs rename to StgSharp.RegularAnalysis/Text/RegexHIR.cs index 46498c96..11e44c35 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexIR.cs +++ b/StgSharp.RegularAnalysis/Text/RegexHIR.cs @@ -32,25 +32,24 @@ using System.Runtime.CompilerServices; using System.Text; using System.Threading.Tasks; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { internal enum RegexIRCommand { - Search_Seq = 0, - Search_Set = 1, - SearchChar = 2, - ConfigFail = 3, - Condition = 4, - Try = 5, - Count = 6, - Count_Seq = 7, - Count_Set = 8, - Pop = 10 + Match_Seq = 0, + Match_Set = 1, + Condition = 3, + Try = 4, + Count = 5, + Count_Seq = 6, + Count_Set = 7, + Pop = 8, + Pack = 9, } - internal abstract class RegexIR + internal abstract class RegexHIR { internal abstract RegexIRCommand Command { get; } @@ -64,70 +63,46 @@ namespace StgSharp.RegularAnalysis.TextRegex /// /// Searches for a certain string pattern or charset from current position. /// - internal sealed class RegexMatchSeqIR( + internal sealed class RegexMatchSeqHIR( string pattern - ) : RegexIR + ) : RegexHIR { public string Pattern { get; } = pattern; - internal override RegexIRCommand Command => RegexIRCommand.Search_Seq; + internal override RegexIRCommand Command => RegexIRCommand.Match_Seq; internal override string AsCommand( int lineNum ) { - return $"[{lineNum}]\t SEARCH_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "" : Pattern)}\""; + return $"[{lineNum}]\t MATCH_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "" : Pattern)}\""; } } - internal sealed class RegexMatchSetIR( + internal sealed class RegexMatchSetHIR( string charset - ) : RegexIR + ) : RegexHIR { public string Charset { get; } = charset; - internal override RegexIRCommand Command => RegexIRCommand.Search_Set; + internal override RegexIRCommand Command => RegexIRCommand.Match_Set; internal override string AsCommand( int lineNum ) { - return $"[{lineNum}]\t SEARCH_SET \"{Charset}\""; + return $"[{lineNum}]\t MATCH_SET \"{Charset}\""; } } - /// - /// Configures the failure jump of current position. The failMod ranges 0-2 - /// - /// - /// - /// - internal sealed class RegexConfigFailIR( - int failMod - ) : RegexIR - { - - public int FailMod { get; } = failMod & 0x3; - - internal override RegexIRCommand Command => RegexIRCommand.ConfigFail; - - internal override string AsCommand( - int lineNum - ) - { - return $"[{lineNum}]\t CONFIG_FAIL {FailMod}"; - } - - } - - internal sealed class RegexConditionIR( + internal sealed class RegexConditionHIR( int success, int fail - ) : RegexIR + ) : RegexHIR { public int SuccessCase { get; } = success; @@ -145,12 +120,33 @@ namespace StgSharp.RegularAnalysis.TextRegex } - internal sealed class RegexCountIR( + internal sealed class RegexPackHIR( + bool condition, + string name = "" + ) : RegexHIR + { + + public bool Condition { get; } = condition; + + public string GroupName { get; } = name; + + internal override RegexIRCommand Command => RegexIRCommand.Pack; + + internal override string AsCommand( + int lineNum + ) + { + return $"[{lineNum}]\t PACK {(Condition ? "SUCCESS" : "FAIL")} GROUP: \"{GroupName}\""; + } + + } + + internal sealed class RegexCountHIR( int min, int max, bool isGreedy, int operation - ) : RegexIR + ) : RegexHIR { public bool IsGreedy { get; } = isGreedy; @@ -172,12 +168,12 @@ namespace StgSharp.RegularAnalysis.TextRegex } - internal sealed class RegexCountSeqIR( + internal sealed class RegexCountSeqHIR( int min, int max, bool isGreedy, string seq - ) : RegexIR + ) : RegexHIR { public bool IsGreedy { get; } = isGreedy; @@ -199,12 +195,12 @@ namespace StgSharp.RegularAnalysis.TextRegex } - internal sealed class RegexCountSetIR( + internal sealed class RegexCountSetHIR( int min, int max, bool isGreedy, string set - ) : RegexIR + ) : RegexHIR { public bool IsGreedy { get; } = isGreedy; @@ -226,9 +222,9 @@ namespace StgSharp.RegularAnalysis.TextRegex } - internal sealed class RegexTryIR( + internal sealed class RegexTryHIR( int count - ) : RegexIR + ) : RegexHIR { public int CountOfCommand { get; } = count; @@ -244,14 +240,14 @@ namespace StgSharp.RegularAnalysis.TextRegex } - internal sealed class RegexPopIR( + internal sealed class RegexPopHIR( bool isSuccess - ) : RegexIR + ) : RegexHIR { public bool IsSuccess { get; } = isSuccess; - internal override RegexIRCommand Command => RegexIRCommand.Try; + internal override RegexIRCommand Command => RegexIRCommand.Pop; internal override string AsCommand( int lineNum diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexIRGenerator.cs b/StgSharp.RegularAnalysis/Text/RegexHIRGenerator.cs similarity index 79% rename from StgSharp.RegularAnalysis/TextRegex/RegexIRGenerator.cs rename to StgSharp.RegularAnalysis/Text/RegexHIRGenerator.cs index e5afb6ee..b1720138 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexIRGenerator.cs +++ b/StgSharp.RegularAnalysis/Text/RegexHIRGenerator.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexIRGenerator" +// file="RegexHIRGenerator" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -32,15 +32,15 @@ using System.Runtime.CompilerServices; using System.Text; using System.Threading.Tasks; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { - internal class RegexIRGenerator + internal class RegexHIRGenerator { private Node _head = new(); private Node _tail; - public RegexIRGenerator() + public RegexHIRGenerator() { _tail = _head; } @@ -52,15 +52,7 @@ namespace StgSharp.RegularAnalysis.TextRegex int fail ) { - _tail.Add(new RegexConditionIR(success, fail)); - Count++; - } - - public void EmitConfigFail( - int fail - ) - { - _tail.Add(new RegexConfigFailIR(fail)); + _tail.Add(new RegexConditionHIR(success, fail)); Count++; } @@ -71,7 +63,7 @@ namespace StgSharp.RegularAnalysis.TextRegex int operation ) { - _tail.Add(new RegexCountIR(min, max, isGreedy, operation)); + _tail.Add(new RegexCountHIR(min, max, isGreedy, operation)); Count++; } @@ -82,7 +74,7 @@ namespace StgSharp.RegularAnalysis.TextRegex string seq ) { - _tail.Add(new RegexCountSeqIR(min, max, isGreedy, seq)); + _tail.Add(new RegexCountSeqHIR(min, max, isGreedy, seq)); Count++; } @@ -93,12 +85,12 @@ namespace StgSharp.RegularAnalysis.TextRegex string set ) { - _tail.Add(new RegexCountSetIR(min, max, isGreedy, set)); + _tail.Add(new RegexCountSetHIR(min, max, isGreedy, set)); Count++; } public void EmitIRStream( - RegexIRGenerator stream + RegexHIRGenerator stream ) { if (stream is null || stream.Count == 0) { @@ -113,7 +105,7 @@ namespace StgSharp.RegularAnalysis.TextRegex string unit ) { - _tail.Add(new RegexMatchSeqIR(unit)); + _tail.Add(new RegexMatchSeqHIR(unit)); Count++; } @@ -121,7 +113,16 @@ namespace StgSharp.RegularAnalysis.TextRegex string unit ) { - _tail.Add(new RegexMatchSetIR(unit)); + _tail.Add(new RegexMatchSetHIR(unit)); + Count++; + } + + public void EmitPack( + bool condition, + string name = "" + ) + { + _tail.Add(new RegexPackHIR(condition, name)); Count++; } @@ -129,7 +130,7 @@ namespace StgSharp.RegularAnalysis.TextRegex bool isSuccess ) { - _tail.Add(new RegexPopIR(isSuccess)); + _tail.Add(new RegexPopHIR(isSuccess)); Count++; } @@ -137,13 +138,13 @@ namespace StgSharp.RegularAnalysis.TextRegex int count ) { - _tail.Add(new RegexTryIR(count)); + _tail.Add(new RegexTryHIR(count)); Count++; } - public List ToList() + public List ToList() { - List result = new(Count); + List result = new(Count); Node current = _head; while (current != null) { @@ -160,11 +161,11 @@ namespace StgSharp.RegularAnalysis.TextRegex public int Count => IRList.Count; - public List IRList { get; } = []; + public List IRList { get; } = []; [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Add( - RegexIR ir + RegexHIR ir ) { IRList.Add(ir); diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs similarity index 99% rename from StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs rename to StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs index 6e7bbb80..36398174 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs @@ -37,7 +37,7 @@ using System.Text; using System.Threading.Tasks; using System.Xml.Linq; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { public partial class RegexInterpreter { diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.IRGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.HIRGenerate.cs similarity index 88% rename from StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.IRGenerate.cs rename to StgSharp.RegularAnalysis/Text/RegexInterpreter.HIRGenerate.cs index 7c7e77a2..bc42c472 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.IRGenerate.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.HIRGenerate.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexInterpreter.IRGenerate" +// file="RegexInterpreter.HIRGenerate" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -38,13 +38,13 @@ using System.Text; using System.Threading.Tasks; using System.Xml; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { public partial class RegexInterpreter { - internal static List GenerateIR( - AbstractSyntaxTree tree + internal static List GenerateIR( + AbstractSyntaxTree tree ) { return GenerateIRRecursion(tree.Root, 0).ToList(); @@ -53,7 +53,7 @@ namespace StgSharp.RegularAnalysis.TextRegex [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void EmitCount( int curDepth, - RegexIRGenerator ir, + RegexHIRGenerator ir, int min, int max, bool isGreedy, @@ -66,7 +66,7 @@ namespace StgSharp.RegularAnalysis.TextRegex if (isRightOperator) // seq to be count is complex { ir.EmitCount(min, max, isGreedy, 1); - RegexIRGenerator rightIR = GenerateIRRecursion(right, curDepth + 1); + RegexHIRGenerator rightIR = GenerateIRRecursion(right, curDepth + 1); ir.EmitTry(rightIR.Count); ir.EmitIRStream(rightIR); } else @@ -89,9 +89,9 @@ namespace StgSharp.RegularAnalysis.TextRegex } } - private static RegexIRGenerator GenerateIRRecursion( - RegexAstNode node, - int depth + private static RegexHIRGenerator GenerateIRRecursion( + RegexAstNode node, + int depth ) { #pragma warning disable IDE0010 @@ -102,8 +102,8 @@ namespace StgSharp.RegularAnalysis.TextRegex int curDepth = depth; RegexAstNode left = node.Left; RegexAstNode right = node.Right; - RegexIRGenerator leftIR; - RegexIRGenerator rightIR; + RegexHIRGenerator leftIR; + RegexHIRGenerator rightIR; Token token = node.Source; @@ -116,15 +116,16 @@ namespace StgSharp.RegularAnalysis.TextRegex bool is_group = group_source.Length == 1; // simple group: (pattern) string group_name = group_source.StartsWith(@"(?<", StringComparison.Ordinal) ? - group_source[3..^2] : /* named group (?) */ + group_source[3..^1] : /* named group (?) */ string.Empty; /* unnamed group (?:) */ - RegexIRGenerator groupIR = new(); + RegexHIRGenerator groupIR = new(); rightIR = GenerateIRRecursion(right, curDepth + 1); groupIR.EmitTry(rightIR.Count); groupIR.EmitIRStream(rightIR); groupIR.EmitCondition(2, 1); - /*generate pack ir here*/ + groupIR.EmitPop(false); + groupIR.EmitPack(true, group_name); return groupIR; case RegexElementLabel.COUNT: string count_source = token.Value; @@ -143,14 +144,14 @@ namespace StgSharp.RegularAnalysis.TextRegex _ => throw new InvalidOperationException("Unexpected sequence mask for count operator"), }; ReadOnlySpan count_source_span = count_source; - RegexIRGenerator countIR = new(); + RegexHIRGenerator countIR = new(); if (count_source_span[0] == '*') // infinite count { - EmitCount(curDepth, countIR, 0, int.MaxValue, isGreedy, seq); + EmitCount(curDepth, countIR, 0, -1, isGreedy, seq); } else if (count_source_span[0] == '+') // at least one { - EmitCount(curDepth, countIR, 1, int.MaxValue, isGreedy, seq); + EmitCount(curDepth, countIR, 1, -1, isGreedy, seq); } else if (count_source_span[0] == '?') // zero or one { EmitCount(curDepth, countIR, 0, 1, isGreedy, seq); @@ -159,7 +160,7 @@ namespace StgSharp.RegularAnalysis.TextRegex int comma_pos = count_source_span.IndexOf(','); if (comma_pos == -1) { - if (int.TryParse(count_source[1..(isGreedy ? ^3 : ^2)], out int count)) + if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count)) { EmitCount(curDepth, countIR, count, count, isGreedy, seq); } else @@ -169,7 +170,7 @@ namespace StgSharp.RegularAnalysis.TextRegex } else { if (int.TryParse(count_source[1..comma_pos], out int min) && - int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^3 : ^2)], out int max)) + int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max)) { EmitCount(curDepth, countIR, min, max, isGreedy, seq); } else @@ -184,7 +185,7 @@ namespace StgSharp.RegularAnalysis.TextRegex return countIR; case RegexElementLabel.CONCAT: - RegexIRGenerator concat = new(); + RegexHIRGenerator concat = new(); // process left node if (RegexAstNode.IsNullOrEmpty(left)) @@ -237,13 +238,13 @@ namespace StgSharp.RegularAnalysis.TextRegex } return concat; case RegexElementLabel.ALT: - RegexIRGenerator alt = new(); - List casesIRList = []; + RegexHIRGenerator alt = new(); + List casesIRList = []; RegexAstNode cur_node = right; int ir_count = 0; while (!RegexAstNode.IsNullOrEmpty(cur_node)) { - RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1); + RegexHIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1); casesIRList.Add(_case); ir_count += _case.Count; cur_node = cur_node.Next; @@ -251,7 +252,7 @@ namespace StgSharp.RegularAnalysis.TextRegex int case_count = casesIRList.Count; while (case_count != 0) { - RegexIRGenerator cur_ir = casesIRList[^case_count]; + RegexHIRGenerator cur_ir = casesIRList[^case_count]; ir_count -= cur_ir.Count; case_count--; alt.EmitTry(cur_ir.Count); @@ -264,11 +265,11 @@ namespace StgSharp.RegularAnalysis.TextRegex case RegexElementLabel.UNIT_SPAN: goto case RegexElementLabel.UNIT; case RegexElementLabel.UNIT: - RegexIRGenerator unit_seq = new(); + RegexHIRGenerator unit_seq = new(); unit_seq.EmitMatchSeq(node.Source.Value); return unit_seq; case RegexElementLabel.UNIT_SET: - RegexIRGenerator unit_set = new(); + RegexHIRGenerator unit_set = new(); unit_set.EmitMatchSet(node.Source.Value); return unit_set; default: diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRScan.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRScan.cs new file mode 100644 index 00000000..61c7b28c --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRScan.cs @@ -0,0 +1,69 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="RegexInterpreter.IRScan" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + public partial class RegexInterpreter + { + + internal static void ScanAndMarkIROptimization( + RegexHIR ir + ) + { + switch (ir.Command) + { + case RegexIRCommand.Match_Seq: + break; + case RegexIRCommand.Match_Set: + break; + case RegexIRCommand.Pack: + break; + case RegexIRCommand.Condition: + break; + case RegexIRCommand.Try: + break; + case RegexIRCommand.Count: + break; + case RegexIRCommand.Count_Seq: + break; + case RegexIRCommand.Count_Set: + break; + case RegexIRCommand.Pop: + break; + default: + break; + } + } + + } +} diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs new file mode 100644 index 00000000..67d84307 --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs @@ -0,0 +1,51 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="RegexInterpreter.SourceGenerate" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + public partial class RegexInterpreter + { + + internal static string GetSourceCode( + IEnumerable irs + ) + { + StringBuilder sb = new StringBuilder(); + foreach (RegexHIR ir in irs) { + ScanAndMarkIROptimization(ir); + } + return sb.ToString(); + } + + } +} diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.cs similarity index 99% rename from StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs rename to StgSharp.RegularAnalysis/Text/RegexInterpreter.cs index 0a5cfbdc..f7f1c02d 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.cs @@ -35,7 +35,7 @@ using System.Text; using System.Threading.Tasks; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { public partial class RegexInterpreter { diff --git a/StgSharp.RegularAnalysis/Text/RegexLIR.cs b/StgSharp.RegularAnalysis/Text/RegexLIR.cs new file mode 100644 index 00000000..272de4f3 --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/RegexLIR.cs @@ -0,0 +1,60 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="RegexLIR" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + internal enum LIRCommand + { + + Search = 0, + Count = 1, + Try = 2, + + } + + internal abstract class RegexLIR + { + + public abstract LIRCommand Command { get; } + + } + + internal enum RegexSearchMethod + { + + Match, + Find, + Assert, + + } +} diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs b/StgSharp.RegularAnalysis/Text/RegexTokenParser.cs similarity index 98% rename from StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs rename to StgSharp.RegularAnalysis/Text/RegexTokenParser.cs index 452a65a1..f4ee54a3 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs +++ b/StgSharp.RegularAnalysis/Text/RegexTokenParser.cs @@ -34,7 +34,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { internal class RegexTokenParser : TokenParser { diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs b/StgSharp.RegularAnalysis/Text/RegexTokenReader.cs similarity index 99% rename from StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs rename to StgSharp.RegularAnalysis/Text/RegexTokenReader.cs index d57f3518..953b0eb9 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs +++ b/StgSharp.RegularAnalysis/Text/RegexTokenReader.cs @@ -34,7 +34,7 @@ using System.Security.Permissions; using System.Text; using System.Threading.Tasks; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { internal class RegexTokenReader : ITokenReader { diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexTokenUtil.cs b/StgSharp.RegularAnalysis/Text/RegexTokenUtil.cs similarity index 97% rename from StgSharp.RegularAnalysis/TextRegex/RegexTokenUtil.cs rename to StgSharp.RegularAnalysis/Text/RegexTokenUtil.cs index 3d51ab3c..1a32cc24 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexTokenUtil.cs +++ b/StgSharp.RegularAnalysis/Text/RegexTokenUtil.cs @@ -32,7 +32,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { internal static class RegexTokenUtil { diff --git a/StgSharp.RegularAnalysis/Text/TextHelper.cs b/StgSharp.RegularAnalysis/Text/TextHelper.cs new file mode 100644 index 00000000..78d9b4d6 --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/TextHelper.cs @@ -0,0 +1,50 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="TextHelper" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + public static class TextHelper + { + + public static bool TryLastIndexOf( + this ReadOnlySpan span, + ReadOnlySpan value, + out int index + ) + { + index = span.LastIndexOf(value); + return index != -1; + } + + } +} diff --git a/StgSharp.RegularAnalysis/Text/TextRegex.cs b/StgSharp.RegularAnalysis/Text/TextRegex.cs new file mode 100644 index 00000000..b2f4c80d --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/TextRegex.cs @@ -0,0 +1,47 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="TextRegex" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + public abstract class TextRegex + { + + public abstract bool IsMatch( + ReadOnlySpan text + ); + + } + + public ref struct MatchResult { } +} diff --git a/StgSharpDebug/TestRegexAnalyse.cs b/StgSharpDebug/TestRegexAnalyse.cs index 7d9e80d9..16203514 100644 --- a/StgSharpDebug/TestRegexAnalyse.cs +++ b/StgSharpDebug/TestRegexAnalyse.cs @@ -26,7 +26,7 @@ // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- using StgSharp.RegularAnalysis.Abstraction; -using StgSharp.RegularAnalysis.TextRegex; +using StgSharp.RegularAnalysis.Text; using System; using System.Collections.Generic; using System.Linq; @@ -44,7 +44,7 @@ namespace StgSharp @"abc[0-9]x|abc[0-9]y|abd[0-9]x|abd[0-9]y|xyz", @"get_name|get_value|get_type|set_name|set_value|set_type", @"helix|hello|help|hero|hers|her", - @"a*.[abc]*(abcd|a(bc)c)+(?abc)" + @"a*.[abc]*(abcd|a(bc)c)+(?abc{11,13})" ]; public static void Run() @@ -61,7 +61,7 @@ namespace StgSharp Console.Write(new string('|', enumerator.Depth)); Console.WriteLine($"\t{node.Source.Value}\t{node.Source.Flag}"); } - List ir = RegexInterpreter.GenerateIR(tree); + List ir = RegexInterpreter.GenerateIR(tree); Console.WriteLine("-----IR-----"); for (int i = 0; i < ir.Count; i++) { Console.WriteLine(ir[i].AsCommand(i)); diff --git a/future/ngra-regex-ir-review.md b/future/ngra-regex-ir-review.md new file mode 100644 index 00000000..84c2e96a --- /dev/null +++ b/future/ngra-regex-ir-review.md @@ -0,0 +1,1050 @@ +# NGRA 字符串正则 IR 设计补充建议 + +基于当前 `ngra-regex-ir.md` 的设计,我认为整体方向是成立的:它已经不是“完全低级的执行字节码”,而是保留了 `COUNT`、`COUNT_SEQ`、`COUNT_SET`、`TRY`、`PACK` 等高层结构,因此可以继续承担后续 match-opt 的输入。 + +不过,为了让规则更容易扩展、避免语义误判,我建议把优化明确拆成三类,并把当前 IR 分成“语义 IR / HIR”和“计划 IR / LIR”。 + +--- + +## 1. 总体判断 + +当前 IR 最大的优点: + +1. **足够紧凑**:`COUNT_SEQ`、`COUNT_SET`、`TRY` 等复合指令可以避免过早打碎成 `LoadChar + Compare + Branch`。 +2. **保留 regex 结构**:量词、字符集、确定序列、组打包等信息仍然可见。 +3. **适合继续做 match planning**:后续可以从这些指令中识别 `.*literal`、`[set]*`、bounded repeat、固定长度 block 等结构。 + +当前 IR 最大的风险: + +1. `SEARCH` 这个名字容易误导。当前 HIR 中的 `SEARCH` 实际应当是 **MATCH_AT_CURRENT**,不是允许跳过字符的 `FIND`。 +2. 检查点目前只描述为锚点指针栈,但真正的 backtracking frame 至少还要保存 `pos`、capture watermark、symbol watermark、repeat state 和 continuation。 +3. `COUNT_*` 的语义必须明确是否保留 retry/backtracking,否则 `a*.`、`[abc]*abc` 等场景会被错误优化。 +4. 字符集的语义表示和物理表示要分开:语义上 `[0-9a-z]` 是一个 set,物理上可以用 inline set-region 的多条 range-test 表示,避免每个 set 挂 `List`。 + +--- + +## 2. 建议的分层 + +我建议仍然区分 HIR 和 LIR,不是因为机器必须分,而是因为人写规则时边界更清楚。 + +```text +AST + ↓ +AST Opt + ↓ +HIR Gen + ↓ +HIR Analysis + ↓ +HIR → LIR / MatchPlan Lowering + ↓ +LIR + ↓ +SourceGen / RuntimeGen / Interpreter +``` + +### HIR:表达 regex 语义 + +HIR 保留接近正则结构的指令: + +```text +SEARCH_SEQ // 语义上是当前位置 MATCH_SEQ +SEARCH_SET // 语义上是当前位置 MATCH_SET +SEARCH_DOT // 或 UNIT_DOT,不应和 literal "." 混淆 +COUNT +COUNT_SEQ +COUNT_SET +TRY +POP +PACK +``` + +注意:HIR 中可以继续叫 `SEARCH`,但文档里要明确: + +```text +HIR 的 SEARCH_* 不表示向后查找; +它表示从当前活动指针开始匹配。 +真正允许跳过字符的搜索操作只在 LIR / MatchPlan 阶段引入,命名为 FIND_*。 +``` + +### LIR:表达匹配计划 + +LIR 不只是更低级,而是“已经做过策略规划”的 IR: + +```text +MATCH_SEQ +MATCH_CHAR +MATCH_DOT +MATCH_SET + +FIND_SEQ +FIND_CHAR +FIND_SET +FIND_LAST_SEQ +FIND_LAST_SET + +CONSUME_WHILE_SET +CONSUME_UNTIL_CHAR +CONSUME_UNTIL_SET + +COUNT_SET +COUNT_SEQ +COUNT_BLOCK +CHECK_MIN_LENGTH +CHECK_FIXED_OFFSET + +PUSH_BACKTRACK +POP_BACKTRACK +COMMIT +FAIL + +CAPTURE_BEGIN +CAPTURE_END +PACK +``` + +LIR 中应该已经包含: + +```text +LoweredSetId +SearchMode +RepeatMode +RetryMode +CaptureMode +EmitHint +``` + +--- + +## 3. 三类优化策略 + +建议把所有优化分成三类: + +```text +1. Regex algebra rewrite + 改正则结构。 + +2. Match plan rewrite + 改匹配策略。 + +3. Backend / C# span trick + 改 C# 代码写法。 +``` + +这个分类很重要,因为它能直接决定优化应该放在哪个阶段。 + +--- + +# 4. Regex algebra rewrite + +这类优化应该放在 AST Opt 或 HIR 生成前后。它们改变的是正则结构,而不是 C# 写法。 + +## 4.1 Literal 合并 + +```regex +a b c +``` + +变成: + +```regex +abc +``` + +HIR: + +```text +SEARCH_SEQ "abc" +``` + +不要生成三条 `SEARCH_CHAR`。 + +--- + +## 4.2 Concat / Alternation flatten + +```regex +a(?:b(?:c)) +``` + +变成: + +```regex +abc +``` + +```regex +a|(?:b|c) +``` + +变成: + +```regex +a|b|c +``` + +--- + +## 4.3 Alternation factoring / trie + +这个你已经放在 AST Opt 做,位置是对的。 + +```regex +foobar|foobaz|foozip +``` + +变成: + +```regex +foo(?:bar|baz|zip) +``` + +或者 trie AST: + +```text +SEQ "foo" +ALT + SEQ "bar" + SEQ "baz" + SEQ "zip" +``` + +这类结构改写越早越好。后续 IR opt 只需要识别这个结构并决定是否 `switch` / first-set dispatch。 + +--- + +## 4.4 Bounded repeat 展开 + +```regex +a{2,3} +``` + +可以变成: + +```regex +aa a? +``` + +或: + +```text +SEARCH_SEQ "aa" +OPTIONAL SEARCH_CHAR 'a' +``` + +对于 lazy: + +```regex +a{2,3}?b +``` + +要注意尝试顺序。它等价于: + +```regex +aa(?:b|ab) +``` + +而不是: + +```regex +aa(?:a|ab) +``` + +Greedy 版本: + +```regex +a{2,3}b +``` + +如果展开成显式分支,尝试顺序更像: + +```regex +aa(?:ab|b) +``` + +语言集合相同,但 backtracking 尝试顺序不同。 + +--- + +## 4.5 `a+b` 不建议改成 `a*(ab|b)` + +当前文档中写: + +```text +a+b 可以优化为 a*(ab|b) +``` + +这个需要修正。 + +`a+b` 的语言是: + +```text +a^n b, n >= 1 +``` + +而 `a*(ab|b)` 会额外匹配: + +```text +b +``` + +因为第二个分支 `a*b` 可以在 `a*` 取 0 时匹配 `b`。 + +更安全的结构是: + +```regex +aa*b +``` + +或者: + +```text +SEARCH_CHAR 'a' +COUNT_SEQ 'a' 0..∞ +SEARCH_CHAR 'b' +``` + +如果要利用后继 `b` 做策略优化,应在 MatchPlan 阶段处理,而不是做这个不等价的 algebra rewrite。 + +--- + +## 4.6 Charset 合并 + +```regex +[abc]|[def] +``` + +可以变成: + +```regex +[abcdef] +``` + +```regex +[a-f]|[d-z] +``` + +可以变成: + +```regex +[a-z] +``` + +这属于 AST/HIR 层的字符类代数。 + +--- + +## 4.7 Charset 展开为 alternation 要谨慎 + +文档中写: + +```text +abc[de] 可以展开为 (abcd|abce) +``` + +这个是可行的,但不一定总是好。建议规则改成: + +```text +当 charset 很小,并且展开后能形成更长 literal / trie / branch dispatch 时,可以展开; +否则保留为 MATCH_SET。 +``` + +例如: + +```regex +abc[de] +``` + +既可以是: + +```text +SEARCH_SEQ "abc" +SEARCH_SET [de] +``` + +也可以是: + +```text +ALT + SEARCH_SEQ "abcd" + SEARCH_SEQ "abce" +``` + +前者代码更短,后者可能更适合 required literal 或 trie。需要 code size budget。 + +--- + +# 5. Match plan rewrite + +这类优化应该放在 HIR Analysis / HIR → LIR lowering。 + +它们不一定改成另一条正则,而是把匹配过程改成更高效的搜索/消费计划。 + +--- + +## 5.1 长度预测应该在 HIR Analysis 做,SourceGen 只负责发代码 + +当前文档写“长度预测在 source-gen 阶段做”。我建议改成: + +```text +minLength / maxLength 的计算在 HIR Analysis 做; +SourceGen 只负责把 CHECK_MIN_LENGTH 发成 C#。 +``` + +例如: + +```regex +abc[0-9]{2}xyz +``` + +分析结果: + +```text +MinLength = 8 +``` + +LIR: + +```text +CHECK_MIN_LENGTH 8 +MATCH_SEQ "abc" +... +``` + +SourceGen: + +```csharp +if ((uint)(end - pos) < 8) goto Fail; +``` + +--- + +## 5.2 `.*` 优化 + +HIR: + +```text +COUNT_DOT min=0 max=∞ greedy=true +MATCH_CHAR 'a' +``` + +可以 lower 成: + +```text +FIND_LAST_CHAR 'a' +``` + +lazy: + +```text +COUNT_DOT min=0 max=∞ greedy=false +MATCH_CHAR 'a' +``` + +lower 成: + +```text +FIND_CHAR 'a' +``` + +但要注意: + +1. 默认 `.` 不匹配 `\n`,搜索范围应限制在当前行。 +2. Singleline 模式下才可以跨整个 span。 +3. 如果 `.*` 内部或后继结构涉及 capture/backtracking,不能简单“删掉 COUNT”,而应转换为带 retry 的 find plan。 +4. 后继如果是 `MATCH_SEQ "abc"`,则变成 `FIND_SEQ` / `FIND_LAST_SEQ`。 +5. 后继如果是 `MATCH_SET`,则变成 `FIND_SET` / `FIND_LAST_SET`。 + +--- + +## 5.3 `[set]*` 优化 + +文档中写 `[set]*` 优化在 source-gen 阶段做。建议改成: + +```text +是否将 COUNT_SET 变成 CONSUME_WHILE / CONSUME_UNTIL,在 lowering 做; +具体使用 IndexOfAnyExcept / SearchValues / manual loop,在 SourceGen 做。 +``` + +例如: + +```regex +[a-f0-9]* +``` + +LIR: + +```text +CONSUME_WHILE_SET LowerHex +``` + +SourceGen: + +```csharp +int n = span.IndexOfAnyExcept(s_lowerHex); +``` + +或者手写 loop / range compare。 + +--- + +## 5.4 First-set 搜索 + +如果非锚定匹配: + +```regex +abc|defg|xyz +``` + +first-set 是: + +```text +'a', 'd', 'x' +``` + +LIR 可以是: + +```text +FIND_SET { 'a', 'd', 'x' } +VERIFY_BRANCH +``` + +你文档里说“这个不好做,因为 IR 直接把 ALT 打碎成 if-else 串了”。这说明 AST Opt / HIR 里最好保留 ALT region 或者至少保留 branch summary。 + +即使 AST 已经 trie 化,IR 也应该尽量不要把它完全打碎成不可恢复的 if-else 串。至少给每个 region 计算 summary: + +```text +FirstSet +MinLength +RequiredLiteral +CanBacktrack +CanCapture +``` + +--- + +## 5.5 Required literal 搜索 + +例如: + +```regex +.*ERROR:[0-9]+ +``` + +可以直接规划: + +```text +FIND_SEQ "ERROR:" +VERIFY_AFTER_ANCHOR +``` + +这通常比只做 first-set 更强。 + +--- + +## 5.6 Fixed-offset check + +```regex +[a-z]{20}XYZ +``` + +可以 lower 成: + +```text +CHECK_MIN_LENGTH 23 +CHECK_AT_OFFSET 20 MATCH_SEQ "XYZ" +MATCH_SET_REPEAT [a-z] count=20 +MATCH_SEQ "XYZ" +``` + +这个属于 match plan,不属于 sourcegen。 + +--- + +## 5.7 Auto-atomic / possessive 化 + +```regex +[0-9]*[A-Z] +``` + +由于: + +```text +[0-9] ∩ [A-Z] = ∅ +``` + +所以 `[0-9]*` 不需要回退,可以 lower 成: + +```text +COUNT_SET [0-9] greedy atomic +MATCH_SET [A-Z] +``` + +如果后继 first-set 和 repeat set 有交集,就不能这么做。 + +--- + +## 5.8 Repeat retry mode + +LIR 应该明确 repeat 的 retry 策略: + +```text +RetryMode.None +RetryMode.BackoffBy1 +RetryMode.BackoffByStride +RetryMode.SavedPositions +RetryMode.GeneralBacktrack +``` + +例如: + +```regex +(abc[cd])+ +``` + +每轮固定长度 4,所以可以: + +```text +RetryMode.BackoffByStride stride=4 +``` + +不需要保存每个内部指令的回溯点。 + +--- + +# 6. Backend / C# span trick + +这类优化只放 SourceGen。它不再改变 regex 或 match plan。 + +--- + +## 6.1 `StartsWith` + +LIR: + +```text +MATCH_SEQ "sample" +``` + +SourceGen 决定: + +```csharp +slice.StartsWith("sample", StringComparison.Ordinal) +``` + +还是短串展开: + +```csharp +slice.Length >= 3 && +slice[0] == 'a' && +slice[1] == 'b' && +slice[2] == 'c' +``` + +建议规则: + +```text +长度 1: + char compare + +长度 2~3: + 直接 char compare + +中长 literal: + StartsWith + +IgnoreCase: + StartsWith(..., OrdinalIgnoreCase) +``` + +--- + +## 6.2 `[0-9]` 不建议写成 `IndexOf("0123456789")` 用于当前位置测试 + +文档中写: + +```text +[0-9] 展开为 IndexOf("0123456789") +``` + +这个要分两种场景: + +### 当前位置匹配 + +如果是: + +```text +MATCH_SET [0-9] +``` + +更好是: + +```csharp +(uint)(ch - '0') <= 9 +``` + +### span 中搜索 + +如果是: + +```text +FIND_SET [0-9] +``` + +可以考虑: + +```csharp +span.IndexOfAnyInRange('0', '9') +``` + +或: + +```csharp +span.IndexOfAny(s_digits) +``` + +所以规则应写成: + +```text +MATCH_SET [0-9] → unsigned range compare +FIND_SET [0-9] → IndexOfAnyInRange / SearchValues +``` + +--- + +## 6.3 SearchValues + +对于多字符集合: + +```text +FIND_SET [0-9a-f] +``` + +SourceGen 可生成: + +```csharp +private static readonly SearchValues s_lowerHex = + SearchValues.Create("0123456789abcdef"); +``` + +然后: + +```csharp +span.IndexOfAny(s_lowerHex); +``` + +当前位置测试不一定要用 `SearchValues`,可能 range compare 或 bitmap 更快。 + +--- + +## 6.4 Inline vs helper + +复杂 Unicode set: + +```text +MATCH_SET UnicodeWord +``` + +SourceGen 决定: + +```csharp +if (!IsUnicodeWord(ch)) goto Fail; +``` + +还是: + +```csharp +if (ch < 128) +{ + if (!AsciiWordContains(ch)) goto Fail; +} +else +{ + if (!IsUnicodeWord(ch)) goto Fail; +} +``` + +这个应由代码尺寸、热度、是否 source-gen 决定。 + +--- + +## 6.5 Code size budget + +SourceGen 应负责最终代码膨胀控制: + +```text +单方法最大语句数 +literal 展开阈值 +bounded repeat 展开阈值 +是否生成 helper +是否抽 cold path +SearchValues 字段数量 +``` + +--- + +# 7. 字符集的物理表示建议 + +为了避免每个 set 挂 `List`,可以用线性 IR region 表示同一个字符测试。 + +例如语义: + +```regex +[0-9a-z] +``` + +物理 IR: + +```text +MATCH_SET_BEGIN default=false end=13 + TEST_RANGE '0' '9' accept=true + TEST_RANGE 'a' 'z' accept=true +MATCH_SET_END +``` + +语义: + +```regex +[^0-9a-z] +``` + +物理 IR: + +```text +MATCH_SET_BEGIN default=true end=13 + TEST_RANGE '0' '9' accept=false + TEST_RANGE 'a' 'z' accept=false +MATCH_SET_END +``` + +要点是: + +```text +这些 TEST_RANGE 测的是同一个字符; +整个 MATCH_SET 只推进 pos 一次。 +``` + +不能把它误解成连续匹配多个字符。 + +复杂 Unicode set 可以走: + +```text +MATCH_SET_REF setId=42 +``` + +`setId=42` 指向全局 pooled table,而不是每个节点一个 List。 + +--- + +# 8. 检查点语义建议 + +当前文档把检查点描述为锚点指针栈,这对 capture 起点有用,但不够覆盖 backtracking。 + +建议把 checkpoint / backtrack frame 定义成: + +```text +BacktrackFrame: + pos + anchor + captureTop + symbolTop + repeatState + continuation +``` + +其中: + +```text +pos: + 活动指针恢复位置 + +anchor: + 区间开始位置 + +captureTop: + 捕获记录栈回滚点 + +symbolTop: + 符号区回滚点 + +repeatState: + 当前 COUNT/loop 的计数、起止位置、retry mode + +continuation: + 失败后跳到哪里继续尝试 +``` + +如果暂时是 tokenizer / no-capture 模式,可以裁掉 captureTop,但 IR 语义最好预留。 + +--- + +# 9. 建议修改原文的几个点 + +## 9.1 “HIT” 应改为 “HIR” + +原文: + +```text +将HIT进行进一步优化 +``` + +应为: + +```text +将 HIR 进行进一步优化 +``` + +## 9.2 `SEARCH` 的定义应改写 + +原文说“向后搜索确定长度序列”。建议改为: + +```text +HIR 中 SEARCH_* 表示从当前活动指针开始匹配; +它不是允许跳过字符的 FIND。 +真正的 FIND_* 在 HIR→LIR 阶段由 match planner 引入。 +``` + +## 9.3 `a+b → a*(ab|b)` 不等价 + +建议删除或改成: + +```regex +a+b → aa*b +``` + +或者: + +```text +MATCH_CHAR 'a' +COUNT_SEQ 'a' 0..∞ +MATCH_CHAR 'b' +``` + +## 9.4 `[0-9] → IndexOf("0123456789")` 应细分 + +改成: + +```text +当前位置 MATCH_SET: + 用 unsigned range compare。 + +span 搜索 FIND_SET: + 用 IndexOfAnyInRange / SearchValues / IndexOfAny。 +``` + +## 9.5 `.*` 优化不要说 “COUNT 消失” + +建议改成: + +```text +COUNT_DOT 与后继匹配项合并为 FIND / FIND_LAST 计划。 +``` + +这样更准确,因为有时还需要保留 retry / range 限制 / newline 限制。 + +--- + +# 10. 推荐最终管线 + +```text +Tokenize + - 区分 Dot 与 EscapedLiteralDot + - 区分 raw charset / literal / quantifier / group + +AST Build + - 建立 regex 语义树 + +AST Opt + - literal merge + - concat flatten + - alternation factoring / trie + - bounded repeat normalization + - simple group elimination + - charset algebra merge + +HIR Gen + - SEARCH_* as MATCH_AT_CURRENT + - COUNT_* + - TRY / POP + - PACK + - raw set / raw dot + +HIR Analysis + - minLength / maxLength + - nullable + - firstSet + - lastSet + - required literal + - fixed length + - capture effect + - repeat overlap + +HIR → LIR / MatchPlan + - raw set → lowered set + - dot → dot mode + - introduce FIND + - introduce CONSUME + - choose retry mode + - choose atomic/possessive + - emit check-min-length / fixed-offset checks + +LIR + - explicit MATCH / FIND / CONSUME + - explicit lowered set + - explicit backtracking plan + - explicit capture plan + +SourceGen + - StartsWith / SequenceEqual + - IndexOf / IndexOfAny / IndexOfAnyExcept + - SearchValues + - range compare / bitmap + - inline/helper split + - code size budget +``` + +--- + +# 11. 总结 + +我建议保留你当前 IR 的核心设计,但把文档里的边界收紧: + +```text +AST Opt: + 做 Regex algebra rewrite。 + +HIR: + 表示从当前位置开始的 regex 语义。 + +HIR Analysis: + 提取结构属性。 + +HIR → LIR: + 做 Match plan rewrite。 + +LIR: + 表示明确的匹配计划。 + +SourceGen: + 做 C# span trick。 +``` + +最重要的改动是: + +```text +SEARCH 在 HIR 中应被定义为 MATCH_AT_CURRENT; +FIND 只能由后续优化阶段引入。 +``` + +以及: + +```text +字符集可以用线性 set-region 表示,避免 List 分配; +但 set-region 内的多条 range-test 必须测试同一个字符,不能变成多个连续 MATCH。 +``` + +这样你的 IR 会同时满足三个目标: + +1. 对人来说可读; +2. 对优化规则来说容易扩展; +3. 对 GC 和 sourcegen 来说物理表示足够紧凑。 diff --git a/future/ngra-regex-ir.md b/future/ngra-regex-ir.md index 7e80e2e9..4d50de5e 100644 --- a/future/ngra-regex-ir.md +++ b/future/ngra-regex-ir.md @@ -14,15 +14,62 @@ 使用DEFINE指令定义的符号定义在符号区中,符号的类型通过第一次赋值确定,源生成模式下,可以通过第一个赋值指令推导其类型 -## 算子 +## HIR算子 1. 搜索算子(SEARCH %str; SEARCH %unicode),向后搜索确定长度序列或者满足规则的字符集,更新活动指针,不更新锚点指针,搜索结果保存在全局上下文中 2. 搜索算子(SEARCH %set_expression),向后匹配一个满足字符集要求的字符,更新活动指针,不更新锚点指针,搜索结果保存在全局上下文中 -3. 失败配置指令(CINFIG_FAIL %fail),指定当前匹配栈后续如何处理一条指令的失败,可选为:FAST,立刻终止当前层级;CONTINUE,记录这个fail,但是继续操作;REVERSE,反过来,成功了才终止 -4. 条件指令(CONDITION %cmd_true %cmd_fail),比较前一条SEARCH指令是否成功,并指定跳转到指定的位置,如果某种条件不需要跳转,可以写1。如果想要将fail向上级继续抛出,可以%cmd_fail写0. -5. 跳转执行指令(TRY %begin %end),尝试idx指向的IR块,并记住执行结果数据 -6. 计数指令(COUNT %min %max %is_greedy %cmd),重复执行指令cmd,直到满足参数配置的数量为止 -7. 快速序列计数指令(COUNT_SEQ %min %max %is_greedy %seq),快速搜索一个确定性序列,直到满足参数配置的数量为止 -8. 快速字符集计数指令(COUNT_SET %min %max %is_greedy %set),快速搜索一个字符集,直到满足参数配置的数量为止 -9. 尝试(TRY %count),对自身向后的一段内容在新的匹配栈层级中进行运行,被分层的指令数量由COUNT指令确定 -10. 弹出(POP %is_success),提前终止一个try块,并用is_success表示弹出行为视为匹配成功还是失败 +3. 条件指令(CONDITION %cmd_true %cmd_fail),比较前一条SEARCH指令是否成功,并指定跳转到指定的位置,如果某种条件不需要跳转,可以写1。如果想要将fail向上级继续抛出,可以%cmd_fail写0. +4. 计数指令(COUNT %min %max %is_greedy %cmd),重复执行指令cmd,直到满足参数配置的数量为止 +5. 快速序列计数指令(COUNT_SEQ %min %max %is_greedy %seq),快速搜索一个确定性序列,直到满足参数配置的数量为止 +6. 快速字符集计数指令(COUNT_SET %min %max %is_greedy %set),快速搜索一个字符集,直到满足参数配置的数量为止 +7. 尝试(TRY %count),对自身向后的一段内容在新的匹配栈层级中进行运行,被分层的指令数量由COUNT指令确定 +8. 弹出(POP %is_success),提前终止一个try块,并用is_success表示弹出行为视为匹配成功还是失败 +9. 打包指令(PACK %is_success %name),检查上一条语句的结果,将匹配结果打包并记录为指定的名字。如果名字为空,则使用默认的序号进行索引 + +## 优化策略 + +应当注意的是,将HIT进行进一步优化或者展开的目的: + +1. 尽可能降低源生成或者运行时执行中,对多指令上下文的依赖,尽可能保证每一个IR可以单独输出最佳的代码 +2. 减少多余的结构,使得一行指令尽可能处理多的数据 + +### 语义改写优化 + +1. 计数器展开 +建议在lower做 +对于形如a+b或者a{2,4}b的正则表达式,可以优化为a*(ab|b)和a{2,3}(ab|b)的写法 + +2. 确定性计数展开 +建议在lower做 +对于形如a{m,n}的正则表达式,将其展开为a{m}a{0,n-m}的形式 + +3. charset合并 +建议在ast-opt做 +对于形如[ abc ]|[ def ]的表达式,将其合并为[ abcdef ] + +4. charset展开 +在lower做 +对于形如abc[ de ]的表达式,如果字符集容量不超过4,可以站卡为(abcd|abce)形式 + +### 匹配策略改变 + +1. 长度预测 +在source-gen阶段做 +对于一个正则表达式,可以计算其所需的最小长度,可以在开始的时候提前进行比较 + +2. .*优化 +在lower阶段做 +对于COUNT_SET min=0 max=-1 "."指令,也就是.*操作,将后续指令从match转为find + +3. [ set ]*优化 +在source-gen阶段做 +对于其他的COUNT_SET min=0 max=-1 "???"指令,使用IndexOf,IndexOfAnyExcept等进行优化 + +4. 首字母检查 +**这个不好做,因为IR直接把ALT打碎成if-else串了** +对于(abc|defg|xyz)表达式,可以检查其首字母 + +### C# span opt trick + +1. Span.StartWith +2. [ 0-9 ]展开为IndexOf("0123456789") \ No newline at end of file