From 7e5e4d1f7a34c2f266b64d9c0ca76825da880c76 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Mon, 22 Jun 2026 18:18:16 +0800 Subject: [PATCH 1/6] discuss regex ir opt --- .../{TextRegex => Text}/RegexAstNode.cs | 2 +- .../{TextRegex => Text}/RegexElementLabel.cs | 2 +- .../RegexIR.cs => Text/RegexHIR.cs} | 110 +- .../RegexHIRGenerator.cs} | 51 +- .../RegexInterpreter.ASTOptimize.cs} | 2 +- .../RegexInterpreter.HIRGenerate.cs} | 53 +- .../Text/RegexInterpreter.IRScan.cs | 69 ++ .../Text/RegexInterpreter.SourceGenerate.cs | 51 + .../{TextRegex => Text}/RegexInterpreter.cs | 2 +- StgSharp.RegularAnalysis/Text/RegexLIR.cs | 60 + .../{TextRegex => Text}/RegexTokenParser.cs | 2 +- .../{TextRegex => Text}/RegexTokenReader.cs | 2 +- .../{TextRegex => Text}/RegexTokenUtil.cs | 2 +- StgSharp.RegularAnalysis/Text/TextHelper.cs | 50 + StgSharp.RegularAnalysis/Text/TextRegex.cs | 47 + StgSharpDebug/TestRegexAnalyse.cs | 6 +- future/ngra-regex-ir-review.md | 1050 +++++++++++++++++ future/ngra-regex-ir.md | 65 +- 18 files changed, 1499 insertions(+), 127 deletions(-) rename StgSharp.RegularAnalysis/{TextRegex => Text}/RegexAstNode.cs (99%) rename StgSharp.RegularAnalysis/{TextRegex => Text}/RegexElementLabel.cs (97%) rename StgSharp.RegularAnalysis/{TextRegex/RegexIR.cs => Text/RegexHIR.cs} (81%) rename StgSharp.RegularAnalysis/{TextRegex/RegexIRGenerator.cs => Text/RegexHIRGenerator.cs} (79%) rename StgSharp.RegularAnalysis/{TextRegex/RegexInterpreter.Optimize.cs => Text/RegexInterpreter.ASTOptimize.cs} (99%) rename StgSharp.RegularAnalysis/{TextRegex/RegexInterpreter.IRGenerate.cs => Text/RegexInterpreter.HIRGenerate.cs} (88%) create mode 100644 StgSharp.RegularAnalysis/Text/RegexInterpreter.IRScan.cs create mode 100644 StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs rename StgSharp.RegularAnalysis/{TextRegex => Text}/RegexInterpreter.cs (99%) create mode 100644 StgSharp.RegularAnalysis/Text/RegexLIR.cs rename StgSharp.RegularAnalysis/{TextRegex => Text}/RegexTokenParser.cs (98%) rename StgSharp.RegularAnalysis/{TextRegex => Text}/RegexTokenReader.cs (99%) rename StgSharp.RegularAnalysis/{TextRegex => Text}/RegexTokenUtil.cs (97%) create mode 100644 StgSharp.RegularAnalysis/Text/TextHelper.cs create mode 100644 StgSharp.RegularAnalysis/Text/TextRegex.cs create mode 100644 future/ngra-regex-ir-review.md diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs b/StgSharp.RegularAnalysis/Text/RegexAstNode.cs similarity index 99% rename from StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs rename to StgSharp.RegularAnalysis/Text/RegexAstNode.cs index 9b6de712..b5238042 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs +++ b/StgSharp.RegularAnalysis/Text/RegexAstNode.cs @@ -28,7 +28,7 @@ using StgSharp.RegularAnalysis.Abstraction; using System.Runtime.CompilerServices; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { internal class RegexAstNode : ISyntaxNode { diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs b/StgSharp.RegularAnalysis/Text/RegexElementLabel.cs similarity index 97% rename from StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs rename to StgSharp.RegularAnalysis/Text/RegexElementLabel.cs index dec57691..96e3b61b 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs +++ b/StgSharp.RegularAnalysis/Text/RegexElementLabel.cs @@ -31,7 +31,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { [Flags] public enum RegexElementLabel diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexIR.cs b/StgSharp.RegularAnalysis/Text/RegexHIR.cs similarity index 81% rename from StgSharp.RegularAnalysis/TextRegex/RegexIR.cs rename to StgSharp.RegularAnalysis/Text/RegexHIR.cs index 46498c96..11e44c35 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexIR.cs +++ b/StgSharp.RegularAnalysis/Text/RegexHIR.cs @@ -32,25 +32,24 @@ using System.Runtime.CompilerServices; using System.Text; using System.Threading.Tasks; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { internal enum RegexIRCommand { - Search_Seq = 0, - Search_Set = 1, - SearchChar = 2, - ConfigFail = 3, - Condition = 4, - Try = 5, - Count = 6, - Count_Seq = 7, - Count_Set = 8, - Pop = 10 + Match_Seq = 0, + Match_Set = 1, + Condition = 3, + Try = 4, + Count = 5, + Count_Seq = 6, + Count_Set = 7, + Pop = 8, + Pack = 9, } - internal abstract class RegexIR + internal abstract class RegexHIR { internal abstract RegexIRCommand Command { get; } @@ -64,70 +63,46 @@ namespace StgSharp.RegularAnalysis.TextRegex /// /// Searches for a certain string pattern or charset from current position. /// - internal sealed class RegexMatchSeqIR( + internal sealed class RegexMatchSeqHIR( string pattern - ) : RegexIR + ) : RegexHIR { public string Pattern { get; } = pattern; - internal override RegexIRCommand Command => RegexIRCommand.Search_Seq; + internal override RegexIRCommand Command => RegexIRCommand.Match_Seq; internal override string AsCommand( int lineNum ) { - return $"[{lineNum}]\t SEARCH_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "" : Pattern)}\""; + return $"[{lineNum}]\t MATCH_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "" : Pattern)}\""; } } - internal sealed class RegexMatchSetIR( + internal sealed class RegexMatchSetHIR( string charset - ) : RegexIR + ) : RegexHIR { public string Charset { get; } = charset; - internal override RegexIRCommand Command => RegexIRCommand.Search_Set; + internal override RegexIRCommand Command => RegexIRCommand.Match_Set; internal override string AsCommand( int lineNum ) { - return $"[{lineNum}]\t SEARCH_SET \"{Charset}\""; + return $"[{lineNum}]\t MATCH_SET \"{Charset}\""; } } - /// - /// Configures the failure jump of current position. The failMod ranges 0-2 - /// - /// - /// - /// - internal sealed class RegexConfigFailIR( - int failMod - ) : RegexIR - { - - public int FailMod { get; } = failMod & 0x3; - - internal override RegexIRCommand Command => RegexIRCommand.ConfigFail; - - internal override string AsCommand( - int lineNum - ) - { - return $"[{lineNum}]\t CONFIG_FAIL {FailMod}"; - } - - } - - internal sealed class RegexConditionIR( + internal sealed class RegexConditionHIR( int success, int fail - ) : RegexIR + ) : RegexHIR { public int SuccessCase { get; } = success; @@ -145,12 +120,33 @@ namespace StgSharp.RegularAnalysis.TextRegex } - internal sealed class RegexCountIR( + internal sealed class RegexPackHIR( + bool condition, + string name = "" + ) : RegexHIR + { + + public bool Condition { get; } = condition; + + public string GroupName { get; } = name; + + internal override RegexIRCommand Command => RegexIRCommand.Pack; + + internal override string AsCommand( + int lineNum + ) + { + return $"[{lineNum}]\t PACK {(Condition ? "SUCCESS" : "FAIL")} GROUP: \"{GroupName}\""; + } + + } + + internal sealed class RegexCountHIR( int min, int max, bool isGreedy, int operation - ) : RegexIR + ) : RegexHIR { public bool IsGreedy { get; } = isGreedy; @@ -172,12 +168,12 @@ namespace StgSharp.RegularAnalysis.TextRegex } - internal sealed class RegexCountSeqIR( + internal sealed class RegexCountSeqHIR( int min, int max, bool isGreedy, string seq - ) : RegexIR + ) : RegexHIR { public bool IsGreedy { get; } = isGreedy; @@ -199,12 +195,12 @@ namespace StgSharp.RegularAnalysis.TextRegex } - internal sealed class RegexCountSetIR( + internal sealed class RegexCountSetHIR( int min, int max, bool isGreedy, string set - ) : RegexIR + ) : RegexHIR { public bool IsGreedy { get; } = isGreedy; @@ -226,9 +222,9 @@ namespace StgSharp.RegularAnalysis.TextRegex } - internal sealed class RegexTryIR( + internal sealed class RegexTryHIR( int count - ) : RegexIR + ) : RegexHIR { public int CountOfCommand { get; } = count; @@ -244,14 +240,14 @@ namespace StgSharp.RegularAnalysis.TextRegex } - internal sealed class RegexPopIR( + internal sealed class RegexPopHIR( bool isSuccess - ) : RegexIR + ) : RegexHIR { public bool IsSuccess { get; } = isSuccess; - internal override RegexIRCommand Command => RegexIRCommand.Try; + internal override RegexIRCommand Command => RegexIRCommand.Pop; internal override string AsCommand( int lineNum diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexIRGenerator.cs b/StgSharp.RegularAnalysis/Text/RegexHIRGenerator.cs similarity index 79% rename from StgSharp.RegularAnalysis/TextRegex/RegexIRGenerator.cs rename to StgSharp.RegularAnalysis/Text/RegexHIRGenerator.cs index e5afb6ee..b1720138 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexIRGenerator.cs +++ b/StgSharp.RegularAnalysis/Text/RegexHIRGenerator.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexIRGenerator" +// file="RegexHIRGenerator" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -32,15 +32,15 @@ using System.Runtime.CompilerServices; using System.Text; using System.Threading.Tasks; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { - internal class RegexIRGenerator + internal class RegexHIRGenerator { private Node _head = new(); private Node _tail; - public RegexIRGenerator() + public RegexHIRGenerator() { _tail = _head; } @@ -52,15 +52,7 @@ namespace StgSharp.RegularAnalysis.TextRegex int fail ) { - _tail.Add(new RegexConditionIR(success, fail)); - Count++; - } - - public void EmitConfigFail( - int fail - ) - { - _tail.Add(new RegexConfigFailIR(fail)); + _tail.Add(new RegexConditionHIR(success, fail)); Count++; } @@ -71,7 +63,7 @@ namespace StgSharp.RegularAnalysis.TextRegex int operation ) { - _tail.Add(new RegexCountIR(min, max, isGreedy, operation)); + _tail.Add(new RegexCountHIR(min, max, isGreedy, operation)); Count++; } @@ -82,7 +74,7 @@ namespace StgSharp.RegularAnalysis.TextRegex string seq ) { - _tail.Add(new RegexCountSeqIR(min, max, isGreedy, seq)); + _tail.Add(new RegexCountSeqHIR(min, max, isGreedy, seq)); Count++; } @@ -93,12 +85,12 @@ namespace StgSharp.RegularAnalysis.TextRegex string set ) { - _tail.Add(new RegexCountSetIR(min, max, isGreedy, set)); + _tail.Add(new RegexCountSetHIR(min, max, isGreedy, set)); Count++; } public void EmitIRStream( - RegexIRGenerator stream + RegexHIRGenerator stream ) { if (stream is null || stream.Count == 0) { @@ -113,7 +105,7 @@ namespace StgSharp.RegularAnalysis.TextRegex string unit ) { - _tail.Add(new RegexMatchSeqIR(unit)); + _tail.Add(new RegexMatchSeqHIR(unit)); Count++; } @@ -121,7 +113,16 @@ namespace StgSharp.RegularAnalysis.TextRegex string unit ) { - _tail.Add(new RegexMatchSetIR(unit)); + _tail.Add(new RegexMatchSetHIR(unit)); + Count++; + } + + public void EmitPack( + bool condition, + string name = "" + ) + { + _tail.Add(new RegexPackHIR(condition, name)); Count++; } @@ -129,7 +130,7 @@ namespace StgSharp.RegularAnalysis.TextRegex bool isSuccess ) { - _tail.Add(new RegexPopIR(isSuccess)); + _tail.Add(new RegexPopHIR(isSuccess)); Count++; } @@ -137,13 +138,13 @@ namespace StgSharp.RegularAnalysis.TextRegex int count ) { - _tail.Add(new RegexTryIR(count)); + _tail.Add(new RegexTryHIR(count)); Count++; } - public List ToList() + public List ToList() { - List result = new(Count); + List result = new(Count); Node current = _head; while (current != null) { @@ -160,11 +161,11 @@ namespace StgSharp.RegularAnalysis.TextRegex public int Count => IRList.Count; - public List IRList { get; } = []; + public List IRList { get; } = []; [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Add( - RegexIR ir + RegexHIR ir ) { IRList.Add(ir); diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs similarity index 99% rename from StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs rename to StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs index 6e7bbb80..36398174 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs @@ -37,7 +37,7 @@ using System.Text; using System.Threading.Tasks; using System.Xml.Linq; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { public partial class RegexInterpreter { diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.IRGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.HIRGenerate.cs similarity index 88% rename from StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.IRGenerate.cs rename to StgSharp.RegularAnalysis/Text/RegexInterpreter.HIRGenerate.cs index 7c7e77a2..bc42c472 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.IRGenerate.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.HIRGenerate.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexInterpreter.IRGenerate" +// file="RegexInterpreter.HIRGenerate" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -38,13 +38,13 @@ using System.Text; using System.Threading.Tasks; using System.Xml; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { public partial class RegexInterpreter { - internal static List GenerateIR( - AbstractSyntaxTree tree + internal static List GenerateIR( + AbstractSyntaxTree tree ) { return GenerateIRRecursion(tree.Root, 0).ToList(); @@ -53,7 +53,7 @@ namespace StgSharp.RegularAnalysis.TextRegex [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void EmitCount( int curDepth, - RegexIRGenerator ir, + RegexHIRGenerator ir, int min, int max, bool isGreedy, @@ -66,7 +66,7 @@ namespace StgSharp.RegularAnalysis.TextRegex if (isRightOperator) // seq to be count is complex { ir.EmitCount(min, max, isGreedy, 1); - RegexIRGenerator rightIR = GenerateIRRecursion(right, curDepth + 1); + RegexHIRGenerator rightIR = GenerateIRRecursion(right, curDepth + 1); ir.EmitTry(rightIR.Count); ir.EmitIRStream(rightIR); } else @@ -89,9 +89,9 @@ namespace StgSharp.RegularAnalysis.TextRegex } } - private static RegexIRGenerator GenerateIRRecursion( - RegexAstNode node, - int depth + private static RegexHIRGenerator GenerateIRRecursion( + RegexAstNode node, + int depth ) { #pragma warning disable IDE0010 @@ -102,8 +102,8 @@ namespace StgSharp.RegularAnalysis.TextRegex int curDepth = depth; RegexAstNode left = node.Left; RegexAstNode right = node.Right; - RegexIRGenerator leftIR; - RegexIRGenerator rightIR; + RegexHIRGenerator leftIR; + RegexHIRGenerator rightIR; Token token = node.Source; @@ -116,15 +116,16 @@ namespace StgSharp.RegularAnalysis.TextRegex bool is_group = group_source.Length == 1; // simple group: (pattern) string group_name = group_source.StartsWith(@"(?<", StringComparison.Ordinal) ? - group_source[3..^2] : /* named group (?) */ + group_source[3..^1] : /* named group (?) */ string.Empty; /* unnamed group (?:) */ - RegexIRGenerator groupIR = new(); + RegexHIRGenerator groupIR = new(); rightIR = GenerateIRRecursion(right, curDepth + 1); groupIR.EmitTry(rightIR.Count); groupIR.EmitIRStream(rightIR); groupIR.EmitCondition(2, 1); - /*generate pack ir here*/ + groupIR.EmitPop(false); + groupIR.EmitPack(true, group_name); return groupIR; case RegexElementLabel.COUNT: string count_source = token.Value; @@ -143,14 +144,14 @@ namespace StgSharp.RegularAnalysis.TextRegex _ => throw new InvalidOperationException("Unexpected sequence mask for count operator"), }; ReadOnlySpan count_source_span = count_source; - RegexIRGenerator countIR = new(); + RegexHIRGenerator countIR = new(); if (count_source_span[0] == '*') // infinite count { - EmitCount(curDepth, countIR, 0, int.MaxValue, isGreedy, seq); + EmitCount(curDepth, countIR, 0, -1, isGreedy, seq); } else if (count_source_span[0] == '+') // at least one { - EmitCount(curDepth, countIR, 1, int.MaxValue, isGreedy, seq); + EmitCount(curDepth, countIR, 1, -1, isGreedy, seq); } else if (count_source_span[0] == '?') // zero or one { EmitCount(curDepth, countIR, 0, 1, isGreedy, seq); @@ -159,7 +160,7 @@ namespace StgSharp.RegularAnalysis.TextRegex int comma_pos = count_source_span.IndexOf(','); if (comma_pos == -1) { - if (int.TryParse(count_source[1..(isGreedy ? ^3 : ^2)], out int count)) + if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count)) { EmitCount(curDepth, countIR, count, count, isGreedy, seq); } else @@ -169,7 +170,7 @@ namespace StgSharp.RegularAnalysis.TextRegex } else { if (int.TryParse(count_source[1..comma_pos], out int min) && - int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^3 : ^2)], out int max)) + int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max)) { EmitCount(curDepth, countIR, min, max, isGreedy, seq); } else @@ -184,7 +185,7 @@ namespace StgSharp.RegularAnalysis.TextRegex return countIR; case RegexElementLabel.CONCAT: - RegexIRGenerator concat = new(); + RegexHIRGenerator concat = new(); // process left node if (RegexAstNode.IsNullOrEmpty(left)) @@ -237,13 +238,13 @@ namespace StgSharp.RegularAnalysis.TextRegex } return concat; case RegexElementLabel.ALT: - RegexIRGenerator alt = new(); - List casesIRList = []; + RegexHIRGenerator alt = new(); + List casesIRList = []; RegexAstNode cur_node = right; int ir_count = 0; while (!RegexAstNode.IsNullOrEmpty(cur_node)) { - RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1); + RegexHIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1); casesIRList.Add(_case); ir_count += _case.Count; cur_node = cur_node.Next; @@ -251,7 +252,7 @@ namespace StgSharp.RegularAnalysis.TextRegex int case_count = casesIRList.Count; while (case_count != 0) { - RegexIRGenerator cur_ir = casesIRList[^case_count]; + RegexHIRGenerator cur_ir = casesIRList[^case_count]; ir_count -= cur_ir.Count; case_count--; alt.EmitTry(cur_ir.Count); @@ -264,11 +265,11 @@ namespace StgSharp.RegularAnalysis.TextRegex case RegexElementLabel.UNIT_SPAN: goto case RegexElementLabel.UNIT; case RegexElementLabel.UNIT: - RegexIRGenerator unit_seq = new(); + RegexHIRGenerator unit_seq = new(); unit_seq.EmitMatchSeq(node.Source.Value); return unit_seq; case RegexElementLabel.UNIT_SET: - RegexIRGenerator unit_set = new(); + RegexHIRGenerator unit_set = new(); unit_set.EmitMatchSet(node.Source.Value); return unit_set; default: diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRScan.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRScan.cs new file mode 100644 index 00000000..61c7b28c --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRScan.cs @@ -0,0 +1,69 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="RegexInterpreter.IRScan" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + public partial class RegexInterpreter + { + + internal static void ScanAndMarkIROptimization( + RegexHIR ir + ) + { + switch (ir.Command) + { + case RegexIRCommand.Match_Seq: + break; + case RegexIRCommand.Match_Set: + break; + case RegexIRCommand.Pack: + break; + case RegexIRCommand.Condition: + break; + case RegexIRCommand.Try: + break; + case RegexIRCommand.Count: + break; + case RegexIRCommand.Count_Seq: + break; + case RegexIRCommand.Count_Set: + break; + case RegexIRCommand.Pop: + break; + default: + break; + } + } + + } +} diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs new file mode 100644 index 00000000..67d84307 --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs @@ -0,0 +1,51 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="RegexInterpreter.SourceGenerate" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + public partial class RegexInterpreter + { + + internal static string GetSourceCode( + IEnumerable irs + ) + { + StringBuilder sb = new StringBuilder(); + foreach (RegexHIR ir in irs) { + ScanAndMarkIROptimization(ir); + } + return sb.ToString(); + } + + } +} diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.cs similarity index 99% rename from StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs rename to StgSharp.RegularAnalysis/Text/RegexInterpreter.cs index 0a5cfbdc..f7f1c02d 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.cs @@ -35,7 +35,7 @@ using System.Text; using System.Threading.Tasks; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { public partial class RegexInterpreter { diff --git a/StgSharp.RegularAnalysis/Text/RegexLIR.cs b/StgSharp.RegularAnalysis/Text/RegexLIR.cs new file mode 100644 index 00000000..272de4f3 --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/RegexLIR.cs @@ -0,0 +1,60 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="RegexLIR" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + internal enum LIRCommand + { + + Search = 0, + Count = 1, + Try = 2, + + } + + internal abstract class RegexLIR + { + + public abstract LIRCommand Command { get; } + + } + + internal enum RegexSearchMethod + { + + Match, + Find, + Assert, + + } +} diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs b/StgSharp.RegularAnalysis/Text/RegexTokenParser.cs similarity index 98% rename from StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs rename to StgSharp.RegularAnalysis/Text/RegexTokenParser.cs index 452a65a1..f4ee54a3 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs +++ b/StgSharp.RegularAnalysis/Text/RegexTokenParser.cs @@ -34,7 +34,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { internal class RegexTokenParser : TokenParser { diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs b/StgSharp.RegularAnalysis/Text/RegexTokenReader.cs similarity index 99% rename from StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs rename to StgSharp.RegularAnalysis/Text/RegexTokenReader.cs index d57f3518..953b0eb9 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs +++ b/StgSharp.RegularAnalysis/Text/RegexTokenReader.cs @@ -34,7 +34,7 @@ using System.Security.Permissions; using System.Text; using System.Threading.Tasks; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { internal class RegexTokenReader : ITokenReader { diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexTokenUtil.cs b/StgSharp.RegularAnalysis/Text/RegexTokenUtil.cs similarity index 97% rename from StgSharp.RegularAnalysis/TextRegex/RegexTokenUtil.cs rename to StgSharp.RegularAnalysis/Text/RegexTokenUtil.cs index 3d51ab3c..1a32cc24 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexTokenUtil.cs +++ b/StgSharp.RegularAnalysis/Text/RegexTokenUtil.cs @@ -32,7 +32,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -namespace StgSharp.RegularAnalysis.TextRegex +namespace StgSharp.RegularAnalysis.Text { internal static class RegexTokenUtil { diff --git a/StgSharp.RegularAnalysis/Text/TextHelper.cs b/StgSharp.RegularAnalysis/Text/TextHelper.cs new file mode 100644 index 00000000..78d9b4d6 --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/TextHelper.cs @@ -0,0 +1,50 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="TextHelper" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + public static class TextHelper + { + + public static bool TryLastIndexOf( + this ReadOnlySpan span, + ReadOnlySpan value, + out int index + ) + { + index = span.LastIndexOf(value); + return index != -1; + } + + } +} diff --git a/StgSharp.RegularAnalysis/Text/TextRegex.cs b/StgSharp.RegularAnalysis/Text/TextRegex.cs new file mode 100644 index 00000000..b2f4c80d --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/TextRegex.cs @@ -0,0 +1,47 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="TextRegex" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + public abstract class TextRegex + { + + public abstract bool IsMatch( + ReadOnlySpan text + ); + + } + + public ref struct MatchResult { } +} diff --git a/StgSharpDebug/TestRegexAnalyse.cs b/StgSharpDebug/TestRegexAnalyse.cs index 7d9e80d9..16203514 100644 --- a/StgSharpDebug/TestRegexAnalyse.cs +++ b/StgSharpDebug/TestRegexAnalyse.cs @@ -26,7 +26,7 @@ // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- using StgSharp.RegularAnalysis.Abstraction; -using StgSharp.RegularAnalysis.TextRegex; +using StgSharp.RegularAnalysis.Text; using System; using System.Collections.Generic; using System.Linq; @@ -44,7 +44,7 @@ namespace StgSharp @"abc[0-9]x|abc[0-9]y|abd[0-9]x|abd[0-9]y|xyz", @"get_name|get_value|get_type|set_name|set_value|set_type", @"helix|hello|help|hero|hers|her", - @"a*.[abc]*(abcd|a(bc)c)+(?abc)" + @"a*.[abc]*(abcd|a(bc)c)+(?abc{11,13})" ]; public static void Run() @@ -61,7 +61,7 @@ namespace StgSharp Console.Write(new string('|', enumerator.Depth)); Console.WriteLine($"\t{node.Source.Value}\t{node.Source.Flag}"); } - List ir = RegexInterpreter.GenerateIR(tree); + List ir = RegexInterpreter.GenerateIR(tree); Console.WriteLine("-----IR-----"); for (int i = 0; i < ir.Count; i++) { Console.WriteLine(ir[i].AsCommand(i)); diff --git a/future/ngra-regex-ir-review.md b/future/ngra-regex-ir-review.md new file mode 100644 index 00000000..84c2e96a --- /dev/null +++ b/future/ngra-regex-ir-review.md @@ -0,0 +1,1050 @@ +# NGRA 字符串正则 IR 设计补充建议 + +基于当前 `ngra-regex-ir.md` 的设计，我认为整体方向是成立的：它已经不是“完全低级的执行字节码”，而是保留了 `COUNT`、`COUNT_SEQ`、`COUNT_SET`、`TRY`、`PACK` 等高层结构，因此可以继续承担后续 match-opt 的输入。 + +不过，为了让规则更容易扩展、避免语义误判，我建议把优化明确拆成三类，并把当前 IR 分成“语义 IR / HIR”和“计划 IR / LIR”。 + +--- + +## 1. 总体判断 + +当前 IR 最大的优点： + +1. **足够紧凑**：`COUNT_SEQ`、`COUNT_SET`、`TRY` 等复合指令可以避免过早打碎成 `LoadChar + Compare + Branch`。 +2. **保留 regex 结构**：量词、字符集、确定序列、组打包等信息仍然可见。 +3. **适合继续做 match planning**：后续可以从这些指令中识别 `.*literal`、`[set]*`、bounded repeat、固定长度 block 等结构。 + +当前 IR 最大的风险： + +1. `SEARCH` 这个名字容易误导。当前 HIR 中的 `SEARCH` 实际应当是 **MATCH_AT_CURRENT**，不是允许跳过字符的 `FIND`。 +2. 检查点目前只描述为锚点指针栈，但真正的 backtracking frame 至少还要保存 `pos`、capture watermark、symbol watermark、repeat state 和 continuation。 +3. `COUNT_*` 的语义必须明确是否保留 retry/backtracking，否则 `a*.`、`[abc]*abc` 等场景会被错误优化。 +4. 字符集的语义表示和物理表示要分开：语义上 `[0-9a-z]` 是一个 set，物理上可以用 inline set-region 的多条 range-test 表示，避免每个 set 挂 `List`。 + +--- + +## 2. 建议的分层 + +我建议仍然区分 HIR 和 LIR，不是因为机器必须分，而是因为人写规则时边界更清楚。 + +```text +AST + ↓ +AST Opt + ↓ +HIR Gen + ↓ +HIR Analysis + ↓ +HIR → LIR / MatchPlan Lowering + ↓ +LIR + ↓ +SourceGen / RuntimeGen / Interpreter +``` + +### HIR：表达 regex 语义 + +HIR 保留接近正则结构的指令： + +```text +SEARCH_SEQ // 语义上是当前位置 MATCH_SEQ +SEARCH_SET // 语义上是当前位置 MATCH_SET +SEARCH_DOT // 或 UNIT_DOT，不应和 literal "." 混淆 +COUNT +COUNT_SEQ +COUNT_SET +TRY +POP +PACK +``` + +注意：HIR 中可以继续叫 `SEARCH`，但文档里要明确： + +```text +HIR 的 SEARCH_* 不表示向后查找； +它表示从当前活动指针开始匹配。 +真正允许跳过字符的搜索操作只在 LIR / MatchPlan 阶段引入，命名为 FIND_*。 +``` + +### LIR：表达匹配计划 + +LIR 不只是更低级，而是“已经做过策略规划”的 IR： + +```text +MATCH_SEQ +MATCH_CHAR +MATCH_DOT +MATCH_SET + +FIND_SEQ +FIND_CHAR +FIND_SET +FIND_LAST_SEQ +FIND_LAST_SET + +CONSUME_WHILE_SET +CONSUME_UNTIL_CHAR +CONSUME_UNTIL_SET + +COUNT_SET +COUNT_SEQ +COUNT_BLOCK +CHECK_MIN_LENGTH +CHECK_FIXED_OFFSET + +PUSH_BACKTRACK +POP_BACKTRACK +COMMIT +FAIL + +CAPTURE_BEGIN +CAPTURE_END +PACK +``` + +LIR 中应该已经包含： + +```text +LoweredSetId +SearchMode +RepeatMode +RetryMode +CaptureMode +EmitHint +``` + +--- + +## 3. 三类优化策略 + +建议把所有优化分成三类： + +```text +1. Regex algebra rewrite + 改正则结构。 + +2. Match plan rewrite + 改匹配策略。 + +3. Backend / C# span trick + 改 C# 代码写法。 +``` + +这个分类很重要，因为它能直接决定优化应该放在哪个阶段。 + +--- + +# 4. Regex algebra rewrite + +这类优化应该放在 AST Opt 或 HIR 生成前后。它们改变的是正则结构，而不是 C# 写法。 + +## 4.1 Literal 合并 + +```regex +a b c +``` + +变成： + +```regex +abc +``` + +HIR： + +```text +SEARCH_SEQ "abc" +``` + +不要生成三条 `SEARCH_CHAR`。 + +--- + +## 4.2 Concat / Alternation flatten + +```regex +a(?:b(?:c)) +``` + +变成： + +```regex +abc +``` + +```regex +a|(?:b|c) +``` + +变成： + +```regex +a|b|c +``` + +--- + +## 4.3 Alternation factoring / trie + +这个你已经放在 AST Opt 做，位置是对的。 + +```regex +foobar|foobaz|foozip +``` + +变成： + +```regex +foo(?:bar|baz|zip) +``` + +或者 trie AST： + +```text +SEQ "foo" +ALT + SEQ "bar" + SEQ "baz" + SEQ "zip" +``` + +这类结构改写越早越好。后续 IR opt 只需要识别这个结构并决定是否 `switch` / first-set dispatch。 + +--- + +## 4.4 Bounded repeat 展开 + +```regex +a{2,3} +``` + +可以变成： + +```regex +aa a? +``` + +或： + +```text +SEARCH_SEQ "aa" +OPTIONAL SEARCH_CHAR 'a' +``` + +对于 lazy： + +```regex +a{2,3}?b +``` + +要注意尝试顺序。它等价于： + +```regex +aa(?:b|ab) +``` + +而不是： + +```regex +aa(?:a|ab) +``` + +Greedy 版本： + +```regex +a{2,3}b +``` + +如果展开成显式分支，尝试顺序更像： + +```regex +aa(?:ab|b) +``` + +语言集合相同，但 backtracking 尝试顺序不同。 + +--- + +## 4.5 `a+b` 不建议改成 `a*(ab|b)` + +当前文档中写： + +```text +a+b 可以优化为 a*(ab|b) +``` + +这个需要修正。 + +`a+b` 的语言是： + +```text +a^n b, n >= 1 +``` + +而 `a*(ab|b)` 会额外匹配： + +```text +b +``` + +因为第二个分支 `a*b` 可以在 `a*` 取 0 时匹配 `b`。 + +更安全的结构是： + +```regex +aa*b +``` + +或者： + +```text +SEARCH_CHAR 'a' +COUNT_SEQ 'a' 0..∞ +SEARCH_CHAR 'b' +``` + +如果要利用后继 `b` 做策略优化，应在 MatchPlan 阶段处理，而不是做这个不等价的 algebra rewrite。 + +--- + +## 4.6 Charset 合并 + +```regex +[abc]|[def] +``` + +可以变成： + +```regex +[abcdef] +``` + +```regex +[a-f]|[d-z] +``` + +可以变成： + +```regex +[a-z] +``` + +这属于 AST/HIR 层的字符类代数。 + +--- + +## 4.7 Charset 展开为 alternation 要谨慎 + +文档中写： + +```text +abc[de] 可以展开为 (abcd|abce) +``` + +这个是可行的，但不一定总是好。建议规则改成： + +```text +当 charset 很小，并且展开后能形成更长 literal / trie / branch dispatch 时，可以展开； +否则保留为 MATCH_SET。 +``` + +例如： + +```regex +abc[de] +``` + +既可以是： + +```text +SEARCH_SEQ "abc" +SEARCH_SET [de] +``` + +也可以是： + +```text +ALT + SEARCH_SEQ "abcd" + SEARCH_SEQ "abce" +``` + +前者代码更短，后者可能更适合 required literal 或 trie。需要 code size budget。 + +--- + +# 5. Match plan rewrite + +这类优化应该放在 HIR Analysis / HIR → LIR lowering。 + +它们不一定改成另一条正则，而是把匹配过程改成更高效的搜索/消费计划。 + +--- + +## 5.1 长度预测应该在 HIR Analysis 做，SourceGen 只负责发代码 + +当前文档写“长度预测在 source-gen 阶段做”。我建议改成： + +```text +minLength / maxLength 的计算在 HIR Analysis 做； +SourceGen 只负责把 CHECK_MIN_LENGTH 发成 C#。 +``` + +例如： + +```regex +abc[0-9]{2}xyz +``` + +分析结果： + +```text +MinLength = 8 +``` + +LIR： + +```text +CHECK_MIN_LENGTH 8 +MATCH_SEQ "abc" +... +``` + +SourceGen： + +```csharp +if ((uint)(end - pos) < 8) goto Fail; +``` + +--- + +## 5.2 `.*` 优化 + +HIR： + +```text +COUNT_DOT min=0 max=∞ greedy=true +MATCH_CHAR 'a' +``` + +可以 lower 成： + +```text +FIND_LAST_CHAR 'a' +``` + +lazy： + +```text +COUNT_DOT min=0 max=∞ greedy=false +MATCH_CHAR 'a' +``` + +lower 成： + +```text +FIND_CHAR 'a' +``` + +但要注意： + +1. 默认 `.` 不匹配 `\n`，搜索范围应限制在当前行。 +2. Singleline 模式下才可以跨整个 span。 +3. 如果 `.*` 内部或后继结构涉及 capture/backtracking，不能简单“删掉 COUNT”，而应转换为带 retry 的 find plan。 +4. 后继如果是 `MATCH_SEQ "abc"`，则变成 `FIND_SEQ` / `FIND_LAST_SEQ`。 +5. 后继如果是 `MATCH_SET`，则变成 `FIND_SET` / `FIND_LAST_SET`。 + +--- + +## 5.3 `[set]*` 优化 + +文档中写 `[set]*` 优化在 source-gen 阶段做。建议改成： + +```text +是否将 COUNT_SET 变成 CONSUME_WHILE / CONSUME_UNTIL，在 lowering 做； +具体使用 IndexOfAnyExcept / SearchValues / manual loop，在 SourceGen 做。 +``` + +例如： + +```regex +[a-f0-9]* +``` + +LIR： + +```text +CONSUME_WHILE_SET LowerHex +``` + +SourceGen： + +```csharp +int n = span.IndexOfAnyExcept(s_lowerHex); +``` + +或者手写 loop / range compare。 + +--- + +## 5.4 First-set 搜索 + +如果非锚定匹配： + +```regex +abc|defg|xyz +``` + +first-set 是： + +```text +'a', 'd', 'x' +``` + +LIR 可以是： + +```text +FIND_SET { 'a', 'd', 'x' } +VERIFY_BRANCH +``` + +你文档里说“这个不好做，因为 IR 直接把 ALT 打碎成 if-else 串了”。这说明 AST Opt / HIR 里最好保留 ALT region 或者至少保留 branch summary。 + +即使 AST 已经 trie 化，IR 也应该尽量不要把它完全打碎成不可恢复的 if-else 串。至少给每个 region 计算 summary： + +```text +FirstSet +MinLength +RequiredLiteral +CanBacktrack +CanCapture +``` + +--- + +## 5.5 Required literal 搜索 + +例如： + +```regex +.*ERROR:[0-9]+ +``` + +可以直接规划： + +```text +FIND_SEQ "ERROR:" +VERIFY_AFTER_ANCHOR +``` + +这通常比只做 first-set 更强。 + +--- + +## 5.6 Fixed-offset check + +```regex +[a-z]{20}XYZ +``` + +可以 lower 成： + +```text +CHECK_MIN_LENGTH 23 +CHECK_AT_OFFSET 20 MATCH_SEQ "XYZ" +MATCH_SET_REPEAT [a-z] count=20 +MATCH_SEQ "XYZ" +``` + +这个属于 match plan，不属于 sourcegen。 + +--- + +## 5.7 Auto-atomic / possessive 化 + +```regex +[0-9]*[A-Z] +``` + +由于： + +```text +[0-9] ∩ [A-Z] = ∅ +``` + +所以 `[0-9]*` 不需要回退，可以 lower 成： + +```text +COUNT_SET [0-9] greedy atomic +MATCH_SET [A-Z] +``` + +如果后继 first-set 和 repeat set 有交集，就不能这么做。 + +--- + +## 5.8 Repeat retry mode + +LIR 应该明确 repeat 的 retry 策略： + +```text +RetryMode.None +RetryMode.BackoffBy1 +RetryMode.BackoffByStride +RetryMode.SavedPositions +RetryMode.GeneralBacktrack +``` + +例如： + +```regex +(abc[cd])+ +``` + +每轮固定长度 4，所以可以： + +```text +RetryMode.BackoffByStride stride=4 +``` + +不需要保存每个内部指令的回溯点。 + +--- + +# 6. Backend / C# span trick + +这类优化只放 SourceGen。它不再改变 regex 或 match plan。 + +--- + +## 6.1 `StartsWith` + +LIR： + +```text +MATCH_SEQ "sample" +``` + +SourceGen 决定： + +```csharp +slice.StartsWith("sample", StringComparison.Ordinal) +``` + +还是短串展开： + +```csharp +slice.Length >= 3 && +slice[0] == 'a' && +slice[1] == 'b' && +slice[2] == 'c' +``` + +建议规则： + +```text +长度 1: + char compare + +长度 2~3: + 直接 char compare + +中长 literal: + StartsWith + +IgnoreCase: + StartsWith(..., OrdinalIgnoreCase) +``` + +--- + +## 6.2 `[0-9]` 不建议写成 `IndexOf("0123456789")` 用于当前位置测试 + +文档中写： + +```text +[0-9] 展开为 IndexOf("0123456789") +``` + +这个要分两种场景： + +### 当前位置匹配 + +如果是： + +```text +MATCH_SET [0-9] +``` + +更好是： + +```csharp +(uint)(ch - '0') <= 9 +``` + +### span 中搜索 + +如果是： + +```text +FIND_SET [0-9] +``` + +可以考虑： + +```csharp +span.IndexOfAnyInRange('0', '9') +``` + +或： + +```csharp +span.IndexOfAny(s_digits) +``` + +所以规则应写成： + +```text +MATCH_SET [0-9] → unsigned range compare +FIND_SET [0-9] → IndexOfAnyInRange / SearchValues +``` + +--- + +## 6.3 SearchValues + +对于多字符集合： + +```text +FIND_SET [0-9a-f] +``` + +SourceGen 可生成： + +```csharp +private static readonly SearchValues s_lowerHex = + SearchValues.Create("0123456789abcdef"); +``` + +然后： + +```csharp +span.IndexOfAny(s_lowerHex); +``` + +当前位置测试不一定要用 `SearchValues`，可能 range compare 或 bitmap 更快。 + +--- + +## 6.4 Inline vs helper + +复杂 Unicode set： + +```text +MATCH_SET UnicodeWord +``` + +SourceGen 决定： + +```csharp +if (!IsUnicodeWord(ch)) goto Fail; +``` + +还是： + +```csharp +if (ch < 128) +{ + if (!AsciiWordContains(ch)) goto Fail; +} +else +{ + if (!IsUnicodeWord(ch)) goto Fail; +} +``` + +这个应由代码尺寸、热度、是否 source-gen 决定。 + +--- + +## 6.5 Code size budget + +SourceGen 应负责最终代码膨胀控制： + +```text +单方法最大语句数 +literal 展开阈值 +bounded repeat 展开阈值 +是否生成 helper +是否抽 cold path +SearchValues 字段数量 +``` + +--- + +# 7. 字符集的物理表示建议 + +为了避免每个 set 挂 `List`，可以用线性 IR region 表示同一个字符测试。 + +例如语义： + +```regex +[0-9a-z] +``` + +物理 IR： + +```text +MATCH_SET_BEGIN default=false end=13 + TEST_RANGE '0' '9' accept=true + TEST_RANGE 'a' 'z' accept=true +MATCH_SET_END +``` + +语义： + +```regex +[^0-9a-z] +``` + +物理 IR： + +```text +MATCH_SET_BEGIN default=true end=13 + TEST_RANGE '0' '9' accept=false + TEST_RANGE 'a' 'z' accept=false +MATCH_SET_END +``` + +要点是： + +```text +这些 TEST_RANGE 测的是同一个字符； +整个 MATCH_SET 只推进 pos 一次。 +``` + +不能把它误解成连续匹配多个字符。 + +复杂 Unicode set 可以走： + +```text +MATCH_SET_REF setId=42 +``` + +`setId=42` 指向全局 pooled table，而不是每个节点一个 List。 + +--- + +# 8. 检查点语义建议 + +当前文档把检查点描述为锚点指针栈，这对 capture 起点有用，但不够覆盖 backtracking。 + +建议把 checkpoint / backtrack frame 定义成： + +```text +BacktrackFrame: + pos + anchor + captureTop + symbolTop + repeatState + continuation +``` + +其中： + +```text +pos: + 活动指针恢复位置 + +anchor: + 区间开始位置 + +captureTop: + 捕获记录栈回滚点 + +symbolTop: + 符号区回滚点 + +repeatState: + 当前 COUNT/loop 的计数、起止位置、retry mode + +continuation: + 失败后跳到哪里继续尝试 +``` + +如果暂时是 tokenizer / no-capture 模式，可以裁掉 captureTop，但 IR 语义最好预留。 + +--- + +# 9. 建议修改原文的几个点 + +## 9.1 “HIT” 应改为 “HIR” + +原文： + +```text +将HIT进行进一步优化 +``` + +应为： + +```text +将 HIR 进行进一步优化 +``` + +## 9.2 `SEARCH` 的定义应改写 + +原文说“向后搜索确定长度序列”。建议改为： + +```text +HIR 中 SEARCH_* 表示从当前活动指针开始匹配； +它不是允许跳过字符的 FIND。 +真正的 FIND_* 在 HIR→LIR 阶段由 match planner 引入。 +``` + +## 9.3 `a+b → a*(ab|b)` 不等价 + +建议删除或改成： + +```regex +a+b → aa*b +``` + +或者： + +```text +MATCH_CHAR 'a' +COUNT_SEQ 'a' 0..∞ +MATCH_CHAR 'b' +``` + +## 9.4 `[0-9] → IndexOf("0123456789")` 应细分 + +改成： + +```text +当前位置 MATCH_SET: + 用 unsigned range compare。 + +span 搜索 FIND_SET: + 用 IndexOfAnyInRange / SearchValues / IndexOfAny。 +``` + +## 9.5 `.*` 优化不要说 “COUNT 消失” + +建议改成： + +```text +COUNT_DOT 与后继匹配项合并为 FIND / FIND_LAST 计划。 +``` + +这样更准确，因为有时还需要保留 retry / range 限制 / newline 限制。 + +--- + +# 10. 推荐最终管线 + +```text +Tokenize + - 区分 Dot 与 EscapedLiteralDot + - 区分 raw charset / literal / quantifier / group + +AST Build + - 建立 regex 语义树 + +AST Opt + - literal merge + - concat flatten + - alternation factoring / trie + - bounded repeat normalization + - simple group elimination + - charset algebra merge + +HIR Gen + - SEARCH_* as MATCH_AT_CURRENT + - COUNT_* + - TRY / POP + - PACK + - raw set / raw dot + +HIR Analysis + - minLength / maxLength + - nullable + - firstSet + - lastSet + - required literal + - fixed length + - capture effect + - repeat overlap + +HIR → LIR / MatchPlan + - raw set → lowered set + - dot → dot mode + - introduce FIND + - introduce CONSUME + - choose retry mode + - choose atomic/possessive + - emit check-min-length / fixed-offset checks + +LIR + - explicit MATCH / FIND / CONSUME + - explicit lowered set + - explicit backtracking plan + - explicit capture plan + +SourceGen + - StartsWith / SequenceEqual + - IndexOf / IndexOfAny / IndexOfAnyExcept + - SearchValues + - range compare / bitmap + - inline/helper split + - code size budget +``` + +--- + +# 11. 总结 + +我建议保留你当前 IR 的核心设计，但把文档里的边界收紧： + +```text +AST Opt: + 做 Regex algebra rewrite。 + +HIR: + 表示从当前位置开始的 regex 语义。 + +HIR Analysis: + 提取结构属性。 + +HIR → LIR: + 做 Match plan rewrite。 + +LIR: + 表示明确的匹配计划。 + +SourceGen: + 做 C# span trick。 +``` + +最重要的改动是： + +```text +SEARCH 在 HIR 中应被定义为 MATCH_AT_CURRENT； +FIND 只能由后续优化阶段引入。 +``` + +以及： + +```text +字符集可以用线性 set-region 表示，避免 List 分配； +但 set-region 内的多条 range-test 必须测试同一个字符，不能变成多个连续 MATCH。 +``` + +这样你的 IR 会同时满足三个目标： + +1. 对人来说可读； +2. 对优化规则来说容易扩展； +3. 对 GC 和 sourcegen 来说物理表示足够紧凑。 diff --git a/future/ngra-regex-ir.md b/future/ngra-regex-ir.md index 7e80e2e9..4d50de5e 100644 --- a/future/ngra-regex-ir.md +++ b/future/ngra-regex-ir.md @@ -14,15 +14,62 @@ 使用DEFINE指令定义的符号定义在符号区中，符号的类型通过第一次赋值确定，源生成模式下，可以通过第一个赋值指令推导其类型 -## 算子 +## HIR算子 1. 搜索算子（SEARCH %str; SEARCH %unicode），向后搜索确定长度序列或者满足规则的字符集，更新活动指针，不更新锚点指针，搜索结果保存在全局上下文中 2. 搜索算子（SEARCH %set_expression），向后匹配一个满足字符集要求的字符，更新活动指针，不更新锚点指针，搜索结果保存在全局上下文中 -3. 失败配置指令（CINFIG_FAIL %fail），指定当前匹配栈后续如何处理一条指令的失败，可选为：FAST，立刻终止当前层级；CONTINUE，记录这个fail，但是继续操作；REVERSE，反过来，成功了才终止 -4. 条件指令（CONDITION %cmd_true %cmd_fail），比较前一条SEARCH指令是否成功，并指定跳转到指定的位置，如果某种条件不需要跳转，可以写1。如果想要将fail向上级继续抛出，可以%cmd_fail写0. -5. 跳转执行指令（TRY %begin %end），尝试idx指向的IR块，并记住执行结果数据 -6. 计数指令（COUNT %min %max %is_greedy %cmd），重复执行指令cmd，直到满足参数配置的数量为止 -7. 快速序列计数指令（COUNT_SEQ %min %max %is_greedy %seq），快速搜索一个确定性序列，直到满足参数配置的数量为止 -8. 快速字符集计数指令（COUNT_SET %min %max %is_greedy %set），快速搜索一个字符集，直到满足参数配置的数量为止 -9. 尝试（TRY %count），对自身向后的一段内容在新的匹配栈层级中进行运行，被分层的指令数量由COUNT指令确定 -10. 弹出（POP %is_success），提前终止一个try块，并用is_success表示弹出行为视为匹配成功还是失败 +3. 条件指令（CONDITION %cmd_true %cmd_fail），比较前一条SEARCH指令是否成功，并指定跳转到指定的位置，如果某种条件不需要跳转，可以写1。如果想要将fail向上级继续抛出，可以%cmd_fail写0. +4. 计数指令（COUNT %min %max %is_greedy %cmd），重复执行指令cmd，直到满足参数配置的数量为止 +5. 快速序列计数指令（COUNT_SEQ %min %max %is_greedy %seq），快速搜索一个确定性序列，直到满足参数配置的数量为止 +6. 快速字符集计数指令（COUNT_SET %min %max %is_greedy %set），快速搜索一个字符集，直到满足参数配置的数量为止 +7. 尝试（TRY %count），对自身向后的一段内容在新的匹配栈层级中进行运行，被分层的指令数量由COUNT指令确定 +8. 弹出（POP %is_success），提前终止一个try块，并用is_success表示弹出行为视为匹配成功还是失败 +9. 打包指令（PACK %is_success %name），检查上一条语句的结果，将匹配结果打包并记录为指定的名字。如果名字为空，则使用默认的序号进行索引 + +## 优化策略 + +应当注意的是，将HIT进行进一步优化或者展开的目的： + +1. 尽可能降低源生成或者运行时执行中，对多指令上下文的依赖，尽可能保证每一个IR可以单独输出最佳的代码 +2. 减少多余的结构，使得一行指令尽可能处理多的数据 + +### 语义改写优化 + +1. 计数器展开 +建议在lower做 +对于形如a+b或者a{2,4}b的正则表达式，可以优化为a*(ab|b)和a{2,3}(ab|b)的写法 + +2. 确定性计数展开 +建议在lower做 +对于形如a{m,n}的正则表达式，将其展开为a{m}a{0,n-m}的形式 + +3. charset合并 +建议在ast-opt做 +对于形如[ abc ]|[ def ]的表达式，将其合并为[ abcdef ] + +4. charset展开 +在lower做 +对于形如abc[ de ]的表达式，如果字符集容量不超过4，可以站卡为(abcd|abce)形式 + +### 匹配策略改变 + +1. 长度预测 +在source-gen阶段做 +对于一个正则表达式，可以计算其所需的最小长度，可以在开始的时候提前进行比较 + +2. .*优化 +在lower阶段做 +对于COUNT_SET min=0 max=-1 "."指令，也就是.*操作，将后续指令从match转为find + +3. [ set ]*优化 +在source-gen阶段做 +对于其他的COUNT_SET min=0 max=-1 "???"指令，使用IndexOf，IndexOfAnyExcept等进行优化 + +4. 首字母检查 +**这个不好做，因为IR直接把ALT打碎成if-else串了** +对于(abc|defg|xyz)表达式，可以检查其首字母 + +### C# span opt trick + +1. Span.StartWith +2. [ 0-9 ]展开为IndexOf("0123456789") \ No newline at end of file From 2b05d626c7af5c497c5c491c370501dc9d7272b6 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Tue, 23 Jun 2026 20:08:33 +0800 Subject: [PATCH 2/6] begin regex ir opt --- .../Text/{RegexHIR.cs => RegexIR.cs} | 57 +++++--- ...gexHIRGenerator.cs => RegexIRGenerator.cs} | 32 ++--- .../Text/RegexInterpreter.ASTOptimize.cs | 4 +- ...rate.cs => RegexInterpreter.IRGenerate.cs} | 38 +++--- .../Text/RegexInterpreter.IROptimize.cs | 126 ++++++++++++++++++ .../Text/RegexInterpreter.IRScan.cs | 69 ---------- .../Text/RegexInterpreter.SourceGenerate.cs | 6 +- .../Text/RegexInterpreter.cs | 8 +- StgSharp.RegularAnalysis/Text/RegexLIR.cs | 60 --------- StgSharpDebug/TestRegexAnalyse.cs | 4 +- 10 files changed, 210 insertions(+), 194 deletions(-) rename StgSharp.RegularAnalysis/Text/{RegexHIR.cs => RegexIR.cs} (86%) rename StgSharp.RegularAnalysis/Text/{RegexHIRGenerator.cs => RegexIRGenerator.cs} (83%) rename StgSharp.RegularAnalysis/Text/{RegexInterpreter.HIRGenerate.cs => RegexInterpreter.IRGenerate.cs} (91%) create mode 100644 StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs delete mode 100644 StgSharp.RegularAnalysis/Text/RegexInterpreter.IRScan.cs delete mode 100644 StgSharp.RegularAnalysis/Text/RegexLIR.cs diff --git a/StgSharp.RegularAnalysis/Text/RegexHIR.cs b/StgSharp.RegularAnalysis/Text/RegexIR.cs similarity index 86% rename from StgSharp.RegularAnalysis/Text/RegexHIR.cs rename to StgSharp.RegularAnalysis/Text/RegexIR.cs index 11e44c35..7dc15ec3 100644 --- a/StgSharp.RegularAnalysis/Text/RegexHIR.cs +++ b/StgSharp.RegularAnalysis/Text/RegexIR.cs @@ -46,10 +46,11 @@ namespace StgSharp.RegularAnalysis.Text Count_Set = 7, Pop = 8, Pack = 9, + Find_Seq = 10, } - internal abstract class RegexHIR + internal abstract class RegexIR { internal abstract RegexIRCommand Command { get; } @@ -60,12 +61,30 @@ namespace StgSharp.RegularAnalysis.Text } + internal sealed class RegexFindSeqIR( + string pattern + ) : RegexIR + { + + public string Pattern { get; } = pattern; + + internal override RegexIRCommand Command => RegexIRCommand.Find_Seq; + + internal override string AsCommand( + int lineNum + ) + { + return $"[{lineNum}]\t FIND_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "" : Pattern)}\""; + } + + } + /// /// Searches for a certain string pattern or charset from current position. /// - internal sealed class RegexMatchSeqHIR( + internal sealed class RegexMatchSeqIR( string pattern - ) : RegexHIR + ) : RegexIR { public string Pattern { get; } = pattern; @@ -81,9 +100,9 @@ namespace StgSharp.RegularAnalysis.Text } - internal sealed class RegexMatchSetHIR( + internal sealed class RegexMatchSetIR( string charset - ) : RegexHIR + ) : RegexIR { public string Charset { get; } = charset; @@ -99,10 +118,10 @@ namespace StgSharp.RegularAnalysis.Text } - internal sealed class RegexConditionHIR( + internal sealed class RegexConditionIR( int success, int fail - ) : RegexHIR + ) : RegexIR { public int SuccessCase { get; } = success; @@ -120,10 +139,10 @@ namespace StgSharp.RegularAnalysis.Text } - internal sealed class RegexPackHIR( + internal sealed class RegexPackIR( bool condition, string name = "" - ) : RegexHIR + ) : RegexIR { public bool Condition { get; } = condition; @@ -141,12 +160,12 @@ namespace StgSharp.RegularAnalysis.Text } - internal sealed class RegexCountHIR( + internal sealed class RegexCountIR( int min, int max, bool isGreedy, int operation - ) : RegexHIR + ) : RegexIR { public bool IsGreedy { get; } = isGreedy; @@ -168,12 +187,12 @@ namespace StgSharp.RegularAnalysis.Text } - internal sealed class RegexCountSeqHIR( + internal sealed class RegexCountSeqIR( int min, int max, bool isGreedy, string seq - ) : RegexHIR + ) : RegexIR { public bool IsGreedy { get; } = isGreedy; @@ -195,12 +214,12 @@ namespace StgSharp.RegularAnalysis.Text } - internal sealed class RegexCountSetHIR( + internal sealed class RegexCountSetIR( int min, int max, bool isGreedy, string set - ) : RegexHIR + ) : RegexIR { public bool IsGreedy { get; } = isGreedy; @@ -222,9 +241,9 @@ namespace StgSharp.RegularAnalysis.Text } - internal sealed class RegexTryHIR( + internal sealed class RegexTryIR( int count - ) : RegexHIR + ) : RegexIR { public int CountOfCommand { get; } = count; @@ -240,9 +259,9 @@ namespace StgSharp.RegularAnalysis.Text } - internal sealed class RegexPopHIR( + internal sealed class RegexPopIR( bool isSuccess - ) : RegexHIR + ) : RegexIR { public bool IsSuccess { get; } = isSuccess; diff --git a/StgSharp.RegularAnalysis/Text/RegexHIRGenerator.cs b/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs similarity index 83% rename from StgSharp.RegularAnalysis/Text/RegexHIRGenerator.cs rename to StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs index b1720138..b3a572a6 100644 --- a/StgSharp.RegularAnalysis/Text/RegexHIRGenerator.cs +++ b/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs @@ -34,13 +34,13 @@ using System.Threading.Tasks; namespace StgSharp.RegularAnalysis.Text { - internal class RegexHIRGenerator + internal class RegexIRGenerator { private Node _head = new(); private Node _tail; - public RegexHIRGenerator() + public RegexIRGenerator() { _tail = _head; } @@ -52,7 +52,7 @@ namespace StgSharp.RegularAnalysis.Text int fail ) { - _tail.Add(new RegexConditionHIR(success, fail)); + _tail.Add(new RegexConditionIR(success, fail)); Count++; } @@ -63,7 +63,7 @@ namespace StgSharp.RegularAnalysis.Text int operation ) { - _tail.Add(new RegexCountHIR(min, max, isGreedy, operation)); + _tail.Add(new RegexCountIR(min, max, isGreedy, operation)); Count++; } @@ -74,7 +74,7 @@ namespace StgSharp.RegularAnalysis.Text string seq ) { - _tail.Add(new RegexCountSeqHIR(min, max, isGreedy, seq)); + _tail.Add(new RegexCountSeqIR(min, max, isGreedy, seq)); Count++; } @@ -85,12 +85,12 @@ namespace StgSharp.RegularAnalysis.Text string set ) { - _tail.Add(new RegexCountSetHIR(min, max, isGreedy, set)); + _tail.Add(new RegexCountSetIR(min, max, isGreedy, set)); Count++; } public void EmitIRStream( - RegexHIRGenerator stream + RegexIRGenerator stream ) { if (stream is null || stream.Count == 0) { @@ -105,7 +105,7 @@ namespace StgSharp.RegularAnalysis.Text string unit ) { - _tail.Add(new RegexMatchSeqHIR(unit)); + _tail.Add(new RegexMatchSeqIR(unit)); Count++; } @@ -113,7 +113,7 @@ namespace StgSharp.RegularAnalysis.Text string unit ) { - _tail.Add(new RegexMatchSetHIR(unit)); + _tail.Add(new RegexMatchSetIR(unit)); Count++; } @@ -122,7 +122,7 @@ namespace StgSharp.RegularAnalysis.Text string name = "" ) { - _tail.Add(new RegexPackHIR(condition, name)); + _tail.Add(new RegexPackIR(condition, name)); Count++; } @@ -130,7 +130,7 @@ namespace StgSharp.RegularAnalysis.Text bool isSuccess ) { - _tail.Add(new RegexPopHIR(isSuccess)); + _tail.Add(new RegexPopIR(isSuccess)); Count++; } @@ -138,13 +138,13 @@ namespace StgSharp.RegularAnalysis.Text int count ) { - _tail.Add(new RegexTryHIR(count)); + _tail.Add(new RegexTryIR(count)); Count++; } - public List ToList() + public List ToList() { - List result = new(Count); + List result = new(Count); Node current = _head; while (current != null) { @@ -161,11 +161,11 @@ namespace StgSharp.RegularAnalysis.Text public int Count => IRList.Count; - public List IRList { get; } = []; + public List IRList { get; } = []; [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Add( - RegexHIR ir + RegexIR ir ) { IRList.Add(ir); diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs index 36398174..038417b8 100644 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexInterpreter.Optimize" +// file="RegexInterpreter.ASTOptimize" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -39,7 +39,7 @@ using System.Xml.Linq; namespace StgSharp.RegularAnalysis.Text { - public partial class RegexInterpreter + public partial class RegexAnalyzer { private static RegexAstNode FlattenAlt( diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.HIRGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs similarity index 91% rename from StgSharp.RegularAnalysis/Text/RegexInterpreter.HIRGenerate.cs rename to StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs index bc42c472..2bcf7000 100644 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.HIRGenerate.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs @@ -40,11 +40,11 @@ using System.Xml; namespace StgSharp.RegularAnalysis.Text { - public partial class RegexInterpreter + public partial class RegexAnalyzer { - internal static List GenerateIR( - AbstractSyntaxTree tree + internal static List GenerateIR( + AbstractSyntaxTree tree ) { return GenerateIRRecursion(tree.Root, 0).ToList(); @@ -53,7 +53,7 @@ namespace StgSharp.RegularAnalysis.Text [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void EmitCount( int curDepth, - RegexHIRGenerator ir, + RegexIRGenerator ir, int min, int max, bool isGreedy, @@ -66,7 +66,7 @@ namespace StgSharp.RegularAnalysis.Text if (isRightOperator) // seq to be count is complex { ir.EmitCount(min, max, isGreedy, 1); - RegexHIRGenerator rightIR = GenerateIRRecursion(right, curDepth + 1); + RegexIRGenerator rightIR = GenerateIRRecursion(right, curDepth + 1); ir.EmitTry(rightIR.Count); ir.EmitIRStream(rightIR); } else @@ -89,9 +89,9 @@ namespace StgSharp.RegularAnalysis.Text } } - private static RegexHIRGenerator GenerateIRRecursion( - RegexAstNode node, - int depth + private static RegexIRGenerator GenerateIRRecursion( + RegexAstNode node, + int depth ) { #pragma warning disable IDE0010 @@ -102,8 +102,8 @@ namespace StgSharp.RegularAnalysis.Text int curDepth = depth; RegexAstNode left = node.Left; RegexAstNode right = node.Right; - RegexHIRGenerator leftIR; - RegexHIRGenerator rightIR; + RegexIRGenerator leftIR; + RegexIRGenerator rightIR; Token token = node.Source; @@ -119,7 +119,7 @@ namespace StgSharp.RegularAnalysis.Text group_source[3..^1] : /* named group (?) */ string.Empty; /* unnamed group (?:) */ - RegexHIRGenerator groupIR = new(); + RegexIRGenerator groupIR = new(); rightIR = GenerateIRRecursion(right, curDepth + 1); groupIR.EmitTry(rightIR.Count); groupIR.EmitIRStream(rightIR); @@ -144,7 +144,7 @@ namespace StgSharp.RegularAnalysis.Text _ => throw new InvalidOperationException("Unexpected sequence mask for count operator"), }; ReadOnlySpan count_source_span = count_source; - RegexHIRGenerator countIR = new(); + RegexIRGenerator countIR = new(); if (count_source_span[0] == '*') // infinite count { @@ -185,7 +185,7 @@ namespace StgSharp.RegularAnalysis.Text return countIR; case RegexElementLabel.CONCAT: - RegexHIRGenerator concat = new(); + RegexIRGenerator concat = new(); // process left node if (RegexAstNode.IsNullOrEmpty(left)) @@ -238,13 +238,13 @@ namespace StgSharp.RegularAnalysis.Text } return concat; case RegexElementLabel.ALT: - RegexHIRGenerator alt = new(); - List casesIRList = []; + RegexIRGenerator alt = new(); + List casesIRList = []; RegexAstNode cur_node = right; int ir_count = 0; while (!RegexAstNode.IsNullOrEmpty(cur_node)) { - RegexHIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1); + RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1); casesIRList.Add(_case); ir_count += _case.Count; cur_node = cur_node.Next; @@ -252,7 +252,7 @@ namespace StgSharp.RegularAnalysis.Text int case_count = casesIRList.Count; while (case_count != 0) { - RegexHIRGenerator cur_ir = casesIRList[^case_count]; + RegexIRGenerator cur_ir = casesIRList[^case_count]; ir_count -= cur_ir.Count; case_count--; alt.EmitTry(cur_ir.Count); @@ -265,11 +265,11 @@ namespace StgSharp.RegularAnalysis.Text case RegexElementLabel.UNIT_SPAN: goto case RegexElementLabel.UNIT; case RegexElementLabel.UNIT: - RegexHIRGenerator unit_seq = new(); + RegexIRGenerator unit_seq = new(); unit_seq.EmitMatchSeq(node.Source.Value); return unit_seq; case RegexElementLabel.UNIT_SET: - RegexHIRGenerator unit_set = new(); + RegexIRGenerator unit_set = new(); unit_set.EmitMatchSet(node.Source.Value); return unit_set; default: diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs new file mode 100644 index 00000000..804e1e0f --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs @@ -0,0 +1,126 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="RegexInterpreter.IROptimize" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + public partial class RegexAnalyzer + { + + internal static RegexInfo ScanAndOptimizeIR( + List baseIR, + out List optIR + ) + { + int index = 0; + List optimizedIR = new(baseIR.Count); + List segment = []; + RegexInfo info = new(); + do + { + // TODO scan and opt here + OptimizeIR(baseIR, ref index, segment, info); + optimizedIR.AddRange(segment); + index++; + } while (index < baseIR.Count); + optIR = optimizedIR; + return info; + } + + private static void OptimizeDot( + RegexCountSetIR root, + List ir, + ref int index, + List optimizedIR, + RegexInfo info + ) + { + int next = index + 1; + if (next >= ir.Count) + { + optimizedIR.Clear(); + optimizedIR.Add(root); + return; + } + RegexIR nextIR = ir[next]; + if (nextIR.Command == RegexIRCommand.Match_Seq) + { + // opt .*abc style + RegexFindSeqIR find = new((nextIR as RegexMatchSeqIR)!.Pattern); + optimizedIR.Add(find); + info.MinLength += find.Pattern.Length; + return; + } else { } + } + + private static void OptimizeIR( + List ir, + ref int index, + List optimizedIR, + RegexInfo info + ) + { + optimizedIR.Clear(); + + // TODO optimize here + RegexIR root = ir[index]; + switch (root.Command) + { + case RegexIRCommand.Count_Set: + RegexCountSetIR count = (root as RegexCountSetIR)!; + if (count.CharSet == ".") + { + // opt for .+ / .* / .{} pattern + OptimizeDot(count, ir, ref index, optimizedIR, info); + } + break; + case RegexIRCommand.Match_Seq: + info.MinLength += (root as RegexMatchSeqIR)!.Pattern.Length; + optimizedIR.Add(root); + break; + default: + optimizedIR.Add(root); + break; + } + } + + internal class RegexInfo + { + + public int MinLength { get; set; } + + public int TryDepth { get; set; } + + } + + } +} diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRScan.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRScan.cs deleted file mode 100644 index 61c7b28c..00000000 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRScan.cs +++ /dev/null @@ -1,69 +0,0 @@ -//----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -// file="RegexInterpreter.IRScan" -// Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. -// -// ----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace StgSharp.RegularAnalysis.Text -{ - public partial class RegexInterpreter - { - - internal static void ScanAndMarkIROptimization( - RegexHIR ir - ) - { - switch (ir.Command) - { - case RegexIRCommand.Match_Seq: - break; - case RegexIRCommand.Match_Set: - break; - case RegexIRCommand.Pack: - break; - case RegexIRCommand.Condition: - break; - case RegexIRCommand.Try: - break; - case RegexIRCommand.Count: - break; - case RegexIRCommand.Count_Seq: - break; - case RegexIRCommand.Count_Set: - break; - case RegexIRCommand.Pop: - break; - default: - break; - } - } - - } -} diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs index 67d84307..b3c73f66 100644 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs @@ -33,15 +33,15 @@ using System.Threading.Tasks; namespace StgSharp.RegularAnalysis.Text { - public partial class RegexInterpreter + public partial class RegexAnalyzer { internal static string GetSourceCode( - IEnumerable irs + IEnumerable irs ) { StringBuilder sb = new StringBuilder(); - foreach (RegexHIR ir in irs) { + foreach (RegexIR ir in irs) { ScanAndMarkIROptimization(ir); } return sb.ToString(); diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.cs index f7f1c02d..c9dc1134 100644 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.cs @@ -37,7 +37,7 @@ using System.Threading.Tasks; namespace StgSharp.RegularAnalysis.Text { - public partial class RegexInterpreter + public partial class RegexAnalyzer { private AbstractSyntaxTree _tree = new(); @@ -45,7 +45,7 @@ namespace StgSharp.RegularAnalysis.Text private Exception _interpreteException; private string _source; - private RegexInterpreter() { } + private RegexAnalyzer() { } internal AbstractSyntaxTree Tree { @@ -58,11 +58,11 @@ namespace StgSharp.RegularAnalysis.Text } } - public static RegexInterpreter Analyze( + public static RegexAnalyzer Analyze( string regex ) { - RegexInterpreter interpreter = new(); + RegexAnalyzer interpreter = new(); interpreter._source = regex; bool isSuccess; try diff --git a/StgSharp.RegularAnalysis/Text/RegexLIR.cs b/StgSharp.RegularAnalysis/Text/RegexLIR.cs deleted file mode 100644 index 272de4f3..00000000 --- a/StgSharp.RegularAnalysis/Text/RegexLIR.cs +++ /dev/null @@ -1,60 +0,0 @@ -//----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -// file="RegexLIR" -// Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. -// -// ----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace StgSharp.RegularAnalysis.Text -{ - internal enum LIRCommand - { - - Search = 0, - Count = 1, - Try = 2, - - } - - internal abstract class RegexLIR - { - - public abstract LIRCommand Command { get; } - - } - - internal enum RegexSearchMethod - { - - Match, - Find, - Assert, - - } -} diff --git a/StgSharpDebug/TestRegexAnalyse.cs b/StgSharpDebug/TestRegexAnalyse.cs index 16203514..4e52d4a2 100644 --- a/StgSharpDebug/TestRegexAnalyse.cs +++ b/StgSharpDebug/TestRegexAnalyse.cs @@ -51,7 +51,7 @@ namespace StgSharp { foreach (string regex in regexSource) { - RegexInterpreter interpreter = RegexInterpreter.Analyze(regex); + RegexAnalyzer interpreter = RegexAnalyzer.Analyze(regex); AbstractSyntaxTree tree = interpreter.Tree; Console.WriteLine($"result of \"{regex}\""); TreeEnumerator enumerator = new(tree); @@ -61,7 +61,7 @@ namespace StgSharp Console.Write(new string('|', enumerator.Depth)); Console.WriteLine($"\t{node.Source.Value}\t{node.Source.Flag}"); } - List ir = RegexInterpreter.GenerateIR(tree); + List ir = RegexAnalyzer.GenerateIR(tree); Console.WriteLine("-----IR-----"); for (int i = 0; i < ir.Count; i++) { Console.WriteLine(ir[i].AsCommand(i)); From 45f9b04b6dbd8f2a276e9251dac599718ad1e725 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Wed, 24 Jun 2026 18:09:05 +0800 Subject: [PATCH 3/6] add more ir opt code, clearified what to opt, so current opt code may need rewrite --- StgSharp.RegularAnalysis/Text/RegexIR.cs | 5 +- .../Text/RegexInterpreter.IROptimize.cs | 117 ++++++++++++++---- StgSharpDebug/TestRegexAnalyse.cs | 2 +- future/ngra-regex-ir.md | 51 +++----- 4 files changed, 121 insertions(+), 54 deletions(-) diff --git a/StgSharp.RegularAnalysis/Text/RegexIR.cs b/StgSharp.RegularAnalysis/Text/RegexIR.cs index 7dc15ec3..446257f4 100644 --- a/StgSharp.RegularAnalysis/Text/RegexIR.cs +++ b/StgSharp.RegularAnalysis/Text/RegexIR.cs @@ -62,10 +62,13 @@ namespace StgSharp.RegularAnalysis.Text } internal sealed class RegexFindSeqIR( - string pattern + string pattern, + bool isFirst ) : RegexIR { + public bool IsFirst { get; } = isFirst; + public string Pattern { get; } = pattern; internal override RegexIRCommand Command => RegexIRCommand.Find_Seq; diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs index 804e1e0f..95b7bd63 100644 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs @@ -28,6 +28,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.InteropServices; using System.Text; using System.Threading.Tasks; @@ -36,19 +37,22 @@ namespace StgSharp.RegularAnalysis.Text public partial class RegexAnalyzer { - internal static RegexInfo ScanAndOptimizeIR( - List baseIR, - out List optIR + // this array is not thread safe so opt process must be done in single thread. + private RegexIR[] _opt_buffer = new RegexIR[16]; + + internal RegexInfo ScanAndOptimizeIR( + List baseIR, + out List optIR ) { int index = 0; + Span origin_span = CollectionsMarshal.AsSpan(baseIR); List optimizedIR = new(baseIR.Count); List segment = []; RegexInfo info = new(); do { - // TODO scan and opt here - OptimizeIR(baseIR, ref index, segment, info); + OptimizeIR(origin_span, ref index, segment, info); optimizedIR.AddRange(segment); index++; } while (index < baseIR.Count); @@ -56,37 +60,106 @@ namespace StgSharp.RegularAnalysis.Text return info; } - private static void OptimizeDot( - RegexCountSetIR root, - List ir, - ref int index, - List optimizedIR, - RegexInfo info + private int ExpandCount( + string pattern, + int min, + int max, + bool is_greedy, + in Span span + ) + { + // expand count operation to less cycles. + + const int fast_seq_length = 32; + + int fast_count = fast_seq_length / pattern.Length; + fast_count = int.Min(fast_count, min); + if (fast_count > 0) + { + string fast_pattern = string.Concat(Enumerable.Repeat(pattern, fast_count)); + + int fast_min = min / fast_count; + int consumed_count = fast_min * fast_count; + int remain_min = min - consumed_count; + int remain_max = max - consumed_count; + remain_max = int.Max(-1, remain_max); //for negative stands for infinitive + if (remain_max == 0) + { + // a{2} + RegexCountSeqIR fast = new(fast_min, fast_min, is_greedy, fast_pattern); + span[0] = fast; + return 1; + } else + { + // a{2,5} + RegexCountSeqIR fast = new(fast_min, fast_min, is_greedy, fast_pattern); + RegexCountSeqIR tail = new(remain_min, remain_max, is_greedy, pattern); + span[0] = fast; + span[1] = tail; + return 2; + } + } else if (fast_count == 0) + { + // least count is 0 + span[0] = new RegexCountSeqIR(0, max, is_greedy, pattern); + return 1; + } else + { + // least count is infinitive + span[0] = new RegexCountSeqIR(-1, -1, is_greedy, pattern); + return 1; + } + } + + private void OptimizeDot( + RegexCountSetIR root, + Span ir, + ref int index, + List optimizedIR, + RegexInfo info ) { int next = index + 1; - if (next >= ir.Count) + optimizedIR.Clear(); + if (next >= ir.Length) { - optimizedIR.Clear(); optimizedIR.Add(root); return; } RegexIR nextIR = ir[next]; - if (nextIR.Command == RegexIRCommand.Match_Seq) + if (nextIR is RegexMatchSeqIR match_seq) { // opt .*abc style - RegexFindSeqIR find = new((nextIR as RegexMatchSeqIR)!.Pattern); + RegexFindSeqIR find = new(match_seq.Pattern); optimizedIR.Add(find); info.MinLength += find.Pattern.Length; return; - } else { } + } else if (nextIR is RegexCountSeqIR count_seq) + { + // opt .* (abc)* or .*(abc){2,5} style + string pattern = count_seq.Sequence; + int min = count_seq.Min; + int max = count_seq.Max; + switch ((min, max)) + { + case (0, _): + + break; + default: + RegexFindSeqIR find = new(pattern); + + // RegexCountSeqIR + + break; + } + } } - private static void OptimizeIR( - List ir, - ref int index, - List optimizedIR, - RegexInfo info + private void OptimizeIR( + Span ir, + ref int index, + List optimizedIR, + RegexInfo info ) { optimizedIR.Clear(); @@ -120,6 +193,8 @@ namespace StgSharp.RegularAnalysis.Text public int TryDepth { get; set; } + public int GroupCount { get; set; } + } } diff --git a/StgSharpDebug/TestRegexAnalyse.cs b/StgSharpDebug/TestRegexAnalyse.cs index 4e52d4a2..907efc05 100644 --- a/StgSharpDebug/TestRegexAnalyse.cs +++ b/StgSharpDebug/TestRegexAnalyse.cs @@ -72,7 +72,7 @@ namespace StgSharp } } - [GeneratedRegex(@"a*.[abc]*(abcd|a(bc)c)+(?abc)", RegexOptions.ExplicitCapture)] + [GeneratedRegex(@".*a*b", RegexOptions.ExplicitCapture)] internal static partial Regex GetRegex(); } diff --git a/future/ngra-regex-ir.md b/future/ngra-regex-ir.md index 4d50de5e..f3ed80d9 100644 --- a/future/ngra-regex-ir.md +++ b/future/ngra-regex-ir.md @@ -33,43 +33,32 @@ 1. 尽可能降低源生成或者运行时执行中，对多指令上下文的依赖，尽可能保证每一个IR可以单独输出最佳的代码 2. 减少多余的结构，使得一行指令尽可能处理多的数据 -### 语义改写优化 +### AST优化器 -1. 计数器展开 -建议在lower做 -对于形如a+b或者a{2,4}b的正则表达式，可以优化为a*(ab|b)和a{2,3}(ab|b)的写法 +1. ALT合并 +将多个ALT CASE合并到一个链表中 -2. 确定性计数展开 -建议在lower做 -对于形如a{m,n}的正则表达式，将其展开为a{m}a{0,n-m}的形式 +2. SPAN合并 +将连续的CHAR CONCAT合并为一个char span -3. charset合并 -建议在ast-opt做 -对于形如[ abc ]|[ def ]的表达式，将其合并为[ abcdef ] +3. alt合并 +将多个alt分支中的公共部分提取出来 -4. charset展开 -在lower做 -对于形如abc[ de ]的表达式，如果字符集容量不超过4，可以站卡为(abcd|abce)形式 +### IR优化器 -### 匹配策略改变 +1. .*a特征，优化为 +FIND_SEQ(a, last) +.*?a特征，优化为 +FIND_SEQ(a, first) -1. 长度预测 -在source-gen阶段做 -对于一个正则表达式，可以计算其所需的最小长度，可以在开始的时候提前进行比较 +2. a+特征，优化为 +MATCH_SEQ(a) +COUNT_SEQ(a,0,+inf,greedy) -2. .*优化 -在lower阶段做 -对于COUNT_SET min=0 max=-1 "."指令，也就是.*操作，将后续指令从match转为find +3. a{1} -> a -3. [ set ]*优化 -在source-gen阶段做 -对于其他的COUNT_SET min=0 max=-1 "???"指令，使用IndexOf，IndexOfAnyExcept等进行优化 +4. .\*a\*特征，优化为.* -4. 首字母检查 -**这个不好做，因为IR直接把ALT打碎成if-else串了** -对于(abc|defg|xyz)表达式，可以检查其首字母 - -### C# span opt trick - -1. Span.StartWith -2. [ 0-9 ]展开为IndexOf("0123456789") \ No newline at end of file +5. a{m,n}特征，展开为 +MATCH_SEQ(aa...m个a....aa) +COUNT_SEQ(a,0,n,greedy) From 5cd2b8b4baa042cd72537b985a7e2efa5694be0a Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:52:11 +0800 Subject: [PATCH 4/6] regex ir opt code on rewrite --- .../Collections/CapacityFixedList.cs | 85 ++++-- StgSharp.RegularAnalysis/Text/RegexIR.cs | 66 +++-- .../Text/RegexIRGenerator.cs | 6 +- .../Text/RegexInterpreter.IRGenerate.cs | 39 ++- .../Text/RegexInterpreter.IROptimize.cs | 261 +++++++++--------- .../Text/RegexInterpreter.SourceGenerate.cs | 5 +- future/ngra-regex-ir.md | 4 + 7 files changed, 281 insertions(+), 185 deletions(-) diff --git a/StgSharp.Common/Collections/CapacityFixedList.cs b/StgSharp.Common/Collections/CapacityFixedList.cs index 0f350e27..b62d7ed9 100644 --- a/StgSharp.Common/Collections/CapacityFixedList.cs +++ b/StgSharp.Common/Collections/CapacityFixedList.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="CapacityFixedList" // Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -35,7 +35,9 @@ namespace StgSharp.Collections public static class CapacityFixedListBuilder { - internal static CapacityFixedList Create(ReadOnlySpan values) + internal static CapacityFixedList Create( + ReadOnlySpan values + ) { return new CapacityFixedList(values); } @@ -50,28 +52,39 @@ namespace StgSharp.Collections private readonly int _capacity; private int _index; - public CapacityFixedList(int size) + public CapacityFixedList( + int size + ) { _values = new T[size]; _capacity = size; } - public CapacityFixedList(ReadOnlySpan values) + public CapacityFixedList( + ReadOnlySpan values + ) { _values = values.ToArray(); } - public T this[int index] + public T this[ + int index + ] { - get => index < _index ? _values[index] : throw new ArgumentOutOfRangeException(nameof(index)); - set => _values[index] = index < _index ? value : throw new ArgumentOutOfRangeException(nameof(index)); + get => index < _index ? + _values[index] : + throw new ArgumentOutOfRangeException(nameof(index)); + set => _values[index] = + index < _index ? value : throw new ArgumentOutOfRangeException(nameof(index)); } public bool IsReadOnly => false; public int Count => _index + 1; - public void Add(T item) + public void Add( + T item + ) { if (_index < _capacity - 1) { @@ -82,12 +95,19 @@ namespace StgSharp.Collections public void Clear() { - _values = new T[_capacity]; + if (RuntimeHelpers.IsReferenceOrContainsReferences()) { + Array.Fill(_values, default); + } _index = 0; } - public bool Contains(T item) + public bool Contains( + T item + ) { + if (item is null) { + return false; + } for (int i = 0; i <= _index; i++) { if (item.Equals(_values[i])) { @@ -97,7 +117,10 @@ namespace StgSharp.Collections return false; } - public void CopyTo(T[] array, int arrayIndex) + public void CopyTo( + T[] array, + int arrayIndex + ) { _values.CopyTo(array, arrayIndex); } @@ -109,7 +132,9 @@ namespace StgSharp.Collections } } - public int IndexOf(T item) + public int IndexOf( + T item + ) { T v; for (int i = 0; i <= _index; i++) @@ -121,7 +146,9 @@ namespace StgSharp.Collections return -1; } - public bool Remove(T item) + public bool Remove( + T item + ) { T v; for (int i = 0; i <= _index; i++) @@ -136,8 +163,13 @@ namespace StgSharp.Collections return false; } - public void RemoveAll(Predicate predicate) + public void RemoveAll( + Predicate predicate + ) { + if (predicate is null) { + return; + } for (int i = _index; i > 0; i--) { if (predicate(_values[i])) @@ -148,26 +180,41 @@ namespace StgSharp.Collections } } - public void RemoveAt(int index) + public void RemoveAt( + int index + ) { if (index <= _index) { - _values[index] = _values[_index]; + Array.Copy(_values, index + 1, _values, index, _index - index); + if (RuntimeHelpers.IsReferenceOrContainsReferences()) { + _values[_index] = default!; + } _index--; } } - public void Resize(int newSize) + public void Resize( + int newSize + ) { Array.Resize(ref _values, newSize); } + internal Span ToSpan() + { + return _values; + } + IEnumerator IEnumerable.GetEnumerator() { yield return GetEnumerator(); } - void IList.Insert(int index, T item) + void IList.Insert( + int index, + T item + ) { throw new NotSupportedException(); } diff --git a/StgSharp.RegularAnalysis/Text/RegexIR.cs b/StgSharp.RegularAnalysis/Text/RegexIR.cs index 446257f4..0b1fc8f4 100644 --- a/StgSharp.RegularAnalysis/Text/RegexIR.cs +++ b/StgSharp.RegularAnalysis/Text/RegexIR.cs @@ -37,16 +37,34 @@ namespace StgSharp.RegularAnalysis.Text internal enum RegexIRCommand { - Match_Seq = 0, - Match_Set = 1, - Condition = 3, - Try = 4, - Count = 5, - Count_Seq = 6, - Count_Set = 7, - Pop = 8, - Pack = 9, - Find_Seq = 10, + Match_Seq = 1 << 0, + Match_Set = 1 << 1, + Condition = 1 << 2, + Try = 1 << 3, + Count_Complex = 1 << 4, + Count_Seq = 1 << 5, + Count_Set = 1 << 6, + Pop = 1 << 7, + Pack = 1 << 8, + Find_Seq = 1 << 9, + + Match = Match_Seq | Match_Set, + Count = Count_Seq | Count_Set | Count_Complex, + + } + + internal interface IRegexCount + { + + bool IsGreedy { get; } + + bool IsStar { get; } + + bool IsPlus { get; } + + int Min { get; } + + int Max { get; } } @@ -163,29 +181,33 @@ namespace StgSharp.RegularAnalysis.Text } - internal sealed class RegexCountIR( + internal sealed class RegexCountComplexIR( int min, int max, bool isGreedy, int operation - ) : RegexIR + ) : RegexIR, IRegexCount { public bool IsGreedy { get; } = isGreedy; + public bool IsStar => Min == 0 && Max < 0; + + public bool IsPlus => Min == 1 && Max < 0; + public int Min { get; } = min; public int Max { get; } = max; - public int OperationIndex { get; } = operation; + public int OperationCount { get; } = operation; - internal override RegexIRCommand Command { get; } = RegexIRCommand.Count; + internal override RegexIRCommand Command { get; } = RegexIRCommand.Count_Complex; internal override string AsCommand( int lineNum ) { - return $"[{lineNum}]\t COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {OperationIndex}"; + return $"[{lineNum}]\t COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {OperationCount}"; } } @@ -195,11 +217,15 @@ namespace StgSharp.RegularAnalysis.Text int max, bool isGreedy, string seq - ) : RegexIR + ) : RegexIR, IRegexCount { public bool IsGreedy { get; } = isGreedy; + public bool IsStar => Min == 0 && Max < 0; + + public bool IsPlus => Min == 1 && Max < 0; + public int Min { get; } = min; public int Max { get; } = max; @@ -222,11 +248,17 @@ namespace StgSharp.RegularAnalysis.Text int max, bool isGreedy, string set - ) : RegexIR + ) : RegexIR, IRegexCount { public bool IsGreedy { get; } = isGreedy; + public bool IsDotStar => CharSet == "." && Min == 0 && Max < 0; + + public bool IsStar => Min == 0 && Max < 0; + + public bool IsPlus => Min == 1 && Max < 0; + public int Min { get; } = min; public int Max { get; } = max; diff --git a/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs b/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs index b3a572a6..190bd505 100644 --- a/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs +++ b/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexHIRGenerator" +// file="RegexIRGenerator" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -56,14 +56,14 @@ namespace StgSharp.RegularAnalysis.Text Count++; } - public void EmitCount( + public void EmitCountComplex( int min, int max, bool isGreedy, int operation ) { - _tail.Add(new RegexCountIR(min, max, isGreedy, operation)); + _tail.Add(new RegexCountComplexIR(min, max, isGreedy, operation)); Count++; } diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs index 2bcf7000..e1fcdd94 100644 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexInterpreter.HIRGenerate" +// file="RegexInterpreter.IRGenerate" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -57,31 +57,30 @@ namespace StgSharp.RegularAnalysis.Text int min, int max, bool isGreedy, - RegexAstNode right + RegexAstNode body ) { - bool isRightOperator = !RegexAstNode.IsNullOrEmpty(right) && - (right.Source.Flag & RegexElementLabel.OPERATOR) != 0; + bool is_operator = !RegexAstNode.IsNullOrEmpty(body) && + (body.Source.Flag & RegexElementLabel.OPERATOR) != 0; - if (isRightOperator) // seq to be count is complex + if (is_operator) // seq to be count is complex { - ir.EmitCount(min, max, isGreedy, 1); - RegexIRGenerator rightIR = GenerateIRRecursion(right, curDepth + 1); - ir.EmitTry(rightIR.Count); - ir.EmitIRStream(rightIR); + RegexIRGenerator body_ir = GenerateIRRecursion(body, curDepth + 1); + ir.EmitCountComplex(min, max, isGreedy, body_ir.Count); + ir.EmitIRStream(body_ir); } else { - RegexElementLabel count_source_mask = right.Source.Flag; + RegexElementLabel count_source_mask = body.Source.Flag; switch (count_source_mask) { case RegexElementLabel.UNIT: - ir.EmitCountSeq(min, max, isGreedy, right.Source.Value); + ir.EmitCountSeq(min, max, isGreedy, body.Source.Value); break; case RegexElementLabel.UNIT_SET: - ir.EmitCountSet(min, max, isGreedy, right.Source.Value); + ir.EmitCountSet(min, max, isGreedy, body.Source.Value); break; case RegexElementLabel.UNIT_SPAN: - ir.EmitCountSeq(min, max, isGreedy, right.Source.Value); + ir.EmitCountSeq(min, max, isGreedy, body.Source.Value); break; default: throw new InvalidOperationException($"Unexpected regex element type for count operator: {count_source_mask}"); @@ -144,17 +143,17 @@ namespace StgSharp.RegularAnalysis.Text _ => throw new InvalidOperationException("Unexpected sequence mask for count operator"), }; ReadOnlySpan count_source_span = count_source; - RegexIRGenerator countIR = new(); + RegexIRGenerator count_ir = new(); if (count_source_span[0] == '*') // infinite count { - EmitCount(curDepth, countIR, 0, -1, isGreedy, seq); + EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq); } else if (count_source_span[0] == '+') // at least one { - EmitCount(curDepth, countIR, 1, -1, isGreedy, seq); + EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq); } else if (count_source_span[0] == '?') // zero or one { - EmitCount(curDepth, countIR, 0, 1, isGreedy, seq); + EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq); } else if (count_source_span[0] == '{') // zero or one, non-greedy { int comma_pos = count_source_span.IndexOf(','); @@ -162,7 +161,7 @@ namespace StgSharp.RegularAnalysis.Text { if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count)) { - EmitCount(curDepth, countIR, count, count, isGreedy, seq); + EmitCount(curDepth, count_ir, count, count, isGreedy, seq); } else { throw new InvalidCastException($"Invalid count format: {count_source}"); @@ -172,7 +171,7 @@ namespace StgSharp.RegularAnalysis.Text if (int.TryParse(count_source[1..comma_pos], out int min) && int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max)) { - EmitCount(curDepth, countIR, min, max, isGreedy, seq); + EmitCount(curDepth, count_ir, min, max, isGreedy, seq); } else { throw new InvalidCastException($"Invalid count format: {count_source}"); @@ -183,7 +182,7 @@ namespace StgSharp.RegularAnalysis.Text throw new InvalidOperationException(); } // unknown count format - return countIR; + return count_ir; case RegexElementLabel.CONCAT: RegexIRGenerator concat = new(); diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs index 95b7bd63..6a5de231 100644 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs @@ -25,9 +25,12 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- +using StgSharp.Collections; using System; using System.Collections.Generic; using System.Linq; +using System.Numerics; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; using System.Threading.Tasks; @@ -37,154 +40,164 @@ namespace StgSharp.RegularAnalysis.Text public partial class RegexAnalyzer { - // this array is not thread safe so opt process must be done in single thread. - private RegexIR[] _opt_buffer = new RegexIR[16]; + [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal RegexInfo ScanAndOptimizeIR( - List baseIR, - out List optIR + internal static RegexInfo ScanAndOptimizeIR( + List baseIR, + out List optIR ) { int index = 0; Span origin_span = CollectionsMarshal.AsSpan(baseIR); - List optimizedIR = new(baseIR.Count); - List segment = []; + List opt_ir = new(baseIR.Count); + CapacityFixedList opt_window = new(8); + Span opt_win_sp = [.. opt_window]; RegexInfo info = new(); do { - OptimizeIR(origin_span, ref index, segment, info); - optimizedIR.AddRange(segment); - index++; + int process_count = 0; + int last = int.Min(origin_span.Length, index + 8); + origin_span[index..last].CopyTo(opt_win_sp); + + do + { + // plus_expand: + int expand_result = TryExpandPlus(opt_win_sp, opt_ir); + if (expand_result == 0) + { + opt_win_sp[0] = opt_ir[^1]; + process_count++; + + // goto dot_count_count: + } else if (expand_result > 0) + { + // meets a count-complex + RegexCountComplexIR cnt_cmp_ir = Unsafe.As(opt_win_sp[0]); + if (TryExpandComplexPlus(cnt_cmp_ir.IsGreedy, origin_span[(index + 1)..(index + expand_result)], opt_ir)) + { + process_count += expand_result + 1; + break; + } + } + + // dot_count_count: + if (TryOptDotCountCount(opt_win_sp, opt_ir)) + { + process_count += expand_result == 0 ? 1 : 2; + break; + } + + if (TryFlatternCount(opt_win_sp, opt_ir)) { } + } while (false); + + index += process_count; } while (index < baseIR.Count); - optIR = optimizedIR; + optIR = opt_ir; return info; } - private int ExpandCount( - string pattern, - int min, - int max, - bool is_greedy, - in Span span + private static bool TryExpandComplexPlus( + bool is_greedy, + Span complex, + List opt ) { - // expand count operation to less cycles. + int count = complex.Length; + opt.AddRange(complex); + opt.Add(new RegexCountComplexIR(0, -1, is_greedy, count)); + return true; + } - const int fast_seq_length = 32; + #region expand plus - int fast_count = fast_seq_length / pattern.Length; - fast_count = int.Min(fast_count, min); - if (fast_count > 0) + private static int TryExpandPlus( + Span window, + List opt + ) + { + /* a+ + * convert to + * aa* + */ + + RegexIR ir = window[0]; + if (ir is not IRegexCount { IsPlus: true } count) { + return -1; + } + bool is_greedy = count.IsGreedy; + string pattern; + switch (ir.Command & RegexIRCommand.Count) { - string fast_pattern = string.Concat(Enumerable.Repeat(pattern, fast_count)); + case RegexIRCommand.Count_Complex: + RegexCountComplexIR complex = Unsafe.As(ir); + return complex.OperationCount; + case RegexIRCommand.Count_Seq: + RegexCountSeqIR seq = Unsafe.As(ir); + pattern = seq.Sequence; + opt.Add(new RegexMatchSeqIR(pattern)); + opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern)); + return 0; + case RegexIRCommand.Count_Set: + RegexCountSeqIR set = Unsafe.As(ir); + pattern = set.Sequence; + opt.Add(new RegexMatchSeqIR(pattern)); + opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern)); + return 0; + default: + return -1; + } + } - int fast_min = min / fast_count; - int consumed_count = fast_min * fast_count; - int remain_min = min - consumed_count; - int remain_max = max - consumed_count; - remain_max = int.Max(-1, remain_max); //for negative stands for infinitive - if (remain_max == 0) - { - // a{2} - RegexCountSeqIR fast = new(fast_min, fast_min, is_greedy, fast_pattern); - span[0] = fast; - return 1; - } else - { - // a{2,5} - RegexCountSeqIR fast = new(fast_min, fast_min, is_greedy, fast_pattern); - RegexCountSeqIR tail = new(remain_min, remain_max, is_greedy, pattern); - span[0] = fast; - span[1] = tail; - return 2; - } - } else if (fast_count == 0) + #endregion + + private static bool TryFlatternCount( + Span window, + List opt + ) + { + RegexIR ir = window[0]; + if (ir is not RegexCountSeqIR seq) { + return false; + } + return true; + } + + #region dot count count + + private static int TryOptDotCountCount( + Span window, + List opt + ) + { + /* .* * + * convert to + * .* + */ + RegexIR cur_ir = window[0]; + if (cur_ir is not RegexCountSetIR { IsDotStar: true } dot) { + return -1; + } + if (window[1] is not IRegexCount { IsStar: true } after) { + return -1; + } + if (dot.IsGreedy) { - // least count is 0 - span[0] = new RegexCountSeqIR(0, max, is_greedy, pattern); - return 1; + opt.Add(window[0]); + return 0; } else { - // least count is infinitive - span[0] = new RegexCountSeqIR(-1, -1, is_greedy, pattern); - return 1; + /* + * .* ?a* is different + * it is something like + * FIND(a)?a* :.* + */ + RegexIRGenerator g = new(); + g.EmitTry(1); } + return -1; } - private void OptimizeDot( - RegexCountSetIR root, - Span ir, - ref int index, - List optimizedIR, - RegexInfo info - ) - { - int next = index + 1; - optimizedIR.Clear(); - if (next >= ir.Length) - { - optimizedIR.Add(root); - return; - } - RegexIR nextIR = ir[next]; - if (nextIR is RegexMatchSeqIR match_seq) - { - // opt .*abc style - RegexFindSeqIR find = new(match_seq.Pattern); - optimizedIR.Add(find); - info.MinLength += find.Pattern.Length; - return; - } else if (nextIR is RegexCountSeqIR count_seq) - { - // opt .* (abc)* or .*(abc){2,5} style - string pattern = count_seq.Sequence; - int min = count_seq.Min; - int max = count_seq.Max; - switch ((min, max)) - { - case (0, _): - - break; - default: - RegexFindSeqIR find = new(pattern); - - // RegexCountSeqIR - - break; - } - } - } - - private void OptimizeIR( - Span ir, - ref int index, - List optimizedIR, - RegexInfo info - ) - { - optimizedIR.Clear(); - - // TODO optimize here - RegexIR root = ir[index]; - switch (root.Command) - { - case RegexIRCommand.Count_Set: - RegexCountSetIR count = (root as RegexCountSetIR)!; - if (count.CharSet == ".") - { - // opt for .+ / .* / .{} pattern - OptimizeDot(count, ir, ref index, optimizedIR, info); - } - break; - case RegexIRCommand.Match_Seq: - info.MinLength += (root as RegexMatchSeqIR)!.Pattern.Length; - optimizedIR.Add(root); - break; - default: - optimizedIR.Add(root); - break; - } - } + #endregion internal class RegexInfo { diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs index b3c73f66..1abf47d5 100644 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs @@ -41,8 +41,9 @@ namespace StgSharp.RegularAnalysis.Text ) { StringBuilder sb = new StringBuilder(); - foreach (RegexIR ir in irs) { - ScanAndMarkIROptimization(ir); + foreach (RegexIR ir in irs) + { + // ScanAndMarkIROptimization(ir); } return sb.ToString(); } diff --git a/future/ngra-regex-ir.md b/future/ngra-regex-ir.md index f3ed80d9..6e1e8224 100644 --- a/future/ngra-regex-ir.md +++ b/future/ngra-regex-ir.md @@ -55,10 +55,14 @@ FIND_SEQ(a, first) MATCH_SEQ(a) COUNT_SEQ(a,0,+inf,greedy) +完成 + 3. a{1} -> a 4. .\*a\*特征，优化为.* +完成，最优先 + 5. a{m,n}特征，展开为 MATCH_SEQ(aa...m个a....aa) COUNT_SEQ(a,0,n,greedy) From 13fcfe4a4c31b5f1c481cf9d5d2b21d172494795 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Fri, 26 Jun 2026 18:41:50 +0800 Subject: [PATCH 5/6] regex ir opt code rewrite gen2, window scanning --- StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs | 9 +++++++++ .../Text/RegexInterpreter.IROptimize.cs | 4 ++-- future/ngra-regex-ir.md | 4 ++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs b/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs index 0d816001..86ab7806 100644 --- a/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs +++ b/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs @@ -36,6 +36,15 @@ using System.Text; namespace StgSharp.RegularAnalysis.Abstraction { + public interface ITrinitySyntaxNode : ISyntaxNode + where TNode : ISyntaxNode + where TLabel : unmanaged + { + + TNode Third { get; set; } + + } + public interface ISyntaxNode where TNode : ISyntaxNode where TLabel : unmanaged { diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs index 6a5de231..2e517d2d 100644 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs @@ -87,7 +87,7 @@ namespace StgSharp.RegularAnalysis.Text break; } - if (TryFlatternCount(opt_win_sp, opt_ir)) { } + if (TryFlattenCount(opt_win_sp, opt_ir)) { } } while (false); index += process_count; @@ -150,7 +150,7 @@ namespace StgSharp.RegularAnalysis.Text #endregion - private static bool TryFlatternCount( + private static bool TryFlattenCount( Span window, List opt ) diff --git a/future/ngra-regex-ir.md b/future/ngra-regex-ir.md index 6e1e8224..de81e805 100644 --- a/future/ngra-regex-ir.md +++ b/future/ngra-regex-ir.md @@ -66,3 +66,7 @@ COUNT_SEQ(a,0,+inf,greedy) 5. a{m,n}特征，展开为 MATCH_SEQ(aa...m个a....aa) COUNT_SEQ(a,0,n,greedy) + +6. .*?a*$ + +7. 对于一般的.*?，应该将后续的所有东西全都放进一个巨大的try，code gen时使用最慢的一位一位offset直到直到第一个匹配rest的东西 \ No newline at end of file From 359dd6728716c1c0014d64b72ba70c6c65bb8cf8 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:07:16 +0800 Subject: [PATCH 6/6] regex ir opt code rewrite gen2, window scanning --- StgSharp.RegularAnalysis/Text/RegexIR.cs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/StgSharp.RegularAnalysis/Text/RegexIR.cs b/StgSharp.RegularAnalysis/Text/RegexIR.cs index 0b1fc8f4..6a28274f 100644 --- a/StgSharp.RegularAnalysis/Text/RegexIR.cs +++ b/StgSharp.RegularAnalysis/Text/RegexIR.cs @@ -47,6 +47,8 @@ namespace StgSharp.RegularAnalysis.Text Pop = 1 << 7, Pack = 1 << 8, Find_Seq = 1 << 9, + Find_Set = 1 << 10, + Find_Complex = 1 << 11, Match = Match_Seq | Match_Set, Count = Count_Seq | Count_Set | Count_Complex, @@ -68,6 +70,27 @@ namespace StgSharp.RegularAnalysis.Text } + internal sealed class RegexFindComplexIR( + bool nearest, + int count + ) : RegexIR + { + + internal override RegexIRCommand Command => RegexIRCommand.Find_Complex; + + private bool IsNearest { get; } = nearest; + + private int CommandCount { get; } = count; + + internal override string AsCommand( + int lineNum + ) + { + return $"[{lineNum}]\t FIND_COMPLEX {(IsNearest ? "NEAREST" : "FURTHEST")} COUNT: {CommandCount}"; + } + + } + internal abstract class RegexIR {