From 7edb94943d4880bce78b10164cbb96a6d7face95 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Fri, 3 Jul 2026 18:05:10 +0800 Subject: [PATCH] add more rules for ir opt, fix wrong process to dot --- StgSharp.RegularAnalysis/SourceGenerate.cs | 12 ++ .../StgSharp.RegularAnalysis.csproj | 21 +++- ...=> RegexAnalyzerBackEnd.CharSetEmitter.cs} | 30 +++-- ...rceGenerate.cs => RegexAnalyzerBackEnd.cs} | 16 +-- ...s => RegexAnalyzerFrontEnd.ASTGenerate.cs} | 2 +- ...s => RegexAnalyzerFrontEnd.ASTOptimize.cs} | 2 +- ...cs => RegexAnalyzerFrontEnd.IRGenerate.cs} | 2 +- ...cs => RegexAnalyzerFrontEnd.IROptimize.cs} | 107 ++++++++++++++++-- ...exAnalyzer.cs => RegexAnalyzerFrontEnd.cs} | 2 +- StgSharp.RegularAnalysis/Text/RegexIR.cs | 4 +- .../Text/RegexTokenParser.cs | 1 - .../Text/RegexTokenReader.cs | 45 +++++--- StgSharpDebug/TestRegexAnalyse.cs | 8 +- 13 files changed, 200 insertions(+), 52 deletions(-) create mode 100644 StgSharp.RegularAnalysis/SourceGenerate.cs rename StgSharp.RegularAnalysis/Text/{CharSetEmitter.cs => RegexAnalyzerBackEnd.CharSetEmitter.cs} (79%) rename StgSharp.RegularAnalysis/Text/{RegexAnalyzer.SourceGenerate.cs => RegexAnalyzerBackEnd.cs} (81%) rename StgSharp.RegularAnalysis/Text/{RegexAnalyzer.ASTGenerate.cs => RegexAnalyzerFrontEnd.ASTGenerate.cs} (99%) rename StgSharp.RegularAnalysis/Text/{RegexAnalyzer.ASTOptimize.cs => RegexAnalyzerFrontEnd.ASTOptimize.cs} (99%) rename StgSharp.RegularAnalysis/Text/{RegexAnalyzer.IRGenerate.cs => RegexAnalyzerFrontEnd.IRGenerate.cs} (99%) rename StgSharp.RegularAnalysis/Text/{RegexAnalyzer.IROptimize.cs => RegexAnalyzerFrontEnd.IROptimize.cs} (81%) rename StgSharp.RegularAnalysis/Text/{RegexAnalyzer.cs => RegexAnalyzerFrontEnd.cs} (98%) diff --git a/StgSharp.RegularAnalysis/SourceGenerate.cs b/StgSharp.RegularAnalysis/SourceGenerate.cs new file mode 100644 index 00000000..4830e842 --- /dev/null +++ b/StgSharp.RegularAnalysis/SourceGenerate.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis +{ + internal class SourceGenerate + { + } +} diff --git a/StgSharp.RegularAnalysis/StgSharp.RegularAnalysis.csproj b/StgSharp.RegularAnalysis/StgSharp.RegularAnalysis.csproj index d4495f72..0e71b385 100644 --- a/StgSharp.RegularAnalysis/StgSharp.RegularAnalysis.csproj +++ b/StgSharp.RegularAnalysis/StgSharp.RegularAnalysis.csproj @@ -1,7 +1,26 @@ - + + + + true + true + true + $(BaseIntermediateOutputPath)Generated + + + + + + + + + + + + + diff --git a/StgSharp.RegularAnalysis/Text/CharSetEmitter.cs b/StgSharp.RegularAnalysis/Text/RegexAnalyzerBackEnd.CharSetEmitter.cs similarity index 79% rename from StgSharp.RegularAnalysis/Text/CharSetEmitter.cs rename to StgSharp.RegularAnalysis/Text/RegexAnalyzerBackEnd.CharSetEmitter.cs index f0105bac..bc7dc2c5 100644 --- a/StgSharp.RegularAnalysis/Text/CharSetEmitter.cs +++ b/StgSharp.RegularAnalysis/Text/RegexAnalyzerBackEnd.CharSetEmitter.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="CharSetEmitter" +// file="RegexAnalyzerBackEnd.CharSetEmitter" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -44,7 +44,7 @@ namespace StgSharp.RegularAnalysis.Text } - internal record struct CharSet( + internal record struct RegexCharSet( string value, CharSetType t, bool accept @@ -57,13 +57,18 @@ namespace StgSharp.RegularAnalysis.Text public string Value { get; set; } = value; + public override readonly string ToString() + { + return $"{{value:{value}, type:{Type}, accept:{IsAcceptable}}}"; + } + } - internal class CharSetEmitter + internal static partial class RegexAnalyzerBackEnd { - public static CharSet[] FigureCharSetType( - ReadOnlySpan source + public static RegexCharSet[] FigureCharSetType( + ReadOnlySpan source ) { // range or single @@ -76,19 +81,24 @@ namespace StgSharp.RegularAnalysis.Text ReadOnlySpan seq = source.Slice(idx, source.Length - idx - 1); if (seq.IndexOf('-') == -1) { - return [new CharSet(seq.ToString(), CharSetType.Single, accept)]; + return [new RegexCharSet(seq.ToString(), CharSetType.Single, accept)]; } else { - List list = []; - for (int i = 0; i < seq.Length && seq.Length > 0; ) + List list = []; + for (; seq.Length > 0; ) { int pos = seq.IndexOf('-'); + if (pos == -1) + { + RegexCharSet single = new(seq.ToString(), CharSetType.Single, accept); + list.Add(single); + break; + } else if (pos > 1) { list.Add(new(seq[0..(pos - 1)].ToString(), CharSetType.Single, accept)); } list.Add(new($"{seq[pos - 1]}{seq[pos + 1]}", CharSetType.Range, accept)); - i = pos + 2; - seq = seq[i..]; + seq = seq[(pos + 2)..]; } return [.. list]; } diff --git a/StgSharp.RegularAnalysis/Text/RegexAnalyzer.SourceGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexAnalyzerBackEnd.cs similarity index 81% rename from StgSharp.RegularAnalysis/Text/RegexAnalyzer.SourceGenerate.cs rename to StgSharp.RegularAnalysis/Text/RegexAnalyzerBackEnd.cs index 1abf47d5..b06af26a 100644 --- a/StgSharp.RegularAnalysis/Text/RegexAnalyzer.SourceGenerate.cs +++ b/StgSharp.RegularAnalysis/Text/RegexAnalyzerBackEnd.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexInterpreter.SourceGenerate" +// file="RegexAnalyzerBackEnd" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -25,6 +25,7 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- +using Microsoft.CodeAnalysis; using System; using System.Collections.Generic; using System.Linq; @@ -33,19 +34,14 @@ using System.Threading.Tasks; namespace StgSharp.RegularAnalysis.Text { - public partial class RegexAnalyzer + internal static partial class RegexAnalyzerBackEnd { - internal static string GetSourceCode( - IEnumerable irs + internal static string GenerateSource( + List ir ) { - StringBuilder sb = new StringBuilder(); - foreach (RegexIR ir in irs) - { - // ScanAndMarkIROptimization(ir); - } - return sb.ToString(); + return string.Empty; } } diff --git a/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexAnalyzerFrontEnd.ASTGenerate.cs similarity index 99% rename from StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs rename to StgSharp.RegularAnalysis/Text/RegexAnalyzerFrontEnd.ASTGenerate.cs index cd0730b4..cdfd36f9 100644 --- a/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs +++ b/StgSharp.RegularAnalysis/Text/RegexAnalyzerFrontEnd.ASTGenerate.cs @@ -35,7 +35,7 @@ using System.Threading.Tasks; namespace StgSharp.RegularAnalysis.Text { - public partial class RegexAnalyzer + public partial class RegexAnalyzerFrontEnd { private static AbstractSyntaxTree GenerateAST( diff --git a/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs b/StgSharp.RegularAnalysis/Text/RegexAnalyzerFrontEnd.ASTOptimize.cs similarity index 99% rename from StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs rename to StgSharp.RegularAnalysis/Text/RegexAnalyzerFrontEnd.ASTOptimize.cs index 0197a4f5..875b18f9 100644 --- a/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs +++ b/StgSharp.RegularAnalysis/Text/RegexAnalyzerFrontEnd.ASTOptimize.cs @@ -39,7 +39,7 @@ using System.Xml.Linq; namespace StgSharp.RegularAnalysis.Text { - public partial class RegexAnalyzer + public partial class RegexAnalyzerFrontEnd { private static RegexAstNode FlattenAlt( diff --git a/StgSharp.RegularAnalysis/Text/RegexAnalyzer.IRGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexAnalyzerFrontEnd.IRGenerate.cs similarity index 99% rename from StgSharp.RegularAnalysis/Text/RegexAnalyzer.IRGenerate.cs rename to StgSharp.RegularAnalysis/Text/RegexAnalyzerFrontEnd.IRGenerate.cs index 7080e962..c46902d2 100644 --- a/StgSharp.RegularAnalysis/Text/RegexAnalyzer.IRGenerate.cs +++ b/StgSharp.RegularAnalysis/Text/RegexAnalyzerFrontEnd.IRGenerate.cs @@ -40,7 +40,7 @@ using System.Xml; namespace StgSharp.RegularAnalysis.Text { - public partial class RegexAnalyzer + public partial class RegexAnalyzerFrontEnd { internal static List GenerateIR( diff --git a/StgSharp.RegularAnalysis/Text/RegexAnalyzer.IROptimize.cs b/StgSharp.RegularAnalysis/Text/RegexAnalyzerFrontEnd.IROptimize.cs similarity index 81% rename from StgSharp.RegularAnalysis/Text/RegexAnalyzer.IROptimize.cs rename to StgSharp.RegularAnalysis/Text/RegexAnalyzerFrontEnd.IROptimize.cs index a9a15ada..02c7392e 100644 --- a/StgSharp.RegularAnalysis/Text/RegexAnalyzer.IROptimize.cs +++ b/StgSharp.RegularAnalysis/Text/RegexAnalyzerFrontEnd.IROptimize.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexAnalyzer.IROptimize" +// file="RegexAnalyzerFrontEnd.IROptimize" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -37,7 +37,7 @@ using System.Threading.Tasks; namespace StgSharp.RegularAnalysis.Text { - public partial class RegexAnalyzer + public partial class RegexAnalyzerFrontEnd { internal static void ScanAndOptimizeIR( @@ -50,7 +50,10 @@ namespace StgSharp.RegularAnalysis.Text List opt_ir = new(origin_ir.Count); RegexIR[] opt_window = new RegexIR[8]; Span opt_win_sp = opt_window; - opt_cycle: + int result; + + // phase 0: common opt + phase_zero: opt_count = 0; int index = 0; Span origin_span = CollectionsMarshal.AsSpan(origin_ir); @@ -62,7 +65,6 @@ namespace StgSharp.RegularAnalysis.Text opt_win_sp.Clear(); origin_span[index..last].CopyTo(opt_win_sp); - int result; if ((result = TryExpandPlus(opt_win_sp, opt_ir)) >= 0) { process_count++; @@ -130,12 +132,14 @@ namespace StgSharp.RegularAnalysis.Text } (origin_ir, opt_ir) = (opt_ir, origin_ir); opt_ir.Clear(); - goto opt_cycle; + goto phase_zero; } - /**/ + + // phase 1: dot star replace (origin_ir, opt_ir) = (opt_ir, origin_ir); opt_ir.Clear(); index = 0; + opt_count = 0; origin_span = CollectionsMarshal.AsSpan(origin_ir); do { @@ -150,8 +154,10 @@ namespace StgSharp.RegularAnalysis.Text process_count += count + 1; Span opt_block = origin_span.Slice(index, count + 1); opt_ir.AddRange(opt_block); - } else if (TryReplaceDotStar(opt_win_sp, opt_ir) >= 0) + } else + if (TryReplaceDotStar(opt_win_sp, opt_ir) >= 0) { + opt_count++; process_count++; } else { @@ -161,7 +167,53 @@ namespace StgSharp.RegularAnalysis.Text index += process_count; } while (index < origin_ir.Count); - /**/ + if (opt_count == 0) + { + optIR = opt_ir; + return; + } + + // phase 2: find-1 merge and try-find-1 rotate + do + { + (origin_ir, opt_ir) = (opt_ir, origin_ir); + opt_ir.Clear(); + opt_count = 0; + index = 0; + origin_span = CollectionsMarshal.AsSpan(origin_ir); + do + { + int process_count = 0; + int last = int.Min(origin_span.Length, index + 8); + opt_win_sp.Clear(); + origin_span[index..last].CopyTo(opt_win_sp); + + if (opt_win_sp[0] is IRegexComplex { IsFullyOptimized: true, CommandCount: > 0 } skip) + { + int count = skip.CommandCount; + process_count += count + 1; + Span opt_block = origin_span.Slice(index, count + 1); + opt_ir.AddRange(opt_block); + } else if (TyrMergeFindRestTwice(opt_win_sp, opt_ir) == 0) + { + process_count += 2; + opt_count++; + } else if ((result = TyrMergeFindRestTry(opt_win_sp, opt_ir)) >= 0) + { + Span opt_block = origin_span.Slice(index + 2, result); + opt_ir.AddRange(opt_block); + process_count += 2; + process_count += result; + opt_count++; + } else + { + opt_ir.Add(opt_win_sp[0]); + process_count++; + } + + index += process_count; + } while (index < origin_ir.Count); + } while (opt_count > 0); optIR = opt_ir; } @@ -285,6 +337,45 @@ namespace StgSharp.RegularAnalysis.Text } + #region merge find and try-find + + private static int TyrMergeFindRestTwice( + Span window, + List opt + ) + { + if (window[0] is not RegexFindComplexIR { CommandCount: -1 } find_1 || + window[1] is not RegexFindComplexIR find_2) { + return -1; + } + RegexFindComplexIR find_try = new(find_1.IsNearest && find_2.IsNearest, find_2.CommandCount) + { + IsFullyOptimized = true + }; + opt.Add(find_try); + return 0; + } + + private static int TyrMergeFindRestTry( + Span window, + List opt + ) + { + if (window[0] is not RegexFindComplexIR { CommandCount: -1 } _find || + window[1] is not RegexTryIR _try) { + return -1; + } + RegexFindComplexIR find_try = new(_find.IsNearest, _try.CommandCount) + { + IsFullyOptimized = true + }; + opt.Add(find_try); + return find_try.CommandCount; + } + + #endregion + + #region expand once private static void TryExpandComplex( diff --git a/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs b/StgSharp.RegularAnalysis/Text/RegexAnalyzerFrontEnd.cs similarity index 98% rename from StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs rename to StgSharp.RegularAnalysis/Text/RegexAnalyzerFrontEnd.cs index 63eba341..92d26a0d 100644 --- a/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs +++ b/StgSharp.RegularAnalysis/Text/RegexAnalyzerFrontEnd.cs @@ -67,7 +67,7 @@ namespace StgSharp.RegularAnalysis.Text } - public static partial class RegexAnalyzer + public static partial class RegexAnalyzerFrontEnd { public static TextRegexSource Analyze( diff --git a/StgSharp.RegularAnalysis/Text/RegexIR.cs b/StgSharp.RegularAnalysis/Text/RegexIR.cs index ede82aab..4bcf2fec 100644 --- a/StgSharp.RegularAnalysis/Text/RegexIR.cs +++ b/StgSharp.RegularAnalysis/Text/RegexIR.cs @@ -87,12 +87,12 @@ namespace StgSharp.RegularAnalysis.Text public bool IsFullyOptimized { get; set; } + public bool IsNearest { get; } = nearest; + public int CommandCount { get; set; } = count; internal override RegexIRCommand Command => RegexIRCommand.Find_Complex; - private bool IsNearest { get; } = nearest; - internal override string AsCommand( int lineNum ) diff --git a/StgSharp.RegularAnalysis/Text/RegexTokenParser.cs b/StgSharp.RegularAnalysis/Text/RegexTokenParser.cs index f4ee54a3..556bb4b7 100644 --- a/StgSharp.RegularAnalysis/Text/RegexTokenParser.cs +++ b/StgSharp.RegularAnalysis/Text/RegexTokenParser.cs @@ -68,7 +68,6 @@ namespace StgSharp.RegularAnalysis.Text } } else { - Console.WriteLine("returns empty"); return Token.Empty; } } diff --git a/StgSharp.RegularAnalysis/Text/RegexTokenReader.cs b/StgSharp.RegularAnalysis/Text/RegexTokenReader.cs index ada15d54..9c8220a8 100644 --- a/StgSharp.RegularAnalysis/Text/RegexTokenReader.cs +++ b/StgSharp.RegularAnalysis/Text/RegexTokenReader.cs @@ -45,7 +45,9 @@ namespace StgSharp.RegularAnalysis.Text public RegexTokenReader() { } - public RegexTokenReader(string source) + public RegexTokenReader( + string source + ) { _source = source; } @@ -65,9 +67,11 @@ namespace StgSharp.RegularAnalysis.Text { string value = _source[(pos + 1)..(pos + 3)]; pos += 4; - if (uint.TryParse(value, out uint code) && Rune.TryCreate(code, out Rune result)) + if (uint.TryParse(value, out uint code) && + Rune.TryCreate(code, out Rune result)) { - return new Token(result.ToString(), 0, pos, RegexElementLabel.UNIT); + return new Token(result.ToString(), 0, pos, + RegexElementLabel.UNIT); } else { throw new InvalidCastException($"Failed to convert {value} to a valid ascii code point."); @@ -76,9 +80,11 @@ namespace StgSharp.RegularAnalysis.Text { string value = _source[(pos + 1)..(pos + 5)]; pos += 6; - if (uint.TryParse(value, out uint code) && Rune.TryCreate(code, out Rune result)) + if (uint.TryParse(value, out uint code) && + Rune.TryCreate(code, out Rune result)) { - return new Token(result.ToString(), 0, pos, RegexElementLabel.UNIT); + return new Token(result.ToString(), 0, pos, + RegexElementLabel.UNIT); } else { throw new InvalidCastException($"Failed to convert {value} to a valid unicode code point."); @@ -99,6 +105,10 @@ namespace StgSharp.RegularAnalysis.Text } } throw new InvalidOperationException("Unclosed Unicode property escape sequence."); + } else if (_source[pos + 1] == '.') + { + position = pos + 2; + return new Token(".", 0, pos, RegexElementLabel.UNIT); } else { position = pos + 2; @@ -112,21 +122,24 @@ namespace StgSharp.RegularAnalysis.Text { int length = (i - pos - 1); position = i + 1; - return new Token(_source[(pos)..(i + 1)], 0, position, + return new Token(_source[(pos)..(i + 1)], 0, + position, RegexElementLabel.UNIT_SET); } } throw new InvalidOperationException("Unclosed character set"); case '|': position = pos + 1; - return new Token(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.ALT); + return new Token(_source[(pos)..(pos + 1)], 0, pos, + RegexElementLabel.ALT); case '(': // very comlex here return ReadGroupBeginToken(); case ')': position = pos + 1; - return new Token(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.GROUP_END); + return new Token(_source[(pos)..(pos + 1)], 0, pos, + RegexElementLabel.GROUP_END); case '{': for (int i = pos + 1; i < _source.Length; i++) { @@ -141,20 +154,26 @@ namespace StgSharp.RegularAnalysis.Text case '*' or '+': int count = pos + 1 >= _source.Length ? 1 : (_source[pos + 1] == '?' ? 2 : 1); position = pos + count; - return new Token(_source[(pos)..(pos + count)], 0, pos, RegexElementLabel.COUNT); + return new Token(_source[(pos)..(pos + count)], 0, pos, + RegexElementLabel.COUNT); case '?': position = pos + 1; - return new Token(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.COUNT); + return new Token(_source[(pos)..(pos + 1)], 0, pos, + RegexElementLabel.COUNT); case '.': position = pos + 1; - return new Token(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.UNIT); + return new Token(_source[(pos)..(pos + 1)], 0, pos, + RegexElementLabel.UNIT_SET); default: position = pos + 1; - return new Token(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.UNIT); + return new Token(_source[(pos)..(pos + 1)], 0, pos, + RegexElementLabel.UNIT); } } - public bool TryReadToken(out Token t) + public bool TryReadToken( + out Token t + ) { if (IsEmpty) { diff --git a/StgSharpDebug/TestRegexAnalyse.cs b/StgSharpDebug/TestRegexAnalyse.cs index 83d26af1..22b78fc4 100644 --- a/StgSharpDebug/TestRegexAnalyse.cs +++ b/StgSharpDebug/TestRegexAnalyse.cs @@ -40,19 +40,21 @@ namespace StgSharp { private static string[] regexSource = [ - + @"[.]*", + /* @"http|https|ftp|ftps", @"abc[0-9]x|abc[0-9]y|abd[0-9]x|abd[0-9]y|xyz", @"get_name|get_value|get_type|set_name|set_value|set_type", @"helix|hello|help|hero|hers|her", - @"a*.[abc]*(abcd|a(bc)c)+(?abc{11,13})" + @"a*.[abc]*(abcd|a(bc)c)+(?.*)234" +/**/ ]; public static void Run() { foreach (string regex in regexSource) { - TextRegexSource interpreterSource = RegexAnalyzer.Analyze(regex); + TextRegexSource interpreterSource = RegexAnalyzerFrontEnd.Analyze(regex); AbstractSyntaxTree tree = interpreterSource.Ast; Console.WriteLine($"result of \"{regex}\""); TreeEnumerator enumerator = new(tree);