From 5cd2b8b4baa042cd72537b985a7e2efa5694be0a Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:52:11 +0800 Subject: [PATCH] regex ir opt code on rewrite --- .../Collections/CapacityFixedList.cs | 85 ++++-- StgSharp.RegularAnalysis/Text/RegexIR.cs | 66 +++-- .../Text/RegexIRGenerator.cs | 6 +- .../Text/RegexInterpreter.IRGenerate.cs | 39 ++- .../Text/RegexInterpreter.IROptimize.cs | 261 +++++++++--------- .../Text/RegexInterpreter.SourceGenerate.cs | 5 +- future/ngra-regex-ir.md | 4 + 7 files changed, 281 insertions(+), 185 deletions(-) diff --git a/StgSharp.Common/Collections/CapacityFixedList.cs b/StgSharp.Common/Collections/CapacityFixedList.cs index 0f350e27..b62d7ed9 100644 --- a/StgSharp.Common/Collections/CapacityFixedList.cs +++ b/StgSharp.Common/Collections/CapacityFixedList.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="CapacityFixedList" // Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -35,7 +35,9 @@ namespace StgSharp.Collections public static class CapacityFixedListBuilder { - internal static CapacityFixedList Create(ReadOnlySpan values) + internal static CapacityFixedList Create( + ReadOnlySpan values + ) { return new CapacityFixedList(values); } @@ -50,28 +52,39 @@ namespace StgSharp.Collections private readonly int _capacity; private int _index; - public CapacityFixedList(int size) + public CapacityFixedList( + int size + ) { _values = new T[size]; _capacity = size; } - public CapacityFixedList(ReadOnlySpan values) + public CapacityFixedList( + ReadOnlySpan values + ) { _values = values.ToArray(); } - public T this[int index] + public T this[ + int index + ] { - get => index < _index ? _values[index] : throw new ArgumentOutOfRangeException(nameof(index)); - set => _values[index] = index < _index ? value : throw new ArgumentOutOfRangeException(nameof(index)); + get => index < _index ? + _values[index] : + throw new ArgumentOutOfRangeException(nameof(index)); + set => _values[index] = + index < _index ? value : throw new ArgumentOutOfRangeException(nameof(index)); } public bool IsReadOnly => false; public int Count => _index + 1; - public void Add(T item) + public void Add( + T item + ) { if (_index < _capacity - 1) { @@ -82,12 +95,19 @@ namespace StgSharp.Collections public void Clear() { - _values = new T[_capacity]; + if (RuntimeHelpers.IsReferenceOrContainsReferences()) { + Array.Fill(_values, default); + } _index = 0; } - public bool Contains(T item) + public bool Contains( + T item + ) { + if (item is null) { + return false; + } for (int i = 0; i <= _index; i++) { if (item.Equals(_values[i])) { @@ -97,7 +117,10 @@ namespace StgSharp.Collections return false; } - public void CopyTo(T[] array, int arrayIndex) + public void CopyTo( + T[] array, + int arrayIndex + ) { _values.CopyTo(array, arrayIndex); } @@ -109,7 +132,9 @@ namespace StgSharp.Collections } } - public int IndexOf(T item) + public int IndexOf( + T item + ) { T v; for (int i = 0; i <= _index; i++) @@ -121,7 +146,9 @@ namespace StgSharp.Collections return -1; } - public bool Remove(T item) + public bool Remove( + T item + ) { T v; for (int i = 0; i <= _index; i++) @@ -136,8 +163,13 @@ namespace StgSharp.Collections return false; } - public void RemoveAll(Predicate predicate) + public void RemoveAll( + Predicate predicate + ) { + if (predicate is null) { + return; + } for (int i = _index; i > 0; i--) { if (predicate(_values[i])) @@ -148,26 +180,41 @@ namespace StgSharp.Collections } } - public void RemoveAt(int index) + public void RemoveAt( + int index + ) { if (index <= _index) { - _values[index] = _values[_index]; + Array.Copy(_values, index + 1, _values, index, _index - index); + if (RuntimeHelpers.IsReferenceOrContainsReferences()) { + _values[_index] = default!; + } _index--; } } - public void Resize(int newSize) + public void Resize( + int newSize + ) { Array.Resize(ref _values, newSize); } + internal Span ToSpan() + { + return _values; + } + IEnumerator IEnumerable.GetEnumerator() { yield return GetEnumerator(); } - void IList.Insert(int index, T item) + void IList.Insert( + int index, + T item + ) { throw new NotSupportedException(); } diff --git a/StgSharp.RegularAnalysis/Text/RegexIR.cs b/StgSharp.RegularAnalysis/Text/RegexIR.cs index 446257f4..0b1fc8f4 100644 --- a/StgSharp.RegularAnalysis/Text/RegexIR.cs +++ b/StgSharp.RegularAnalysis/Text/RegexIR.cs @@ -37,16 +37,34 @@ namespace StgSharp.RegularAnalysis.Text internal enum RegexIRCommand { - Match_Seq = 0, - Match_Set = 1, - Condition = 3, - Try = 4, - Count = 5, - Count_Seq = 6, - Count_Set = 7, - Pop = 8, - Pack = 9, - Find_Seq = 10, + Match_Seq = 1 << 0, + Match_Set = 1 << 1, + Condition = 1 << 2, + Try = 1 << 3, + Count_Complex = 1 << 4, + Count_Seq = 1 << 5, + Count_Set = 1 << 6, + Pop = 1 << 7, + Pack = 1 << 8, + Find_Seq = 1 << 9, + + Match = Match_Seq | Match_Set, + Count = Count_Seq | Count_Set | Count_Complex, + + } + + internal interface IRegexCount + { + + bool IsGreedy { get; } + + bool IsStar { get; } + + bool IsPlus { get; } + + int Min { get; } + + int Max { get; } } @@ -163,29 +181,33 @@ namespace StgSharp.RegularAnalysis.Text } - internal sealed class RegexCountIR( + internal sealed class RegexCountComplexIR( int min, int max, bool isGreedy, int operation - ) : RegexIR + ) : RegexIR, IRegexCount { public bool IsGreedy { get; } = isGreedy; + public bool IsStar => Min == 0 && Max < 0; + + public bool IsPlus => Min == 1 && Max < 0; + public int Min { get; } = min; public int Max { get; } = max; - public int OperationIndex { get; } = operation; + public int OperationCount { get; } = operation; - internal override RegexIRCommand Command { get; } = RegexIRCommand.Count; + internal override RegexIRCommand Command { get; } = RegexIRCommand.Count_Complex; internal override string AsCommand( int lineNum ) { - return $"[{lineNum}]\t COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {OperationIndex}"; + return $"[{lineNum}]\t COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {OperationCount}"; } } @@ -195,11 +217,15 @@ namespace StgSharp.RegularAnalysis.Text int max, bool isGreedy, string seq - ) : RegexIR + ) : RegexIR, IRegexCount { public bool IsGreedy { get; } = isGreedy; + public bool IsStar => Min == 0 && Max < 0; + + public bool IsPlus => Min == 1 && Max < 0; + public int Min { get; } = min; public int Max { get; } = max; @@ -222,11 +248,17 @@ namespace StgSharp.RegularAnalysis.Text int max, bool isGreedy, string set - ) : RegexIR + ) : RegexIR, IRegexCount { public bool IsGreedy { get; } = isGreedy; + public bool IsDotStar => CharSet == "." && Min == 0 && Max < 0; + + public bool IsStar => Min == 0 && Max < 0; + + public bool IsPlus => Min == 1 && Max < 0; + public int Min { get; } = min; public int Max { get; } = max; diff --git a/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs b/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs index b3a572a6..190bd505 100644 --- a/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs +++ b/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexHIRGenerator" +// file="RegexIRGenerator" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -56,14 +56,14 @@ namespace StgSharp.RegularAnalysis.Text Count++; } - public void EmitCount( + public void EmitCountComplex( int min, int max, bool isGreedy, int operation ) { - _tail.Add(new RegexCountIR(min, max, isGreedy, operation)); + _tail.Add(new RegexCountComplexIR(min, max, isGreedy, operation)); Count++; } diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs index 2bcf7000..e1fcdd94 100644 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexInterpreter.HIRGenerate" +// file="RegexInterpreter.IRGenerate" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -57,31 +57,30 @@ namespace StgSharp.RegularAnalysis.Text int min, int max, bool isGreedy, - RegexAstNode right + RegexAstNode body ) { - bool isRightOperator = !RegexAstNode.IsNullOrEmpty(right) && - (right.Source.Flag & RegexElementLabel.OPERATOR) != 0; + bool is_operator = !RegexAstNode.IsNullOrEmpty(body) && + (body.Source.Flag & RegexElementLabel.OPERATOR) != 0; - if (isRightOperator) // seq to be count is complex + if (is_operator) // seq to be count is complex { - ir.EmitCount(min, max, isGreedy, 1); - RegexIRGenerator rightIR = GenerateIRRecursion(right, curDepth + 1); - ir.EmitTry(rightIR.Count); - ir.EmitIRStream(rightIR); + RegexIRGenerator body_ir = GenerateIRRecursion(body, curDepth + 1); + ir.EmitCountComplex(min, max, isGreedy, body_ir.Count); + ir.EmitIRStream(body_ir); } else { - RegexElementLabel count_source_mask = right.Source.Flag; + RegexElementLabel count_source_mask = body.Source.Flag; switch (count_source_mask) { case RegexElementLabel.UNIT: - ir.EmitCountSeq(min, max, isGreedy, right.Source.Value); + ir.EmitCountSeq(min, max, isGreedy, body.Source.Value); break; case RegexElementLabel.UNIT_SET: - ir.EmitCountSet(min, max, isGreedy, right.Source.Value); + ir.EmitCountSet(min, max, isGreedy, body.Source.Value); break; case RegexElementLabel.UNIT_SPAN: - ir.EmitCountSeq(min, max, isGreedy, right.Source.Value); + ir.EmitCountSeq(min, max, isGreedy, body.Source.Value); break; default: throw new InvalidOperationException($"Unexpected regex element type for count operator: {count_source_mask}"); @@ -144,17 +143,17 @@ namespace StgSharp.RegularAnalysis.Text _ => throw new InvalidOperationException("Unexpected sequence mask for count operator"), }; ReadOnlySpan count_source_span = count_source; - RegexIRGenerator countIR = new(); + RegexIRGenerator count_ir = new(); if (count_source_span[0] == '*') // infinite count { - EmitCount(curDepth, countIR, 0, -1, isGreedy, seq); + EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq); } else if (count_source_span[0] == '+') // at least one { - EmitCount(curDepth, countIR, 1, -1, isGreedy, seq); + EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq); } else if (count_source_span[0] == '?') // zero or one { - EmitCount(curDepth, countIR, 0, 1, isGreedy, seq); + EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq); } else if (count_source_span[0] == '{') // zero or one, non-greedy { int comma_pos = count_source_span.IndexOf(','); @@ -162,7 +161,7 @@ namespace StgSharp.RegularAnalysis.Text { if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count)) { - EmitCount(curDepth, countIR, count, count, isGreedy, seq); + EmitCount(curDepth, count_ir, count, count, isGreedy, seq); } else { throw new InvalidCastException($"Invalid count format: {count_source}"); @@ -172,7 +171,7 @@ namespace StgSharp.RegularAnalysis.Text if (int.TryParse(count_source[1..comma_pos], out int min) && int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max)) { - EmitCount(curDepth, countIR, min, max, isGreedy, seq); + EmitCount(curDepth, count_ir, min, max, isGreedy, seq); } else { throw new InvalidCastException($"Invalid count format: {count_source}"); @@ -183,7 +182,7 @@ namespace StgSharp.RegularAnalysis.Text throw new InvalidOperationException(); } // unknown count format - return countIR; + return count_ir; case RegexElementLabel.CONCAT: RegexIRGenerator concat = new(); diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs index 95b7bd63..6a5de231 100644 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs @@ -25,9 +25,12 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- +using StgSharp.Collections; using System; using System.Collections.Generic; using System.Linq; +using System.Numerics; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; using System.Threading.Tasks; @@ -37,154 +40,164 @@ namespace StgSharp.RegularAnalysis.Text public partial class RegexAnalyzer { - // this array is not thread safe so opt process must be done in single thread. - private RegexIR[] _opt_buffer = new RegexIR[16]; + [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal RegexInfo ScanAndOptimizeIR( - List baseIR, - out List optIR + internal static RegexInfo ScanAndOptimizeIR( + List baseIR, + out List optIR ) { int index = 0; Span origin_span = CollectionsMarshal.AsSpan(baseIR); - List optimizedIR = new(baseIR.Count); - List segment = []; + List opt_ir = new(baseIR.Count); + CapacityFixedList opt_window = new(8); + Span opt_win_sp = [.. opt_window]; RegexInfo info = new(); do { - OptimizeIR(origin_span, ref index, segment, info); - optimizedIR.AddRange(segment); - index++; + int process_count = 0; + int last = int.Min(origin_span.Length, index + 8); + origin_span[index..last].CopyTo(opt_win_sp); + + do + { + // plus_expand: + int expand_result = TryExpandPlus(opt_win_sp, opt_ir); + if (expand_result == 0) + { + opt_win_sp[0] = opt_ir[^1]; + process_count++; + + // goto dot_count_count: + } else if (expand_result > 0) + { + // meets a count-complex + RegexCountComplexIR cnt_cmp_ir = Unsafe.As(opt_win_sp[0]); + if (TryExpandComplexPlus(cnt_cmp_ir.IsGreedy, origin_span[(index + 1)..(index + expand_result)], opt_ir)) + { + process_count += expand_result + 1; + break; + } + } + + // dot_count_count: + if (TryOptDotCountCount(opt_win_sp, opt_ir)) + { + process_count += expand_result == 0 ? 1 : 2; + break; + } + + if (TryFlatternCount(opt_win_sp, opt_ir)) { } + } while (false); + + index += process_count; } while (index < baseIR.Count); - optIR = optimizedIR; + optIR = opt_ir; return info; } - private int ExpandCount( - string pattern, - int min, - int max, - bool is_greedy, - in Span span + private static bool TryExpandComplexPlus( + bool is_greedy, + Span complex, + List opt ) { - // expand count operation to less cycles. + int count = complex.Length; + opt.AddRange(complex); + opt.Add(new RegexCountComplexIR(0, -1, is_greedy, count)); + return true; + } - const int fast_seq_length = 32; + #region expand plus - int fast_count = fast_seq_length / pattern.Length; - fast_count = int.Min(fast_count, min); - if (fast_count > 0) + private static int TryExpandPlus( + Span window, + List opt + ) + { + /* a+ + * convert to + * aa* + */ + + RegexIR ir = window[0]; + if (ir is not IRegexCount { IsPlus: true } count) { + return -1; + } + bool is_greedy = count.IsGreedy; + string pattern; + switch (ir.Command & RegexIRCommand.Count) { - string fast_pattern = string.Concat(Enumerable.Repeat(pattern, fast_count)); + case RegexIRCommand.Count_Complex: + RegexCountComplexIR complex = Unsafe.As(ir); + return complex.OperationCount; + case RegexIRCommand.Count_Seq: + RegexCountSeqIR seq = Unsafe.As(ir); + pattern = seq.Sequence; + opt.Add(new RegexMatchSeqIR(pattern)); + opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern)); + return 0; + case RegexIRCommand.Count_Set: + RegexCountSeqIR set = Unsafe.As(ir); + pattern = set.Sequence; + opt.Add(new RegexMatchSeqIR(pattern)); + opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern)); + return 0; + default: + return -1; + } + } - int fast_min = min / fast_count; - int consumed_count = fast_min * fast_count; - int remain_min = min - consumed_count; - int remain_max = max - consumed_count; - remain_max = int.Max(-1, remain_max); //for negative stands for infinitive - if (remain_max == 0) - { - // a{2} - RegexCountSeqIR fast = new(fast_min, fast_min, is_greedy, fast_pattern); - span[0] = fast; - return 1; - } else - { - // a{2,5} - RegexCountSeqIR fast = new(fast_min, fast_min, is_greedy, fast_pattern); - RegexCountSeqIR tail = new(remain_min, remain_max, is_greedy, pattern); - span[0] = fast; - span[1] = tail; - return 2; - } - } else if (fast_count == 0) + #endregion + + private static bool TryFlatternCount( + Span window, + List opt + ) + { + RegexIR ir = window[0]; + if (ir is not RegexCountSeqIR seq) { + return false; + } + return true; + } + + #region dot count count + + private static int TryOptDotCountCount( + Span window, + List opt + ) + { + /* .* * + * convert to + * .* + */ + RegexIR cur_ir = window[0]; + if (cur_ir is not RegexCountSetIR { IsDotStar: true } dot) { + return -1; + } + if (window[1] is not IRegexCount { IsStar: true } after) { + return -1; + } + if (dot.IsGreedy) { - // least count is 0 - span[0] = new RegexCountSeqIR(0, max, is_greedy, pattern); - return 1; + opt.Add(window[0]); + return 0; } else { - // least count is infinitive - span[0] = new RegexCountSeqIR(-1, -1, is_greedy, pattern); - return 1; + /* + * .* ?a* is different + * it is something like + * FIND(a)?a* :.* + */ + RegexIRGenerator g = new(); + g.EmitTry(1); } + return -1; } - private void OptimizeDot( - RegexCountSetIR root, - Span ir, - ref int index, - List optimizedIR, - RegexInfo info - ) - { - int next = index + 1; - optimizedIR.Clear(); - if (next >= ir.Length) - { - optimizedIR.Add(root); - return; - } - RegexIR nextIR = ir[next]; - if (nextIR is RegexMatchSeqIR match_seq) - { - // opt .*abc style - RegexFindSeqIR find = new(match_seq.Pattern); - optimizedIR.Add(find); - info.MinLength += find.Pattern.Length; - return; - } else if (nextIR is RegexCountSeqIR count_seq) - { - // opt .* (abc)* or .*(abc){2,5} style - string pattern = count_seq.Sequence; - int min = count_seq.Min; - int max = count_seq.Max; - switch ((min, max)) - { - case (0, _): - - break; - default: - RegexFindSeqIR find = new(pattern); - - // RegexCountSeqIR - - break; - } - } - } - - private void OptimizeIR( - Span ir, - ref int index, - List optimizedIR, - RegexInfo info - ) - { - optimizedIR.Clear(); - - // TODO optimize here - RegexIR root = ir[index]; - switch (root.Command) - { - case RegexIRCommand.Count_Set: - RegexCountSetIR count = (root as RegexCountSetIR)!; - if (count.CharSet == ".") - { - // opt for .+ / .* / .{} pattern - OptimizeDot(count, ir, ref index, optimizedIR, info); - } - break; - case RegexIRCommand.Match_Seq: - info.MinLength += (root as RegexMatchSeqIR)!.Pattern.Length; - optimizedIR.Add(root); - break; - default: - optimizedIR.Add(root); - break; - } - } + #endregion internal class RegexInfo { diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs index b3c73f66..1abf47d5 100644 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs @@ -41,8 +41,9 @@ namespace StgSharp.RegularAnalysis.Text ) { StringBuilder sb = new StringBuilder(); - foreach (RegexIR ir in irs) { - ScanAndMarkIROptimization(ir); + foreach (RegexIR ir in irs) + { + // ScanAndMarkIROptimization(ir); } return sb.ToString(); } diff --git a/future/ngra-regex-ir.md b/future/ngra-regex-ir.md index f3ed80d9..6e1e8224 100644 --- a/future/ngra-regex-ir.md +++ b/future/ngra-regex-ir.md @@ -55,10 +55,14 @@ FIND_SEQ(a, first) MATCH_SEQ(a) COUNT_SEQ(a,0,+inf,greedy) +完成 + 3. a{1} -> a 4. .\*a\*特征，优化为.* +完成，最优先 + 5. a{m,n}特征，展开为 MATCH_SEQ(aa...m个a....aa) COUNT_SEQ(a,0,n,greedy)