diff --git a/StgSharp.Common/Collections/CapacityFixedList.cs b/StgSharp.Common/Collections/CapacityFixedList.cs index 0f350e27..b62d7ed9 100644 --- a/StgSharp.Common/Collections/CapacityFixedList.cs +++ b/StgSharp.Common/Collections/CapacityFixedList.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="CapacityFixedList" // Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -35,7 +35,9 @@ namespace StgSharp.Collections public static class CapacityFixedListBuilder { - internal static CapacityFixedList Create(ReadOnlySpan values) + internal static CapacityFixedList Create( + ReadOnlySpan values + ) { return new CapacityFixedList(values); } @@ -50,28 +52,39 @@ namespace StgSharp.Collections private readonly int _capacity; private int _index; - public CapacityFixedList(int size) + public CapacityFixedList( + int size + ) { _values = new T[size]; _capacity = size; } - public CapacityFixedList(ReadOnlySpan values) + public CapacityFixedList( + ReadOnlySpan values + ) { _values = values.ToArray(); } - public T this[int index] + public T this[ + int index + ] { - get => index < _index ? _values[index] : throw new ArgumentOutOfRangeException(nameof(index)); - set => _values[index] = index < _index ? value : throw new ArgumentOutOfRangeException(nameof(index)); + get => index < _index ? + _values[index] : + throw new ArgumentOutOfRangeException(nameof(index)); + set => _values[index] = + index < _index ? value : throw new ArgumentOutOfRangeException(nameof(index)); } public bool IsReadOnly => false; public int Count => _index + 1; - public void Add(T item) + public void Add( + T item + ) { if (_index < _capacity - 1) { @@ -82,12 +95,19 @@ namespace StgSharp.Collections public void Clear() { - _values = new T[_capacity]; + if (RuntimeHelpers.IsReferenceOrContainsReferences()) { + Array.Fill(_values, default); + } _index = 0; } - public bool Contains(T item) + public bool Contains( + T item + ) { + if (item is null) { + return false; + } for (int i = 0; i <= _index; i++) { if (item.Equals(_values[i])) { @@ -97,7 +117,10 @@ namespace StgSharp.Collections return false; } - public void CopyTo(T[] array, int arrayIndex) + public void CopyTo( + T[] array, + int arrayIndex + ) { _values.CopyTo(array, arrayIndex); } @@ -109,7 +132,9 @@ namespace StgSharp.Collections } } - public int IndexOf(T item) + public int IndexOf( + T item + ) { T v; for (int i = 0; i <= _index; i++) @@ -121,7 +146,9 @@ namespace StgSharp.Collections return -1; } - public bool Remove(T item) + public bool Remove( + T item + ) { T v; for (int i = 0; i <= _index; i++) @@ -136,8 +163,13 @@ namespace StgSharp.Collections return false; } - public void RemoveAll(Predicate predicate) + public void RemoveAll( + Predicate predicate + ) { + if (predicate is null) { + return; + } for (int i = _index; i > 0; i--) { if (predicate(_values[i])) @@ -148,26 +180,41 @@ namespace StgSharp.Collections } } - public void RemoveAt(int index) + public void RemoveAt( + int index + ) { if (index <= _index) { - _values[index] = _values[_index]; + Array.Copy(_values, index + 1, _values, index, _index - index); + if (RuntimeHelpers.IsReferenceOrContainsReferences()) { + _values[_index] = default!; + } _index--; } } - public void Resize(int newSize) + public void Resize( + int newSize + ) { Array.Resize(ref _values, newSize); } + internal Span ToSpan() + { + return _values; + } + IEnumerator IEnumerable.GetEnumerator() { yield return GetEnumerator(); } - void IList.Insert(int index, T item) + void IList.Insert( + int index, + T item + ) { throw new NotSupportedException(); } diff --git a/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs b/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs index 0d816001..86ab7806 100644 --- a/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs +++ b/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs @@ -36,6 +36,15 @@ using System.Text; namespace StgSharp.RegularAnalysis.Abstraction { + public interface ITrinitySyntaxNode : ISyntaxNode + where TNode : ISyntaxNode + where TLabel : unmanaged + { + + TNode Third { get; set; } + + } + public interface ISyntaxNode where TNode : ISyntaxNode where TLabel : unmanaged { diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs b/StgSharp.RegularAnalysis/Text/RegexAstNode.cs similarity index 100% rename from StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs rename to StgSharp.RegularAnalysis/Text/RegexAstNode.cs diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs b/StgSharp.RegularAnalysis/Text/RegexElementLabel.cs similarity index 100% rename from StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs rename to StgSharp.RegularAnalysis/Text/RegexElementLabel.cs diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexIR.cs b/StgSharp.RegularAnalysis/Text/RegexIR.cs similarity index 68% rename from StgSharp.RegularAnalysis/TextRegex/RegexIR.cs rename to StgSharp.RegularAnalysis/Text/RegexIR.cs index ecea4f6e..6a28274f 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexIR.cs +++ b/StgSharp.RegularAnalysis/Text/RegexIR.cs @@ -37,16 +37,57 @@ namespace StgSharp.RegularAnalysis.Text internal enum RegexIRCommand { - Search_Seq = 0, - Search_Set = 1, - SearchChar = 2, - ConfigFail = 3, - Condition = 4, - Try = 5, - Count = 6, - Count_Seq = 7, - Count_Set = 8, - Pop = 10 + Match_Seq = 1 << 0, + Match_Set = 1 << 1, + Condition = 1 << 2, + Try = 1 << 3, + Count_Complex = 1 << 4, + Count_Seq = 1 << 5, + Count_Set = 1 << 6, + Pop = 1 << 7, + Pack = 1 << 8, + Find_Seq = 1 << 9, + Find_Set = 1 << 10, + Find_Complex = 1 << 11, + + Match = Match_Seq | Match_Set, + Count = Count_Seq | Count_Set | Count_Complex, + + } + + internal interface IRegexCount + { + + bool IsGreedy { get; } + + bool IsStar { get; } + + bool IsPlus { get; } + + int Min { get; } + + int Max { get; } + + } + + internal sealed class RegexFindComplexIR( + bool nearest, + int count + ) : RegexIR + { + + internal override RegexIRCommand Command => RegexIRCommand.Find_Complex; + + private bool IsNearest { get; } = nearest; + + private int CommandCount { get; } = count; + + internal override string AsCommand( + int lineNum + ) + { + return $"[{lineNum}]\t FIND_COMPLEX {(IsNearest ? "NEAREST" : "FURTHEST")} COUNT: {CommandCount}"; + } } @@ -61,6 +102,27 @@ namespace StgSharp.RegularAnalysis.Text } + internal sealed class RegexFindSeqIR( + string pattern, + bool isFirst + ) : RegexIR + { + + public bool IsFirst { get; } = isFirst; + + public string Pattern { get; } = pattern; + + internal override RegexIRCommand Command => RegexIRCommand.Find_Seq; + + internal override string AsCommand( + int lineNum + ) + { + return $"[{lineNum}]\t FIND_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "" : Pattern)}\""; + } + + } + /// /// Searches for a certain string pattern or charset from current position. /// @@ -71,13 +133,13 @@ namespace StgSharp.RegularAnalysis.Text public string Pattern { get; } = pattern; - internal override RegexIRCommand Command => RegexIRCommand.Search_Seq; + internal override RegexIRCommand Command => RegexIRCommand.Match_Seq; internal override string AsCommand( int lineNum ) { - return $"[{lineNum}]\t SEARCH_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "" : Pattern)}\""; + return $"[{lineNum}]\t MATCH_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "" : Pattern)}\""; } } @@ -89,37 +151,13 @@ namespace StgSharp.RegularAnalysis.Text public string Charset { get; } = charset; - internal override RegexIRCommand Command => RegexIRCommand.Search_Set; + internal override RegexIRCommand Command => RegexIRCommand.Match_Set; internal override string AsCommand( int lineNum ) { - return $"[{lineNum}]\t SEARCH_SET \"{Charset}\""; - } - - } - - /// - /// Configures the failure jump of current position. The failMod ranges 0-2 - /// - /// - /// - /// - internal sealed class RegexConfigFailIR( - int failMod - ) : RegexIR - { - - public int FailMod { get; } = failMod & 0x3; - - internal override RegexIRCommand Command => RegexIRCommand.ConfigFail; - - internal override string AsCommand( - int lineNum - ) - { - return $"[{lineNum}]\t CONFIG_FAIL {FailMod}"; + return $"[{lineNum}]\t MATCH_SET \"{Charset}\""; } } @@ -145,29 +183,54 @@ namespace StgSharp.RegularAnalysis.Text } - internal sealed class RegexCountIR( - int min, - int max, - bool isGreedy, - int operation + internal sealed class RegexPackIR( + bool condition, + string name = "" ) : RegexIR { - public bool IsGreedy { get; } = isGreedy; + public bool Condition { get; } = condition; - public int Min { get; } = min; + public string GroupName { get; } = name; - public int Max { get; } = max; - - public int OperationIndex { get; } = operation; - - internal override RegexIRCommand Command { get; } = RegexIRCommand.Count; + internal override RegexIRCommand Command => RegexIRCommand.Pack; internal override string AsCommand( int lineNum ) { - return $"[{lineNum}]\t COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {OperationIndex}"; + return $"[{lineNum}]\t PACK {(Condition ? "SUCCESS" : "FAIL")} GROUP: \"{GroupName}\""; + } + + } + + internal sealed class RegexCountComplexIR( + int min, + int max, + bool isGreedy, + int operation + ) : RegexIR, IRegexCount + { + + public bool IsGreedy { get; } = isGreedy; + + public bool IsStar => Min == 0 && Max < 0; + + public bool IsPlus => Min == 1 && Max < 0; + + public int Min { get; } = min; + + public int Max { get; } = max; + + public int OperationCount { get; } = operation; + + internal override RegexIRCommand Command { get; } = RegexIRCommand.Count_Complex; + + internal override string AsCommand( + int lineNum + ) + { + return $"[{lineNum}]\t COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {OperationCount}"; } } @@ -177,11 +240,15 @@ namespace StgSharp.RegularAnalysis.Text int max, bool isGreedy, string seq - ) : RegexIR + ) : RegexIR, IRegexCount { public bool IsGreedy { get; } = isGreedy; + public bool IsStar => Min == 0 && Max < 0; + + public bool IsPlus => Min == 1 && Max < 0; + public int Min { get; } = min; public int Max { get; } = max; @@ -204,11 +271,17 @@ namespace StgSharp.RegularAnalysis.Text int max, bool isGreedy, string set - ) : RegexIR + ) : RegexIR, IRegexCount { public bool IsGreedy { get; } = isGreedy; + public bool IsDotStar => CharSet == "." && Min == 0 && Max < 0; + + public bool IsStar => Min == 0 && Max < 0; + + public bool IsPlus => Min == 1 && Max < 0; + public int Min { get; } = min; public int Max { get; } = max; @@ -251,7 +324,7 @@ namespace StgSharp.RegularAnalysis.Text public bool IsSuccess { get; } = isSuccess; - internal override RegexIRCommand Command => RegexIRCommand.Try; + internal override RegexIRCommand Command => RegexIRCommand.Pop; internal override string AsCommand( int lineNum diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexIRGenerator.cs b/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs similarity index 94% rename from StgSharp.RegularAnalysis/TextRegex/RegexIRGenerator.cs rename to StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs index 541a02f3..190bd505 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexIRGenerator.cs +++ b/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs @@ -56,22 +56,14 @@ namespace StgSharp.RegularAnalysis.Text Count++; } - public void EmitConfigFail( - int fail - ) - { - _tail.Add(new RegexConfigFailIR(fail)); - Count++; - } - - public void EmitCount( + public void EmitCountComplex( int min, int max, bool isGreedy, int operation ) { - _tail.Add(new RegexCountIR(min, max, isGreedy, operation)); + _tail.Add(new RegexCountComplexIR(min, max, isGreedy, operation)); Count++; } @@ -125,6 +117,15 @@ namespace StgSharp.RegularAnalysis.Text Count++; } + public void EmitPack( + bool condition, + string name = "" + ) + { + _tail.Add(new RegexPackIR(condition, name)); + Count++; + } + public void EmitPop( bool isSuccess ) diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs similarity index 99% rename from StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs rename to StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs index 36398174..038417b8 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.Optimize.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexInterpreter.Optimize" +// file="RegexInterpreter.ASTOptimize" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -39,7 +39,7 @@ using System.Xml.Linq; namespace StgSharp.RegularAnalysis.Text { - public partial class RegexInterpreter + public partial class RegexAnalyzer { private static RegexAstNode FlattenAlt( diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.IREmit.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IREmit.cs similarity index 86% rename from StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.IREmit.cs rename to StgSharp.RegularAnalysis/Text/RegexInterpreter.IREmit.cs index e3e8b1f6..af8a9397 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.IREmit.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IREmit.cs @@ -40,7 +40,7 @@ using System.Xml; namespace StgSharp.RegularAnalysis.Text { - public partial class RegexInterpreter + public partial class RegexAnalyzer { internal static List GenerateIR(AbstractSyntaxTree tree) @@ -55,30 +55,34 @@ namespace StgSharp.RegularAnalysis.Text int min, int max, bool isGreedy, +<<<<<<<< HEAD:StgSharp.RegularAnalysis/Text/RegexInterpreter.IREmit.cs RegexAstNode right) +======== + RegexAstNode body + ) +>>>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs { - bool isRightOperator = !RegexAstNode.IsNullOrEmpty(right) && - (right.Source.Flag & RegexElementLabel.OPERATOR) != 0; + bool is_operator = !RegexAstNode.IsNullOrEmpty(body) && + (body.Source.Flag & RegexElementLabel.OPERATOR) != 0; - if (isRightOperator) // seq to be count is complex + if (is_operator) // seq to be count is complex { - ir.EmitCount(min, max, isGreedy, 1); - RegexIRGenerator rightIR = GenerateIRRecursion(right, curDepth + 1); - ir.EmitTry(rightIR.Count); - ir.EmitIRStream(rightIR); + RegexIRGenerator body_ir = GenerateIRRecursion(body, curDepth + 1); + ir.EmitCountComplex(min, max, isGreedy, body_ir.Count); + ir.EmitIRStream(body_ir); } else { - RegexElementLabel count_source_mask = right.Source.Flag; + RegexElementLabel count_source_mask = body.Source.Flag; switch (count_source_mask) { case RegexElementLabel.UNIT: - ir.EmitCountSeq(min, max, isGreedy, right.Source.Value); + ir.EmitCountSeq(min, max, isGreedy, body.Source.Value); break; case RegexElementLabel.UNIT_SET: - ir.EmitCountSet(min, max, isGreedy, right.Source.Value); + ir.EmitCountSet(min, max, isGreedy, body.Source.Value); break; case RegexElementLabel.UNIT_SPAN: - ir.EmitCountSeq(min, max, isGreedy, right.Source.Value); + ir.EmitCountSeq(min, max, isGreedy, body.Source.Value); break; default: throw new InvalidOperationException($"Unexpected regex element type for count operator: {count_source_mask}"); @@ -110,7 +114,7 @@ namespace StgSharp.RegularAnalysis.Text bool is_group = group_source.Length == 1; // simple group: (pattern) string group_name = group_source.StartsWith(@"(?<", StringComparison.Ordinal) ? - group_source[3..^2] : /* named group (?) */ + group_source[3..^1] : /* named group (?) */ string.Empty; /* unnamed group (?:) */ RegexIRGenerator groupIR = new(); @@ -118,7 +122,8 @@ namespace StgSharp.RegularAnalysis.Text groupIR.EmitTry(rightIR.Count); groupIR.EmitIRStream(rightIR); groupIR.EmitCondition(2, 1); - /*generate pack ir here*/ + groupIR.EmitPop(false); + groupIR.EmitPack(true, group_name); return groupIR; case RegexElementLabel.COUNT: string count_source = token.Value; @@ -137,25 +142,25 @@ namespace StgSharp.RegularAnalysis.Text _ => throw new InvalidOperationException("Unexpected sequence mask for count operator"), }; ReadOnlySpan count_source_span = count_source; - RegexIRGenerator countIR = new(); + RegexIRGenerator count_ir = new(); if (count_source_span[0] == '*') // infinite count { - EmitCount(curDepth, countIR, 0, int.MaxValue, isGreedy, seq); + EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq); } else if (count_source_span[0] == '+') // at least one { - EmitCount(curDepth, countIR, 1, int.MaxValue, isGreedy, seq); + EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq); } else if (count_source_span[0] == '?') // zero or one { - EmitCount(curDepth, countIR, 0, 1, isGreedy, seq); + EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq); } else if (count_source_span[0] == '{') // zero or one, non-greedy { int comma_pos = count_source_span.IndexOf(','); if (comma_pos == -1) { - if (int.TryParse(count_source[1..(isGreedy ? ^3 : ^2)], out int count)) + if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count)) { - EmitCount(curDepth, countIR, count, count, isGreedy, seq); + EmitCount(curDepth, count_ir, count, count, isGreedy, seq); } else { throw new InvalidCastException($"Invalid count format: {count_source}"); @@ -163,9 +168,9 @@ namespace StgSharp.RegularAnalysis.Text } else { if (int.TryParse(count_source[1..comma_pos], out int min) && - int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^3 : ^2)], out int max)) + int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max)) { - EmitCount(curDepth, countIR, min, max, isGreedy, seq); + EmitCount(curDepth, count_ir, min, max, isGreedy, seq); } else { throw new InvalidCastException($"Invalid count format: {count_source}"); @@ -176,7 +181,7 @@ namespace StgSharp.RegularAnalysis.Text throw new InvalidOperationException(); } // unknown count format - return countIR; + return count_ir; case RegexElementLabel.CONCAT: RegexIRGenerator concat = new(); diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs new file mode 100644 index 00000000..af8a9397 --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs @@ -0,0 +1,280 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="RegexInterpreter.IREmit" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using StgSharp.RegularAnalysis.Abstraction; +using System; +using System.Collections.Generic; +using System.ComponentModel.Design; +using System.Data; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Reflection.Emit; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; +using System.Xml; + +namespace StgSharp.RegularAnalysis.Text +{ + public partial class RegexAnalyzer + { + + internal static List GenerateIR(AbstractSyntaxTree tree) + { + return GenerateIRRecursion(tree.Root, 0).ToList(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void EmitCount( + int curDepth, + RegexIRGenerator ir, + int min, + int max, + bool isGreedy, +<<<<<<<< HEAD:StgSharp.RegularAnalysis/Text/RegexInterpreter.IREmit.cs + RegexAstNode right) +======== + RegexAstNode body + ) +>>>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs + { + bool is_operator = !RegexAstNode.IsNullOrEmpty(body) && + (body.Source.Flag & RegexElementLabel.OPERATOR) != 0; + + if (is_operator) // seq to be count is complex + { + RegexIRGenerator body_ir = GenerateIRRecursion(body, curDepth + 1); + ir.EmitCountComplex(min, max, isGreedy, body_ir.Count); + ir.EmitIRStream(body_ir); + } else + { + RegexElementLabel count_source_mask = body.Source.Flag; + switch (count_source_mask) + { + case RegexElementLabel.UNIT: + ir.EmitCountSeq(min, max, isGreedy, body.Source.Value); + break; + case RegexElementLabel.UNIT_SET: + ir.EmitCountSet(min, max, isGreedy, body.Source.Value); + break; + case RegexElementLabel.UNIT_SPAN: + ir.EmitCountSeq(min, max, isGreedy, body.Source.Value); + break; + default: + throw new InvalidOperationException($"Unexpected regex element type for count operator: {count_source_mask}"); + } + } + } + + private static RegexIRGenerator GenerateIRRecursion(RegexAstNode node, int depth) + { +#pragma warning disable IDE0010 + if (depth > 1024) + { + throw new OverflowException("Regex parse exceeds maximum recursion depth"); + } + int curDepth = depth; + RegexAstNode left = node.Left; + RegexAstNode right = node.Right; + RegexIRGenerator leftIR; + RegexIRGenerator rightIR; + + Token token = node.Source; + + switch (node.Source.Flag) + { + case RegexElementLabel.GROUP_BEGIN: + string group_source = token.Value; + + // process group type + bool is_group = group_source.Length == 1; // simple group: (pattern) + string group_name = + group_source.StartsWith(@"(?<", StringComparison.Ordinal) ? + group_source[3..^1] : /* named group (?) */ + string.Empty; /* unnamed group (?:) */ + + RegexIRGenerator groupIR = new(); + rightIR = GenerateIRRecursion(right, curDepth + 1); + groupIR.EmitTry(rightIR.Count); + groupIR.EmitIRStream(rightIR); + groupIR.EmitCondition(2, 1); + groupIR.EmitPop(false); + groupIR.EmitPack(true, group_name); + return groupIR; + case RegexElementLabel.COUNT: + string count_source = token.Value; + bool isGreedy = !(count_source.Length > 1 && count_source.EndsWith('?')); + int seq_mask = (RegexAstNode.IsNullOrEmpty(left) ? 0 : 2) + (RegexAstNode.IsNullOrEmpty(right) ? + 0 : + 1); + RegexAstNode seq = seq_mask switch + { + // no seq, invalid count operator + 0 => throw new InvalidOperationException("Count operator must have a sequence to operate on"), + 1 => right,// right seq + 2 => left,// left seq + // both sides, invalid count operator + 3 => throw new InvalidOperationException("Count operator cannot have two sequences to operate on"), + _ => throw new InvalidOperationException("Unexpected sequence mask for count operator"), + }; + ReadOnlySpan count_source_span = count_source; + RegexIRGenerator count_ir = new(); + + if (count_source_span[0] == '*') // infinite count + { + EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq); + } else if (count_source_span[0] == '+') // at least one + { + EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq); + } else if (count_source_span[0] == '?') // zero or one + { + EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq); + } else if (count_source_span[0] == '{') // zero or one, non-greedy + { + int comma_pos = count_source_span.IndexOf(','); + if (comma_pos == -1) + { + if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count)) + { + EmitCount(curDepth, count_ir, count, count, isGreedy, seq); + } else + { + throw new InvalidCastException($"Invalid count format: {count_source}"); + } + } else + { + if (int.TryParse(count_source[1..comma_pos], out int min) && + int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max)) + { + EmitCount(curDepth, count_ir, min, max, isGreedy, seq); + } else + { + throw new InvalidCastException($"Invalid count format: {count_source}"); + } + } + } else + { + throw new InvalidOperationException(); + } // unknown count format + + return count_ir; + case RegexElementLabel.CONCAT: + RegexIRGenerator concat = new(); + + // process left node + if (RegexAstNode.IsNullOrEmpty(left)) + { + // do nothing + } else if ((left.Source.Flag & RegexElementLabel.SEQUENCE) != 0) + { + switch (left.Source.Flag) + { + case RegexElementLabel.UNIT_SPAN: + goto case RegexElementLabel.UNIT; + case RegexElementLabel.UNIT: + concat.EmitMatchSeq(left.Source.Value); + break; + case RegexElementLabel.UNIT_SET: + concat.EmitMatchSet(left.Source.Value); + break; + default: + break; + } + } else if ((left.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0) + { + leftIR = GenerateIRRecursion(left, curDepth + 1); + concat.EmitIRStream(leftIR); + } + + // process right node + if (RegexAstNode.IsNullOrEmpty(right)) + { + // do nothing + } else if ((right.Source.Flag & RegexElementLabel.SEQUENCE) != 0) + { + switch (right.Source.Flag) + { + case RegexElementLabel.UNIT_SPAN: + goto case RegexElementLabel.UNIT; + case RegexElementLabel.UNIT: + concat.EmitMatchSeq(right.Source.Value); + break; + case RegexElementLabel.UNIT_SET: + concat.EmitMatchSet(right.Source.Value); + break; + default: + break; + } + } else if ((right.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0) + { + rightIR = GenerateIRRecursion(right, curDepth + 1); + concat.EmitIRStream(rightIR); + } + return concat; + case RegexElementLabel.ALT: + RegexIRGenerator alt = new(); + List casesIRList = []; + RegexAstNode cur_node = right; + int ir_count = 0; + while (!RegexAstNode.IsNullOrEmpty(cur_node)) + { + RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1); + casesIRList.Add(_case); + ir_count += _case.Count; + cur_node = cur_node.Next; + } + int case_count = casesIRList.Count; + while (case_count != 0) + { + RegexIRGenerator cur_ir = casesIRList[^case_count]; + ir_count -= cur_ir.Count; + case_count--; + alt.EmitTry(cur_ir.Count); + alt.EmitIRStream(cur_ir); + alt.EmitCondition(1/* need to calc here */, 2); + alt.EmitPop(true); + } + alt.EmitPop(false); + return alt; + case RegexElementLabel.UNIT_SPAN: + goto case RegexElementLabel.UNIT; + case RegexElementLabel.UNIT: + RegexIRGenerator unit_seq = new(); + unit_seq.EmitMatchSeq(node.Source.Value); + return unit_seq; + case RegexElementLabel.UNIT_SET: + RegexIRGenerator unit_set = new(); + unit_set.EmitMatchSet(node.Source.Value); + return unit_set; + default: + return null!; + } +#pragma warning restore IDE0010 + } + + } +} diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs new file mode 100644 index 00000000..2e517d2d --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs @@ -0,0 +1,214 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="RegexInterpreter.IROptimize" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using StgSharp.Collections; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + public partial class RegexAnalyzer + { + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + + internal static RegexInfo ScanAndOptimizeIR( + List baseIR, + out List optIR + ) + { + int index = 0; + Span origin_span = CollectionsMarshal.AsSpan(baseIR); + List opt_ir = new(baseIR.Count); + CapacityFixedList opt_window = new(8); + Span opt_win_sp = [.. opt_window]; + RegexInfo info = new(); + do + { + int process_count = 0; + int last = int.Min(origin_span.Length, index + 8); + origin_span[index..last].CopyTo(opt_win_sp); + + do + { + // plus_expand: + int expand_result = TryExpandPlus(opt_win_sp, opt_ir); + if (expand_result == 0) + { + opt_win_sp[0] = opt_ir[^1]; + process_count++; + + // goto dot_count_count: + } else if (expand_result > 0) + { + // meets a count-complex + RegexCountComplexIR cnt_cmp_ir = Unsafe.As(opt_win_sp[0]); + if (TryExpandComplexPlus(cnt_cmp_ir.IsGreedy, origin_span[(index + 1)..(index + expand_result)], opt_ir)) + { + process_count += expand_result + 1; + break; + } + } + + // dot_count_count: + if (TryOptDotCountCount(opt_win_sp, opt_ir)) + { + process_count += expand_result == 0 ? 1 : 2; + break; + } + + if (TryFlattenCount(opt_win_sp, opt_ir)) { } + } while (false); + + index += process_count; + } while (index < baseIR.Count); + optIR = opt_ir; + return info; + } + + private static bool TryExpandComplexPlus( + bool is_greedy, + Span complex, + List opt + ) + { + int count = complex.Length; + opt.AddRange(complex); + opt.Add(new RegexCountComplexIR(0, -1, is_greedy, count)); + return true; + } + + #region expand plus + + private static int TryExpandPlus( + Span window, + List opt + ) + { + /* a+ + * convert to + * aa* + */ + + RegexIR ir = window[0]; + if (ir is not IRegexCount { IsPlus: true } count) { + return -1; + } + bool is_greedy = count.IsGreedy; + string pattern; + switch (ir.Command & RegexIRCommand.Count) + { + case RegexIRCommand.Count_Complex: + RegexCountComplexIR complex = Unsafe.As(ir); + return complex.OperationCount; + case RegexIRCommand.Count_Seq: + RegexCountSeqIR seq = Unsafe.As(ir); + pattern = seq.Sequence; + opt.Add(new RegexMatchSeqIR(pattern)); + opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern)); + return 0; + case RegexIRCommand.Count_Set: + RegexCountSeqIR set = Unsafe.As(ir); + pattern = set.Sequence; + opt.Add(new RegexMatchSeqIR(pattern)); + opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern)); + return 0; + default: + return -1; + } + } + + #endregion + + private static bool TryFlattenCount( + Span window, + List opt + ) + { + RegexIR ir = window[0]; + if (ir is not RegexCountSeqIR seq) { + return false; + } + return true; + } + + #region dot count count + + private static int TryOptDotCountCount( + Span window, + List opt + ) + { + /* .* * + * convert to + * .* + */ + RegexIR cur_ir = window[0]; + if (cur_ir is not RegexCountSetIR { IsDotStar: true } dot) { + return -1; + } + if (window[1] is not IRegexCount { IsStar: true } after) { + return -1; + } + if (dot.IsGreedy) + { + opt.Add(window[0]); + return 0; + } else + { + /* + * .* ?a* is different + * it is something like + * FIND(a)?a* :.* + */ + RegexIRGenerator g = new(); + g.EmitTry(1); + } + return -1; + } + + #endregion + + internal class RegexInfo + { + + public int MinLength { get; set; } + + public int TryDepth { get; set; } + + public int GroupCount { get; set; } + + } + + } +} diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.SourceEmit.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceEmit.cs similarity index 100% rename from StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.SourceEmit.cs rename to StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceEmit.cs diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs new file mode 100644 index 00000000..1abf47d5 --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs @@ -0,0 +1,52 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="RegexInterpreter.SourceGenerate" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + public partial class RegexAnalyzer + { + + internal static string GetSourceCode( + IEnumerable irs + ) + { + StringBuilder sb = new StringBuilder(); + foreach (RegexIR ir in irs) + { + // ScanAndMarkIROptimization(ir); + } + return sb.ToString(); + } + + } +} diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.cs similarity index 96% rename from StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs rename to StgSharp.RegularAnalysis/Text/RegexInterpreter.cs index 21f2119f..141df63b 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs +++ b/StgSharp.RegularAnalysis/Text/RegexInterpreter.cs @@ -37,7 +37,7 @@ using System.Threading.Tasks; namespace StgSharp.RegularAnalysis.Text { - public partial class RegexInterpreter + public partial class RegexAnalyzer { private AbstractSyntaxTree _tree = new(); @@ -45,7 +45,7 @@ namespace StgSharp.RegularAnalysis.Text private Exception _interpreteException; private string _source; - private RegexInterpreter() { } + private RegexAnalyzer() { } internal AbstractSyntaxTree Tree { @@ -58,9 +58,15 @@ namespace StgSharp.RegularAnalysis.Text } } +<<<<<<< HEAD:StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs public static RegexInterpreter Analyze(string regex) +======= + public static RegexAnalyzer Analyze( + string regex + ) +>>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/RegexInterpreter.cs { - RegexInterpreter interpreter = new(); + RegexAnalyzer interpreter = new(); interpreter._source = regex; bool isSuccess; try diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs b/StgSharp.RegularAnalysis/Text/RegexTokenParser.cs similarity index 100% rename from StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs rename to StgSharp.RegularAnalysis/Text/RegexTokenParser.cs diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs b/StgSharp.RegularAnalysis/Text/RegexTokenReader.cs similarity index 100% rename from StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs rename to StgSharp.RegularAnalysis/Text/RegexTokenReader.cs diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexTokenUtil.cs b/StgSharp.RegularAnalysis/Text/RegexTokenUtil.cs similarity index 100% rename from StgSharp.RegularAnalysis/TextRegex/RegexTokenUtil.cs rename to StgSharp.RegularAnalysis/Text/RegexTokenUtil.cs diff --git a/StgSharp.RegularAnalysis/Text/TextHelper.cs b/StgSharp.RegularAnalysis/Text/TextHelper.cs new file mode 100644 index 00000000..78d9b4d6 --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/TextHelper.cs @@ -0,0 +1,50 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="TextHelper" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + public static class TextHelper + { + + public static bool TryLastIndexOf( + this ReadOnlySpan span, + ReadOnlySpan value, + out int index + ) + { + index = span.LastIndexOf(value); + return index != -1; + } + + } +} diff --git a/StgSharp.RegularAnalysis/TextRegex/TextRegex.cs b/StgSharp.RegularAnalysis/Text/TextRegex.cs similarity index 78% rename from StgSharp.RegularAnalysis/TextRegex/TextRegex.cs rename to StgSharp.RegularAnalysis/Text/TextRegex.cs index deafa82e..04194dd4 100644 --- a/StgSharp.RegularAnalysis/TextRegex/TextRegex.cs +++ b/StgSharp.RegularAnalysis/Text/TextRegex.cs @@ -28,6 +28,10 @@ using System; using System.Collections.Generic; using System.Linq; +<<<<<<< HEAD:StgSharp.RegularAnalysis/TextRegex/TextRegex.cs +======= +using System.Runtime.CompilerServices; +>>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/TextRegex.cs using System.Text; using System.Threading.Tasks; @@ -35,6 +39,7 @@ namespace StgSharp.RegularAnalysis.Text { public abstract class TextRegex { +<<<<<<< HEAD:StgSharp.RegularAnalysis/TextRegex/TextRegex.cs public ref struct Match { private ReadOnlySpan _sequence; @@ -42,4 +47,14 @@ namespace StgSharp.RegularAnalysis.Text } } +======= + + public abstract bool IsMatch( + ReadOnlySpan text + ); + + } + + public ref struct MatchResult { } +>>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/TextRegex.cs } diff --git a/StgSharpDebug/TestRegexAnalyse.cs b/StgSharpDebug/TestRegexAnalyse.cs index 0215c18c..907efc05 100644 --- a/StgSharpDebug/TestRegexAnalyse.cs +++ b/StgSharpDebug/TestRegexAnalyse.cs @@ -44,13 +44,14 @@ namespace StgSharp @"abc[0-9]x|abc[0-9]y|abd[0-9]x|abd[0-9]y|xyz", @"get_name|get_value|get_type|set_name|set_value|set_type", @"helix|hello|help|hero|hers|her", - @"a*.[abc]*(abcd|a(bc)c)+(?abc)"]; + @"a*.[abc]*(abcd|a(bc)c)+(?abc{11,13})" + ]; public static void Run() { foreach (string regex in regexSource) { - RegexInterpreter interpreter = RegexInterpreter.Analyze(regex); + RegexAnalyzer interpreter = RegexAnalyzer.Analyze(regex); AbstractSyntaxTree tree = interpreter.Tree; Console.WriteLine($"result of \"{regex}\""); TreeEnumerator enumerator = new(tree); @@ -60,7 +61,7 @@ namespace StgSharp Console.Write(new string('|', enumerator.Depth)); Console.WriteLine($"\t{node.Source.Value}\t{node.Source.Flag}"); } - List ir = RegexInterpreter.GenerateIR(tree); + List ir = RegexAnalyzer.GenerateIR(tree); Console.WriteLine("-----IR-----"); for (int i = 0; i < ir.Count; i++) { Console.WriteLine(ir[i].AsCommand(i)); @@ -71,7 +72,7 @@ namespace StgSharp } } - [GeneratedRegex(@"a*.[abc]*(abcd|a(bc)c)+(?abc)", RegexOptions.ExplicitCapture)] + [GeneratedRegex(@".*a*b", RegexOptions.ExplicitCapture)] internal static partial Regex GetRegex(); } diff --git a/future/ngra-regex-ir-review.md b/future/ngra-regex-ir-review.md new file mode 100644 index 00000000..84c2e96a --- /dev/null +++ b/future/ngra-regex-ir-review.md @@ -0,0 +1,1050 @@ +# NGRA 字符串正则 IR 设计补充建议 + +基于当前 `ngra-regex-ir.md` 的设计,我认为整体方向是成立的:它已经不是“完全低级的执行字节码”,而是保留了 `COUNT`、`COUNT_SEQ`、`COUNT_SET`、`TRY`、`PACK` 等高层结构,因此可以继续承担后续 match-opt 的输入。 + +不过,为了让规则更容易扩展、避免语义误判,我建议把优化明确拆成三类,并把当前 IR 分成“语义 IR / HIR”和“计划 IR / LIR”。 + +--- + +## 1. 总体判断 + +当前 IR 最大的优点: + +1. **足够紧凑**:`COUNT_SEQ`、`COUNT_SET`、`TRY` 等复合指令可以避免过早打碎成 `LoadChar + Compare + Branch`。 +2. **保留 regex 结构**:量词、字符集、确定序列、组打包等信息仍然可见。 +3. **适合继续做 match planning**:后续可以从这些指令中识别 `.*literal`、`[set]*`、bounded repeat、固定长度 block 等结构。 + +当前 IR 最大的风险: + +1. `SEARCH` 这个名字容易误导。当前 HIR 中的 `SEARCH` 实际应当是 **MATCH_AT_CURRENT**,不是允许跳过字符的 `FIND`。 +2. 检查点目前只描述为锚点指针栈,但真正的 backtracking frame 至少还要保存 `pos`、capture watermark、symbol watermark、repeat state 和 continuation。 +3. `COUNT_*` 的语义必须明确是否保留 retry/backtracking,否则 `a*.`、`[abc]*abc` 等场景会被错误优化。 +4. 字符集的语义表示和物理表示要分开:语义上 `[0-9a-z]` 是一个 set,物理上可以用 inline set-region 的多条 range-test 表示,避免每个 set 挂 `List`。 + +--- + +## 2. 建议的分层 + +我建议仍然区分 HIR 和 LIR,不是因为机器必须分,而是因为人写规则时边界更清楚。 + +```text +AST + ↓ +AST Opt + ↓ +HIR Gen + ↓ +HIR Analysis + ↓ +HIR → LIR / MatchPlan Lowering + ↓ +LIR + ↓ +SourceGen / RuntimeGen / Interpreter +``` + +### HIR:表达 regex 语义 + +HIR 保留接近正则结构的指令: + +```text +SEARCH_SEQ // 语义上是当前位置 MATCH_SEQ +SEARCH_SET // 语义上是当前位置 MATCH_SET +SEARCH_DOT // 或 UNIT_DOT,不应和 literal "." 混淆 +COUNT +COUNT_SEQ +COUNT_SET +TRY +POP +PACK +``` + +注意:HIR 中可以继续叫 `SEARCH`,但文档里要明确: + +```text +HIR 的 SEARCH_* 不表示向后查找; +它表示从当前活动指针开始匹配。 +真正允许跳过字符的搜索操作只在 LIR / MatchPlan 阶段引入,命名为 FIND_*。 +``` + +### LIR:表达匹配计划 + +LIR 不只是更低级,而是“已经做过策略规划”的 IR: + +```text +MATCH_SEQ +MATCH_CHAR +MATCH_DOT +MATCH_SET + +FIND_SEQ +FIND_CHAR +FIND_SET +FIND_LAST_SEQ +FIND_LAST_SET + +CONSUME_WHILE_SET +CONSUME_UNTIL_CHAR +CONSUME_UNTIL_SET + +COUNT_SET +COUNT_SEQ +COUNT_BLOCK +CHECK_MIN_LENGTH +CHECK_FIXED_OFFSET + +PUSH_BACKTRACK +POP_BACKTRACK +COMMIT +FAIL + +CAPTURE_BEGIN +CAPTURE_END +PACK +``` + +LIR 中应该已经包含: + +```text +LoweredSetId +SearchMode +RepeatMode +RetryMode +CaptureMode +EmitHint +``` + +--- + +## 3. 三类优化策略 + +建议把所有优化分成三类: + +```text +1. Regex algebra rewrite + 改正则结构。 + +2. Match plan rewrite + 改匹配策略。 + +3. Backend / C# span trick + 改 C# 代码写法。 +``` + +这个分类很重要,因为它能直接决定优化应该放在哪个阶段。 + +--- + +# 4. Regex algebra rewrite + +这类优化应该放在 AST Opt 或 HIR 生成前后。它们改变的是正则结构,而不是 C# 写法。 + +## 4.1 Literal 合并 + +```regex +a b c +``` + +变成: + +```regex +abc +``` + +HIR: + +```text +SEARCH_SEQ "abc" +``` + +不要生成三条 `SEARCH_CHAR`。 + +--- + +## 4.2 Concat / Alternation flatten + +```regex +a(?:b(?:c)) +``` + +变成: + +```regex +abc +``` + +```regex +a|(?:b|c) +``` + +变成: + +```regex +a|b|c +``` + +--- + +## 4.3 Alternation factoring / trie + +这个你已经放在 AST Opt 做,位置是对的。 + +```regex +foobar|foobaz|foozip +``` + +变成: + +```regex +foo(?:bar|baz|zip) +``` + +或者 trie AST: + +```text +SEQ "foo" +ALT + SEQ "bar" + SEQ "baz" + SEQ "zip" +``` + +这类结构改写越早越好。后续 IR opt 只需要识别这个结构并决定是否 `switch` / first-set dispatch。 + +--- + +## 4.4 Bounded repeat 展开 + +```regex +a{2,3} +``` + +可以变成: + +```regex +aa a? +``` + +或: + +```text +SEARCH_SEQ "aa" +OPTIONAL SEARCH_CHAR 'a' +``` + +对于 lazy: + +```regex +a{2,3}?b +``` + +要注意尝试顺序。它等价于: + +```regex +aa(?:b|ab) +``` + +而不是: + +```regex +aa(?:a|ab) +``` + +Greedy 版本: + +```regex +a{2,3}b +``` + +如果展开成显式分支,尝试顺序更像: + +```regex +aa(?:ab|b) +``` + +语言集合相同,但 backtracking 尝试顺序不同。 + +--- + +## 4.5 `a+b` 不建议改成 `a*(ab|b)` + +当前文档中写: + +```text +a+b 可以优化为 a*(ab|b) +``` + +这个需要修正。 + +`a+b` 的语言是: + +```text +a^n b, n >= 1 +``` + +而 `a*(ab|b)` 会额外匹配: + +```text +b +``` + +因为第二个分支 `a*b` 可以在 `a*` 取 0 时匹配 `b`。 + +更安全的结构是: + +```regex +aa*b +``` + +或者: + +```text +SEARCH_CHAR 'a' +COUNT_SEQ 'a' 0..∞ +SEARCH_CHAR 'b' +``` + +如果要利用后继 `b` 做策略优化,应在 MatchPlan 阶段处理,而不是做这个不等价的 algebra rewrite。 + +--- + +## 4.6 Charset 合并 + +```regex +[abc]|[def] +``` + +可以变成: + +```regex +[abcdef] +``` + +```regex +[a-f]|[d-z] +``` + +可以变成: + +```regex +[a-z] +``` + +这属于 AST/HIR 层的字符类代数。 + +--- + +## 4.7 Charset 展开为 alternation 要谨慎 + +文档中写: + +```text +abc[de] 可以展开为 (abcd|abce) +``` + +这个是可行的,但不一定总是好。建议规则改成: + +```text +当 charset 很小,并且展开后能形成更长 literal / trie / branch dispatch 时,可以展开; +否则保留为 MATCH_SET。 +``` + +例如: + +```regex +abc[de] +``` + +既可以是: + +```text +SEARCH_SEQ "abc" +SEARCH_SET [de] +``` + +也可以是: + +```text +ALT + SEARCH_SEQ "abcd" + SEARCH_SEQ "abce" +``` + +前者代码更短,后者可能更适合 required literal 或 trie。需要 code size budget。 + +--- + +# 5. Match plan rewrite + +这类优化应该放在 HIR Analysis / HIR → LIR lowering。 + +它们不一定改成另一条正则,而是把匹配过程改成更高效的搜索/消费计划。 + +--- + +## 5.1 长度预测应该在 HIR Analysis 做,SourceGen 只负责发代码 + +当前文档写“长度预测在 source-gen 阶段做”。我建议改成: + +```text +minLength / maxLength 的计算在 HIR Analysis 做; +SourceGen 只负责把 CHECK_MIN_LENGTH 发成 C#。 +``` + +例如: + +```regex +abc[0-9]{2}xyz +``` + +分析结果: + +```text +MinLength = 8 +``` + +LIR: + +```text +CHECK_MIN_LENGTH 8 +MATCH_SEQ "abc" +... +``` + +SourceGen: + +```csharp +if ((uint)(end - pos) < 8) goto Fail; +``` + +--- + +## 5.2 `.*` 优化 + +HIR: + +```text +COUNT_DOT min=0 max=∞ greedy=true +MATCH_CHAR 'a' +``` + +可以 lower 成: + +```text +FIND_LAST_CHAR 'a' +``` + +lazy: + +```text +COUNT_DOT min=0 max=∞ greedy=false +MATCH_CHAR 'a' +``` + +lower 成: + +```text +FIND_CHAR 'a' +``` + +但要注意: + +1. 默认 `.` 不匹配 `\n`,搜索范围应限制在当前行。 +2. Singleline 模式下才可以跨整个 span。 +3. 如果 `.*` 内部或后继结构涉及 capture/backtracking,不能简单“删掉 COUNT”,而应转换为带 retry 的 find plan。 +4. 后继如果是 `MATCH_SEQ "abc"`,则变成 `FIND_SEQ` / `FIND_LAST_SEQ`。 +5. 后继如果是 `MATCH_SET`,则变成 `FIND_SET` / `FIND_LAST_SET`。 + +--- + +## 5.3 `[set]*` 优化 + +文档中写 `[set]*` 优化在 source-gen 阶段做。建议改成: + +```text +是否将 COUNT_SET 变成 CONSUME_WHILE / CONSUME_UNTIL,在 lowering 做; +具体使用 IndexOfAnyExcept / SearchValues / manual loop,在 SourceGen 做。 +``` + +例如: + +```regex +[a-f0-9]* +``` + +LIR: + +```text +CONSUME_WHILE_SET LowerHex +``` + +SourceGen: + +```csharp +int n = span.IndexOfAnyExcept(s_lowerHex); +``` + +或者手写 loop / range compare。 + +--- + +## 5.4 First-set 搜索 + +如果非锚定匹配: + +```regex +abc|defg|xyz +``` + +first-set 是: + +```text +'a', 'd', 'x' +``` + +LIR 可以是: + +```text +FIND_SET { 'a', 'd', 'x' } +VERIFY_BRANCH +``` + +你文档里说“这个不好做,因为 IR 直接把 ALT 打碎成 if-else 串了”。这说明 AST Opt / HIR 里最好保留 ALT region 或者至少保留 branch summary。 + +即使 AST 已经 trie 化,IR 也应该尽量不要把它完全打碎成不可恢复的 if-else 串。至少给每个 region 计算 summary: + +```text +FirstSet +MinLength +RequiredLiteral +CanBacktrack +CanCapture +``` + +--- + +## 5.5 Required literal 搜索 + +例如: + +```regex +.*ERROR:[0-9]+ +``` + +可以直接规划: + +```text +FIND_SEQ "ERROR:" +VERIFY_AFTER_ANCHOR +``` + +这通常比只做 first-set 更强。 + +--- + +## 5.6 Fixed-offset check + +```regex +[a-z]{20}XYZ +``` + +可以 lower 成: + +```text +CHECK_MIN_LENGTH 23 +CHECK_AT_OFFSET 20 MATCH_SEQ "XYZ" +MATCH_SET_REPEAT [a-z] count=20 +MATCH_SEQ "XYZ" +``` + +这个属于 match plan,不属于 sourcegen。 + +--- + +## 5.7 Auto-atomic / possessive 化 + +```regex +[0-9]*[A-Z] +``` + +由于: + +```text +[0-9] ∩ [A-Z] = ∅ +``` + +所以 `[0-9]*` 不需要回退,可以 lower 成: + +```text +COUNT_SET [0-9] greedy atomic +MATCH_SET [A-Z] +``` + +如果后继 first-set 和 repeat set 有交集,就不能这么做。 + +--- + +## 5.8 Repeat retry mode + +LIR 应该明确 repeat 的 retry 策略: + +```text +RetryMode.None +RetryMode.BackoffBy1 +RetryMode.BackoffByStride +RetryMode.SavedPositions +RetryMode.GeneralBacktrack +``` + +例如: + +```regex +(abc[cd])+ +``` + +每轮固定长度 4,所以可以: + +```text +RetryMode.BackoffByStride stride=4 +``` + +不需要保存每个内部指令的回溯点。 + +--- + +# 6. Backend / C# span trick + +这类优化只放 SourceGen。它不再改变 regex 或 match plan。 + +--- + +## 6.1 `StartsWith` + +LIR: + +```text +MATCH_SEQ "sample" +``` + +SourceGen 决定: + +```csharp +slice.StartsWith("sample", StringComparison.Ordinal) +``` + +还是短串展开: + +```csharp +slice.Length >= 3 && +slice[0] == 'a' && +slice[1] == 'b' && +slice[2] == 'c' +``` + +建议规则: + +```text +长度 1: + char compare + +长度 2~3: + 直接 char compare + +中长 literal: + StartsWith + +IgnoreCase: + StartsWith(..., OrdinalIgnoreCase) +``` + +--- + +## 6.2 `[0-9]` 不建议写成 `IndexOf("0123456789")` 用于当前位置测试 + +文档中写: + +```text +[0-9] 展开为 IndexOf("0123456789") +``` + +这个要分两种场景: + +### 当前位置匹配 + +如果是: + +```text +MATCH_SET [0-9] +``` + +更好是: + +```csharp +(uint)(ch - '0') <= 9 +``` + +### span 中搜索 + +如果是: + +```text +FIND_SET [0-9] +``` + +可以考虑: + +```csharp +span.IndexOfAnyInRange('0', '9') +``` + +或: + +```csharp +span.IndexOfAny(s_digits) +``` + +所以规则应写成: + +```text +MATCH_SET [0-9] → unsigned range compare +FIND_SET [0-9] → IndexOfAnyInRange / SearchValues +``` + +--- + +## 6.3 SearchValues + +对于多字符集合: + +```text +FIND_SET [0-9a-f] +``` + +SourceGen 可生成: + +```csharp +private static readonly SearchValues s_lowerHex = + SearchValues.Create("0123456789abcdef"); +``` + +然后: + +```csharp +span.IndexOfAny(s_lowerHex); +``` + +当前位置测试不一定要用 `SearchValues`,可能 range compare 或 bitmap 更快。 + +--- + +## 6.4 Inline vs helper + +复杂 Unicode set: + +```text +MATCH_SET UnicodeWord +``` + +SourceGen 决定: + +```csharp +if (!IsUnicodeWord(ch)) goto Fail; +``` + +还是: + +```csharp +if (ch < 128) +{ + if (!AsciiWordContains(ch)) goto Fail; +} +else +{ + if (!IsUnicodeWord(ch)) goto Fail; +} +``` + +这个应由代码尺寸、热度、是否 source-gen 决定。 + +--- + +## 6.5 Code size budget + +SourceGen 应负责最终代码膨胀控制: + +```text +单方法最大语句数 +literal 展开阈值 +bounded repeat 展开阈值 +是否生成 helper +是否抽 cold path +SearchValues 字段数量 +``` + +--- + +# 7. 字符集的物理表示建议 + +为了避免每个 set 挂 `List`,可以用线性 IR region 表示同一个字符测试。 + +例如语义: + +```regex +[0-9a-z] +``` + +物理 IR: + +```text +MATCH_SET_BEGIN default=false end=13 + TEST_RANGE '0' '9' accept=true + TEST_RANGE 'a' 'z' accept=true +MATCH_SET_END +``` + +语义: + +```regex +[^0-9a-z] +``` + +物理 IR: + +```text +MATCH_SET_BEGIN default=true end=13 + TEST_RANGE '0' '9' accept=false + TEST_RANGE 'a' 'z' accept=false +MATCH_SET_END +``` + +要点是: + +```text +这些 TEST_RANGE 测的是同一个字符; +整个 MATCH_SET 只推进 pos 一次。 +``` + +不能把它误解成连续匹配多个字符。 + +复杂 Unicode set 可以走: + +```text +MATCH_SET_REF setId=42 +``` + +`setId=42` 指向全局 pooled table,而不是每个节点一个 List。 + +--- + +# 8. 检查点语义建议 + +当前文档把检查点描述为锚点指针栈,这对 capture 起点有用,但不够覆盖 backtracking。 + +建议把 checkpoint / backtrack frame 定义成: + +```text +BacktrackFrame: + pos + anchor + captureTop + symbolTop + repeatState + continuation +``` + +其中: + +```text +pos: + 活动指针恢复位置 + +anchor: + 区间开始位置 + +captureTop: + 捕获记录栈回滚点 + +symbolTop: + 符号区回滚点 + +repeatState: + 当前 COUNT/loop 的计数、起止位置、retry mode + +continuation: + 失败后跳到哪里继续尝试 +``` + +如果暂时是 tokenizer / no-capture 模式,可以裁掉 captureTop,但 IR 语义最好预留。 + +--- + +# 9. 建议修改原文的几个点 + +## 9.1 “HIT” 应改为 “HIR” + +原文: + +```text +将HIT进行进一步优化 +``` + +应为: + +```text +将 HIR 进行进一步优化 +``` + +## 9.2 `SEARCH` 的定义应改写 + +原文说“向后搜索确定长度序列”。建议改为: + +```text +HIR 中 SEARCH_* 表示从当前活动指针开始匹配; +它不是允许跳过字符的 FIND。 +真正的 FIND_* 在 HIR→LIR 阶段由 match planner 引入。 +``` + +## 9.3 `a+b → a*(ab|b)` 不等价 + +建议删除或改成: + +```regex +a+b → aa*b +``` + +或者: + +```text +MATCH_CHAR 'a' +COUNT_SEQ 'a' 0..∞ +MATCH_CHAR 'b' +``` + +## 9.4 `[0-9] → IndexOf("0123456789")` 应细分 + +改成: + +```text +当前位置 MATCH_SET: + 用 unsigned range compare。 + +span 搜索 FIND_SET: + 用 IndexOfAnyInRange / SearchValues / IndexOfAny。 +``` + +## 9.5 `.*` 优化不要说 “COUNT 消失” + +建议改成: + +```text +COUNT_DOT 与后继匹配项合并为 FIND / FIND_LAST 计划。 +``` + +这样更准确,因为有时还需要保留 retry / range 限制 / newline 限制。 + +--- + +# 10. 推荐最终管线 + +```text +Tokenize + - 区分 Dot 与 EscapedLiteralDot + - 区分 raw charset / literal / quantifier / group + +AST Build + - 建立 regex 语义树 + +AST Opt + - literal merge + - concat flatten + - alternation factoring / trie + - bounded repeat normalization + - simple group elimination + - charset algebra merge + +HIR Gen + - SEARCH_* as MATCH_AT_CURRENT + - COUNT_* + - TRY / POP + - PACK + - raw set / raw dot + +HIR Analysis + - minLength / maxLength + - nullable + - firstSet + - lastSet + - required literal + - fixed length + - capture effect + - repeat overlap + +HIR → LIR / MatchPlan + - raw set → lowered set + - dot → dot mode + - introduce FIND + - introduce CONSUME + - choose retry mode + - choose atomic/possessive + - emit check-min-length / fixed-offset checks + +LIR + - explicit MATCH / FIND / CONSUME + - explicit lowered set + - explicit backtracking plan + - explicit capture plan + +SourceGen + - StartsWith / SequenceEqual + - IndexOf / IndexOfAny / IndexOfAnyExcept + - SearchValues + - range compare / bitmap + - inline/helper split + - code size budget +``` + +--- + +# 11. 总结 + +我建议保留你当前 IR 的核心设计,但把文档里的边界收紧: + +```text +AST Opt: + 做 Regex algebra rewrite。 + +HIR: + 表示从当前位置开始的 regex 语义。 + +HIR Analysis: + 提取结构属性。 + +HIR → LIR: + 做 Match plan rewrite。 + +LIR: + 表示明确的匹配计划。 + +SourceGen: + 做 C# span trick。 +``` + +最重要的改动是: + +```text +SEARCH 在 HIR 中应被定义为 MATCH_AT_CURRENT; +FIND 只能由后续优化阶段引入。 +``` + +以及: + +```text +字符集可以用线性 set-region 表示,避免 List 分配; +但 set-region 内的多条 range-test 必须测试同一个字符,不能变成多个连续 MATCH。 +``` + +这样你的 IR 会同时满足三个目标: + +1. 对人来说可读; +2. 对优化规则来说容易扩展; +3. 对 GC 和 sourcegen 来说物理表示足够紧凑。 diff --git a/future/ngra-regex-ir.md b/future/ngra-regex-ir.md index 7e80e2e9..de81e805 100644 --- a/future/ngra-regex-ir.md +++ b/future/ngra-regex-ir.md @@ -14,15 +14,59 @@ 使用DEFINE指令定义的符号定义在符号区中,符号的类型通过第一次赋值确定,源生成模式下,可以通过第一个赋值指令推导其类型 -## 算子 +## HIR算子 1. 搜索算子(SEARCH %str; SEARCH %unicode),向后搜索确定长度序列或者满足规则的字符集,更新活动指针,不更新锚点指针,搜索结果保存在全局上下文中 2. 搜索算子(SEARCH %set_expression),向后匹配一个满足字符集要求的字符,更新活动指针,不更新锚点指针,搜索结果保存在全局上下文中 -3. 失败配置指令(CINFIG_FAIL %fail),指定当前匹配栈后续如何处理一条指令的失败,可选为:FAST,立刻终止当前层级;CONTINUE,记录这个fail,但是继续操作;REVERSE,反过来,成功了才终止 -4. 条件指令(CONDITION %cmd_true %cmd_fail),比较前一条SEARCH指令是否成功,并指定跳转到指定的位置,如果某种条件不需要跳转,可以写1。如果想要将fail向上级继续抛出,可以%cmd_fail写0. -5. 跳转执行指令(TRY %begin %end),尝试idx指向的IR块,并记住执行结果数据 -6. 计数指令(COUNT %min %max %is_greedy %cmd),重复执行指令cmd,直到满足参数配置的数量为止 -7. 快速序列计数指令(COUNT_SEQ %min %max %is_greedy %seq),快速搜索一个确定性序列,直到满足参数配置的数量为止 -8. 快速字符集计数指令(COUNT_SET %min %max %is_greedy %set),快速搜索一个字符集,直到满足参数配置的数量为止 -9. 尝试(TRY %count),对自身向后的一段内容在新的匹配栈层级中进行运行,被分层的指令数量由COUNT指令确定 -10. 弹出(POP %is_success),提前终止一个try块,并用is_success表示弹出行为视为匹配成功还是失败 +3. 条件指令(CONDITION %cmd_true %cmd_fail),比较前一条SEARCH指令是否成功,并指定跳转到指定的位置,如果某种条件不需要跳转,可以写1。如果想要将fail向上级继续抛出,可以%cmd_fail写0. +4. 计数指令(COUNT %min %max %is_greedy %cmd),重复执行指令cmd,直到满足参数配置的数量为止 +5. 快速序列计数指令(COUNT_SEQ %min %max %is_greedy %seq),快速搜索一个确定性序列,直到满足参数配置的数量为止 +6. 快速字符集计数指令(COUNT_SET %min %max %is_greedy %set),快速搜索一个字符集,直到满足参数配置的数量为止 +7. 尝试(TRY %count),对自身向后的一段内容在新的匹配栈层级中进行运行,被分层的指令数量由COUNT指令确定 +8. 弹出(POP %is_success),提前终止一个try块,并用is_success表示弹出行为视为匹配成功还是失败 +9. 打包指令(PACK %is_success %name),检查上一条语句的结果,将匹配结果打包并记录为指定的名字。如果名字为空,则使用默认的序号进行索引 + +## 优化策略 + +应当注意的是,将HIT进行进一步优化或者展开的目的: + +1. 尽可能降低源生成或者运行时执行中,对多指令上下文的依赖,尽可能保证每一个IR可以单独输出最佳的代码 +2. 减少多余的结构,使得一行指令尽可能处理多的数据 + +### AST优化器 + +1. ALT合并 +将多个ALT CASE合并到一个链表中 + +2. SPAN合并 +将连续的CHAR CONCAT合并为一个char span + +3. alt合并 +将多个alt分支中的公共部分提取出来 + +### IR优化器 + +1. .*a特征,优化为 +FIND_SEQ(a, last) +.*?a特征,优化为 +FIND_SEQ(a, first) + +2. a+特征,优化为 +MATCH_SEQ(a) +COUNT_SEQ(a,0,+inf,greedy) + +完成 + +3. a{1} -> a + +4. .\*a\*特征,优化为.* + +完成,最优先 + +5. a{m,n}特征,展开为 +MATCH_SEQ(aa...m个a....aa) +COUNT_SEQ(a,0,n,greedy) + +6. .*?a*$ + +7. 对于一般的.*?,应该将后续的所有东西全都放进一个巨大的try,code gen时使用最慢的一位一位offset直到直到第一个匹配rest的东西 \ No newline at end of file