diff --git a/src/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs b/src/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs index 36ea99fe..9e560d25 100644 --- a/src/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs +++ b/src/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs @@ -16,9 +16,8 @@ using System.Text; namespace StgSharp.RegularAnalysis.Abstraction { - public interface ITrinitySyntaxNode : ISyntaxNode - where TNode : ISyntaxNode - where TPayload : ISyntaxPayload + public interface ITrinitySyntaxNode : ISyntaxNode + where TNode : ISyntaxNode where TLabel : unmanaged { @@ -26,12 +25,12 @@ namespace StgSharp.RegularAnalysis.Abstraction } - public interface ISyntaxNode - where TNode : ISyntaxNode - where TPayload : ISyntaxPayload + public interface ISyntaxNode where TNode : ISyntaxNode where TLabel : unmanaged { + ISyntaxPayload Source { get; } + long NodeFlag { get; } TLabel EqualityTypeConvert { get; } @@ -48,8 +47,6 @@ namespace StgSharp.RegularAnalysis.Abstraction static abstract TNode Empty { get; } - TPayload Source { get; } - protected internal int EnumState { get; set; } void AppendNode( diff --git a/src/StgSharp.RegularAnalysis/Abstraction/ISyntaxPayload.cs b/src/StgSharp.RegularAnalysis/Abstraction/ISyntaxPayload.cs index f6f1f25c..ef74c10d 100644 --- a/src/StgSharp.RegularAnalysis/Abstraction/ISyntaxPayload.cs +++ b/src/StgSharp.RegularAnalysis/Abstraction/ISyntaxPayload.cs @@ -13,20 +13,16 @@ using System.Threading.Tasks; namespace StgSharp.RegularAnalysis.Abstraction { - // TODO 不对劲，TCarrier是个昏招，还是去掉吧 - public interface ISyntaxPayload - where TSelf : ISyntaxPayload - where TCarrier : ISyntaxNode - where TLabel : unmanaged + public interface ISyntaxPayload where TLabel : unmanaged { - int Column { get; set; } + int Column { get; } - int Row { get; set; } + int Row { get; } - string Source { get; set; } + string Source { get; } - TLabel Flag { get; set; } + TLabel Flag { get; } } } diff --git a/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs b/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs index 09ad20cb..a23b7830 100644 --- a/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs +++ b/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs @@ -15,7 +15,7 @@ using System.Threading.Tasks; namespace StgSharp.RegularAnalysis.Text { - public partial class RegexAnalyzer + public static partial class RegexAnalyzer { private static AbstractSyntaxTree GenerateAST( @@ -50,10 +50,10 @@ namespace StgSharp.RegularAnalysis.Text } _stack.DecreaseDepth(); op = _stack.PopOperator(); - if (op.Source.Flag != RegexElementLabel.GROUP_BEGIN) { + if (op.Label != RegexElementLabel.GROUP_BEGIN) { throw new InvalidOperationException("Unmatched group begin and end symbol"); } - if (op.Source.Value.Length > 1) + if (op.Payload.Source.Length > 1) { op.Right = _stack.PopOperand(); _stack.PushOperand(op); @@ -69,7 +69,7 @@ namespace StgSharp.RegularAnalysis.Text // process operator by precedence while (_stack.TryPeekOperator(out RegexAstNode topOp)) { - (bool Valid, int Value) cmp = RegexAstNode.ComparePrecedence(topOp.Source.Flag, token.Flag); + (bool Valid, int Value) cmp = RegexAstNode.ComparePrecedence(topOp.Label, token.Flag); if (!cmp.Valid) { throw new InvalidOperationException("Operators with same precedence are not supported."); diff --git a/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTNodeOptimize.cs b/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTNodeOptimize.cs index 1105fa9f..04d77000 100644 --- a/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTNodeOptimize.cs +++ b/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTNodeOptimize.cs @@ -10,7 +10,7 @@ using System; namespace StgSharp.RegularAnalysis.Text { - internal partial class RegexAnalyzer + public static partial class RegexAnalyzer { private static bool TryFlattenCount( diff --git a/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs b/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs index a57446b0..310b5fec 100644 --- a/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs +++ b/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs @@ -19,7 +19,7 @@ using System.Xml.Linq; namespace StgSharp.RegularAnalysis.Text { - public partial class RegexAnalyzer + public static partial class RegexAnalyzer { private static RegexAstNode FlattenAlt( @@ -128,11 +128,11 @@ namespace StgSharp.RegularAnalysis.Text offset += buffer[i].Length; } }); - root.Left = new RegexAstNode(new Token(str, left.Source.Line, - left.Source.Column, - str.Length == 1 ? - RegexElementLabel.UNIT : - RegexElementLabel.UNIT_SPAN)); + root.Left = new RegexAstNode(new RegexLiteralPayload(str, str, left.Payload.Row, + left.Payload.Column, + str.Length == 1 ? + RegexElementLabel.UNIT : + RegexElementLabel.UNIT_SPAN)); return true; } @@ -262,15 +262,15 @@ namespace StgSharp.RegularAnalysis.Text int level, RegexAstNode node ) - : base(node.Source) + : base(node.Payload) { Level = level; Cases = []; RegexAstNode right = IsNullOrEmpty(node.Right) ? new RegexAstNode( - new Token(node.Value, 0, node.Source.Column, - RegexElementLabel.UNIT) + new RegexLiteralPayload(node.Value, node.Value, node.Payload.Row, + node.Payload.Column, RegexElementLabel.UNIT) ) : node.Right; Cases.Add(right); @@ -285,7 +285,7 @@ namespace StgSharp.RegularAnalysis.Text int level, List c ) - : base(c[0].Source) + : base(c[0].Payload) { Cases = c; Level = level; @@ -312,7 +312,7 @@ namespace StgSharp.RegularAnalysis.Text if ((Cases[i].EqualityTypeConvert & RegexElementLabel.SEQUENCE) != 0) { RegexAstNode concat = new(new Token(string.Empty, 0, - Cases[i].Source.Column, + Cases[i].Payload.Column, RegexElementLabel.CONCAT)) { Left = Cases[i] @@ -385,14 +385,14 @@ namespace StgSharp.RegularAnalysis.Text if (IsNullOrEmpty(UnionEnd)) { branch = new RegexAstNode(new Token(string.Empty, 0, - Source.Column, + Payload.Column, RegexElementLabel.ALT)); UnionBegin = branch; UnionEnd = branch; } else { branch = new RegexAstNode(new Token(string.Empty, 0, - UnionEnd.Source.Column, + UnionEnd.Payload.Column, RegexElementLabel.ALT)); UnionEnd.Right = branch; branch.Parent = UnionEnd; @@ -415,8 +415,8 @@ namespace StgSharp.RegularAnalysis.Text { RegexAstNode right = IsNullOrEmpty(_case.Right) ? new RegexAstNode( - new Token(_case.Value, 0, _case.Source.Column, - RegexElementLabel.UNIT) + new RegexLiteralPayload(_case.Value, _case.Value, _case.Payload.Row, + _case.Payload.Column, RegexElementLabel.UNIT) ) : _case.Right; Cases.Add(right); @@ -432,14 +432,14 @@ namespace StgSharp.RegularAnalysis.Text { // then we suppose that alt has only one are same return origin .EqualityTypeConvert == RegexElementLabel.CONCAT && - (origin.Left.EqualityTypeConvert & RegexElementLabel.SINGLE) != 0 && + (origin.Left.EqualityTypeConvert & RegexElementLabel.UNIT_SINGLE) != 0 && origin.Left.EqualityTypeConvert == alt.UnionEnd.Left.EqualityTypeConvert && origin.Left.Value == alt.UnionEnd.Left.Value; } else { return origin.EqualityTypeConvert == RegexElementLabel.CONCAT && target.EqualityTypeConvert == RegexElementLabel.CONCAT && - (origin.Left.EqualityTypeConvert & RegexElementLabel.SINGLE) != 0 && + (origin.Left.EqualityTypeConvert & RegexElementLabel.UNIT_SINGLE) != 0 && origin.Left.EqualityTypeConvert == target.Left.EqualityTypeConvert && origin.Left.Value == target.Left.Value; } diff --git a/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs b/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs index a48ea10f..ee5c141d 100644 --- a/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs +++ b/src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs @@ -24,8 +24,6 @@ namespace StgSharp.RegularAnalysis.Text internal TextRegexSource( string pattern, AbstractSyntaxTree ast, - List base_ir, - List ir, RegexInfo info, Exception? analyseException = null @@ -33,8 +31,6 @@ namespace StgSharp.RegularAnalysis.Text { Pattern = pattern; Ast = ast; - IRs = ir; - BaseIR = base_ir; AnalyseException = analyseException; Info = info; } @@ -44,11 +40,6 @@ namespace StgSharp.RegularAnalysis.Text public string Pattern { get; } internal AbstractSyntaxTree Ast { get; } - /* - internal List BaseIR { get; } - - internal List IRs { get; } - /**/ internal RegexInfo Info { get; } @@ -58,9 +49,7 @@ namespace StgSharp.RegularAnalysis.Text { public static TextRegexSource Analyze( - string regex, - bool require_ast = false, - bool require_origin_ir = false + string regex ) { #pragma warning disable CA1031 @@ -72,10 +61,11 @@ namespace StgSharp.RegularAnalysis.Text { tree = GenerateAST(regex); OptimizeTree(tree); + return new TextRegexSource(regex, tree, info, null); } catch (Exception ex) { - return new TextRegexSource(regex, null, null, null, info, ex); + return new TextRegexSource(regex, null, info, ex); } /* // List base_ir; diff --git a/src/StgSharp.RegularAnalysis/Text/RegexAstNode.AdditionalPayload.cs b/src/StgSharp.RegularAnalysis/Text/RegexAstNode.AdditionalPayload.cs index 12301acd..7c0740a0 100644 --- a/src/StgSharp.RegularAnalysis/Text/RegexAstNode.AdditionalPayload.cs +++ b/src/StgSharp.RegularAnalysis/Text/RegexAstNode.AdditionalPayload.cs @@ -8,27 +8,323 @@ using StgSharp.RegularAnalysis.Abstraction; using System; using System.Collections.Generic; -using System.Linq; +using System.Globalization; using System.Text; -using System.Threading.Tasks; namespace StgSharp.RegularAnalysis.Text { - public class RegexCountPayload : ISyntaxPayload + public abstract record RegexAstPayload( + string Source, + int Row, + int Column, + RegexElementLabel Flag + ) : ISyntaxPayload; + + public sealed record RegexEmptyPayload() : RegexAstPayload(string.Empty, 0, 0, + RegexElementLabel.NONE); + + /// + /// A deterministic literal sequence. Escapes represented as literals are consumed here. + /// + public sealed record RegexLiteralPayload( + string Text, + string Source, + int Row, + int Column, + RegexElementLabel Flag + ) : RegexAstPayload(Source, Row, Column, Flag); + + public enum RegexCharSetType { - public int Min { get; set; } - - public int Max { get; set; } - - public string Source { get; set; } + None, + Single, + Range, + Set, + Any } - public class RegexCommonPayload : IRegexAstNodeAdditionalPayload + public sealed record RegexCharSet( + string Value, + RegexCharSetType Type, + bool Accept + ); + + /// + /// A character-set atom expanded into the rules formerly produced by FigureCharSetType. + /// + public sealed record RegexCharSetPayload( + IReadOnlyList Set, + bool Accept, + string Source, + int Row, + int Column + ) : RegexAstPayload(Source, Row, Column, RegexElementLabel.UNIT_SET) { - public string Source { get; set; } = string.Empty; + public bool IsAny => Set.Count == 1 && Set[0].Type == RegexCharSetType.Any; + + } + + public sealed record RegexCountPayload( + int Min, + int Max, + bool IsGreedy, + string Source, + int Row, + int Column + ) : RegexAstPayload(Source, Row, Column, RegexElementLabel.COUNT) + { + + public bool IsStar => Min == 0 && Max < 0; + + public bool IsPlus => Min == 1 && Max < 0; + + } + + public sealed record RegexGroupPayload( + string Name, + bool IsCapturing, + string Source, + int Row, + int Column + ) : RegexAstPayload(Source, Row, Column, RegexElementLabel.GROUP_BEGIN); + + public sealed record RegexCommonPayload( + string Source, + int Row, + int Column, + RegexElementLabel Flag + ) : RegexAstPayload(Source, Row, Column, Flag) + { + + public static RegexAstPayload FromToken( + Token token + ) + { + return token.Flag switch + { + RegexElementLabel.UNIT or RegexElementLabel.UNIT_SPAN => new RegexLiteralPayload(DecodeLiteral(token.Value), + token.Value, + token.Line, + token.Column, + token.Flag), + RegexElementLabel.UNIT_SET => ParseCharSet(token), + RegexElementLabel.COUNT => ParseCount(token), + RegexElementLabel.GROUP_BEGIN => ParseGroup(token), + _ => new RegexCommonPayload(token.Value, token.Line, token.Column, token.Flag) + }; + } + + private static void AddEscape( + char escaped, + List rules + ) + { + bool ruleAccept = char.IsAsciiLetterLower(escaped); + char kind = char.ToLower(escaped, CultureInfo.InvariantCulture); + if (!IsNamedSet(escaped)) { + throw new InvalidCastException($"Unknown character set escape: \\{escaped}"); + } + rules.Add(new(kind.ToString(), RegexCharSetType.Set, ruleAccept)); + } + + private static char DecodeEscapedChar( + char value + ) + { + return value switch + { + 'n' or 'r' => throw new NotSupportedException( + "Multi-line regular expressions are not supported yet."), + 't' => '\t', + '0' => '\0', + _ => value + }; + } + + private static string DecodeLiteral( + string source + ) + { + if (source.Length != 2 || source[0] != '\\') { + return source; + } + return source[1] switch + { + 'n' => "\n", + 'r' => "\r", + 't' => "\t", + '0' => "\0", + _ => source[1].ToString() + }; + } + + private static bool IsNamedSet( + char value + ) + { + return + char.ToLower(value, CultureInfo.InvariantCulture) is 's' or 'w' or 'd'; + } + + private static void ParseBracketContent( + ReadOnlySpan content, + List rules + ) + { + HashSet singles = []; + int index = 0; + while (index < content.Length) + { + if (content[index] is '\r' or '\n') { + throw new NotSupportedException( + "Multi-line regular expressions are not supported yet."); + } + if (content[index] == '\\') + { + if (++index >= content.Length) { + throw new InvalidCastException("Unclosed escape in character set."); + } + char escaped = content[index++]; + if (IsNamedSet(escaped)) + { + AddEscape(escaped, rules); + } else + { + _ = singles.Add(DecodeEscapedChar(escaped)); + } + continue; + } + + char first = content[index++]; + if (index < content.Length - 0 && + content[index] == '-' && + index + 1 < content.Length) + { + index++; + char last; + if (content[index] == '\\') + { + if (++index >= content.Length) { + throw new InvalidCastException("Unclosed range in character set."); + } + last = DecodeEscapedChar(content[index++]); + } else + { + last = content[index++]; + } + if (last < first) { + throw new InvalidCastException($"Invalid character range: {first}-{last}"); + } + rules.Add(new(content[first..last].ToString(), RegexCharSetType.Range, true)); + } else + { + _ = singles.Add(first); + } + } + + if (singles.Count > 0) + { + StringBuilder text = new(singles.Count); + foreach (char value in singles) { + _ = text.Append(value); + } + rules.Add(new(text.ToString(), RegexCharSetType.Single, true)); + } + } + + private static RegexCharSetPayload ParseCharSet( + Token token + ) + { + string source = token.Value; + List rules = []; + bool accept = true; + + if (source == ".") + { + rules.Add(new(string.Empty, RegexCharSetType.Any, true)); + } else if (source.Length >= 2 && source[0] == '[' && source[^1] == ']') + { + int start = source.Length > 2 && source[1] == '^' ? 2 : 1; + accept = start == 1; + ParseBracketContent(source.AsSpan(start, source.Length - start - 1), rules); + } else if (source.Length == 2 && source[0] == '\\') + { + AddEscape(source[1], rules); + } else + { + throw new InvalidCastException($"Unknown char set: {source}"); + } + + return new RegexCharSetPayload(rules.ToArray(), accept, source, token.Line, + token.Column); + } + + private static RegexCountPayload ParseCount( + Token token + ) + { + string source = token.Value; + bool greedy = source.Length == 1 || source[^1] != '?'; + ReadOnlySpan count = greedy ? + source.AsSpan() : + source.AsSpan(0, source.Length - 1); + return count[0] switch + { + '*' => new RegexCountPayload(0, -1, greedy, source, token.Line, token.Column), + '+' => new RegexCountPayload(1, -1, greedy, source, token.Line, token.Column), + '?' => new RegexCountPayload(0, 1, greedy, source, token.Line, token.Column), + '{' => ParseRange(count, greedy, token), + _ => throw new InvalidOperationException($"Invalid count format: {source}") + }; + } + + private static RegexGroupPayload ParseGroup( + Token token + ) + { + string source = token.Value; + bool named = source.StartsWith("(?<", StringComparison.Ordinal); + string name = named ? source[3..^1] : string.Empty; + return new RegexGroupPayload(name, source != "(?:", source, token.Line, token.Column); + } + + private static RegexCountPayload ParseRange( + ReadOnlySpan count, + bool greedy, + Token token + ) + { + if (count.Length < 3 || count[^1] != '}') { + throw new InvalidOperationException($"Invalid count format: {token.Value}"); + } + ReadOnlySpan body = count[1..^1]; + int comma = body.IndexOf(','); + if (comma < 0) + { + if (!int.TryParse(body, out int exact)) { + throw new InvalidOperationException($"Invalid count format: {token.Value}"); + } + return new RegexCountPayload(exact, exact, greedy, token.Value, token.Line, + token.Column); + } + if (!int.TryParse(body[..comma], out int min)) { + throw new InvalidOperationException($"Invalid count format: {token.Value}"); + } + ReadOnlySpan maxText = body[(comma + 1)..]; + int max = maxText.IsEmpty ? + -1 : + int.TryParse(maxText, out int parsed) ? + parsed : + throw new InvalidOperationException($"Invalid count format: {token.Value}"); + if (max >= 0 && max < min) { + throw new InvalidOperationException($"Invalid count range: {token.Value}"); + } + return new RegexCountPayload(min, max, greedy, token.Value, token.Line, token.Column); + } } } diff --git a/src/StgSharp.RegularAnalysis/Text/RegexAstNode.cs b/src/StgSharp.RegularAnalysis/Text/RegexAstNode.cs index 80af460f..f77b10fa 100644 --- a/src/StgSharp.RegularAnalysis/Text/RegexAstNode.cs +++ b/src/StgSharp.RegularAnalysis/Text/RegexAstNode.cs @@ -19,14 +19,20 @@ namespace StgSharp.RegularAnalysis.Text public RegexAstNode( Token source + ) + : this(RegexCommonPayload.FromToken(source)) + { } + + public RegexAstNode( + RegexAstPayload payload ) { - Source = source; + Payload = payload; } public int EnumState { get; set; } - public long NodeFlag => (long)Source.Flag; + public long NodeFlag => (long)Label; public static RegexAstNode Empty { get; } = new EmptyRegexAstNode(); @@ -63,11 +69,24 @@ namespace StgSharp.RegularAnalysis.Text } } - public RegexElementLabel EqualityTypeConvert => Source.Flag; + public RegexElementLabel EqualityTypeConvert => Label; - public string Value => Source.Value; + public RegexElementLabel Label => Payload.Flag; - public Token Source { get; private protected set; } + public string Value => Payload switch + { + RegexLiteralPayload literal => literal.Text, + RegexCharSetPayload set => set.Source, + _ => Payload.Source + }; + + public RegexAstPayload Payload { get; private protected set; } + + public ISyntaxPayload Source => Payload; + + internal TPayload PayloadAs() where TPayload : RegexAstPayload => + Payload as TPayload ?? throw new InvalidOperationException( + $"Node {Label} does not carry {typeof(TPayload).Name}."); public void AppendNode( RegexAstNode nextToken @@ -135,12 +154,12 @@ namespace StgSharp.RegularAnalysis.Text { if (useLeft) { - Source = Left.Source; + Payload = Left.Payload; Left.Parent = Empty; Left = Empty; } else { - Source = Right.Source; + Payload = Right.Payload; Right.Parent = Empty; Right = Empty; } @@ -160,7 +179,7 @@ namespace StgSharp.RegularAnalysis.Text RegexAstNode node ) { - Source = node.Source; + Payload = node.Payload; Left = node.Left; Right = node.Right; if (!IsNullOrEmpty(Left)) { @@ -186,7 +205,7 @@ namespace StgSharp.RegularAnalysis.Text { - public EmptyRegexAstNode() : base(Token.Empty) { } + public EmptyRegexAstNode() : base(new RegexEmptyPayload()) { } } diff --git a/src/StgSharp.RegularAnalysis/Text/RegexElementLabel.cs b/src/StgSharp.RegularAnalysis/Text/RegexElementLabel.cs index e3246596..39044b88 100644 --- a/src/StgSharp.RegularAnalysis/Text/RegexElementLabel.cs +++ b/src/StgSharp.RegularAnalysis/Text/RegexElementLabel.cs @@ -27,7 +27,7 @@ namespace StgSharp.RegularAnalysis.Text CONCAT = 64, ALT = 128, SEQUENCE = UNIT | UNIT_SPAN | UNIT_SET, - SINGLE = UNIT | UNIT_SET, + UNIT_SINGLE = UNIT | UNIT_SET, OPERATOR = COUNT | ALT | CONCAT, VAST_OPERATOR = OPERATOR | GROUP_BEGIN, ATOM_BEGIN = SEQUENCE | GROUP_BEGIN , diff --git a/src/StgSharp.RegularAnalysis/Text/RegexTokenReader.cs b/src/StgSharp.RegularAnalysis/Text/RegexTokenReader.cs index 1e0a20e1..8743bea5 100644 --- a/src/StgSharp.RegularAnalysis/Text/RegexTokenReader.cs +++ b/src/StgSharp.RegularAnalysis/Text/RegexTokenReader.cs @@ -37,6 +37,8 @@ namespace StgSharp.RegularAnalysis.Text int pos = position; switch (_source[pos]) { + case '\r' or '\n': + throw new NotSupportedException("Multi-line regular expressions are not supported yet."); case '\\': if (pos + 1 == _source.Length) { throw new InvalidOperationException("Unclosed char"); @@ -85,6 +87,9 @@ namespace StgSharp.RegularAnalysis.Text return new Token(".", 0, pos, RegexElementLabel.UNIT); } else { + if (_source[pos + 1] is 'n' or 'r') { + throw new NotSupportedException("Multi-line regular expressions are not supported yet."); + } position = pos + 2; return new Token(_source[(pos)..(pos + 2)], 0, pos, RegexElementLabel.UNIT_SET); @@ -116,8 +121,12 @@ namespace StgSharp.RegularAnalysis.Text { if (_source[i] == '}') { - position = i + 1; - return new Token(_source[(pos)..(i + 1)], 0, pos, + int end = i + 1; + if (end < _source.Length && _source[end] == '?') { + end++; + } + position = end; + return new Token(_source[pos..end], 0, pos, RegexElementLabel.COUNT); } } @@ -127,8 +136,10 @@ namespace StgSharp.RegularAnalysis.Text position = pos + count; return new Token(_source[(pos)..(pos + count)], 0, pos, RegexElementLabel.COUNT); case '?': - position = pos + 1; - return new Token(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.COUNT); + int optionalCount = pos + 1 < _source.Length && _source[pos + 1] == '?' ? 2 : 1; + position = pos + optionalCount; + return new Token(_source[pos..(pos + optionalCount)], 0, pos, + RegexElementLabel.COUNT); case '.': position = pos + 1; return new Token(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.UNIT_SET); diff --git a/src/StgSharp.RegularAnalysis/Text/TextRegex.cs b/src/StgSharp.RegularAnalysis/Text/TextRegex.cs index 374c0ac0..d14bddc3 100644 --- a/src/StgSharp.RegularAnalysis/Text/TextRegex.cs +++ b/src/StgSharp.RegularAnalysis/Text/TextRegex.cs @@ -19,7 +19,7 @@ namespace StgSharp.RegularAnalysis.Text { private const int WordCategoriesMask = - 1 << (int)UnicodeCategory.UppercaseLetter | (1 << (int)UnicodeCategory.LowercaseLetter) | (1 << (int)UnicodeCategory.TitlecaseLetter) | 1 << (int)UnicodeCategory.ModifierLetter | 1 << (int)UnicodeCategory.OtherLetter | 1 << (int)UnicodeCategory.NonSpacingMark | 1 << (int)UnicodeCategory.DecimalDigitNumber | 1 << (int)UnicodeCategory.ConnectorPunctuation; + (1 << (int)UnicodeCategory.UppercaseLetter) | (1 << (int)UnicodeCategory.LowercaseLetter) | (1 << (int)UnicodeCategory.TitlecaseLetter) | 1 << (int)UnicodeCategory.ModifierLetter | 1 << (int)UnicodeCategory.OtherLetter | 1 << (int)UnicodeCategory.NonSpacingMark | 1 << (int)UnicodeCategory.DecimalDigitNumber | 1 << (int)UnicodeCategory.ConnectorPunctuation; public abstract MatchResult Match( ReadOnlySpan text @@ -84,17 +84,17 @@ namespace StgSharp.RegularAnalysis.Text /// . /// /// - /// /// This method operates on a single , and therefore /// on a - /// single UTF-16 code unit rather than on a complete Unicode scalar /// value. /// - /// /// Characters in the Unicode Basic Multilingual Plane are classified - /// /// normally. A supplementary-plane character is represented by a /// surrogate pair - /// and is therefore presented to this method as two /// separate surrogate code units. - /// Neither surrogate is independently /// treated as a word character. This follows the - /// char-based behavior of /// the corresponding .NET Regex helper. /// /// - /// For ASCII input, the Unicode-category lookup is avoided. In ASCII, /// the accepted - /// set is exactly: /// [A-Za-z0-9_] . /// /// This method + /// /// This method operates on a single , and therefore /// on + /// a single UTF-16 code unit rather than on a complete Unicode scalar /// value. + /// /// /// Characters in the Unicode Basic Multilingual Plane are + /// classified /// normally. A supplementary-plane character is represented by a /// + /// surrogate pair and is therefore presented to this method as two /// separate surrogate + /// code units. Neither surrogate is independently /// treated as a word character. This + /// follows the char-based behavior of /// the corresponding .NET Regex helper. /// + /// /// For ASCII input, the Unicode-category lookup is avoided. In ASCII, /// the + /// accepted set is exactly: /// [A-Za-z0-9_] . /// /// This method /// implements ordinary .NET \w semantics. It does /// not implement the ASCII- - /// oriented behavior enabled by /// RegexOptions.ECMAScript . /// + /// oriented behavior enabled by /// RegexOptions.ECMAScript . /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] protected static bool IsCharWord( diff --git a/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs b/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs index 007646b6..fc7312cb 100644 --- a/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs +++ b/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs @@ -10,6 +10,7 @@ using StgSharp.RegularAnalysis.Abstraction; using System; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -18,443 +19,133 @@ namespace StgSharp.RegularAnalysis.Text { internal partial class TextRegexSourceGen { - /* - * --------------------------------------------------------------------------------------- - * NGRA text regex source generation backend - * --------------------------------------------------------------------------------------- - * - * The backend turns a fully prefixed IR stream into the body of one Match method. It is - * organised in layers, ordered by how many IR entries a single generator consumes: - * - * L0 whole IR stream method skeleton, slot declarations, entry guards - * L1 several regions ALT chain, group tail, complex count, complex find - * L2 one region the block a TRY lowers to - * L3 two to eight IR fixed windows worth collapsing into better code - * L4 one IR the fallback, one generator per RegexIRCommand - * L5 zero IR prefix decoration wrapped around L1 to L4 output - * - * File layout: - * - * TextRegexSourceGen.GenClass.cs this file, L0 and shared helpers - * TextRegexSourceGen.GenClass.MultiIR.cs L1, L2 and L3 - * TextRegexSourceGen.GenClass.SingleIR.cs L4 - * TextRegexSourceGen.GenClass.Prefix.cs L5 - * TextRegexSourceGen.GenContext.cs variable naming and slot allocation - * - * Contract every generator honours: - * - * 1. A TryGenerateXxx returns how many IR entries it consumed, or -1 when the window - * does not match its shape. It returns an entry count, never a character count or a - * pattern length. - * 2. The dispatch chain offers a window to L1, then L2, then L3, then L4. Higher - * consumption wins, so a shape spanning a whole region plus a tail is never split by - * a smaller generator. - * 3. L2, L4, L5 and the required entries of L1 together are sufficient for correctness. - * The flattening entries of L1 and all of L3 only improve the shape of what those - * would already emit, so an unimplemented one must return -1 cleanly rather than - * emit a partial result. - * 4. A prefix consumes no IR. L1 to L4 describe what an instruction does, L5 decides - * when it runs and who reads the outcome. - * 5. Scope follows Prefix.WorkingRegion, not expression nesting. - * - * Control flow shape: - * - * A region lowers to do { ... } while(false), and POP lowers to break. The goto plus - * label scheme used by the .NET generated regex backend is deliberately not adopted. - * - * A FIND lowers to a candidate retry loop nested inside the region block. That loop is - * nearer than the region do block, so a POP inside it needs the __rN_done flag described - * in TextRegexSourceGen.GenClass.MultiIR.cs rather than a plain break. - * - * Generated variable names are owned by SourceGenContext. No generator invents one. - * - * The per shape checklist this backend is rebuilt against lives in - * future/ngra-regex-sourcegen-goldens.md, and the handwritten target code for each shape - * lives in test/SingleFile/regex.cs. - */ - // private const string current_ptr = "cur_ptr_global"; // global match cursor - // private const string offset = "offset"; // - // private const string orig_text = "text"; // + internal static void GenerateMethodSource( + RegexAstNode ast, + in SequenceEmitter sc, + in List> func_list, + bool is_find = false - #region L0 method level - - /// - /// Generates one Match method from a fully prefixed IR stream. - /// - /// - /// Not implemented yet. Returns an empty emitter so the generator pipeline stays runnable - /// while the backend is being rebuilt. - /// - internal static SequenceEmitter GenerateSource( - TextRegexSource source ) { - // - // L0, method skeleton - // - // TODO L0.1 Emit the Match signature from ROS_char and SourceGenContext.InputSpan. - // TODO L0.2 Emit region anchor and region result slots, count from RegexInfo.RegionCount. - // TODO L0.3 Emit line result slots, only for lines an Observe_Line prefix refers to. - // TODO L0.5 Emit the cursor, the step result and any scratch declaration the context collected. - // - - /* - * Slot demand is only known once the body exists, so the body has to be generated - * into a detached emitter first and spliced in after the declaration block. - * - * The dispatch chain that walks the IR stream belongs here too. It offers each window - * to L1, L2, L3 then L4, and adds the returned consumption count to the cursor. An - * unclaimed window must produce a hard build break rather than silently emit nothing, - * because code that compiles but matches incorrectly is far harder to diagnose. - */ - - SequenceEmitter se = new(); - List> sub_match_list = []; - SourceGenContext sc = new(); - - RegexInfo info = source.Info; - List ir = source.IRs; - ReadOnlySpan ir_sp = CollectionsMarshal.AsSpan(ir); - - SequenceEmitter match_e = new(); - - sc.RegionExtent = MeasureRegionExtents(ir_sp, info.RegionCount); - - // begin a very big do while(0) - _ = se.AppendLine(sc.BeginRegion()); - - _ = GenerateBlock(0, ir_sp, sc, match_e, sub_match_list); - - _ = se.AppendLine($@"public override MatchResult Match({ROS_char} text)")// - .AppendLine(@"{"); - - int min_predict_length = info.MinPredictLength; - if (min_predict_length > 0) { - _ = se.AppendLine($@"if (text.Length < {min_predict_length})") - .AppendLine(@"{") - .AppendLine(@"return default;") - .AppendLine(@"}") - .AppendLine(); - } - - int region_count = info.RegionCount; - for (int i = 0; i < region_count; i++) + RegexAstNode root = ast; + switch (root.Label & (RegexElementLabel.VAST_OPERATOR)) { - _ = se.AppendLine($@"int region_{i}_start = -1;")// - .AppendLine($@"int region_{i}_end = -1;") - .AppendLine($@"bool region_{i}_success;"); + case RegexElementLabel.CONCAT: + GenerateConcat(root, sc, func_list); + break; + case RegexElementLabel.COUNT: + break; + case RegexElementLabel.GROUP_BEGIN: + break; + case RegexElementLabel.ALT: + break; } - /* - _ = se.AppendLine($@"int {current_ptr} = 0;") - .AppendLine($@"int {offset} = 0;") - .AppendLine(); - /**/ - - // _ = sc.GenVarDefine(se); - - _ = se.Append(match_e); - - // end of the biggest region - _ = se.AppendLine(sc.CloseRegion()); - - // close the entire match result and return - - foreach (SequenceEmitter _sub in sub_match_list) - { - _ = se.AppendLine() // - .Append(_sub); - } - - _ = se.Append(@"}"); - - return se; } - /// - /// Recursive dispatch entry. Walks one contiguous IR block and offers each window to the - /// generators in descending consumption order. Region bodies, count bodies and find - /// bodies recurse into this same method. - /// - /// - /// Total IR entries consumed, always ir_span.Length . - /// - private static int GenerateBlock( - int outer_level, - ReadOnlySpan ir_span, - SourceGenContext sc, - in SequenceEmitter code, - in List> local_func + #region supporting method + + private static bool IsNamedFind( + RegexAstNode node ) { - int index = 0; - while (index < ir_span.Length) + // (?.*)abc + + if (node is null) { - ReadOnlySpan rest = ir_span[index..]; - - // ground entries pass one by one, a foreign region id opens a mountain; - // carving always lands on a mountain head, so the extent is the carve length - int id = rest[0].Prefix.WorkingRegion; - int carve = id == outer_level ? 1 : sc.RegionExtent[id]; - Debug.Assert(id == outer_level || carve == FindRegionEnd(rest)); - - index += GenerateRegion(outer_level, rest[..carve], sc, code, local_func); + return false; } - return index; - } - - /// - /// Processes exactly one carved region, or one ground instruction. Generators that own a - /// body re-enter , which is what closes the recursion. - /// - /// - /// IR entries consumed, at least 1. - /// - private static int GenerateRegion( - int outer_level, - ReadOnlySpan region, - SourceGenContext sc, - in SequenceEmitter code, - in List> local_func - ) - { - int consumed; - - // L1, several regions plus a tail - if ((consumed = TryGenerateAltChain(outer_level, region, sc, code, local_func)) > 0) { - return consumed; + if ((node.Label & RegexElementLabel.CONCAT) != 0) { + return false; } - if ((consumed = TryGenerateNamedCountAny(outer_level, region, sc, code, - local_func)) > 0) { - return consumed; + RegexAstNode group = node.Left; + if ((group.Label & RegexElementLabel.GROUP_BEGIN) == 0) { + return false; } + RegexAstNode count = group.Right; - // L2, one region - if ((consumed = TryGenerateTryBlock(outer_level, region, sc, code, local_func)) > 0) { - return consumed; - } - - // L4, single instruction fallback - if ((consumed = TryGenerateSingleIR(outer_level, region, sc, code, local_func)) > 0) { - return consumed; - } - if ((consumed = TryGeneratePack(outer_level, region, sc, code, local_func)) > 0) { - return consumed; - } - - // unclaimed window must break the build of the generated file - _ = code.AppendLine($@"#error NGRA has no source generation rule for {region[0]}"); - return 1; + // .* + return (count.Label & RegexElementLabel.COUNT) != 0 && + (count.Right.Label & RegexElementLabel.UNIT_SINGLE) != 0 && + count.Right.Source.Source == "."; } #endregion - private enum RegionExitMethod - { + #region verb generate - Break, - Continue + private static void GenerateAlt( + RegexAstNode node, + [NotNull] in SequenceEmitter sc, + in List> func_list + ) { } - } - - #region shared helpers - - // backward scan; the final write per id happens at its first entry, leaving full extent - private static int[] MeasureRegionExtents( - ReadOnlySpan ir_sp, - int regionCount + private static void GenerateConcat( + RegexAstNode node, + [NotNull] in SequenceEmitter sc, + in List> func_list ) { - int[] last_seen = new int[regionCount + 1]; - int[] extent = new int[regionCount + 1]; - Array.Fill(last_seen, -1); - for (int i = ir_sp.Length - 1; i >= 0; i--) + RegexAstNode left = node.Left; + RegexAstNode right = node.Right; + + if (left is not null) { - int id = ir_sp[i].Prefix.WorkingRegion; - if (last_seen[id] < 0) { - last_seen[id] = i; - } - extent[id] = last_seen[id] - i + 1; - } - return extent; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int FindRegionEnd( - ReadOnlySpan ir_sp - ) - { - return FindRegionEnd(ir_sp, ir_sp[0].Prefix.WorkingRegion); - } - - // exclusive bound of the mountain opened at ir_sp[0], sub peaks included - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int FindRegionEnd( - ReadOnlySpan ir_sp, - int level - ) - { - for (int end = ir_sp.Length - 1; end > 0; end--) - { - if (ir_sp[end].Prefix.WorkingRegion == level) { - return end + 1; - } - } - return 1; - } - - private static string FormatStringLiteral( - string str - ) - { - return SyntaxFactory.Literal(str).Text; - } - - private static string FormatCharLiteral( - char value - ) - { - return SyntaxFactory.Literal(value).Text; - } - - #endregion - - #region charset config - - internal enum CharSetType - { - - None, - Single, - Range, - Set - - } - - internal record class RegexCharSet( - string Value, - CharSetType Type, - bool Accept - ) - { - - public override string ToString() - { - return $"{{value:{Value}, type:{Type}, accept:{Accept}}}"; - } - - } - - private static RegexCharSet[] FigureCharSetType( - ReadOnlySpan source, - out bool accept - ) - { - // range or single - HashSet single_char = []; - if (source[0] == '[') - { - int idx; - accept = source[1] != '^'; - idx = accept ? 1 : 2; - List list = []; - ReadOnlySpan seq = source.Slice(idx, source.Length - idx - 1); - if (seq.IndexOf('-') == -1) + // find + if ((left.Label & RegexElementLabel.COUNT) == RegexElementLabel.COUNT && + (left.Right.Label & RegexElementLabel.UNIT_SINGLE) == RegexElementLabel.UNIT_SINGLE && + left.Right.Source.Source == "." && + right is not null) { - SplitSet(single_char, seq, list); - StringBuilder sb = new(); - foreach (char c in single_char) { - _ = sb.Append(c); + // TODO use find + GenerateMethodSource(right, sc, func_list, true); + } else if (IsNamedFind(node)) + { + if (right is not null) + { + // TODO named find + GenerateMethodSource(right, sc, func_list, true); + } else + { + // TODO pack rest of string to group } - list.Add(new(sb.ToString(), CharSetType.Single, true)); - return [.. list]; } else { - for (; seq.Length > 0; ) - { - int pos = seq.IndexOf('-'); - if (pos == -1) - { - // has no range - SplitSet(single_char, seq, list); - break; - } else if (pos > 1) - { - // has other sequence before range - SplitSet(single_char, seq[0..(pos - 1)], list); - } - list.Add(new($"{seq[pos - 1]}{seq[pos + 1]}", CharSetType.Range, true)); - seq = seq[(pos + 2)..]; + GenerateMethodSource(left, sc, func_list); + if (right is not null) { + GenerateMethodSource(right, sc, func_list); } - StringBuilder sb = new(); - foreach (char c in single_char) { - _ = sb.Append(c); - } - list.Add(new(sb.ToString(), CharSetType.Single, true)); - return [.. list]; } - } else if (source[0] == '\\') + } + if (right is not null) { + GenerateMethodSource(right, sc, func_list); + } + } + + private static void GenerateCount( + RegexAstNode node, + [NotNull] in SequenceEmitter sc, + in List> func_list + ) + { + RegexAstNode value = node.Left ?? node.Right; + if ((value.Label & RegexElementLabel.SEQUENCE) == 0) { - // set - accept = char.IsAsciiLetterUpper(source[0]); - return char.ToLower(source[0], CultureInfo.InvariantCulture) - switch - { - 's' => [new("s", CharSetType.Set, accept)], - 'w' => [new("w", CharSetType.Set, accept)], - 'd' => [new("d", CharSetType.Set, accept)], - _ => throw new NotSupportedException() - }; + GenerateMethodSource(value, sc, func_list); } else { - // unknown, throw - throw new InvalidCastException($"Unknown char set: {source}"); - } - - static void SplitSet( - HashSet single_char, - ReadOnlySpan seq, - List sets - ) - { - if (seq.Length == 0 || sets == null) { - return; - } - int pos; - ReadOnlySpan remain = seq; - while ((pos = remain.IndexOf('\\')) != -1 || remain.Length > 0) + RegexCountPayload payload = value.PayloadAs(); + int min = payload.Min; + int max = payload.Max; + bool is_greedy = payload.IsGreedy; + switch (value.Label & RegexElementLabel.SEQUENCE) { - if (pos != 0) - { - ReadOnlySpan singles = remain[0..pos]; - sets.Add(new RegexCharSet(singles.ToString(), CharSetType.Single, true)); - } - bool accept = char.IsAsciiLetterUpper(remain[pos + 1]); - - switch (char.ToLower(remain[pos + 1], CultureInfo.InvariantCulture)) - { - case 's': - sets.Add(new("s", CharSetType.Set, accept)); - break; - case 'w': - sets.Add(new("w", CharSetType.Set, accept)); - break; - case 'd': - sets.Add(new("d", CharSetType.Set, accept)); - break; - case '[' or '(' or ')' or ']' or '\\' or '\"': - _ = single_char.Add(remain[pos + 1]); - break; - default: - throw new InvalidCastException($"Unknown char or charset {remain[0..1]}"); - } - remain = remain[(pos + 2)..]; - } - if (remain.Length > 0) - { - foreach (char c in remain) { - _ = single_char.Add(c); - } + case RegexElementLabel.UNIT: + break; + case RegexElementLabel.UNIT_SET: + break; + case RegexElementLabel.UNIT_SPAN: + break; } } }