add base func of ast2code generator, entry to baseic verbs

src/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs

@@ -16,9 +16,8 @@ using System.Text;
 
 namespace StgSharp.RegularAnalysis.Abstraction
 {
-    public interface ITrinitySyntaxNode<TNode, TPayload, TLabel> : ISyntaxNode<TNode, TPayload, TLabel>
+    public interface ITrinitySyntaxNode<TNode, TLabel> : ISyntaxNode<TNode, TLabel>
-        where TNode : ISyntaxNode<TNode, TPayload, TLabel>
+        where TNode : ISyntaxNode<TNode, TLabel>
-        where TPayload : ISyntaxPayload<TPayload, TNode, TLabel>
         where TLabel : unmanaged
     {
 
@@ -26,12 +25,12 @@ namespace StgSharp.RegularAnalysis.Abstraction
 
     }
 
-    public interface ISyntaxNode<TNode, TPayload, TLabel>
+    public interface ISyntaxNode<TNode, TLabel> where TNode : ISyntaxNode<TNode, TLabel>
-        where TNode : ISyntaxNode<TNode, TPayload, TLabel>
-        where TPayload : ISyntaxPayload<TPayload, TNode, TLabel>
         where TLabel : unmanaged
     {
 
+        ISyntaxPayload<TLabel> Source { get; }
+
         long NodeFlag { get; }
 
         TLabel EqualityTypeConvert { get; }
@@ -48,8 +47,6 @@ namespace StgSharp.RegularAnalysis.Abstraction
 
         static abstract TNode Empty { get; }
 
-        TPayload Source { get; }
-
         protected internal int EnumState { get; set; }
 
         void AppendNode(

src/StgSharp.RegularAnalysis/Abstraction/ISyntaxPayload.cs

@@ -13,20 +13,16 @@ using System.Threading.Tasks;
 
 namespace StgSharp.RegularAnalysis.Abstraction
 {
-    // TODO 不对劲，TCarrier是个昏招，还是去掉吧
+    public interface ISyntaxPayload<TLabel> where TLabel : unmanaged
-    public interface ISyntaxPayload<TSelf, TCarrier, TLabel>
-        where TSelf : ISyntaxPayload<TSelf, TCarrier, TLabel>
-        where TCarrier : ISyntaxNode<TCarrier, TSelf, TLabel>
-        where TLabel : unmanaged
     {
 
-        int Column { get; set; }
+        int Column { get; }
 
-        int Row { get; set; }
+        int Row { get; }
 
-        string Source { get; set; }
+        string Source { get; }
 
-        TLabel Flag { get; set; }
+        TLabel Flag { get; }
 
     }
 }

src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs

@@ -15,7 +15,7 @@ using System.Threading.Tasks;
 
 namespace StgSharp.RegularAnalysis.Text
 {
-    public partial class RegexAnalyzer
+    public static partial class RegexAnalyzer
     {
 
         private static AbstractSyntaxTree<RegexAstNode, RegexElementLabel> GenerateAST(
@@ -50,10 +50,10 @@ namespace StgSharp.RegularAnalysis.Text
                     }
                     _stack.DecreaseDepth();
                     op = _stack.PopOperator();
-                    if (op.Source.Flag != RegexElementLabel.GROUP_BEGIN) {
+                    if (op.Label != RegexElementLabel.GROUP_BEGIN) {
                         throw new InvalidOperationException("Unmatched group begin and end symbol");
                     }
-                    if (op.Source.Value.Length > 1)
+                    if (op.Payload.Source.Length > 1)
                     {
                         op.Right = _stack.PopOperand();
                         _stack.PushOperand(op);
@@ -69,7 +69,7 @@ namespace StgSharp.RegularAnalysis.Text
                         // process operator by precedence
                         while (_stack.TryPeekOperator(out RegexAstNode topOp))
                         {
-                            (bool Valid, int Value) cmp = RegexAstNode.ComparePrecedence(topOp.Source.Flag, token.Flag);
+                            (bool Valid, int Value) cmp = RegexAstNode.ComparePrecedence(topOp.Label, token.Flag);
 
                             if (!cmp.Valid) {
                                 throw new InvalidOperationException("Operators with same precedence are not supported.");

src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTNodeOptimize.cs

@@ -10,7 +10,7 @@ using System;
 
 namespace StgSharp.RegularAnalysis.Text
 {
-    internal partial class RegexAnalyzer
+    public static partial class RegexAnalyzer
     {
 
         private static bool TryFlattenCount(

src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs

@@ -19,7 +19,7 @@ using System.Xml.Linq;
 
 namespace StgSharp.RegularAnalysis.Text
 {
-    public partial class RegexAnalyzer
+    public static partial class RegexAnalyzer
     {
 
         private static RegexAstNode FlattenAlt(
@@ -128,11 +128,11 @@ namespace StgSharp.RegularAnalysis.Text
                     offset += buffer[i].Length;
                 }
             });
-            root.Left = new RegexAstNode(new Token<RegexElementLabel>(str, left.Source.Line,
+            root.Left = new RegexAstNode(new RegexLiteralPayload(str, str, left.Payload.Row,
-                                                                      left.Source.Column,
+                                                                 left.Payload.Column,
-                                                                      str.Length == 1 ?
+                                                                 str.Length == 1 ?
-                                                                      RegexElementLabel.UNIT :
+                                                                 RegexElementLabel.UNIT :
-                                                                      RegexElementLabel.UNIT_SPAN));
+                                                                 RegexElementLabel.UNIT_SPAN));
             return true;
         }
 
@@ -262,15 +262,15 @@ namespace StgSharp.RegularAnalysis.Text
                     int level,
                     RegexAstNode node
             )
-                : base(node.Source)
+                : base(node.Payload)
             {
                 Level = level;
                 Cases = [];
 
                 RegexAstNode right = IsNullOrEmpty(node.Right) ?
                                      new RegexAstNode(
-                    new Token<RegexElementLabel>(node.Value, 0, node.Source.Column,
+                    new RegexLiteralPayload(node.Value, node.Value, node.Payload.Row,
-                                                 RegexElementLabel.UNIT)
+                                            node.Payload.Column, RegexElementLabel.UNIT)
                     ) :
                                      node.Right;
                 Cases.Add(right);
@@ -285,7 +285,7 @@ namespace StgSharp.RegularAnalysis.Text
                    int level,
                    List<RegexAstNode> c
             )
-                : base(c[0].Source)
+                : base(c[0].Payload)
             {
                 Cases = c;
                 Level = level;
@@ -312,7 +312,7 @@ namespace StgSharp.RegularAnalysis.Text
                     if ((Cases[i].EqualityTypeConvert & RegexElementLabel.SEQUENCE) != 0)
                     {
                         RegexAstNode concat = new(new Token<RegexElementLabel>(string.Empty, 0,
-                                                                               Cases[i].Source.Column,
+                                                                               Cases[i].Payload.Column,
                                                                                RegexElementLabel.CONCAT))
                         {
                             Left = Cases[i]
@@ -385,14 +385,14 @@ namespace StgSharp.RegularAnalysis.Text
                 if (IsNullOrEmpty(UnionEnd))
                 {
                     branch = new RegexAstNode(new Token<RegexElementLabel>(string.Empty, 0,
-                                                                           Source.Column,
+                                                                           Payload.Column,
                                                                            RegexElementLabel.ALT));
                     UnionBegin = branch;
                     UnionEnd = branch;
                 } else
                 {
                     branch = new RegexAstNode(new Token<RegexElementLabel>(string.Empty, 0,
-                                                                           UnionEnd.Source.Column,
+                                                                           UnionEnd.Payload.Column,
                                                                            RegexElementLabel.ALT));
                     UnionEnd.Right = branch;
                     branch.Parent = UnionEnd;
@@ -415,8 +415,8 @@ namespace StgSharp.RegularAnalysis.Text
             {
                 RegexAstNode right = IsNullOrEmpty(_case.Right) ?
                                      new RegexAstNode(
-                    new Token<RegexElementLabel>(_case.Value, 0, _case.Source.Column,
+                    new RegexLiteralPayload(_case.Value, _case.Value, _case.Payload.Row,
-                                                 RegexElementLabel.UNIT)
+                                            _case.Payload.Column, RegexElementLabel.UNIT)
                     ) :
                                      _case.Right;
                 Cases.Add(right);
@@ -432,14 +432,14 @@ namespace StgSharp.RegularAnalysis.Text
                 {
                     // then we suppose that alt has only one are same
                     return origin .EqualityTypeConvert == RegexElementLabel.CONCAT &&
-                           (origin.Left.EqualityTypeConvert & RegexElementLabel.SINGLE) != 0 &&
+                           (origin.Left.EqualityTypeConvert & RegexElementLabel.UNIT_SINGLE) != 0 &&
                            origin.Left.EqualityTypeConvert == alt.UnionEnd.Left.EqualityTypeConvert &&
                            origin.Left.Value == alt.UnionEnd.Left.Value;
                 } else
                 {
                     return origin.EqualityTypeConvert == RegexElementLabel.CONCAT &&
                            target.EqualityTypeConvert == RegexElementLabel.CONCAT &&
-                           (origin.Left.EqualityTypeConvert & RegexElementLabel.SINGLE) != 0 &&
+                           (origin.Left.EqualityTypeConvert & RegexElementLabel.UNIT_SINGLE) != 0 &&
                            origin.Left.EqualityTypeConvert == target.Left.EqualityTypeConvert &&
                            origin.Left.Value == target.Left.Value;
                 }

src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs

@@ -24,8 +24,6 @@ namespace StgSharp.RegularAnalysis.Text
         internal TextRegexSource(
                  string pattern,
                  AbstractSyntaxTree<RegexAstNode, RegexElementLabel> ast,
-                 List<RegexIR> base_ir,
-                 List<RegexIR> ir,
                  RegexInfo info,
                  Exception? analyseException = null
 
@@ -33,8 +31,6 @@ namespace StgSharp.RegularAnalysis.Text
         {
             Pattern = pattern;
             Ast = ast;
-            IRs = ir;
-            BaseIR = base_ir;
             AnalyseException = analyseException;
             Info = info;
         }
@@ -44,11 +40,6 @@ namespace StgSharp.RegularAnalysis.Text
         public string Pattern { get; }
 
         internal AbstractSyntaxTree<RegexAstNode, RegexElementLabel> Ast { get; }
-        /*
-        internal List<RegexIR> BaseIR { get; }
-
-        internal List<RegexIR> IRs { get; }
-        /**/
 
         internal RegexInfo Info { get; }
 
@@ -58,9 +49,7 @@ namespace StgSharp.RegularAnalysis.Text
     {
 
         public static TextRegexSource Analyze(
-                                      string regex,
+                                      string regex
-                                      bool require_ast = false,
-                                      bool require_origin_ir = false
         )
         {
 #pragma warning disable CA1031
@@ -72,10 +61,11 @@ namespace StgSharp.RegularAnalysis.Text
             {
                 tree = GenerateAST(regex);
                 OptimizeTree(tree);
+                return new TextRegexSource(regex, tree, info, null);
             }
             catch (Exception ex)
             {
-                return new TextRegexSource(regex, null, null, null, info, ex);
+                return new TextRegexSource(regex, null, info, ex);
             }
             /*
             // List<RegexIR> base_ir;

src/StgSharp.RegularAnalysis/Text/RegexAstNode.AdditionalPayload.cs

@@ -8,27 +8,323 @@
 using StgSharp.RegularAnalysis.Abstraction;
 using System;
 using System.Collections.Generic;
-using System.Linq;
+using System.Globalization;
 using System.Text;
-using System.Threading.Tasks;
 
 namespace StgSharp.RegularAnalysis.Text
 {
-    public class RegexCountPayload : ISyntaxPayload<RegexCountPayload, RegexAstNode, RegexElementLabel>
+    public abstract record RegexAstPayload(
+                           string Source,
+                           int Row,
+                           int Column,
+                           RegexElementLabel Flag
+    ) : ISyntaxPayload<RegexElementLabel>;
+
+    public sealed record RegexEmptyPayload() : RegexAstPayload(string.Empty, 0, 0,
+                                                               RegexElementLabel.NONE);
+
+    /// <summary>
+    ///   A deterministic literal sequence. Escapes represented as literals are consumed here.
+    /// </summary>
+    public sealed record RegexLiteralPayload(
+                         string Text,
+                         string Source,
+                         int Row,
+                         int Column,
+                         RegexElementLabel Flag
+    ) : RegexAstPayload(Source, Row, Column, Flag);
+
+    public enum RegexCharSetType
     {
 
-        public int Min { get; set; }
+        None,
-
+        Single,
-        public int Max { get; set; }
+        Range,
-
+        Set,
-        public string Source { get; set; }
+        Any
 
     }
 
-    public class RegexCommonPayload : IRegexAstNodeAdditionalPayload
+    public sealed record RegexCharSet(
+                         string Value,
+                         RegexCharSetType Type,
+                         bool Accept
+    );
+
+    /// <summary>
+    ///   A character-set atom expanded into the rules formerly produced by FigureCharSetType.
+    /// </summary>
+    public sealed record RegexCharSetPayload(
+                         IReadOnlyList<RegexCharSet> Set,
+                         bool Accept,
+                         string Source,
+                         int Row,
+                         int Column
+    ) : RegexAstPayload(Source, Row, Column, RegexElementLabel.UNIT_SET)
     {
 
-        public string Source { get; set; } = string.Empty;
+        public bool IsAny => Set.Count == 1 && Set[0].Type == RegexCharSetType.Any;
+
+    }
+
+    public sealed record RegexCountPayload(
+                         int Min,
+                         int Max,
+                         bool IsGreedy,
+                         string Source,
+                         int Row,
+                         int Column
+    ) : RegexAstPayload(Source, Row, Column, RegexElementLabel.COUNT)
+    {
+
+        public bool IsStar => Min == 0 && Max < 0;
+
+        public bool IsPlus => Min == 1 && Max < 0;
+
+    }
+
+    public sealed record RegexGroupPayload(
+                         string Name,
+                         bool IsCapturing,
+                         string Source,
+                         int Row,
+                         int Column
+    ) : RegexAstPayload(Source, Row, Column, RegexElementLabel.GROUP_BEGIN);
+
+    public sealed record RegexCommonPayload(
+                         string Source,
+                         int Row,
+                         int Column,
+                         RegexElementLabel Flag
+    ) : RegexAstPayload(Source, Row, Column, Flag)
+    {
+
+        public static RegexAstPayload FromToken(
+                                      Token<RegexElementLabel> token
+        )
+        {
+            return token.Flag switch
+            {
+                RegexElementLabel.UNIT or RegexElementLabel.UNIT_SPAN => new RegexLiteralPayload(DecodeLiteral(token.Value),
+                                                                                                 token.Value,
+                                                                                                 token.Line,
+                                                                                                 token.Column,
+                                                                                                 token.Flag),
+                RegexElementLabel.UNIT_SET => ParseCharSet(token),
+                RegexElementLabel.COUNT => ParseCount(token),
+                RegexElementLabel.GROUP_BEGIN => ParseGroup(token),
+                _ => new RegexCommonPayload(token.Value, token.Line, token.Column, token.Flag)
+            };
+        }
+
+        private static void AddEscape(
+                            char escaped,
+                            List<RegexCharSet> rules
+        )
+        {
+            bool ruleAccept = char.IsAsciiLetterLower(escaped);
+            char kind = char.ToLower(escaped, CultureInfo.InvariantCulture);
+            if (!IsNamedSet(escaped)) {
+                throw new InvalidCastException($"Unknown character set escape: \\{escaped}");
+            }
+            rules.Add(new(kind.ToString(), RegexCharSetType.Set, ruleAccept));
+        }
+
+        private static char DecodeEscapedChar(
+                            char value
+        )
+        {
+            return value switch
+            {
+                'n' or 'r' => throw new NotSupportedException(
+                    "Multi-line regular expressions are not supported yet."),
+                't' => '\t',
+                '0' => '\0',
+                _ => value
+            };
+        }
+
+        private static string DecodeLiteral(
+                              string source
+        )
+        {
+            if (source.Length != 2 || source[0] != '\\') {
+                return source;
+            }
+            return source[1] switch
+            {
+                'n' => "\n",
+                'r' => "\r",
+                't' => "\t",
+                '0' => "\0",
+                _ => source[1].ToString()
+            };
+        }
+
+        private static bool IsNamedSet(
+                            char value
+        )
+        {
+            return
+            char.ToLower(value, CultureInfo.InvariantCulture) is 's' or 'w' or 'd';
+        }
+
+        private static void ParseBracketContent(
+                            ReadOnlySpan<char> content,
+                            List<RegexCharSet> rules
+        )
+        {
+            HashSet<char> singles = [];
+            int index = 0;
+            while (index < content.Length)
+            {
+                if (content[index] is '\r' or '\n') {
+                    throw new NotSupportedException(
+                        "Multi-line regular expressions are not supported yet.");
+                }
+                if (content[index] == '\\')
+                {
+                    if (++index >= content.Length) {
+                        throw new InvalidCastException("Unclosed escape in character set.");
+                    }
+                    char escaped = content[index++];
+                    if (IsNamedSet(escaped))
+                    {
+                        AddEscape(escaped, rules);
+                    } else
+                    {
+                        _ = singles.Add(DecodeEscapedChar(escaped));
+                    }
+                    continue;
+                }
+
+                char first = content[index++];
+                if (index < content.Length - 0 &&
+                    content[index] == '-' &&
+                    index + 1 < content.Length)
+                {
+                    index++;
+                    char last;
+                    if (content[index] == '\\')
+                    {
+                        if (++index >= content.Length) {
+                            throw new InvalidCastException("Unclosed range in character set.");
+                        }
+                        last = DecodeEscapedChar(content[index++]);
+                    } else
+                    {
+                        last = content[index++];
+                    }
+                    if (last < first) {
+                        throw new InvalidCastException($"Invalid character range: {first}-{last}");
+                    }
+                    rules.Add(new(content[first..last].ToString(), RegexCharSetType.Range, true));
+                } else
+                {
+                    _ = singles.Add(first);
+                }
+            }
+
+            if (singles.Count > 0)
+            {
+                StringBuilder text = new(singles.Count);
+                foreach (char value in singles) {
+                    _ = text.Append(value);
+                }
+                rules.Add(new(text.ToString(), RegexCharSetType.Single, true));
+            }
+        }
+
+        private static RegexCharSetPayload ParseCharSet(
+                                           Token<RegexElementLabel> token
+        )
+        {
+            string source = token.Value;
+            List<RegexCharSet> rules = [];
+            bool accept = true;
+
+            if (source == ".")
+            {
+                rules.Add(new(string.Empty, RegexCharSetType.Any, true));
+            } else if (source.Length >= 2 && source[0] == '[' && source[^1] == ']')
+            {
+                int start = source.Length > 2 && source[1] == '^' ? 2 : 1;
+                accept = start == 1;
+                ParseBracketContent(source.AsSpan(start, source.Length - start - 1), rules);
+            } else if (source.Length == 2 && source[0] == '\\')
+            {
+                AddEscape(source[1], rules);
+            } else
+            {
+                throw new InvalidCastException($"Unknown char set: {source}");
+            }
+
+            return new RegexCharSetPayload(rules.ToArray(), accept, source, token.Line,
+                                           token.Column);
+        }
+
+        private static RegexCountPayload ParseCount(
+                                         Token<RegexElementLabel> token
+        )
+        {
+            string source = token.Value;
+            bool greedy = source.Length == 1 || source[^1] != '?';
+            ReadOnlySpan<char> count = greedy ?
+                                       source.AsSpan() :
+                                       source.AsSpan(0, source.Length - 1);
+            return count[0] switch
+            {
+                '*' => new RegexCountPayload(0, -1, greedy, source, token.Line, token.Column),
+                '+' => new RegexCountPayload(1, -1, greedy, source, token.Line, token.Column),
+                '?' => new RegexCountPayload(0, 1, greedy, source, token.Line, token.Column),
+                '{' => ParseRange(count, greedy, token),
+                _ => throw new InvalidOperationException($"Invalid count format: {source}")
+            };
+        }
+
+        private static RegexGroupPayload ParseGroup(
+                                         Token<RegexElementLabel> token
+        )
+        {
+            string source = token.Value;
+            bool named = source.StartsWith("(?<", StringComparison.Ordinal);
+            string name = named ? source[3..^1] : string.Empty;
+            return new RegexGroupPayload(name, source != "(?:", source, token.Line, token.Column);
+        }
+
+        private static RegexCountPayload ParseRange(
+                                         ReadOnlySpan<char> count,
+                                         bool greedy,
+                                         Token<RegexElementLabel> token
+        )
+        {
+            if (count.Length < 3 || count[^1] != '}') {
+                throw new InvalidOperationException($"Invalid count format: {token.Value}");
+            }
+            ReadOnlySpan<char> body = count[1..^1];
+            int comma = body.IndexOf(',');
+            if (comma < 0)
+            {
+                if (!int.TryParse(body, out int exact)) {
+                    throw new InvalidOperationException($"Invalid count format: {token.Value}");
+                }
+                return new RegexCountPayload(exact, exact, greedy, token.Value, token.Line,
+                                             token.Column);
+            }
+            if (!int.TryParse(body[..comma], out int min)) {
+                throw new InvalidOperationException($"Invalid count format: {token.Value}");
+            }
+            ReadOnlySpan<char> maxText = body[(comma + 1)..];
+            int max = maxText.IsEmpty ?
+                      -1 :
+                      int.TryParse(maxText, out int parsed) ?
+                      parsed :
+                      throw new InvalidOperationException($"Invalid count format: {token.Value}");
+            if (max >= 0 && max < min) {
+                throw new InvalidOperationException($"Invalid count range: {token.Value}");
+            }
+            return new RegexCountPayload(min, max, greedy, token.Value, token.Line, token.Column);
+        }
 
     }
 }

src/StgSharp.RegularAnalysis/Text/RegexAstNode.cs

@@ -19,14 +19,20 @@ namespace StgSharp.RegularAnalysis.Text
 
         public RegexAstNode(
                Token<RegexElementLabel> source
+        )
+            : this(RegexCommonPayload.FromToken(source))
+        { }
+
+        public RegexAstNode(
+               RegexAstPayload payload
         )
         {
-            Source = source;
+            Payload = payload;
         }
 
         public int EnumState { get; set; }
 
-        public long NodeFlag => (long)Source.Flag;
+        public long NodeFlag => (long)Label;
 
         public static RegexAstNode Empty { get; }
         = new EmptyRegexAstNode();
@@ -63,11 +69,24 @@ namespace StgSharp.RegularAnalysis.Text
             }
         }
 
-        public RegexElementLabel EqualityTypeConvert => Source.Flag;
+        public RegexElementLabel EqualityTypeConvert => Label;
 
-        public string Value => Source.Value;
+        public RegexElementLabel Label => Payload.Flag;
 
-        public Token<RegexElementLabel> Source { get; private protected set; }
+        public string Value => Payload switch
+        {
+            RegexLiteralPayload literal => literal.Text,
+            RegexCharSetPayload set => set.Source,
+            _ => Payload.Source
+        };
+
+        public RegexAstPayload Payload { get; private protected set; }
+
+        public ISyntaxPayload<RegexElementLabel> Source => Payload;
+
+        internal TPayload PayloadAs<TPayload>() where TPayload : RegexAstPayload =>
+            Payload as TPayload ?? throw new InvalidOperationException(
+                $"Node {Label} does not carry {typeof(TPayload).Name}.");
 
         public void AppendNode(
                     RegexAstNode nextToken
@@ -135,12 +154,12 @@ namespace StgSharp.RegularAnalysis.Text
         {
             if (useLeft)
             {
-                Source = Left.Source;
+                Payload = Left.Payload;
                 Left.Parent = Empty;
                 Left = Empty;
             } else
             {
-                Source = Right.Source;
+                Payload = Right.Payload;
                 Right.Parent = Empty;
                 Right = Empty;
             }
@@ -160,7 +179,7 @@ namespace StgSharp.RegularAnalysis.Text
                       RegexAstNode node
         )
         {
-            Source = node.Source;
+            Payload = node.Payload;
             Left = node.Left;
             Right = node.Right;
             if (!IsNullOrEmpty(Left)) {
@@ -186,7 +205,7 @@ namespace StgSharp.RegularAnalysis.Text
 
         {
 
-            public EmptyRegexAstNode() : base(Token<RegexElementLabel>.Empty) { }
+            public EmptyRegexAstNode() : base(new RegexEmptyPayload()) { }
 
         }
 

src/StgSharp.RegularAnalysis/Text/RegexElementLabel.cs

@@ -27,7 +27,7 @@ namespace StgSharp.RegularAnalysis.Text
         CONCAT = 64,
         ALT = 128,
         SEQUENCE = UNIT | UNIT_SPAN | UNIT_SET,
-        SINGLE = UNIT | UNIT_SET,
+        UNIT_SINGLE = UNIT | UNIT_SET,
         OPERATOR = COUNT | ALT | CONCAT,
         VAST_OPERATOR = OPERATOR | GROUP_BEGIN,
         ATOM_BEGIN = SEQUENCE | GROUP_BEGIN ,

src/StgSharp.RegularAnalysis/Text/RegexTokenReader.cs

@@ -37,6 +37,8 @@ namespace StgSharp.RegularAnalysis.Text
             int pos = position;
             switch (_source[pos])
             {
+                case '\r' or '\n':
+                    throw new NotSupportedException("Multi-line regular expressions are not supported yet.");
                 case '\\':
                     if (pos + 1 == _source.Length) {
                         throw new InvalidOperationException("Unclosed char");
@@ -85,6 +87,9 @@ namespace StgSharp.RegularAnalysis.Text
                         return new Token<RegexElementLabel>(".", 0, pos, RegexElementLabel.UNIT);
                     } else
                     {
+                        if (_source[pos + 1] is 'n' or 'r') {
+                            throw new NotSupportedException("Multi-line regular expressions are not supported yet.");
+                        }
                         position = pos + 2;
                         return new Token<RegexElementLabel>(_source[(pos)..(pos + 2)], 0, pos,
                                                             RegexElementLabel.UNIT_SET);
@@ -116,8 +121,12 @@ namespace StgSharp.RegularAnalysis.Text
                     {
                         if (_source[i] == '}')
                         {
-                            position = i + 1;
+                            int end = i + 1;
-                            return new Token<RegexElementLabel>(_source[(pos)..(i + 1)], 0, pos,
+                            if (end < _source.Length && _source[end] == '?') {
+                                end++;
+                            }
+                            position = end;
+                            return new Token<RegexElementLabel>(_source[pos..end], 0, pos,
                                                                 RegexElementLabel.COUNT);
                         }
                     }
@@ -127,8 +136,10 @@ namespace StgSharp.RegularAnalysis.Text
                     position = pos + count;
                     return new Token<RegexElementLabel>(_source[(pos)..(pos + count)], 0, pos, RegexElementLabel.COUNT);
                 case '?':
-                    position = pos + 1;
+                    int optionalCount = pos + 1 < _source.Length && _source[pos + 1] == '?' ? 2 : 1;
-                    return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.COUNT);
+                    position = pos + optionalCount;
+                    return new Token<RegexElementLabel>(_source[pos..(pos + optionalCount)], 0, pos,
+                                                        RegexElementLabel.COUNT);
                 case '.':
                     position = pos + 1;
                     return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.UNIT_SET);

src/StgSharp.RegularAnalysis/Text/TextRegex.cs

@@ -19,7 +19,7 @@ namespace StgSharp.RegularAnalysis.Text
     {
 
         private const int WordCategoriesMask =
-    1 << (int)UnicodeCategory.UppercaseLetter | (1 << (int)UnicodeCategory.LowercaseLetter) | (1 << (int)UnicodeCategory.TitlecaseLetter) | 1 << (int)UnicodeCategory.ModifierLetter | 1 << (int)UnicodeCategory.OtherLetter | 1 << (int)UnicodeCategory.NonSpacingMark | 1 << (int)UnicodeCategory.DecimalDigitNumber | 1 << (int)UnicodeCategory.ConnectorPunctuation;
+    (1 << (int)UnicodeCategory.UppercaseLetter) | (1 << (int)UnicodeCategory.LowercaseLetter) | (1 << (int)UnicodeCategory.TitlecaseLetter) | 1 << (int)UnicodeCategory.ModifierLetter | 1 << (int)UnicodeCategory.OtherLetter | 1 << (int)UnicodeCategory.NonSpacingMark | 1 << (int)UnicodeCategory.DecimalDigitNumber | 1 << (int)UnicodeCategory.ConnectorPunctuation;
 
         public abstract MatchResult Match(
                                     ReadOnlySpan<char> text
@@ -84,17 +84,17 @@ namespace StgSharp.RegularAnalysis.Text
         ///   <see langword="false" />.
         /// </returns>
         /// <remarks>
-        ///   <para>/// This method operates on a single <see cref="char" />, and therefore /// on a
+        ///   <para> /// This method operates on a single <see cref="char" />, and therefore /// on
-        ///   single UTF-16 code unit rather than on a complete Unicode scalar /// value. ///
+        ///   a single UTF-16 code unit rather than on a complete Unicode scalar /// value.
-        ///   </para> <para>/// Characters in the Unicode Basic Multilingual Plane are classified
+        ///   ///</para> <para> /// Characters in the Unicode Basic Multilingual Plane are
-        ///   /// normally. A supplementary-plane character is represented by a /// surrogate pair
+        ///   classified /// normally. A supplementary-plane character is represented by a ///
-        ///   and is therefore presented to this method as two /// separate surrogate code units.
+        ///   surrogate pair and is therefore presented to this method as two /// separate surrogate
-        ///   Neither surrogate is independently /// treated as a word character. This follows the
+        ///   code units. Neither surrogate is independently /// treated as a word character. This
-        ///   char-based behavior of /// the corresponding .NET Regex helper. /// </para> <para>///
+        ///   follows the char-based behavior of /// the corresponding .NET Regex helper. ///</para>
-        ///   For ASCII input, the Unicode-category lookup is avoided. In ASCII, /// the accepted
+        ///   <para> /// For ASCII input, the Unicode-category lookup is avoided. In ASCII, /// the
-        ///   set is exactly: /// <c> [A-Za-z0-9_] </c>. /// </para> <para>/// This method
+        ///   accepted set is exactly: /// <c> [A-Za-z0-9_] </c>. ///</para> <para> /// This method
         ///   implements ordinary .NET <c> \w </c> semantics. It does /// not implement the ASCII-
-        ///   oriented behavior enabled by /// <c> RegexOptions.ECMAScript </c>. /// </para>
+        ///   oriented behavior enabled by /// <c> RegexOptions.ECMAScript </c>. ///</para>
         /// </remarks>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         protected static bool IsCharWord(

src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs

@@ -10,6 +10,7 @@ using StgSharp.RegularAnalysis.Abstraction;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
@@ -18,443 +19,133 @@ namespace StgSharp.RegularAnalysis.Text
 {
     internal partial class TextRegexSourceGen
     {
-        /*
-         * ---------------------------------------------------------------------------------------
-         * NGRA text regex source generation backend
-         * ---------------------------------------------------------------------------------------
-         *
-         * The backend turns a fully prefixed IR stream into the body of one Match method. It is
-         * organised in layers, ordered by how many IR entries a single generator consumes:
-         *
-         *   L0  whole IR stream     method skeleton, slot declarations, entry guards
-         *   L1  several regions     ALT chain, group tail, complex count, complex find
-         *   L2  one region          the block a TRY lowers to
-         *   L3  two to eight IR     fixed windows worth collapsing into better code
-         *   L4  one IR              the fallback, one generator per RegexIRCommand
-         *   L5  zero IR             prefix decoration wrapped around L1 to L4 output
-         *
-         * File layout:
-         *
-         *   TextRegexSourceGen.GenClass.cs           this file, L0 and shared helpers
-         *   TextRegexSourceGen.GenClass.MultiIR.cs   L1, L2 and L3
-         *   TextRegexSourceGen.GenClass.SingleIR.cs  L4
-         *   TextRegexSourceGen.GenClass.Prefix.cs    L5
-         *   TextRegexSourceGen.GenContext.cs         variable naming and slot allocation
-         *
-         * Contract every generator honours:
-         *
-         *   1. A TryGenerateXxx returns how many IR entries it consumed, or -1 when the window
-         *      does not match its shape. It returns an entry count, never a character count or a
-         *      pattern length.
-         *   2. The dispatch chain offers a window to L1, then L2, then L3, then L4. Higher
-         *      consumption wins, so a shape spanning a whole region plus a tail is never split by
-         *      a smaller generator.
-         *   3. L2, L4, L5 and the required entries of L1 together are sufficient for correctness.
-         *      The flattening entries of L1 and all of L3 only improve the shape of what those
-         *      would already emit, so an unimplemented one must return -1 cleanly rather than
-         *      emit a partial result.
-         *   4. A prefix consumes no IR. L1 to L4 describe what an instruction does, L5 decides
-         *      when it runs and who reads the outcome.
-         *   5. Scope follows Prefix.WorkingRegion, not expression nesting.
-         *
-         * Control flow shape:
-         *
-         *   A region lowers to do { ... } while(false), and POP lowers to break. The goto plus
-         *   label scheme used by the .NET generated regex backend is deliberately not adopted.
-         *
-         *   A FIND lowers to a candidate retry loop nested inside the region block. That loop is
-         *   nearer than the region do block, so a POP inside it needs the __rN_done flag described
-         *   in TextRegexSourceGen.GenClass.MultiIR.cs rather than a plain break.
-         *
-         * Generated variable names are owned by SourceGenContext. No generator invents one.
-         *
-         * The per shape checklist this backend is rebuilt against lives in
-         * future/ngra-regex-sourcegen-goldens.md, and the handwritten target code for each shape
-         * lives in test/SingleFile/regex.cs.
-         */
 
-        // private const string current_ptr = "cur_ptr_global";                // global match cursor
+        internal static void GenerateMethodSource(
-        // private const string offset = "offset";                             // 
+                             RegexAstNode ast,
-        // private const string orig_text = "text";                            // 
+                             in SequenceEmitter<string> sc,
+                             in List<SequenceEmitter<string>> func_list,
+                             bool is_find = false
 
-        #region L0 method level
-
-        /// <summary>
-        ///   Generates one <c> Match </c> method from a fully prefixed IR stream.
-        /// </summary>
-        /// <remarks>
-        ///   Not implemented yet. Returns an empty emitter so the generator pipeline stays runnable
-        ///   while the backend is being rebuilt.
-        /// </remarks>
-        internal static SequenceEmitter<string> GenerateSource(
-                                                TextRegexSource source
         )
         {
-            //
+            RegexAstNode root = ast;
-            // L0, method skeleton
+            switch (root.Label & (RegexElementLabel.VAST_OPERATOR))
-            //
-            // TODO  L0.1  Emit the Match signature from ROS_char and SourceGenContext.InputSpan.
-            // TODO  L0.2  Emit region anchor and region result slots, count from RegexInfo.RegionCount.
-            // TODO  L0.3  Emit line result slots, only for lines an Observe_Line prefix refers to.
-            // TODO  L0.5  Emit the cursor, the step result and any scratch declaration the context collected.
-            //
-
-            /*
-             * Slot demand is only known once the body exists, so the body has to be generated
-             * into a detached emitter first and spliced in after the declaration block.
-             *
-             * The dispatch chain that walks the IR stream belongs here too. It offers each window
-             * to L1, L2, L3 then L4, and adds the returned consumption count to the cursor. An
-             * unclaimed window must produce a hard build break rather than silently emit nothing,
-             * because code that compiles but matches incorrectly is far harder to diagnose.
-             */
-
-            SequenceEmitter<string> se = new();
-            List<SequenceEmitter<string>> sub_match_list = [];
-            SourceGenContext sc = new();
-
-            RegexInfo info = source.Info;
-            List<RegexIR> ir = source.IRs;
-            ReadOnlySpan<RegexIR> ir_sp = CollectionsMarshal.AsSpan(ir);
-
-            SequenceEmitter<string> match_e = new();
-
-            sc.RegionExtent = MeasureRegionExtents(ir_sp, info.RegionCount);
-
-            // begin a very big do while(0)
-            _ = se.AppendLine(sc.BeginRegion());
-
-            _ = GenerateBlock(0, ir_sp, sc, match_e, sub_match_list);
-
-            _ = se.AppendLine($@"public override MatchResult Match({ROS_char} text)")//
-                  .AppendLine(@"{");
-
-            int min_predict_length = info.MinPredictLength;
-            if (min_predict_length > 0) {
-                _ = se.AppendLine($@"if (text.Length < {min_predict_length})")
-                      .AppendLine(@"{")
-                      .AppendLine(@"return default;")
-                      .AppendLine(@"}")
-                      .AppendLine();
-            }
-
-            int region_count = info.RegionCount;
-            for (int i = 0; i < region_count; i++)
             {
-                _ = se.AppendLine($@"int region_{i}_start = -1;")//
+                case RegexElementLabel.CONCAT:
-                      .AppendLine($@"int region_{i}_end = -1;")
+                    GenerateConcat(root, sc, func_list);
-                      .AppendLine($@"bool region_{i}_success;");
+                    break;
+                case RegexElementLabel.COUNT:
+                    break;
+                case RegexElementLabel.GROUP_BEGIN:
+                    break;
+                case RegexElementLabel.ALT:
+                    break;
             }
-            /*
-            _ = se.AppendLine($@"int {current_ptr} = 0;")
-                  .AppendLine($@"int {offset} = 0;")
-                  .AppendLine();
-            /**/
-
-            // _ = sc.GenVarDefine(se);
-
-            _ = se.Append(match_e);
-
-            // end of the biggest region
-            _ = se.AppendLine(sc.CloseRegion());
-
-            // close the entire match result and return 
-
-            foreach (SequenceEmitter<string> _sub in sub_match_list)
-            {
-                _ = se.AppendLine()                     //
-                      .Append(_sub);
-            }
-
-            _ = se.Append(@"}");
-
-            return se;
         }
 
-        /// <summary>
+        #region supporting method
-        ///   Recursive dispatch entry. Walks one contiguous IR block and offers each window to the
+
-        ///   generators in descending consumption order. Region bodies, count bodies and find
+        private static bool IsNamedFind(
-        ///   bodies recurse into this same method.
+                            RegexAstNode node
-        /// </summary>
-        /// <returns>
-        ///   Total IR entries consumed, always <c> ir_span.Length </c>.
-        /// </returns>
-        private static int GenerateBlock(
-                           int outer_level,
-                           ReadOnlySpan<RegexIR> ir_span,
-                           SourceGenContext sc,
-                           in SequenceEmitter<string> code,
-                           in List<SequenceEmitter<string>> local_func
         )
         {
-            int index = 0;
+            // (?<name>.*)abc
-            while (index < ir_span.Length)
+
+            if (node is null)
             {
-                ReadOnlySpan<RegexIR> rest = ir_span[index..];
+                return false;
-
-                // ground entries pass one by one, a foreign region id opens a mountain;
-                // carving always lands on a mountain head, so the extent is the carve length
-                int id = rest[0].Prefix.WorkingRegion;
-                int carve = id == outer_level ? 1 : sc.RegionExtent[id];
-                Debug.Assert(id == outer_level || carve == FindRegionEnd(rest));
-
-                index += GenerateRegion(outer_level, rest[..carve], sc, code, local_func);
             }
-            return index;
+            if ((node.Label & RegexElementLabel.CONCAT) != 0) {
-        }
+                return false;
-
-        /// <summary>
-        ///   Processes exactly one carved region, or one ground instruction. Generators that own a
-        ///   body re-enter <see cref="GenerateBlock" />, which is what closes the recursion.
-        /// </summary>
-        /// <returns>
-        ///   IR entries consumed, at least 1.
-        /// </returns>
-        private static int GenerateRegion(
-                           int outer_level,
-                           ReadOnlySpan<RegexIR> region,
-                           SourceGenContext sc,
-                           in SequenceEmitter<string> code,
-                           in List<SequenceEmitter<string>> local_func
-        )
-        {
-            int consumed;
-
-            // L1, several regions plus a tail
-            if ((consumed = TryGenerateAltChain(outer_level, region, sc, code, local_func)) > 0) {
-                return consumed;
             }
-            if ((consumed = TryGenerateNamedCountAny(outer_level, region, sc, code,
+            RegexAstNode group = node.Left;
-                                                     local_func)) > 0) {
+            if ((group.Label & RegexElementLabel.GROUP_BEGIN) == 0) {
-                return consumed;
+                return false;
             }
+            RegexAstNode count = group.Right;
 
-            // L2, one region
+            // .*
-            if ((consumed = TryGenerateTryBlock(outer_level, region, sc, code, local_func)) > 0) {
+            return (count.Label & RegexElementLabel.COUNT) != 0 &&
-                return consumed;
+                   (count.Right.Label & RegexElementLabel.UNIT_SINGLE) != 0 &&
-            }
+                   count.Right.Source.Source == ".";
-
-            // L4, single instruction fallback
-            if ((consumed = TryGenerateSingleIR(outer_level, region, sc, code, local_func)) > 0) {
-                return consumed;
-            }
-            if ((consumed = TryGeneratePack(outer_level, region, sc, code, local_func)) > 0) {
-                return consumed;
-            }
-
-            // unclaimed window must break the build of the generated file
-            _ = code.AppendLine($@"#error NGRA has no source generation rule for {region[0]}");
-            return 1;
         }
 
             #endregion
 
-        private enum RegionExitMethod
+        #region verb generate
-        {
 
-            Break,
+        private static void GenerateAlt(
-            Continue
+                            RegexAstNode node,
+                            [NotNull] in SequenceEmitter<string> sc,
+                            in List<SequenceEmitter<string>> func_list
+        ) { }
 
-        }
+        private static void GenerateConcat(
-
+                            RegexAstNode node,
-        #region shared helpers
+                            [NotNull] in SequenceEmitter<string> sc,
-
+                            in List<SequenceEmitter<string>> func_list
-        // backward scan; the final write per id happens at its first entry, leaving full extent
-        private static int[] MeasureRegionExtents(
-                             ReadOnlySpan<RegexIR> ir_sp,
-                             int regionCount
         )
         {
-            int[] last_seen = new int[regionCount + 1];
+            RegexAstNode left = node.Left;
-            int[] extent = new int[regionCount + 1];
+            RegexAstNode right = node.Right;
-            Array.Fill(last_seen, -1);
+
-            for (int i = ir_sp.Length - 1; i >= 0; i--)
+            if (left is not null)
             {
-                int id = ir_sp[i].Prefix.WorkingRegion;
+                // find
-                if (last_seen[id] < 0) {
+                if ((left.Label & RegexElementLabel.COUNT) == RegexElementLabel.COUNT &&
-                    last_seen[id] = i;
+                    (left.Right.Label & RegexElementLabel.UNIT_SINGLE) == RegexElementLabel.UNIT_SINGLE &&
-                }
+                    left.Right.Source.Source == "." &&
-                extent[id] = last_seen[id] - i + 1;
+                    right is not null)
-            }
-            return extent;
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int FindRegionEnd(
-                           ReadOnlySpan<RegexIR> ir_sp
-        )
-        {
-            return FindRegionEnd(ir_sp, ir_sp[0].Prefix.WorkingRegion);
-        }
-
-        // exclusive bound of the mountain opened at ir_sp[0], sub peaks included
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int FindRegionEnd(
-                           ReadOnlySpan<RegexIR> ir_sp,
-                           int level
-        )
-        {
-            for (int end = ir_sp.Length - 1; end > 0; end--)
-            {
-                if (ir_sp[end].Prefix.WorkingRegion == level) {
-                    return end + 1;
-                }
-            }
-            return 1;
-        }
-
-        private static string FormatStringLiteral(
-                              string str
-        )
-        {
-            return SyntaxFactory.Literal(str).Text;
-        }
-
-        private static string FormatCharLiteral(
-                              char value
-        )
-        {
-            return SyntaxFactory.Literal(value).Text;
-        }
-
-            #endregion
-
-        #region charset config
-
-        internal enum CharSetType
-        {
-
-            None,
-            Single,
-            Range,
-            Set
-
-        }
-
-        internal record class RegexCharSet(
-                              string Value,
-                              CharSetType Type,
-                              bool Accept
-        )
-        {
-
-            public override string ToString()
-            {
-                return $"{{value:{Value}, type:{Type}, accept:{Accept}}}";
-            }
-
-        }
-
-        private static RegexCharSet[] FigureCharSetType(
-                                      ReadOnlySpan<char> source,
-                                      out bool accept
-        )
-        {
-            // range or single
-            HashSet<char> single_char = [];
-            if (source[0] == '[')
-            {
-                int idx;
-                accept = source[1] != '^';
-                idx = accept ? 1 : 2;
-                List<RegexCharSet> list = [];
-                ReadOnlySpan<char> seq = source.Slice(idx, source.Length - idx - 1);
-                if (seq.IndexOf('-') == -1)
                 {
-                    SplitSet(single_char, seq, list);
+                    // TODO use find
-                    StringBuilder sb = new();
+                    GenerateMethodSource(right, sc, func_list, true);
-                    foreach (char c in single_char) {
+                } else if (IsNamedFind(node))
-                        _ = sb.Append(c);
+                {
+                    if (right is not null)
+                    {
+                        // TODO named find
+                        GenerateMethodSource(right, sc, func_list, true);
+                    } else
+                    {
+                        // TODO pack rest of string to group
                     }
-                    list.Add(new(sb.ToString(), CharSetType.Single, true));
-                    return [.. list];
                 } else
                 {
-                    for (; seq.Length > 0; )
+                    GenerateMethodSource(left, sc, func_list);
-                    {
+                    if (right is not null) {
-                        int pos = seq.IndexOf('-');
+                        GenerateMethodSource(right, sc, func_list);
-                        if (pos == -1)
-                        {
-                            // has no range
-                            SplitSet(single_char, seq, list);
-                            break;
-                        } else if (pos > 1)
-                        {
-                            // has other sequence before range
-                            SplitSet(single_char, seq[0..(pos - 1)], list);
-                        }
-                        list.Add(new($"{seq[pos - 1]}{seq[pos + 1]}", CharSetType.Range, true));
-                        seq = seq[(pos + 2)..];
                     }
-                    StringBuilder sb = new();
-                    foreach (char c in single_char) {
-                        _ = sb.Append(c);
-                    }
-                    list.Add(new(sb.ToString(), CharSetType.Single, true));
-                    return [.. list];
                 }
-            } else if (source[0] == '\\')
+            }
+            if (right is not null) {
+                GenerateMethodSource(right, sc, func_list);
+            }
+        }
+
+        private static void GenerateCount(
+                            RegexAstNode node,
+                            [NotNull] in SequenceEmitter<string> sc,
+                            in List<SequenceEmitter<string>> func_list
+        )
+        {
+            RegexAstNode value = node.Left ?? node.Right;
+            if ((value.Label & RegexElementLabel.SEQUENCE) == 0)
             {
-                // set
+                GenerateMethodSource(value, sc, func_list);
-                accept = char.IsAsciiLetterUpper(source[0]);
-                return char.ToLower(source[0], CultureInfo.InvariantCulture)
-                    switch
-                {
-                    's' => [new("s", CharSetType.Set, accept)],
-                    'w' => [new("w", CharSetType.Set, accept)],
-                    'd' => [new("d", CharSetType.Set, accept)],
-                    _ => throw new NotSupportedException()
-                };
             } else
             {
-                // unknown, throw
+                RegexCountPayload payload = value.PayloadAs<RegexCountPayload>();
-                throw new InvalidCastException($"Unknown char set: {source}");
+                int min = payload.Min;
-            }
+                int max = payload.Max;
-
+                bool is_greedy = payload.IsGreedy;
-            static void SplitSet(
+                switch (value.Label & RegexElementLabel.SEQUENCE)
-                        HashSet<char> single_char,
-                        ReadOnlySpan<char> seq,
-                        List<RegexCharSet> sets
-            )
-            {
-                if (seq.Length == 0 || sets == null) {
-                    return;
-                }
-                int pos;
-                ReadOnlySpan<char> remain = seq;
-                while ((pos = remain.IndexOf('\\')) != -1 || remain.Length > 0)
                 {
-                    if (pos != 0)
+                    case RegexElementLabel.UNIT:
-                    {
+                        break;
-                        ReadOnlySpan<char> singles = remain[0..pos];
+                    case RegexElementLabel.UNIT_SET:
-                        sets.Add(new RegexCharSet(singles.ToString(), CharSetType.Single, true));
+                        break;
-                    }
+                    case RegexElementLabel.UNIT_SPAN:
-                    bool accept = char.IsAsciiLetterUpper(remain[pos + 1]);
+                        break;
-
-                    switch (char.ToLower(remain[pos + 1], CultureInfo.InvariantCulture))
-                    {
-                        case 's':
-                            sets.Add(new("s", CharSetType.Set, accept));
-                            break;
-                        case 'w':
-                            sets.Add(new("w", CharSetType.Set, accept));
-                            break;
-                        case 'd':
-                            sets.Add(new("d", CharSetType.Set, accept));
-                            break;
-                        case '[' or '(' or ')' or ']' or '\\' or '\"':
-                            _ = single_char.Add(remain[pos + 1]);
-                            break;
-                        default:
-                            throw new InvalidCastException($"Unknown char or charset {remain[0..1]}");
-                    }
-                    remain = remain[(pos + 2)..];
-                }
-                if (remain.Length > 0)
-                {
-                    foreach (char c in remain) {
-                        _ = single_char.Add(c);
-                    }
                 }
             }
         }