text regex source gen framework is briefly done. Going on ir2code

2026-07-14 17:44:29 +08:00
parent bc90e340df
commit 4d2a4eeb18
32 changed files with 276 additions and 140 deletions
@@ -3,9 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.14.36705.20
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GenerateGL", "StgSharp.GenerateGL\GenerateGL.csproj", "{9FD27EE6-E94D-4E64-A13B-5A349DEA215A}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GenerateGL", "test\StgSharp.GenerateGL\GenerateGL.csproj", "{9FD27EE6-E94D-4E64-A13B-5A349DEA215A}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StgSharp.Common", "StgSharp.Common\StgSharp.Common.csproj", "{5D646C0E-CCF8-B5B4-7DE8-1C37542E7B4D}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StgSharp.Common", "src\StgSharp.Common\StgSharp.Common.csproj", "{5D646C0E-CCF8-B5B4-7DE8-1C37542E7B4D}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -48,6 +48,15 @@ namespace StgSharp.RegularAnalysis.Abstraction
        private readonly Node _head;
        private Node _tail;

+        private SequenceEmitter(
+                Node node
+        )
+        {
+            _isRoot = true;
+            _head = node;
+            _tail = _head;
+        }
+
        public SequenceEmitter()
        {
            _isRoot = true;
@@ -57,8 +66,6 @@ namespace StgSharp.RegularAnalysis.Abstraction

        public int Count { get; private set; }

-        private static bool IsString { get; } = typeof(T) == typeof(string);
-
        public SequenceEmitter<T> Append(
                                  T unit
        )
@@ -92,6 +99,16 @@ namespace StgSharp.RegularAnalysis.Abstraction
            return this;
        }

+        public SequenceEmitter<T> Clone()
+        {
+            if (!_isRoot) {
+                throw new InvalidOperationException("This emitter has been appended to another emitter. It is now readonly.");
+            }
+            SequenceEmitter<T> clone = new();
+            Node current = _head;
+            return clone;
+        }
+
        public Enumerator GetEnumerator()
        {
            return _isRoot ?
@@ -173,6 +190,18 @@ namespace StgSharp.RegularAnalysis.Abstraction
        private sealed class Node
        {

+            public Node()
+            {
+                IRList = [];
+            }
+
+            public Node(
+                   int capacity
+            )
+            {
+                IRList = new(capacity);
+            }
+
            public T this[
                     int index
            ]
@@ -185,7 +214,7 @@ namespace StgSharp.RegularAnalysis.Abstraction

            public int Count => IRList.Count;

-            public List<T> IRList { get; } = [];
+            public List<T> IRList { get; }

            [MethodImpl(MethodImplOptions.AggressiveInlining)]
            public void Add(
@@ -195,6 +224,13 @@ namespace StgSharp.RegularAnalysis.Abstraction
                IRList.Add(ir);
            }

+            public void AddRange(
+                        IEnumerable<T> irs
+            )
+            {
+                IRList.AddRange(irs);
+            }
+
        }

    }
@@ -1,105 +0,0 @@
-// -----------------------------------------------------------------------------
-// file="RegexAnalyzer.SourceGen.CharSetEmitter"
-// Project: StgSharp
-// Copyright (c) Nitload.
-// SPDX-License-Identifier: MIT
-// -----------------------------------------------------------------------------
-
-using System;
-using System.Collections.Generic;
-using System.Globalization;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace StgSharp.RegularAnalysis.Text
-{
-    internal enum CharSetType
-    {
-
-        None,
-        Single,
-        Range,
-        Set
-
-    }
-
-    internal record struct RegexCharSet(
-                           string value,
-                           CharSetType t,
-                           bool accept
-    )
-    {
-
-        public bool IsAcceptable { get; set; } = accept;
-
-        public CharSetType Type { get; set; } = t;
-
-        public string Value { get; set; } = value;
-
-        public override readonly string ToString()
-        {
-            return $"{{value:{value}, type:{Type}, accept:{IsAcceptable}}}";
-        }
-
-    }
-
-    internal static partial class RegexAnalyzerBackEnd
-    {
-
-        public static RegexCharSet[] FigureCharSetType(
-                                     ReadOnlySpan<char> source
-        )
-        {
-            // range or single
-            bool accept;
-            if (source[0] == '[')
-            {
-                int idx;
-                accept = source[1] != '^';
-                idx = accept ? 1 : 2;
-                ReadOnlySpan<char> seq = source.Slice(idx, source.Length - idx - 1);
-                if (seq.IndexOf('-') == -1)
-                {
-                    return [new RegexCharSet(seq.ToString(), CharSetType.Single, accept)];
-                } else
-                {
-                    List<RegexCharSet> list = [];
-                    for (; seq.Length > 0; )
-                    {
-                        int pos = seq.IndexOf('-');
-                        if (pos == -1)
-                        {
-                            RegexCharSet single = new(seq.ToString(), CharSetType.Single, accept);
-                            list.Add(single);
-                            break;
-                        } else
-                        if (pos > 1) {
-                            list.Add(new(seq[0..(pos - 1)].ToString(), CharSetType.Single, accept));
-                        }
-                        list.Add(new($"{seq[pos - 1]}{seq[pos + 1]}", CharSetType.Range, accept));
-                        seq = seq[(pos + 2)..];
-                    }
-                    return [.. list];
-                }
-            } else if (source[0] == '\\')
-            {
-                // set
-                accept = char.IsAsciiLetterUpper(source[0]);
-                return char.ToLower(source[0], CultureInfo.InvariantCulture)
-                switch
-                {
-                    's' => [new("s", CharSetType.Set, accept)],
-                    'w' => [new("w", CharSetType.Set, accept)],
-                    'd' => [new("d", CharSetType.Set, accept)],
-                    _ => throw new NotSupportedException()
-                };
-            } else
-            {
-                // unknown, throw
-                throw new InvalidCastException($"Unknown char set: {source}");
-            }
-        }
-
-    }
-}
@@ -24,6 +24,13 @@ namespace StgSharp.RegularAnalysis.Text
                                                                              category:"NGRA.TextRegex",
                                                                              isEnabledByDefault:true);

+        private static readonly DiagnosticDescriptor CompileFailDiag = new(id:"NGRA005",
+                                                                           title:"Unexpected exception when compiling",
+                                                                           messageFormat:"Unexpected exception handles when compiling expression \"{0}\":{1}",
+                                                                           defaultSeverity:DiagnosticSeverity.Error,
+                                                                           category:"NGRA.TextRegex",
+                                                                           isEnabledByDefault:true);
+
        private static readonly DiagnosticDescriptor IncorrectContainingTypeDiag = new(id:"NGRA005",
                                                                                       title:$"Containing type can only be struct or class",
                                                                                       messageFormat:$"Containing type: {{0}} is a {{1}}, but not class or struct",
@@ -1,26 +1,26 @@
 // -----------------------------------------------------------------------------
-// file="RegexAnalyzer.SourceGen"
+// file="TextRegexSourceGen.GenClass"
 // Project: StgSharp
 // Copyright (c) Nitload.
 // SPDX-License-Identifier: MIT
 // -----------------------------------------------------------------------------

-using Microsoft.CodeAnalysis;
 using StgSharp.RegularAnalysis.Abstraction;
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Text;
 using System.Threading.Tasks;

 namespace StgSharp.RegularAnalysis.Text
 {
-    public partial class RegexAnalyzer
+    internal partial class TextRegexSourceGen
    {

-        internal static SequenceEmitter<string> GenerateSource(TextRegexSource source)
+        internal static SequenceEmitter<string> GenerateSource(
+                                                TextRegexSource source
+        )
        {
            SequenceEmitter<string> se = new();
            List<SequenceEmitter<string>> sub_match_list = [];
@@ -33,7 +33,7 @@ namespace StgSharp.RegularAnalysis.Text

            const string current_ptr = "current_ptr";

-            _ = se.AppendLine(@"public override MatchResult Match(ReadOnlySpan<char> text)")//
+            _ = se.AppendLine($@"public override MatchResult Match({ROS_char} text)")//
                  .AppendLine(@"{");

            int region_count = info.RegionCount;
@@ -51,6 +51,7 @@ namespace StgSharp.RegularAnalysis.Text
            {
                int count = int.Min(8, ir_sp.Length - index);
                ir_sp.Slice(index, count).CopyTo(scan_win);
+                if (TryGenerateMatch(scan_win, se) > 0) { }
            } while (index < ir.Count);

            foreach (SequenceEmitter<string> _sub in sub_match_list)
@@ -63,14 +64,17 @@ namespace StgSharp.RegularAnalysis.Text

            return se;
            /**/
-            int TryGenerateMatch(Span<RegexIR> scan_win, SequenceEmitter<string> sb)
+            int TryGenerateMatch(
+                Span<RegexIR> scan_win,
+                SequenceEmitter<string> sb
+            )
            {
                if (scan_win[0] is not RegexMatchSeqIR seq) {
                    return -1;
                }
                string pattern = seq.Pattern;
                int length = pattern.Length;
-                _ = sb.Append(string.Empty)
+                _ = sb.AppendLine()
                      .AppendLine($@"if(text[{current_ptr}..({current_ptr} + {length})] == {pattern})")
                      .AppendLine($@"{{")
                      .AppendLine($@"{current_ptr} += {length};")
@@ -83,4 +87,3 @@ namespace StgSharp.RegularAnalysis.Text

    }
 }
-
@@ -0,0 +1,133 @@
+// -----------------------------------------------------------------------------
+// file="TextRegexSourceGen.GenFile"
+// Project: StgSharp
+// Copyright (c) Nitload.
+// SPDX-License-Identifier: MIT
+// -----------------------------------------------------------------------------
+
+using Microsoft.CodeAnalysis;
+using StgSharp.RegularAnalysis.Abstraction;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StgSharp.RegularAnalysis.Text
+{
+    internal partial class TextRegexSourceGen
+    {
+
+        public const string ROS_char = "global::System.ReadOnlySpan<char>";
+        public const string Text_Regex = "global::StgSharp.RegularAnalysis.Text.TextRegex";
+
+        private static readonly SymbolDisplayFormat TypeSourceFormat =
+            SymbolDisplayFormat.FullyQualifiedFormat
+                               .AddMiscellaneousOptions(
+                                    SymbolDisplayMiscellaneousOptions.IncludeNullableReferenceTypeModifier);
+
+        private static string AsSource<T>()
+        {
+            Type t = typeof(T);
+            return t.IsGenericType ? throw new InvalidOperationException() : $"global::{t.FullName}";
+        }
+
+        private static void GenerateRegexSourceFile(
+                            SourceProductionContext context,
+                            TextRegexBinding binding
+        )
+        {
+            StringBuilder file_name_sb = new();
+            SequenceEmitter<string> ce = new();
+            SourceRecord gen_source = binding.Source;
+            TypeSource type = binding.Source.ContainingType;
+
+            _ = file_name_sb.Append(CultureInfo.InvariantCulture, $"{gen_source.NamespaceName}.");
+            _ = ce.AppendLine($@"// <auto-generated/>")
+                  .AppendLine($@"#nullable enable")
+                  .AppendLine($@"using StgSharp.RegularAnalysis.Text;")
+                  .AppendLine($@"using StgSharp.RegularAnalysis.Abstraction;")
+                  .AppendLine($@"using System;")
+                  .AppendLine($@"using System.Text;")
+                  .AppendLine($@"using System.Threading.Tasks;")
+                  .AppendLine()
+                  .AppendLine($@"namespace {gen_source.NamespaceName};");
+
+            string static_modifier;
+
+            int brace_count = 0;
+            do
+            {
+                _ = file_name_sb.Append(CultureInfo.InvariantCulture, $@"{type.TypeName}.");
+                string declare_type = type.DeclType switch
+                {
+                    TypeKind.Class => "class",
+                    TypeKind.Struct => "struct",
+                    _ => throw new InvalidOperationException($"Unexpected type kind: {type.DeclType}"),
+                };
+                static_modifier = type.IsStatic ? "static" : string.Empty;
+                _ = ce.AppendLine($@"{ToKeyword(type.Accessibility)} {static_modifier} partial {declare_type} {type.TypeName}")
+                      .AppendLine("{");
+                brace_count++;
+                type = type.InnerType!;
+            } while (type is not null);
+
+            static_modifier = gen_source.IsMethodStatic ? "static" : string.Empty;
+            _ = ce.AppendLine(@$"{ToKeyword(gen_source.MethodAccessibility)} {static_modifier} {Text_Regex} {gen_source.MethodName}()")
+                  .AppendLine("{")
+                  /**/ //TODO return single instance of the TextRegex object
+                  .AppendLine("}");
+
+            // TODO inject method source code here
+
+            for (int i = 0; i < brace_count; i++)
+            {
+                _ = ce.AppendLine("}");
+            }
+
+            StringBuilder code_source = new();
+            brace_count = 0;
+            foreach (string line in ce)
+            {
+                bool is_right_brace = line == "}";
+                brace_count -= is_right_brace ? 1 : 0;
+                for (int i = 0; i < brace_count; i++) {
+                    _ = code_source.Append("    ");
+                }
+                _ = code_source.AppendLine(line);
+                brace_count += is_right_brace ? 0 : (line == "{" ? 1 : 0);
+            }
+
+            _ = file_name_sb.Append(CultureInfo.InvariantCulture, $@"{binding.Source.MethodName}.g.cs");
+            string file_name = file_name_sb.ToString();
+            context.AddSource(file_name, code_source.ToString());
+
+            foreach (Diagnostic diag in binding.RegexDiagnostic) {
+                context.ReportDiagnostic(diag);
+            }
+        }
+
+        private static string ToKeyword(
+                              Accessibility accessibility
+        )
+        {
+            return accessibility switch
+            {
+                Accessibility.Public => "public",
+                Accessibility.Private => "private",
+                Accessibility.Internal => "internal",
+                Accessibility.Protected => "protected",
+
+                Accessibility.ProtectedAndInternal => "private protected",
+
+                Accessibility.ProtectedOrInternal => "protected internal",
+
+                _ => throw new InvalidOperationException(
+                    $"Unexpected accessibility: {accessibility}"),
+            };
+        }
+
+    }
+}
@@ -7,6 +7,7 @@

 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
+using StgSharp.RegularAnalysis.Abstraction;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -18,19 +19,21 @@ namespace StgSharp.RegularAnalysis.Text
    internal partial class TextRegexSourceGen
    {

-        private record struct SourceRecord(
+        private readonly record struct SourceRecord(

-                              bool IsClassStatic,
-                              bool IsMethodStatic,
-                              string Pattern,
-                              string MethodName,
-                              string NamespaceName,
-                              Accessibility MethodAccessibility,
-                              TypeSource ContainingType
+                                       bool IsClassStatic,
+                                       bool IsMethodStatic,
+                                       string Pattern,
+                                       string MethodName,
+                                       string NamespaceName,
+                                       Accessibility MethodAccessibility,
+                                       TypeSource ContainingType,
+                                       Location PatternDefineLocation
        );

        private sealed record TypeSource(
                              bool IsStatic,
+                              string TypeDisplayName,
                              string TypeName,
                              TypeKind DeclType,
                              Accessibility Accessibility,
@@ -46,6 +49,8 @@ namespace StgSharp.RegularAnalysis.Text

            public bool IsMethodStatic { get; set; }

+            public Location PatternDefineLocation { get; set; }
+
            public string NameSpaceName { get; set; }

            public string Pattern { get; set; }
@@ -57,7 +62,8 @@ namespace StgSharp.RegularAnalysis.Text
            public readonly SourceRecord AsRecord()
            {
                return new SourceRecord(IsClassStatic, IsMethodStatic, Pattern, MethodName,
-                                        NameSpaceName, MethodAccessibility, ContainingType);
+                                        NameSpaceName, MethodAccessibility, ContainingType,
+                                        PatternDefineLocation);
            }

        }
@@ -67,10 +73,28 @@ namespace StgSharp.RegularAnalysis.Text

            private readonly List<Diagnostic> _diagnostics = [];

+            private TextRegexBinding(
+                    List<Diagnostic> diagnostics,
+                    SequenceEmitter<string> generatedSource,
+                    SourceRecord source
+            )
+            {
+                _diagnostics = diagnostics ?? [];
+                GeneratedSource = generatedSource ?? new();
+                Source = source;
+            }
+
+            public TextRegexBinding()
+            {
+                _diagnostics = [];
+            }
+
            public bool IsValid { get; private set; } = true;

            public IReadOnlyList<Diagnostic> RegexDiagnostic => _diagnostics;

+            public SequenceEmitter<string> GeneratedSource { get; set; }
+
            public SourceRecord Source { get; set; }

            public void AddDiagnostic(
@@ -89,16 +113,15 @@ namespace StgSharp.RegularAnalysis.Text
            {
                Diagnostic diag = Diagnostic.Create(descriptor, location, messageArgs);
                _diagnostics.Add(diag);
-                IsValid = IsValid && (diag.DefaultSeverity != DiagnosticSeverity.Error);
+                IsValid = IsValid && (diag.Severity != DiagnosticSeverity.Error);
            }

            public TextRegexBinding Clone()
            {
-                TextRegexBinding binding = new()
-                {
-                    IsValid = this.IsValid,
-                };
-                binding._diagnostics.AddRange(this._diagnostics);
+                SequenceEmitter<string> code = GeneratedSource.Clone();
+                List<Diagnostic> diag = new(_diagnostics.Count);
+                diag.AddRange(this._diagnostics);
+                TextRegexBinding binding = new(diag, code, Source with { });
                return binding;
            }

@@ -8,10 +8,12 @@
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
+using StgSharp.RegularAnalysis.Abstraction;
 using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Reflection;
+using System.Security.AccessControl;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -21,7 +23,7 @@ namespace StgSharp.RegularAnalysis.Text
    internal partial class TextRegexSourceGen : IIncrementalGenerator
    {

-        private static readonly string TextRegexTypeName = typeof(TextRegex).FullName ??
+        private static readonly string TextRegexTypeName = typeof(TextRegexAttribute).FullName ??
            string.Empty;

        public void Initialize(
@@ -37,6 +39,12 @@ namespace StgSharp.RegularAnalysis.Text
                ) => true, BindTextRegexCandidate);

            IncrementalValuesProvider<TextRegexBinding> match = RegisterDiagnosticBypass(context, provider);
+
+            IncrementalValuesProvider<TextRegexBinding> compile = match.Select(CompileRegexMethodSource);
+
+            IncrementalValuesProvider<TextRegexBinding> compile_success = RegisterDiagnosticBypass(context, compile);
+
+            context.RegisterSourceOutput(compile_success, GenerateRegexSourceFile);
        }

        private static TextRegexBinding BindTextRegexCandidate(
@@ -84,7 +92,7 @@ namespace StgSharp.RegularAnalysis.Text

            INamedTypeSymbol contain = method_symbol.ContainingType;
            INamespaceSymbol namespace_symbol = contain.ContainingNamespace;
-            TypeSource type_source = null;
+            TypeSource type_source = null!;

            while (contain is not null)
            {
@@ -98,22 +106,53 @@ namespace StgSharp.RegularAnalysis.Text
                if (!bind.IsValid) {
                    return bind;
                }
-                TypeSource t = new(contain.IsStatic, contain.Name, contain.TypeKind,
+                string display = contain.ToDisplayString(TypeSourceFormat);
+                TypeSource t = new(contain.IsStatic, display, contain.Name, contain.TypeKind,
                                   contain.DeclaredAccessibility, type_source);
                type_source = t;
-                contain = contain.BaseType!;
+                contain = contain.ContainingType!;
            }
            Source s = new()
            {
                MethodName = method_symbol.Name,
-                NameSpaceName = namespace_symbol.Name,
+                NameSpaceName = namespace_symbol.ToDisplayString(),
+                PatternDefineLocation = method_decl.GetLocation(),
+                IsMethodStatic = method_decl.Modifiers.Any(SyntaxKind.StaticKeyword),
+                ContainingType = type_source,
+                MethodAccessibility = method_symbol.DeclaredAccessibility,
            };
-
+            foreach (KeyValuePair<string, TypedConstant> item in attribute.NamedArguments)
+            {
+                if (item.Key == nameof(TextRegexAttribute.Pattern)) {
+                    s.Pattern = item.Value.Value as string ?? string.Empty;
+                }
+            }

            bind.Source = s.AsRecord();
            return bind;
        }

+        private static TextRegexBinding CompileRegexMethodSource(
+                                        TextRegexBinding source,
+                                        CancellationToken ct
+        )
+        {
+            ct.ThrowIfCancellationRequested();
+            TextRegexBinding result = source.Clone();
+            string pattern = result.Source.Pattern;
+            TextRegexSource analyze_result = RegexAnalyzer.Analyze(pattern);
+            if (analyze_result.AnalyseException is not null)
+            {
+                result.AddDiagnostic(CompileFailDiag, null, analyze_result.AnalyseException.Message);
+            } else
+            {
+                SequenceEmitter<string> method_source = GenerateSource(analyze_result);
+                result.GeneratedSource = method_source;
+            }
+
+            return result;
+        }
+
        private static IncrementalValuesProvider<TextRegexBinding> RegisterDiagnosticBypass(
                                                                   IncrementalGeneratorInitializationContext context,
                                                                   IncrementalValuesProvider<TextRegexBinding> bindings