From b9dab207529e36286f78579d47e3582bc5c07f24 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Fri, 17 Jul 2026 18:03:53 +0800 Subject: [PATCH] mainly theoratical work --- .../Text/TextRegex.cs | 117 ++++++++++++++++++ .../Text/TextRegexSourceGen.GenClass.cs | 32 +++-- 2 files changed, 141 insertions(+), 8 deletions(-) diff --git a/src/StgSharp.RegularAnalysis/Text/TextRegex.cs b/src/StgSharp.RegularAnalysis/Text/TextRegex.cs index 089991df..374c0ac0 100644 --- a/src/StgSharp.RegularAnalysis/Text/TextRegex.cs +++ b/src/StgSharp.RegularAnalysis/Text/TextRegex.cs @@ -7,7 +7,9 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; +using System.Runtime.CompilerServices; using System.Text; using System.Threading.Tasks; @@ -16,10 +18,125 @@ namespace StgSharp.RegularAnalysis.Text public abstract class TextRegex { + private const int WordCategoriesMask = + 1 << (int)UnicodeCategory.UppercaseLetter | (1 << (int)UnicodeCategory.LowercaseLetter) | (1 << (int)UnicodeCategory.TitlecaseLetter) | 1 << (int)UnicodeCategory.ModifierLetter | 1 << (int)UnicodeCategory.OtherLetter | 1 << (int)UnicodeCategory.NonSpacingMark | 1 << (int)UnicodeCategory.DecimalDigitNumber | 1 << (int)UnicodeCategory.ConnectorPunctuation; + public abstract MatchResult Match( ReadOnlySpan text ); + #region is char a word + + /* + * Semantic reference: + * + * .NET regular-expression word character: \w + * + * A character is treated as a word character when its Unicode general + * category is one of: + * + * Lu UppercaseLetter + * Ll LowercaseLetter + * Lt TitlecaseLetter + * Lm ModifierLetter + * Lo OtherLetter + * Mn NonSpacingMark + * Nd DecimalDigitNumber + * Pc ConnectorPunctuation + * + * The behavior and the ASCII-fast-path design are based on: + * + * dotnet/runtime + * System.Text.RegularExpressions.RegexCharClass.IsWordChar + * + * Source: + * https://github.com/dotnet/runtime/blob/main/src/libraries/ + * System.Text.RegularExpressions/src/System/Text/RegularExpressions/ + * RegexCharClass.cs + * + * dotnet/runtime is licensed under the MIT License. + * + * This implementation is written for NGRA rather than copied verbatim, + * but the source is recorded here because its intended semantics are + * explicitly compatible with .NET Regex. + */ + + /// + /// Bit mask containing every Unicode general category accepted by the default .NET + /// regular-expression \w character class. + /// + /// + /// values are small integer identifiers, so each category + /// can be represented by one bit. + /// + private const uint WordCategoryMask = + (1u << (int)UnicodeCategory.UppercaseLetter) | (1u << (int)UnicodeCategory.LowercaseLetter) | (1u << (int)UnicodeCategory.TitlecaseLetter) | (1u << (int)UnicodeCategory.ModifierLetter) | (1u << (int)UnicodeCategory.OtherLetter) | (1u << (int)UnicodeCategory.NonSpacingMark) | (1u << (int)UnicodeCategory.DecimalDigitNumber) | (1u << (int)UnicodeCategory.ConnectorPunctuation); + + /// + /// Determines whether one UTF-16 code unit belongs to the default .NET regular-expression + /// \w character class. + /// + /// + /// The UTF-16 code unit to classify. + /// + /// + /// when is a word character; otherwise, + /// . + /// + /// + /// /// This method operates on a single , and therefore /// on a + /// single UTF-16 code unit rather than on a complete Unicode scalar /// value. /// + /// /// Characters in the Unicode Basic Multilingual Plane are classified + /// /// normally. A supplementary-plane character is represented by a /// surrogate pair + /// and is therefore presented to this method as two /// separate surrogate code units. + /// Neither surrogate is independently /// treated as a word character. This follows the + /// char-based behavior of /// the corresponding .NET Regex helper. /// /// + /// For ASCII input, the Unicode-category lookup is avoided. In ASCII, /// the accepted + /// set is exactly: /// [A-Za-z0-9_] . /// /// This method + /// implements ordinary .NET \w semantics. It does /// not implement the ASCII- + /// oriented behavior enabled by /// RegexOptions.ECMAScript . /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected static bool IsCharWord( + char value + ) + { + /* + * ASCII fast path. + * + * Unsigned subtraction combines the lower-bound and upper-bound + * checks into one comparison: + * + * value in ['A', 'Z'] + * => value - 'A' is in [0, 25] + * + * Values below 'A' underflow after conversion to uint and therefore + * also fail the comparison. + */ + if (value <= '\x7F') + { + return + (uint)(value - 'A') <= 'Z' - 'A' || + (uint)(value - 'a') <= 'z' - 'a' || + (uint)(value - '0') <= '9' - '0' || + value == '_'; + } + + /* + * Non-ASCII path. + * + * CharUnicodeInfo classifies the current UTF-16 code unit using + * Unicode general categories. The resulting category selects one + * bit from WordCategoryMask. + */ + UnicodeCategory category = + CharUnicodeInfo.GetUnicodeCategory(value); + + return + (WordCategoryMask & (1u << (int)category)) != 0; + } + + #endregion } public ref struct MatchResult { } diff --git a/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs b/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs index a915af4d..0b4249da 100644 --- a/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs +++ b/src/StgSharp.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs @@ -5,6 +5,7 @@ // SPDX-License-Identifier: MIT // ----------------------------------------------------------------------------- +using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; using StgSharp.RegularAnalysis.Abstraction; using System; @@ -90,6 +91,20 @@ namespace StgSharp.RegularAnalysis.Text return se; } + private static string FormatStringLiteral( + string str + ) + { + return SyntaxFactory.Literal(str).Text; + } + + private static string FormatStringLiteral( + char str + ) + { + return SyntaxFactory.Literal(str).Text; + } + #region sub generator private static int TryGenerateFindSeq( @@ -122,7 +137,7 @@ namespace StgSharp.RegularAnalysis.Text if (scan_win[0] is not RegexFindSeqIR seq) { return -1; } - string pattern = seq.Pattern; + string pattern = FormatStringLiteral(seq.Pattern); int length = pattern.Length; _ = se.AppendLine() @@ -189,13 +204,14 @@ namespace StgSharp.RegularAnalysis.Text { // Handle single character case CharSetType.Single: + string literal = FormatStringLiteral(c.Value); caller = first ? "IndexOfAny" : "LastIndexOfAny"; - return $@"({offset} = {remain}.{caller}(""{c.Value}"") != -1)"; + return $@"({offset} = {remain}.{caller}(""{literal}"") != -1)"; // Handle range of characters case CharSetType.Range: caller = first ? "IndexOfAnyInRange" : "LastIndexOfAnyInRange"; - return $@"({offset} = {remain}.{caller}('{c.Value[0]}', '{c.Value[1]}') != -1)"; + return $@"({offset} = {remain}.{caller}('{FormatStringLiteral(c.Value[0])}', '{FormatStringLiteral(c.Value[1])}') != -1)"; // Handle set of characters case CharSetType.Set: @@ -384,7 +400,7 @@ namespace StgSharp.RegularAnalysis.Text ReadOnlySpan seq = source.Slice(idx, source.Length - idx - 1); if (seq.IndexOf('-') == -1) { - return [ new RegexCharSet(seq.ToString(), CharSetType.Single, true) ]; + return [new RegexCharSet(seq.ToString(), CharSetType.Single, true)]; } else { List list = []; @@ -404,7 +420,7 @@ namespace StgSharp.RegularAnalysis.Text list.Add(new($"{seq[pos - 1]}{seq[pos + 1]}", CharSetType.Range, true)); seq = seq[(pos + 2)..]; } - return [ .. list ]; + return [.. list]; } } else if (source[0] == '\\') { @@ -413,9 +429,9 @@ namespace StgSharp.RegularAnalysis.Text return char.ToLower(source[0], CultureInfo.InvariantCulture) switch { - 's' => [ new("s", CharSetType.Set, accept) ], - 'w' => [ new("w", CharSetType.Set, accept) ], - 'd' => [ new("d", CharSetType.Set, accept) ], + 's' => [new("s", CharSetType.Set, accept)], + 'w' => [new("w", CharSetType.Set, accept)], + 'd' => [new("d", CharSetType.Set, accept)], _ => throw new NotSupportedException() }; } else