mainly theoratical work
This commit is contained in:
Youmu Liu
@@ -7,7 +7,9 @@
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
@@ -16,10 +18,125 @@ namespace StgSharp.RegularAnalysis.Text
|
||||
public abstract class TextRegex
|
||||
{
|
||||
|
||||
private const int WordCategoriesMask =
|
||||
1 << (int)UnicodeCategory.UppercaseLetter | (1 << (int)UnicodeCategory.LowercaseLetter) | (1 << (int)UnicodeCategory.TitlecaseLetter) | 1 << (int)UnicodeCategory.ModifierLetter | 1 << (int)UnicodeCategory.OtherLetter | 1 << (int)UnicodeCategory.NonSpacingMark | 1 << (int)UnicodeCategory.DecimalDigitNumber | 1 << (int)UnicodeCategory.ConnectorPunctuation;
|
||||
|
||||
public abstract MatchResult Match(
|
||||
ReadOnlySpan<char> text
|
||||
);
|
||||
|
||||
#region is char a word
|
||||
|
||||
/*
|
||||
* Semantic reference:
|
||||
*
|
||||
* .NET regular-expression word character: \w
|
||||
*
|
||||
* A character is treated as a word character when its Unicode general
|
||||
* category is one of:
|
||||
*
|
||||
* Lu UppercaseLetter
|
||||
* Ll LowercaseLetter
|
||||
* Lt TitlecaseLetter
|
||||
* Lm ModifierLetter
|
||||
* Lo OtherLetter
|
||||
* Mn NonSpacingMark
|
||||
* Nd DecimalDigitNumber
|
||||
* Pc ConnectorPunctuation
|
||||
*
|
||||
* The behavior and the ASCII-fast-path design are based on:
|
||||
*
|
||||
* dotnet/runtime
|
||||
* System.Text.RegularExpressions.RegexCharClass.IsWordChar
|
||||
*
|
||||
* Source:
|
||||
* https://github.com/dotnet/runtime/blob/main/src/libraries/
|
||||
* System.Text.RegularExpressions/src/System/Text/RegularExpressions/
|
||||
* RegexCharClass.cs
|
||||
*
|
||||
* dotnet/runtime is licensed under the MIT License.
|
||||
*
|
||||
* This implementation is written for NGRA rather than copied verbatim,
|
||||
* but the source is recorded here because its intended semantics are
|
||||
* explicitly compatible with .NET Regex.
|
||||
*/
|
||||
|
||||
/// <summary>
|
||||
/// Bit mask containing every Unicode general category accepted by the default .NET
|
||||
/// regular-expression <c> \w </c> character class.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <see cref="UnicodeCategory" /> values are small integer identifiers, so each category
|
||||
/// can be represented by one bit.
|
||||
/// </remarks>
|
||||
private const uint WordCategoryMask =
|
||||
(1u << (int)UnicodeCategory.UppercaseLetter) | (1u << (int)UnicodeCategory.LowercaseLetter) | (1u << (int)UnicodeCategory.TitlecaseLetter) | (1u << (int)UnicodeCategory.ModifierLetter) | (1u << (int)UnicodeCategory.OtherLetter) | (1u << (int)UnicodeCategory.NonSpacingMark) | (1u << (int)UnicodeCategory.DecimalDigitNumber) | (1u << (int)UnicodeCategory.ConnectorPunctuation);
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether one UTF-16 code unit belongs to the default .NET regular-expression
|
||||
/// <c> \w </c> character class.
|
||||
/// </summary>
|
||||
/// <param name="value">
|
||||
/// The UTF-16 code unit to classify.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// <see langword="true" /> when <paramref name="value" /> is a word character; otherwise,
|
||||
/// <see langword="false" />.
|
||||
/// </returns>
|
||||
/// <remarks>
|
||||
/// <para>/// This method operates on a single <see cref="char" />, and therefore /// on a
|
||||
/// single UTF-16 code unit rather than on a complete Unicode scalar /// value. ///
|
||||
/// </para> <para>/// Characters in the Unicode Basic Multilingual Plane are classified
|
||||
/// /// normally. A supplementary-plane character is represented by a /// surrogate pair
|
||||
/// and is therefore presented to this method as two /// separate surrogate code units.
|
||||
/// Neither surrogate is independently /// treated as a word character. This follows the
|
||||
/// char-based behavior of /// the corresponding .NET Regex helper. /// </para> <para>///
|
||||
/// For ASCII input, the Unicode-category lookup is avoided. In ASCII, /// the accepted
|
||||
/// set is exactly: /// <c> [A-Za-z0-9_] </c>. /// </para> <para>/// This method
|
||||
/// implements ordinary .NET <c> \w </c> semantics. It does /// not implement the ASCII-
|
||||
/// oriented behavior enabled by /// <c> RegexOptions.ECMAScript </c>. /// </para>
|
||||
/// </remarks>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
protected static bool IsCharWord(
|
||||
char value
|
||||
)
|
||||
{
|
||||
/*
|
||||
* ASCII fast path.
|
||||
*
|
||||
* Unsigned subtraction combines the lower-bound and upper-bound
|
||||
* checks into one comparison:
|
||||
*
|
||||
* value in ['A', 'Z']
|
||||
* => value - 'A' is in [0, 25]
|
||||
*
|
||||
* Values below 'A' underflow after conversion to uint and therefore
|
||||
* also fail the comparison.
|
||||
*/
|
||||
if (value <= '\x7F')
|
||||
{
|
||||
return
|
||||
(uint)(value - 'A') <= 'Z' - 'A' ||
|
||||
(uint)(value - 'a') <= 'z' - 'a' ||
|
||||
(uint)(value - '0') <= '9' - '0' ||
|
||||
value == '_';
|
||||
}
|
||||
|
||||
/*
|
||||
* Non-ASCII path.
|
||||
*
|
||||
* CharUnicodeInfo classifies the current UTF-16 code unit using
|
||||
* Unicode general categories. The resulting category selects one
|
||||
* bit from WordCategoryMask.
|
||||
*/
|
||||
UnicodeCategory category =
|
||||
CharUnicodeInfo.GetUnicodeCategory(value);
|
||||
|
||||
return
|
||||
(WordCategoryMask & (1u << (int)category)) != 0;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
public ref struct MatchResult { }
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using Microsoft.CodeAnalysis.CSharp;
|
||||
using Microsoft.CodeAnalysis.CSharp.Syntax;
|
||||
using StgSharp.RegularAnalysis.Abstraction;
|
||||
using System;
|
||||
@@ -90,6 +91,20 @@ namespace StgSharp.RegularAnalysis.Text
|
||||
return se;
|
||||
}
|
||||
|
||||
private static string FormatStringLiteral(
|
||||
string str
|
||||
)
|
||||
{
|
||||
return SyntaxFactory.Literal(str).Text;
|
||||
}
|
||||
|
||||
private static string FormatStringLiteral(
|
||||
char str
|
||||
)
|
||||
{
|
||||
return SyntaxFactory.Literal(str).Text;
|
||||
}
|
||||
|
||||
#region sub generator
|
||||
|
||||
private static int TryGenerateFindSeq(
|
||||
@@ -122,7 +137,7 @@ namespace StgSharp.RegularAnalysis.Text
|
||||
if (scan_win[0] is not RegexFindSeqIR seq) {
|
||||
return -1;
|
||||
}
|
||||
string pattern = seq.Pattern;
|
||||
string pattern = FormatStringLiteral(seq.Pattern);
|
||||
int length = pattern.Length;
|
||||
|
||||
_ = se.AppendLine()
|
||||
@@ -189,13 +204,14 @@ namespace StgSharp.RegularAnalysis.Text
|
||||
{
|
||||
// Handle single character
|
||||
case CharSetType.Single:
|
||||
string literal = FormatStringLiteral(c.Value);
|
||||
caller = first ? "IndexOfAny" : "LastIndexOfAny";
|
||||
return $@"({offset} = {remain}.{caller}(""{c.Value}"") != -1)";
|
||||
return $@"({offset} = {remain}.{caller}(""{literal}"") != -1)";
|
||||
|
||||
// Handle range of characters
|
||||
case CharSetType.Range:
|
||||
caller = first ? "IndexOfAnyInRange" : "LastIndexOfAnyInRange";
|
||||
return $@"({offset} = {remain}.{caller}('{c.Value[0]}', '{c.Value[1]}') != -1)";
|
||||
return $@"({offset} = {remain}.{caller}('{FormatStringLiteral(c.Value[0])}', '{FormatStringLiteral(c.Value[1])}') != -1)";
|
||||
|
||||
// Handle set of characters
|
||||
case CharSetType.Set:
|
||||
@@ -384,7 +400,7 @@ namespace StgSharp.RegularAnalysis.Text
|
||||
ReadOnlySpan<char> seq = source.Slice(idx, source.Length - idx - 1);
|
||||
if (seq.IndexOf('-') == -1)
|
||||
{
|
||||
return [ new RegexCharSet(seq.ToString(), CharSetType.Single, true) ];
|
||||
return [new RegexCharSet(seq.ToString(), CharSetType.Single, true)];
|
||||
} else
|
||||
{
|
||||
List<RegexCharSet> list = [];
|
||||
@@ -404,7 +420,7 @@ namespace StgSharp.RegularAnalysis.Text
|
||||
list.Add(new($"{seq[pos - 1]}{seq[pos + 1]}", CharSetType.Range, true));
|
||||
seq = seq[(pos + 2)..];
|
||||
}
|
||||
return [ .. list ];
|
||||
return [.. list];
|
||||
}
|
||||
} else if (source[0] == '\\')
|
||||
{
|
||||
@@ -413,9 +429,9 @@ namespace StgSharp.RegularAnalysis.Text
|
||||
return char.ToLower(source[0], CultureInfo.InvariantCulture)
|
||||
switch
|
||||
{
|
||||
's' => [ new("s", CharSetType.Set, accept) ],
|
||||
'w' => [ new("w", CharSetType.Set, accept) ],
|
||||
'd' => [ new("d", CharSetType.Set, accept) ],
|
||||
's' => [new("s", CharSetType.Set, accept)],
|
||||
'w' => [new("w", CharSetType.Set, accept)],
|
||||
'd' => [new("d", CharSetType.Set, accept)],
|
||||
_ => throw new NotSupportedException()
|
||||
};
|
||||
} else
|
||||
|
||||
Reference in New Issue
Block a user