From 6292157fc03fc6a0cf158ac0266c82db05c2fd1d Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Wed, 4 Feb 2026 16:30:09 +0800 Subject: [PATCH] add parser for concat --- .../Abstraction/ITokenParser.cs | 51 +++++++++++ ...{IScriptTokenReader.cs => ITokenReader.cs} | 19 +++- StgSharp.RegularAnalysis/Abstraction/Token.cs | 8 ++ .../RegularAST/RegularAST.cs | 13 --- .../StgSharp.RegularAnalysis.csproj | 4 - .../TextRegex/RegexAstNode.cs | 12 +++ ...{RegexAstLabel.cs => RegexElementLabel.cs} | 21 +++-- .../RegexInterpreter.cs} | 16 ++-- .../TextRegex/RegexTokenParser.cs | 88 +++++++++++++++++++ .../{RegexLexer.cs => RegexTokenReader.cs} | 57 ++++++------ 10 files changed, 227 insertions(+), 62 deletions(-) create mode 100644 StgSharp.RegularAnalysis/Abstraction/ITokenParser.cs rename StgSharp.RegularAnalysis/Abstraction/{IScriptTokenReader.cs => ITokenReader.cs} (75%) delete mode 100644 StgSharp.RegularAnalysis/RegularAST/RegularAST.cs create mode 100644 StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs rename StgSharp.RegularAnalysis/TextRegex/{RegexAstLabel.cs => RegexElementLabel.cs} (82%) rename StgSharp.RegularAnalysis/{RegularAST/SequenceElementType.cs => TextRegex/RegexInterpreter.cs} (78%) create mode 100644 StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs rename StgSharp.RegularAnalysis/TextRegex/{RegexLexer.cs => RegexTokenReader.cs} (79%) diff --git a/StgSharp.RegularAnalysis/Abstraction/ITokenParser.cs b/StgSharp.RegularAnalysis/Abstraction/ITokenParser.cs new file mode 100644 index 00000000..1c0aa3c2 --- /dev/null +++ b/StgSharp.RegularAnalysis/Abstraction/ITokenParser.cs @@ -0,0 +1,51 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="ITokenParser" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Abstraction +{ + public abstract class TokenParser : ITokenReader where TIn : unmanaged + where TOut : unmanaged + { + + public abstract bool IsEmpty { get; } + + protected internal ITokenReader Source { get; internal set; } + + public abstract Token ReadToken(); + + public abstract bool TryReadToken( + out Token t + ); + + } +} diff --git a/StgSharp.RegularAnalysis/Abstraction/IScriptTokenReader.cs b/StgSharp.RegularAnalysis/Abstraction/ITokenReader.cs similarity index 75% rename from StgSharp.RegularAnalysis/Abstraction/IScriptTokenReader.cs rename to StgSharp.RegularAnalysis/Abstraction/ITokenReader.cs index cc83abd0..a15872df 100644 --- a/StgSharp.RegularAnalysis/Abstraction/IScriptTokenReader.cs +++ b/StgSharp.RegularAnalysis/Abstraction/ITokenReader.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="IScriptTokenReader" +// file="ITokenReader" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -27,7 +27,22 @@ // ----------------------------------------------------------------------- namespace StgSharp.RegularAnalysis.Abstraction { - public interface IScriptTokenReader where TLabel : unmanaged + public static class TokenReader + { + + public static TokenParser Pipe( + this ITokenReader source, + Func> builder + ) where TIn : unmanaged where TOut : unmanaged + { + TokenParser parser = builder(); + parser.Source = source; + return parser; + } + + } + + public interface ITokenReader where TLabel : unmanaged { bool IsEmpty { get; } diff --git a/StgSharp.RegularAnalysis/Abstraction/Token.cs b/StgSharp.RegularAnalysis/Abstraction/Token.cs index 074c87a8..6fa9766c 100644 --- a/StgSharp.RegularAnalysis/Abstraction/Token.cs +++ b/StgSharp.RegularAnalysis/Abstraction/Token.cs @@ -76,6 +76,14 @@ namespace StgSharp.RegularAnalysis.Abstraction return new Token(Value, line, column, Flag); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Token Remark( + TNewLabel label + ) where TNewLabel : unmanaged + { + return new Token(Value, Line, Column, label); + } + public static bool operator !=( Token left, Token right diff --git a/StgSharp.RegularAnalysis/RegularAST/RegularAST.cs b/StgSharp.RegularAnalysis/RegularAST/RegularAST.cs deleted file mode 100644 index 1bc9da1c..00000000 --- a/StgSharp.RegularAnalysis/RegularAST/RegularAST.cs +++ /dev/null @@ -1,13 +0,0 @@ - -using StgSharp.RegularAnalysis.Abstraction; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -using System.Threading.Tasks; - -namespace StgSharp.RegularAnalysis.RegularAST -{ - internal class RegularAST : AbstractSyntaxTree { } -} diff --git a/StgSharp.RegularAnalysis/StgSharp.RegularAnalysis.csproj b/StgSharp.RegularAnalysis/StgSharp.RegularAnalysis.csproj index e85687f1..0153ae4b 100644 --- a/StgSharp.RegularAnalysis/StgSharp.RegularAnalysis.csproj +++ b/StgSharp.RegularAnalysis/StgSharp.RegularAnalysis.csproj @@ -10,8 +10,4 @@ - - - - diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs b/StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs new file mode 100644 index 00000000..7fc2dacb --- /dev/null +++ b/StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.TextRegex +{ + internal class RegexAstNode + { + } +} diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexAstLabel.cs b/StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs similarity index 82% rename from StgSharp.RegularAnalysis/TextRegex/RegexAstLabel.cs rename to StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs index 921e59ae..47ff50b1 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexAstLabel.cs +++ b/StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexAstLabel" +// file="RegexElementLabel" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -33,15 +33,20 @@ using System.Threading.Tasks; namespace StgSharp.RegularAnalysis.TextRegex { - internal enum RegexAstLabel + [Flags] + internal enum RegexElementLabel { - CHAR, - CHAR_SET, - GROUP_NAME, - GROUP_BEGIN, - GROUP_END, - COUNT, + NONE = 0, + UNIT = 1, + GROUP_BEGIN = 2, + GROUP_END = 4, + COUNT = 8, + ALT = 16, + CONCAT = 32, + OPERATOR = COUNT | ALT | CONCAT, + ATOM_BEGIN = UNIT | GROUP_BEGIN, + ATOM_END = UNIT | GROUP_END | COUNT, } } diff --git a/StgSharp.RegularAnalysis/RegularAST/SequenceElementType.cs b/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs similarity index 78% rename from StgSharp.RegularAnalysis/RegularAST/SequenceElementType.cs rename to StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs index bda189e9..9b7384d3 100644 --- a/StgSharp.RegularAnalysis/RegularAST/SequenceElementType.cs +++ b/StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="SequenceElementType" +// file="RegexInterpreter" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -29,21 +29,21 @@ using StgSharp.RegularAnalysis.Abstraction; using System; using System.Collections.Generic; using System.Linq; -using System.Numerics; using System.Text; + using System.Threading.Tasks; -namespace StgSharp.RegularAnalysis.RegularAST +namespace StgSharp.RegularAnalysis.TextRegex { - internal class SequenceElementType : ITypeSource> - where T : unmanaged, INumber + public class RegexInterpreter { - public bool IsConvertable( - SequenceElementType targetType + public static AbstractSyntaxTree Analyze( + string regex ) { - throw new NotImplementedException(); + RegexTokenReader reader = new RegexTokenReader(regex); + TokenParser lexer = reader.Pipe(() => new RegexTokenParser()); } } diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs b/StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs new file mode 100644 index 00000000..46ab7326 --- /dev/null +++ b/StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs @@ -0,0 +1,88 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="RegexTokenParser" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using StgSharp.RegularAnalysis.Abstraction; +using System; +using System.Collections.Generic; +using System.Diagnostics.Tracing; +using System.IO.Pipes; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.TextRegex +{ + internal class RegexTokenParser : TokenParser + { + + private readonly Queue> _buffer = []; + + private RegexElementLabel _prev; + + public override bool IsEmpty => Source.IsEmpty && _buffer.Count == 0; + + public override Token ReadToken() + { + if (_buffer.TryDequeue(out Token token)) { + return token; + } + if (Source.TryReadToken(out Token orig)) + { + if (((int)(orig.Flag | RegexElementLabel.ATOM_BEGIN) != 0) && ((int)_prev != 0)) + { + Token ret = new Token(string.Empty, orig.Line, + orig.Column, + RegexElementLabel.OPERATOR); + _buffer.Enqueue(orig); + _prev = orig.Flag & RegexElementLabel.ATOM_END; + return ret; + } else + { + _prev = orig.Flag & RegexElementLabel.ATOM_END; + return orig; + } + } else + { + return Token.Empty; + } + } + + public override bool TryReadToken( + out Token t + ) + { + if (IsEmpty) + { + t = Token.Empty; + return false; + } + t = ReadToken(); + return t == Token.Empty; + } + + } +} diff --git a/StgSharp.RegularAnalysis/TextRegex/RegexLexer.cs b/StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs similarity index 79% rename from StgSharp.RegularAnalysis/TextRegex/RegexLexer.cs rename to StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs index ea1af2e8..68749990 100644 --- a/StgSharp.RegularAnalysis/TextRegex/RegexLexer.cs +++ b/StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexLexer" +// file="RegexTokenReader" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -36,16 +36,18 @@ using System.Threading.Tasks; namespace StgSharp.RegularAnalysis.TextRegex { - internal class RegexLexer : IScriptTokenReader + internal class RegexTokenReader : ITokenReader { private static string _source; private int position = 0; + public RegexTokenReader() { } + public bool IsEmpty => string.IsNullOrEmpty(_source) || position >= _source.Length; - public Token ReadToken() + public Token ReadToken() { int pos = position; switch (_source[pos]) @@ -55,8 +57,8 @@ namespace StgSharp.RegularAnalysis.TextRegex throw new InvalidOperationException("Unclosed char"); } position = pos + 2; - return new Token(_source[(pos)..(pos + 2)], 0, pos, - RegexAstLabel.CHAR); + return new Token(_source[(pos)..(pos + 2)], 0, pos, + RegexElementLabel.UNIT); case '[': for (int i = pos + 1; i < _source.Length; i++) { @@ -64,8 +66,9 @@ namespace StgSharp.RegularAnalysis.TextRegex { int length = (i - pos - 1); position = i + 1; - return new Token(_source[(pos - 1)..(i - 1)], 0, - position, RegexAstLabel.CHAR_SET); + return new Token(_source[(pos - 1)..(i - 1)], 0, + position, + RegexElementLabel.CHAR_SET); } } throw new InvalidOperationException("Unclosed character set"); @@ -75,54 +78,54 @@ namespace StgSharp.RegularAnalysis.TextRegex return ReadGroupBeginToken(); case ')': position = pos + 1; - return new Token(_source[(pos)..1], 0, pos, - RegexAstLabel.GROUP_END); + return new Token(_source[(pos)..1], 0, pos, + RegexElementLabel.GROUP_END); case '{': for (int i = pos + 1; i < _source.Length; i++) { if (_source[i] == '}') { position = i + 1; - return new Token(_source[(pos)..(i - 1)], 0, pos, - RegexAstLabel.COUNT); + return new Token(_source[(pos)..(i - 1)], 0, pos, + RegexElementLabel.COUNT); } } throw new InvalidOperationException("Unclosed count specifier"); case '*' or '+': int count = _source[pos + 1] == '?' ? 2 : 1; position = pos + count; - return new Token(_source[(pos)..(pos + count)], 0, pos, - RegexAstLabel.COUNT); + return new Token(_source[(pos)..(pos + count)], 0, pos, + RegexElementLabel.COUNT); case '?': position = pos + 1; - return new Token(_source[(pos)..(pos + 1)], 0, pos, - RegexAstLabel.COUNT); + return new Token(_source[(pos)..(pos + 1)], 0, pos, + RegexElementLabel.COUNT); case '.': - return new Token(_source[(pos)..(pos + 1)], 0, pos, - RegexAstLabel.CHAR_SET); + return new Token(_source[(pos)..(pos + 1)], 0, pos, + RegexElementLabel.UNIT); default: position = pos + 1; - return new Token(_source[(pos)..(pos + 1)], 0, pos, - RegexAstLabel.CHAR); + return new Token(_source[(pos)..(pos + 1)], 0, pos, + RegexElementLabel.UNIT); } } public bool TryReadToken( - out Token t + out Token t ) { if (IsEmpty) { - t = Token.Empty; + t = Token.Empty; return false; } t = ReadToken(); - return Token.Empty == t; + return Token.Empty == t; } - private Token ReadGroupBeginToken() + private Token ReadGroupBeginToken() { - if (_source is null) { + if (IsEmpty) { throw new InvalidOperationException("Source is null."); } @@ -136,7 +139,7 @@ namespace StgSharp.RegularAnalysis.TextRegex if (start + 1 >= _source.Length || _source[start + 1] != '?') { position = start + 1; - return new Token("(", 0, start, RegexAstLabel.GROUP_BEGIN); + return new Token("(", 0, start, RegexElementLabel.GROUP_BEGIN); } // From here we have "(?" @@ -151,7 +154,7 @@ namespace StgSharp.RegularAnalysis.TextRegex if (c2 == ':') { position = start + 3; - return new Token("(?:", 0, start, RegexAstLabel.GROUP_BEGIN); + return new Token("(?:", 0, start, RegexElementLabel.GROUP_BEGIN); } // Assertions (write the first-level cases, but not supported yet) @@ -232,7 +235,7 @@ namespace StgSharp.RegularAnalysis.TextRegex string text = _source.Substring(start, consumedLen); position = start + consumedLen; - return new Token(text, 0, start, RegexAstLabel.GROUP_BEGIN); + return new Token(text, 0, start, RegexElementLabel.GROUP_BEGIN); } // Optional: comments "(?#...)" or inline options "(?i:...)" etc.