add parser for concat

StgSharp.RegularAnalysis/Abstraction/ITokenParser.cs

@@ -0,0 +1,51 @@
+//-----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+// file="ITokenParser"
+// Project: StgSharp
+// AuthorGroup: Nitload
+// Copyright (c) Nitload. All rights reserved.
+//     
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//     
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StgSharp.RegularAnalysis.Abstraction
+{
+    public abstract class TokenParser<TIn, TOut> : ITokenReader<TOut> where TIn : unmanaged
+        where TOut : unmanaged
+    {
+
+        public abstract bool IsEmpty { get; }
+
+        protected internal ITokenReader<TIn> Source { get; internal set; }
+
+        public abstract Token<TOut> ReadToken();
+
+        public abstract bool TryReadToken(
+                             out Token<TOut> t
+        );
+
+    }
+}

StgSharp.RegularAnalysis/Abstraction/ITokenReader.cs

@@ -1,6 +1,6 @@
 //-----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-// file="IScriptTokenReader"
+// file="ITokenReader"
 // Project: StgSharp
 // AuthorGroup: Nitload
 // Copyright (c) Nitload. All rights reserved.
@@ -27,7 +27,22 @@
 // -----------------------------------------------------------------------
 namespace StgSharp.RegularAnalysis.Abstraction
 {
-    public interface IScriptTokenReader<TLabel> where TLabel : unmanaged
+    public static class TokenReader
+    {
+
+        public static TokenParser<TIn, TOut> Pipe<TIn, TOut>(
+                                             this ITokenReader<TIn> source,
+                                             Func<TokenParser<TIn, TOut>> builder
+        ) where TIn : unmanaged where TOut : unmanaged
+        {
+            TokenParser<TIn, TOut> parser = builder();
+            parser.Source = source;
+            return parser;
+        }
+
+    }
+
+    public interface ITokenReader<TLabel> where TLabel : unmanaged
     {
 
         bool IsEmpty { get; }

StgSharp.RegularAnalysis/Abstraction/Token.cs

@@ -76,6 +76,14 @@ namespace StgSharp.RegularAnalysis.Abstraction
             return new Token<TLabel>(Value, line, column, Flag);
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public Token<TNewLabel> Remark<TNewLabel>(
+                                TNewLabel label
+        ) where TNewLabel : unmanaged
+        {
+            return new Token<TNewLabel>(Value, Line, Column, label);
+        }
+
         public static bool operator !=(
                                     Token<TLabel> left,
                                     Token<TLabel> right

StgSharp.RegularAnalysis/RegularAST/RegularAST.cs

@@ -1,13 +0,0 @@
-
-using StgSharp.RegularAnalysis.Abstraction;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-
-using System.Threading.Tasks;
-
-namespace StgSharp.RegularAnalysis.RegularAST
-{
-    internal class RegularAST : AbstractSyntaxTree<RegularSyntaxNode, RegularType> { }
-}

StgSharp.RegularAnalysis/StgSharp.RegularAnalysis.csproj

@@ -10,8 +10,4 @@
     <ProjectReference Include="..\StgSharp.Common\StgSharp.Common.csproj" />
   </ItemGroup>
 
-  <ItemGroup>
-    <Folder Include="Abstraction\" />
-  </ItemGroup>
-
 </Project>

StgSharp.RegularAnalysis/TextRegex/RegexAstNode.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StgSharp.RegularAnalysis.TextRegex
+{
+    internal class RegexAstNode
+    {
+    }
+}

StgSharp.RegularAnalysis/TextRegex/RegexElementLabel.cs

@@ -1,6 +1,6 @@
 //-----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-// file="RegexAstLabel"
+// file="RegexElementLabel"
 // Project: StgSharp
 // AuthorGroup: Nitload
 // Copyright (c) Nitload. All rights reserved.
@@ -33,15 +33,20 @@ using System.Threading.Tasks;
 
 namespace StgSharp.RegularAnalysis.TextRegex
 {
-    internal enum RegexAstLabel
+    [Flags]
+    internal enum RegexElementLabel
     {
 
-        CHAR,
+        NONE = 0,
-        CHAR_SET,
+        UNIT = 1,
-        GROUP_NAME,
+        GROUP_BEGIN = 2,
-        GROUP_BEGIN,
+        GROUP_END = 4,
-        GROUP_END,
+        COUNT = 8,
-        COUNT,
+        ALT = 16,
+        CONCAT = 32,
+        OPERATOR = COUNT | ALT | CONCAT,
+        ATOM_BEGIN = UNIT | GROUP_BEGIN,
+        ATOM_END = UNIT | GROUP_END | COUNT,
 
     }
 }

StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs

@@ -1,6 +1,6 @@
 //-----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-// file="SequenceElementType"
+// file="RegexInterpreter"
 // Project: StgSharp
 // AuthorGroup: Nitload
 // Copyright (c) Nitload. All rights reserved.
@@ -29,21 +29,21 @@ using StgSharp.RegularAnalysis.Abstraction;
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Numerics;
 using System.Text;
+
 using System.Threading.Tasks;
 
-namespace StgSharp.RegularAnalysis.RegularAST
+namespace StgSharp.RegularAnalysis.TextRegex
 {
-    internal class SequenceElementType<T> : ITypeSource<SequenceElementType<T>>
+    public class RegexInterpreter
-        where T : unmanaged, INumber<T>
     {
 
-        public bool IsConvertable(
+        public static AbstractSyntaxTree<RegexAstNode, RegexElementLabel> Analyze(
-                    SequenceElementType<T> targetType
+                                                                          string regex
         )
         {
-            throw new NotImplementedException();
+            RegexTokenReader reader = new RegexTokenReader(regex);
+            TokenParser<RegexElementLabel, RegexElementLabel> lexer = reader.Pipe(() => new RegexTokenParser());
         }
 
     }

StgSharp.RegularAnalysis/TextRegex/RegexTokenParser.cs

@@ -0,0 +1,88 @@
+//-----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+// file="RegexTokenParser"
+// Project: StgSharp
+// AuthorGroup: Nitload
+// Copyright (c) Nitload. All rights reserved.
+//     
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//     
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+using StgSharp.RegularAnalysis.Abstraction;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Tracing;
+using System.IO.Pipes;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StgSharp.RegularAnalysis.TextRegex
+{
+    internal class RegexTokenParser : TokenParser<RegexElementLabel, RegexElementLabel>
+    {
+
+        private readonly Queue<Token<RegexElementLabel>> _buffer = [];
+
+        private RegexElementLabel _prev;
+
+        public override bool IsEmpty => Source.IsEmpty && _buffer.Count == 0;
+
+        public override Token<RegexElementLabel> ReadToken()
+        {
+            if (_buffer.TryDequeue(out Token<RegexElementLabel> token)) {
+                return token;
+            }
+            if (Source.TryReadToken(out Token<RegexElementLabel> orig))
+            {
+                if (((int)(orig.Flag | RegexElementLabel.ATOM_BEGIN) != 0) && ((int)_prev != 0))
+                {
+                    Token<RegexElementLabel> ret = new Token<RegexElementLabel>(string.Empty, orig.Line,
+                                                                            orig.Column,
+                                                                            RegexElementLabel.OPERATOR);
+                    _buffer.Enqueue(orig);
+                    _prev = orig.Flag & RegexElementLabel.ATOM_END;
+                    return ret;
+                } else
+                {
+                    _prev = orig.Flag & RegexElementLabel.ATOM_END;
+                    return orig;
+                }
+            } else
+            {
+                return Token<RegexElementLabel>.Empty;
+            }
+        }
+
+        public override bool TryReadToken(
+                             out Token<RegexElementLabel> t
+        )
+        {
+            if (IsEmpty)
+            {
+                t = Token<RegexElementLabel>.Empty;
+                return false;
+            }
+            t = ReadToken();
+            return t == Token<RegexElementLabel>.Empty;
+        }
+
+    }
+}

StgSharp.RegularAnalysis/TextRegex/RegexTokenReader.cs

@@ -1,6 +1,6 @@
 //-----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-// file="RegexLexer"
+// file="RegexTokenReader"
 // Project: StgSharp
 // AuthorGroup: Nitload
 // Copyright (c) Nitload. All rights reserved.
@@ -36,16 +36,18 @@ using System.Threading.Tasks;
 
 namespace StgSharp.RegularAnalysis.TextRegex
 {
-    internal class RegexLexer : IScriptTokenReader<RegexAstLabel>
+    internal class RegexTokenReader : ITokenReader<RegexElementLabel>
     {
 
         private static string _source;
 
         private int position = 0;
 
+        public RegexTokenReader() { }
+
         public bool IsEmpty => string.IsNullOrEmpty(_source) || position >= _source.Length;
 
-        public Token<RegexAstLabel> ReadToken()
+        public Token<RegexElementLabel> ReadToken()
         {
             int pos = position;
             switch (_source[pos])
@@ -55,8 +57,8 @@ namespace StgSharp.RegularAnalysis.TextRegex
                         throw new InvalidOperationException("Unclosed char");
                     }
                     position = pos + 2;
-                    return new Token<RegexAstLabel>(_source[(pos)..(pos + 2)], 0, pos,
+                    return new Token<RegexElementLabel>(_source[(pos)..(pos + 2)], 0, pos,
-                                                    RegexAstLabel.CHAR);
+                                                        RegexElementLabel.UNIT);
                 case '[':
                     for (int i = pos + 1; i < _source.Length; i++)
                     {
@@ -64,8 +66,9 @@ namespace StgSharp.RegularAnalysis.TextRegex
                         {
                             int length = (i - pos - 1);
                             position = i + 1;
-                            return new Token<RegexAstLabel>(_source[(pos - 1)..(i - 1)], 0,
+                            return new Token<RegexElementLabel>(_source[(pos - 1)..(i - 1)], 0,
-                                                            position, RegexAstLabel.CHAR_SET);
+                                                                position,
+                                                                RegexElementLabel.CHAR_SET);
                         }
                     }
                     throw new InvalidOperationException("Unclosed character set");
@@ -75,54 +78,54 @@ namespace StgSharp.RegularAnalysis.TextRegex
                     return ReadGroupBeginToken();
                 case ')':
                     position = pos + 1;
-                    return new Token<RegexAstLabel>(_source[(pos)..1], 0, pos,
+                    return new Token<RegexElementLabel>(_source[(pos)..1], 0, pos,
-                                                    RegexAstLabel.GROUP_END);
+                                                        RegexElementLabel.GROUP_END);
                 case '{':
                     for (int i = pos + 1; i < _source.Length; i++)
                     {
                         if (_source[i] == '}')
                         {
                             position = i + 1;
-                            return new Token<RegexAstLabel>(_source[(pos)..(i - 1)], 0, pos,
+                            return new Token<RegexElementLabel>(_source[(pos)..(i - 1)], 0, pos,
-                                                            RegexAstLabel.COUNT);
+                                                                RegexElementLabel.COUNT);
                         }
                     }
                     throw new InvalidOperationException("Unclosed count specifier");
                 case '*' or '+':
                     int count = _source[pos + 1] == '?' ? 2 : 1;
                     position = pos + count;
-                    return new Token<RegexAstLabel>(_source[(pos)..(pos + count)], 0, pos,
+                    return new Token<RegexElementLabel>(_source[(pos)..(pos + count)], 0, pos,
-                                                    RegexAstLabel.COUNT);
+                                                        RegexElementLabel.COUNT);
                 case '?':
                     position = pos + 1;
-                    return new Token<RegexAstLabel>(_source[(pos)..(pos + 1)], 0, pos,
+                    return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
-                                                    RegexAstLabel.COUNT);
+                                                        RegexElementLabel.COUNT);
                 case '.':
-                    return new Token<RegexAstLabel>(_source[(pos)..(pos + 1)], 0, pos,
+                    return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
-                                                    RegexAstLabel.CHAR_SET);
+                                                        RegexElementLabel.UNIT);
                 default:
                     position = pos + 1;
-                    return new Token<RegexAstLabel>(_source[(pos)..(pos + 1)], 0, pos,
+                    return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
-                                                    RegexAstLabel.CHAR);
+                                                        RegexElementLabel.UNIT);
             }
         }
 
         public bool TryReadToken(
-                    out Token<RegexAstLabel> t
+                    out Token<RegexElementLabel> t
         )
         {
             if (IsEmpty)
             {
-                t = Token<RegexAstLabel>.Empty;
+                t = Token<RegexElementLabel>.Empty;
                 return false;
             }
             t = ReadToken();
-            return Token<RegexAstLabel>.Empty == t;
+            return Token<RegexElementLabel>.Empty == t;
         }
 
-        private Token<RegexAstLabel> ReadGroupBeginToken()
+        private Token<RegexElementLabel> ReadGroupBeginToken()
         {
-            if (_source is null) {
+            if (IsEmpty) {
                 throw new InvalidOperationException("Source is null.");
             }
 
@@ -136,7 +139,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
             if (start + 1 >= _source.Length || _source[start + 1] != '?')
             {
                 position = start + 1;
-                return new Token<RegexAstLabel>("(", 0, start, RegexAstLabel.GROUP_BEGIN);
+                return new Token<RegexElementLabel>("(", 0, start, RegexElementLabel.GROUP_BEGIN);
             }
 
             // From here we have "(?"
@@ -151,7 +154,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
             if (c2 == ':')
             {
                 position = start + 3;
-                return new Token<RegexAstLabel>("(?:", 0, start, RegexAstLabel.GROUP_BEGIN);
+                return new Token<RegexElementLabel>("(?:", 0, start, RegexElementLabel.GROUP_BEGIN);
             }
 
             // Assertions (write the first-level cases, but not supported yet)
@@ -232,7 +235,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
                 string text = _source.Substring(start, consumedLen);
                 position = start + consumedLen;
 
-                return new Token<RegexAstLabel>(text, 0, start, RegexAstLabel.GROUP_BEGIN);
+                return new Token<RegexElementLabel>(text, 0, start, RegexElementLabel.GROUP_BEGIN);
             }
 
             // Optional: comments "(?#...)" or inline options "(?i:...)" etc.