add parser for concat

StgSharp.RegularAnalysis/Abstraction/ITokenParser.cs

@@ -0,0 +1,51 @@
+//-----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+// file="ITokenParser"
+// Project: StgSharp
+// AuthorGroup: Nitload
+// Copyright (c) Nitload. All rights reserved.
+//     
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//     
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StgSharp.RegularAnalysis.Abstraction
+{
+    public abstract class TokenParser<TIn, TOut> : ITokenReader<TOut> where TIn : unmanaged
+        where TOut : unmanaged
+    {
+
+        public abstract bool IsEmpty { get; }
+
+        protected internal ITokenReader<TIn> Source { get; internal set; }
+
+        public abstract Token<TOut> ReadToken();
+
+        public abstract bool TryReadToken(
+                             out Token<TOut> t
+        );
+
+    }
+}

StgSharp.RegularAnalysis/Abstraction/ITokenReader.cs

@@ -1,6 +1,6 @@
 //-----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-// file="IScriptTokenReader"
+// file="ITokenReader"
 // Project: StgSharp
 // AuthorGroup: Nitload
 // Copyright (c) Nitload. All rights reserved.
@@ -27,7 +27,22 @@
 // -----------------------------------------------------------------------
 namespace StgSharp.RegularAnalysis.Abstraction
 {
-    public interface IScriptTokenReader<TLabel> where TLabel : unmanaged
+    public static class TokenReader
+    {
+
+        public static TokenParser<TIn, TOut> Pipe<TIn, TOut>(
+                                             this ITokenReader<TIn> source,
+                                             Func<TokenParser<TIn, TOut>> builder
+        ) where TIn : unmanaged where TOut : unmanaged
+        {
+            TokenParser<TIn, TOut> parser = builder();
+            parser.Source = source;
+            return parser;
+        }
+
+    }
+
+    public interface ITokenReader<TLabel> where TLabel : unmanaged
     {
 
         bool IsEmpty { get; }

StgSharp.RegularAnalysis/Abstraction/Token.cs

@@ -76,6 +76,14 @@ namespace StgSharp.RegularAnalysis.Abstraction
             return new Token<TLabel>(Value, line, column, Flag);
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public Token<TNewLabel> Remark<TNewLabel>(
+                                TNewLabel label
+        ) where TNewLabel : unmanaged
+        {
+            return new Token<TNewLabel>(Value, Line, Column, label);
+        }
+
         public static bool operator !=(
                                     Token<TLabel> left,
                                     Token<TLabel> right