youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Daily update on 20251020-041041 by Nitload Youmu. Additional Message : Code cleanup and new design of parallel (maybe THE THENTH VERSION? Tough work!).

Browse Source 
Signed-off-by: Nitload <YoumuLiu_NSI@whv0t.onmicrosoft.com>
This commit is contained in:
Nitload
2025-10-20 04:10:41 +08:00
parent 1927f4fdf3
commit 547ba80a01
141 changed files with 5475 additions and 6175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Script/Express/ExpTokenReader.cs
+145 -157
View File
@@ -1,33 +1,30 @@
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
// file="ExpTokenReader.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// -----------------------------------------------------------------------
// file="ExpTokenReader"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using CommunityToolkit.HighPerformance;
using StgSharp.HighPerformance;
@@ -43,12 +40,12 @@ namespace StgSharp.Script.Express
public static ExpTokenReaderPreProcessor Default { get; } = new DefaultProcessor();
public abstract bool Process( string codeLine, int line, ref int current, out Token t );
public abstract bool Process(string codeLine, int line, ref int current, out Token t);
private class DefaultProcessor : ExpTokenReaderPreProcessor
{
public override bool Process( string codeLine, int line, ref int current, out Token t )
public override bool Process(string codeLine, int line, ref int current, out Token t)
{
t = Token.Empty;
return false;
@@ -80,7 +77,7 @@ namespace StgSharp.Script.Express
private string _lineCache;
private Token _last;
public ExpTokenReader( IScriptSourceProvider provider )
public ExpTokenReader(IScriptSourceProvider provider)
{
_provider = provider;
_current = 0;
@@ -90,8 +87,7 @@ namespace StgSharp.Script.Express
public bool IsEmpty
{
get => ( ( string.IsNullOrEmpty( _lineCache ) || _current >= _lineCache.Length ) ) &&
_provider.IsEmpty;
get => ((string.IsNullOrEmpty(_lineCache) || _current >= _lineCache.Length)) && _provider.IsEmpty;
}
public ExpTokenReaderPreProcessor TokenReaderPreProcessor { get; set; } = ExpTokenReaderPreProcessor.Default;
@@ -103,28 +99,28 @@ namespace StgSharp.Script.Express
public unsafe ReadOnlySpan<char> ReadLine()
{
bool lineOver = string.IsNullOrEmpty( _lineCache ) || _current >= _lineCache.Length;
while( lineOver || _isReadingMultiLineComment )
bool lineOver = string.IsNullOrEmpty(_lineCache) || _current >= _lineCache.Length;
while (lineOver || _isReadingMultiLineComment)
{
if( IsEmpty ) {
if (IsEmpty) {
return ReadOnlySpan<char>.Empty;
}
if( lineOver )
if (lineOver)
{
do
{
_lineCache = _provider.ReadLine( out _lineNum );
} while (string.IsNullOrEmpty( _lineCache ) && !_provider.IsEmpty);
_lineCache = _provider.ReadLine(out _lineNum);
} while (string.IsNullOrEmpty(_lineCache) && !_provider.IsEmpty);
_current = 0;
}
if( _isReadingMultiLineComment )
if (_isReadingMultiLineComment)
{
int index;
fixed( char* cPtr = _lineCache.AsSpan( _current ) ) {
fixed (char* cPtr = _lineCache.AsSpan(_current)) {
index = UnsafeCompute.IndexOfCharPair(
cPtr, MultiLineEnd, _lineCache.Length );
cPtr, MultiLineEnd, _lineCache.Length);
}
if( index == -1 )
if (index == -1)
{
_lineCache = string.Empty;
} else
@@ -133,9 +129,9 @@ namespace StgSharp.Script.Express
_current += index + 2;
}
}
lineOver = string.IsNullOrEmpty( _lineCache ) || _current >= _lineCache.Length;
lineOver = string.IsNullOrEmpty(_lineCache) || _current >= _lineCache.Length;
}
return _lineCache.AsSpan().Slice( _current );
return _lineCache.AsSpan().Slice(_current);
}
public unsafe Token ReadToken()
@@ -143,95 +139,95 @@ namespace StgSharp.Script.Express
int begin, end, rest;
readLine:
ReadOnlySpan<char> source;
if( !TryReadLine( out source ) ) {
if (!TryReadLine(out source)) {
return Token.Empty;
}
if( TokenReaderPreProcessor.Process( _lineCache, LineNumber, ref _current,
out Token t ) ) {
if (TokenReaderPreProcessor.Process(_lineCache, LineNumber, ref _current,
out Token t)) {
return t;
}
char c = source[ 0 ];
if( char.IsLetter( c ) )
char c = source[0];
if (char.IsLetter(c))
{
Match match = _nextWordTokenPattern.Match( _lineCache, _current );
if( match.Success )
Match match = _nextWordTokenPattern.Match(_lineCache, _current);
if (match.Success)
{
begin = match.Groups[ TokenMark ].Index;
end = match.Groups[ EndMark ].Index;
rest = match.Groups[ RestMark ].Index;
begin = match.Groups[TokenMark].Index;
end = match.Groups[EndMark].Index;
rest = match.Groups[RestMark].Index;
_current = rest;
_last = new(
ExpressCompile.PoolString( source[ ..( end - begin ) ] ), _lineNum, begin,
TokenFlag.Member );
ExpressCompile.PoolString(source[..(end - begin)]), _lineNum, begin,
TokenFlag.Member);
return _last;
} else
{
throw new ExpInvalidCharException( _current );
throw new ExpInvalidCharException(_current);
}
} else if( char.IsPunctuation( c ) || char.IsSymbol( c ) )
} else if (char.IsPunctuation(c) || char.IsSymbol(c))
{
Match match;
switch( c )
switch (c)
{
case '\'':
char cNext = source[ 1 ];
if( cNext == '\'' ) //empty string
char cNext = source[1];
if (cNext == '\'') //empty string
{
begin = _current;
_current += 2;
return new( string.Empty, _lineNum, begin, TokenFlag.String );
return new(string.Empty, _lineNum, begin, TokenFlag.String);
}
match = _nextStringLiteralPatternWithSingle.Match( _lineCache, _current );
if( match.Success )
match = _nextStringLiteralPatternWithSingle.Match(_lineCache, _current);
if (match.Success)
{
begin = match.Groups[ "token" ].Index;
end = match.Groups[ "end" ].Index;
rest = match.Groups[ "rest" ].Index;
begin = match.Groups["token"].Index;
end = match.Groups["end"].Index;
rest = match.Groups["rest"].Index;
_current = rest;
_last = new(
ExpressCompile.PoolString( source[ 1..( end - begin ) ] ), _lineNum,
begin, TokenFlag.String );
ExpressCompile.PoolString(source[1..(end - begin)]), _lineNum,
begin, TokenFlag.String);
return _last;
} else
{
throw new ExpInvalidCharException( '\"' );
throw new ExpInvalidCharException('\"');
}
case '"':
cNext = source[ 1 ];
if( cNext == '\"' ) //empty string
cNext = source[1];
if (cNext == '\"') //empty string
{
begin = _current;
_current += 2;
return new( string.Empty, _lineNum, begin, TokenFlag.String );
return new(string.Empty, _lineNum, begin, TokenFlag.String);
}
match = _nextStringLiteralPattern.Match( _lineCache, _current );
if( match.Success )
match = _nextStringLiteralPattern.Match(_lineCache, _current);
if (match.Success)
{
begin = match.Groups[ "token" ].Index;
end = match.Groups[ "end" ].Index;
rest = match.Groups[ "rest" ].Index;
begin = match.Groups["token"].Index;
end = match.Groups["end"].Index;
rest = match.Groups["rest"].Index;
_current = rest;
_last = new(
ExpressCompile.PoolString( source[ 1..( end - begin ) ] ), _lineNum,
begin, TokenFlag.String );
ExpressCompile.PoolString(source[1..(end - begin)]), _lineNum,
begin, TokenFlag.String);
return _last;
} else
{
throw new ExpInvalidCharException( '\"' );
throw new ExpInvalidCharException('\"');
}
case '(':
begin = _current;
_current++;
_last = new(
ExpressCompile.PoolString( source[ ..1 ] ), _lineNum, begin,
TokenFlag.Separator_Left );
ExpressCompile.PoolString(source[..1]), _lineNum, begin,
TokenFlag.Separator_Left);
return _last;
case ')' or '}':
begin = _current;
_current++;
_last = new Token(
ExpressCompile.PoolString( source[ ..1 ] ), _lineNum, begin,
TokenFlag.Separator_Right );
ExpressCompile.PoolString(source[..1]), _lineNum, begin,
TokenFlag.Separator_Right);
return _last;
case '-':
@@ -239,24 +235,24 @@ namespace StgSharp.Script.Express
// 2. -- comment
// 3. - number type param(UNARY)
// 4. param - param(BINARY)
cNext = source[ 1 ];
if( char.IsNumber( cNext ) )
cNext = source[1];
if (char.IsNumber(cNext))
{
match = _nextNumberLiteralPattern.Match( _lineCache, _current );
if( match.Success )
match = _nextNumberLiteralPattern.Match(_lineCache, _current);
if (match.Success)
{
begin = match.Groups[ TokenMark ].Index;
end = match.Groups[ EndMark ].Index;
rest = match.Groups[ RestMark ].Index;
begin = match.Groups[TokenMark].Index;
end = match.Groups[EndMark].Index;
rest = match.Groups[RestMark].Index;
_current = rest;
return new(
ExpressCompile.PoolString( source[ ..( end - begin ) ] ),
_lineNum, begin, TokenFlag.Number );
ExpressCompile.PoolString(source[..(end - begin)]),
_lineNum, begin, TokenFlag.Number);
} else
{
throw new ExpInvalidCharException( _current );
throw new ExpInvalidCharException(_current);
}
} else if( cNext == '-' )
} else if (cNext == '-')
{
_lineCache = string.Empty;
_current = 0;
@@ -265,18 +261,18 @@ namespace StgSharp.Script.Express
{
begin = _current;
_current++;
if( _last.Flag == TokenFlag.Member ||
if (_last.Flag == TokenFlag.Member ||
_last.Flag == TokenFlag.Index_Right ||
_last.Flag == TokenFlag.Separator_Right )
_last.Flag == TokenFlag.Separator_Right)
{
_last = new(
ExpressCompile.PoolString( source[ ..1 ] ), _lineNum, begin,
TokenFlag.Symbol_Binary );
ExpressCompile.PoolString(source[..1]), _lineNum, begin,
TokenFlag.Symbol_Binary);
} else
{
_last = new(
ExpressCompile.PoolString( source[ ..1 ] ), _lineNum, begin,
TokenFlag.Symbol_Unary );
ExpressCompile.PoolString(source[..1]), _lineNum, begin,
TokenFlag.Symbol_Unary);
}
return _last;
}
@@ -284,139 +280,131 @@ namespace StgSharp.Script.Express
begin = _current;
_current++;
_last = new Token(
ExpressCompile.PoolString( "{" ), _lineNum, begin,
TokenFlag.Separator_Left );
ExpressCompile.PoolString("{"), _lineNum, begin,
TokenFlag.Separator_Left);
return _last;
case '[':
begin = _current;
_current++;
_last = new Token(
ExpressCompile.PoolString( "[" ), _lineNum, begin,
TokenFlag.Index_Left );
ExpressCompile.PoolString("["), _lineNum, begin,
TokenFlag.Index_Left);
return _last;
case ']':
begin = _current;
_current++;
_last = new Token(
ExpressCompile.PoolString( "]" ), _lineNum, begin,
TokenFlag.Index_Right );
ExpressCompile.PoolString("]"), _lineNum, begin,
TokenFlag.Index_Right);
return _last;
case ';' or ',':
begin = _current;
_current++;
_last = new Token(
ExpressCompile.PoolString( source[ ..1 ] ), _lineNum, begin,
TokenFlag.Separator_Middle );
ExpressCompile.PoolString(source[..1]), _lineNum, begin,
TokenFlag.Separator_Middle);
return _last;
case ':':
match = _nextSymbolPattern.Match( _lineCache, _current );
begin = match.Groups[ TokenMark ].Index;
end = match.Groups[ EndMark ].Index;
rest = match.Groups[ RestMark ].Index;
match = _nextSymbolPattern.Match(_lineCache, _current);
begin = match.Groups[TokenMark].Index;
end = match.Groups[EndMark].Index;
rest = match.Groups[RestMark].Index;
_current = rest;
if( end - begin == 1 )
if (end - begin == 1)
{
_last = new Token(
ExpressCompile.Keyword.Colon, _lineNum, begin,
TokenFlag.Separator_Middle );
TokenFlag.Separator_Middle);
} else
{
_last = new Token(
ExpressCompile.PoolString( source[ ..( end - begin ) ] ), _lineNum,
begin, TokenFlag.Symbol_Binary );
ExpressCompile.PoolString(source[..(end - begin)]), _lineNum,
begin, TokenFlag.Symbol_Binary);
}
return _last;
case '+' or '/' or '\\' or '*' or '.':
match = _nextSymbolPattern.Match( _lineCache, _current );
begin = match.Groups[ TokenMark ].Index;
end = match.Groups[ EndMark ].Index;
rest = match.Groups[ RestMark ].Index;
match = _nextSymbolPattern.Match(_lineCache, _current);
begin = match.Groups[TokenMark].Index;
end = match.Groups[EndMark].Index;
rest = match.Groups[RestMark].Index;
_current = rest;
_last = new Token(
ExpressCompile.PoolString( source[ ..( end - begin ) ] ), _lineNum,
begin, TokenFlag.Symbol_Binary );
ExpressCompile.PoolString(source[..(end - begin)]), _lineNum,
begin, TokenFlag.Symbol_Binary);
return _last;
case '=':
begin = _current;
_current++;
_last = new Token(
ExpressCompile.Keyword.Equal, _lineNum, begin,
TokenFlag.Symbol_Binary );
TokenFlag.Symbol_Binary);
return _last;
default:
throw new ExpInvalidCharException( c );
throw new ExpInvalidCharException(c);
}
} else if( char.IsNumber( c ) )
} else if (char.IsNumber(c))
{
Match match = _nextNumberLiteralPattern.Match( _lineCache, _current );
if( match.Success )
Match match = _nextNumberLiteralPattern.Match(_lineCache, _current);
if (match.Success)
{
begin = match.Groups[ TokenMark ].Index;
end = match.Groups[ EndMark ].Index;
rest = match.Groups[ RestMark ].Index;
begin = match.Groups[TokenMark].Index;
end = match.Groups[EndMark].Index;
rest = match.Groups[RestMark].Index;
_current = rest;
string str = ExpressCompile.PoolString( source[ ..( end - begin ) ] );
return new( str, _lineNum, begin, TokenFlag.Number );
string str = ExpressCompile.PoolString(source[..(end - begin)]);
return new(str, _lineNum, begin, TokenFlag.Number);
} else
{
throw new ExpInvalidCharException( _current );
throw new ExpInvalidCharException(_current);
}
} else if( char.IsWhiteSpace( c ) )
} else if (char.IsWhiteSpace(c))
{
Match match = _skipBlankPattern.Match( _lineCache, _current );
if( match.Success )
Match match = _skipBlankPattern.Match(_lineCache, _current);
if (match.Success)
{
rest = match.Groups[ RestMark ].Index;
rest = match.Groups[RestMark].Index;
_current = rest;
goto readLine;
} else
{
throw new ExpInvalidCharException( _current );
throw new ExpInvalidCharException(_current);
}
} else
{
throw new ExpInvalidCharException( c );
throw new ExpInvalidCharException(c);
}
}
public bool TryReadLine( out ReadOnlySpan<char> line )
public bool TryReadLine(out ReadOnlySpan<char> line)
{
line = ReadLine();
return line != ReadOnlySpan<char>.Empty;
}
public bool TryReadToken( out Token t )
public bool TryReadToken(out Token t)
{
t = ReadToken();
return !IsEmpty;
}
[GeneratedRegex(
@"(?<token>[a-zA-Z][a-zA-Z0-9_]*)(?<end>\s*?)(?<rest>\S|$)",
RegexOptions.Singleline )]
[GeneratedRegex(@"(?<token>[a-zA-Z][a-zA-Z0-9_]*)(?<end>\s*?)(?<rest>\S|$)", RegexOptions.Singleline)]
private static partial Regex GetNextWordTokenPattern();
[GeneratedRegex(
@"(?<token>-?\s*?[0-9]+(?:\.[0-9]*)?E?(?:[+-]?[0-9]+)?)(?<end>\s*?)(?<rest>\S|$)",
RegexOptions.Singleline )]
RegexOptions.Singleline)]
private static partial Regex GetNumberLiteralPattern();
[GeneratedRegex( @"(?:\s+?)(?<rest>\S|$)", RegexOptions.Singleline )]
[GeneratedRegex(@"(?:\s+?)(?<rest>\S|$)", RegexOptions.Singleline)]
private static partial Regex GetSkipBlankPattern();
[GeneratedRegex(
@"(?<token>"")(?>\\\\|\\""|(?!')[^""])*(?<end>""|$)\s*?(?<rest>\S|$)",
RegexOptions.Singleline )]
[GeneratedRegex(@"(?<token>"")(?>\\\\|\\""|(?!')[^""])*(?<end>""|$)\s*?(?<rest>\S|$)", RegexOptions.Singleline)]
private static partial Regex GetStringLiteralPattern();
[GeneratedRegex(
@"(?<token>')(?>\\\\|\\'|''|(?!')[^'])*(?<end>')\s*?(?<rest>\S|$)",
RegexOptions.Singleline )]
[GeneratedRegex(@"(?<token>')(?>\\\\|\\'|''|(?!')[^'])*(?<end>')\s*?(?<rest>\S|$)", RegexOptions.Singleline)]
private static partial Regex GetStringLiteralPatternWithSingle();
[GeneratedRegex(
@"(?<token>[:<>=+\-*\/\.]+)(?<end>\s?)\s*?(?<rest>\S|$)",
RegexOptions.Singleline )]
[GeneratedRegex(@"(?<token>[:<>=+\-*\/\.]+)(?<end>\s?)\s*?(?<rest>\S|$)", RegexOptions.Singleline)]
private static partial Regex GetSymbolPattern();
}