daily update on 20250208 by nitload youmu. lexical analyzer complete but little problem
This commit is contained in:
Nitload
@@ -30,6 +30,7 @@
|
||||
//-----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
|
||||
namespace StgSharp.Script.Express
|
||||
@@ -43,6 +44,24 @@ namespace StgSharp.Script.Express
|
||||
|
||||
}
|
||||
|
||||
public class ExpInvalidCharException : Exception
|
||||
{
|
||||
public ExpInvalidCharException(char c) :
|
||||
base($"An unknown char {c} occurred.")
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public class ExpInvalidTokenException : Exception
|
||||
{
|
||||
public ExpInvalidTokenException(int index) :
|
||||
base($"Cannot read a token from line at {index}")
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public class ExpInvalidTypeDeclareEndingExceptions : Exception
|
||||
{
|
||||
public ExpInvalidTypeDeclareEndingExceptions(
|
||||
|
||||
@@ -34,6 +34,7 @@ using StgSharp.Intrinsic;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Tracing;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
@@ -44,141 +45,198 @@ namespace StgSharp.Script.Express
|
||||
public partial class ExpSourceReader
|
||||
{
|
||||
|
||||
private const int SingleComment = 0x2D2D, // --
|
||||
MultiLineBegin = 0x282A, // (*
|
||||
MultiLineEnd = 0x2A29; // *)
|
||||
private const int
|
||||
MultiLineEnd = 0x0029002A; // *)
|
||||
|
||||
private Regex _nextWordTokenPattern = GetNextWordTokenPattern(),
|
||||
_;
|
||||
private const string Token = "token", End = "end", Rest = "rest";
|
||||
|
||||
private bool _isReadingMultiLineComment;
|
||||
private int _index;
|
||||
private int _current;
|
||||
private IScriptSourceProvider _provider;
|
||||
private string _lineCache;
|
||||
|
||||
public ExpSourceReader( IScriptSourceProvider provider )
|
||||
{
|
||||
_provider = provider;
|
||||
_current = 0;
|
||||
_lineCache = string.Empty;
|
||||
_isReadingMultiLineComment = false;
|
||||
}
|
||||
|
||||
[GeneratedRegex(@"(?<token>[a-zA-Z][a-zA-Z0-9_]*)(?<end>\s|\W)(?<rest>.*)")]
|
||||
internal static partial Regex GetNextWordTokenPattern();
|
||||
public bool IsEmpty
|
||||
{
|
||||
get => (( string.IsNullOrEmpty( _lineCache ) ||
|
||||
_current >= _lineCache.Length )) && _provider.IsEmpty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <para> Read code until end of a line. Lines in multi-line comment will be ignored. Line
|
||||
/// <para>Read code until end of a line. Lines in multi-line comment will be ignored. Line
|
||||
/// begins with part of multi-line comment will automatically remove the comment part, but
|
||||
/// beginning with a full multi-line comment will not causing removing. </para> <para>(*Just
|
||||
/// Like The Comment Here*) Will Not Be Removed</para>
|
||||
/// beginning with a full multi-line comment will not causing removing.</para> <para> (*Just
|
||||
/// Like The Comment Here*) Will Not Be Removed </para>
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public unsafe ReadOnlySpan<char> ReadLine()
|
||||
{
|
||||
while( string.IsNullOrEmpty( _lineCache ) || _index >=
|
||||
_lineCache.Length || _isReadingMultiLineComment ) {
|
||||
_lineCache = _provider.ReadLine();
|
||||
bool lineOver = string.IsNullOrEmpty(_lineCache) || _current >=
|
||||
_lineCache.Length;
|
||||
while( lineOver || _isReadingMultiLineComment ) {
|
||||
if (lineOver)
|
||||
{
|
||||
do
|
||||
{
|
||||
_lineCache = _provider.ReadLine();
|
||||
} while (string.IsNullOrEmpty(_lineCache));
|
||||
_current = 0;
|
||||
}
|
||||
if( _isReadingMultiLineComment ) {
|
||||
int index;
|
||||
fixed( char* cPtr = _lineCache.AsSpan() ) {
|
||||
fixed( char* cPtr = _lineCache.AsSpan(_current) ) {
|
||||
index = UnsafeCompute.IndexOfCharPair(
|
||||
cPtr, MultiLineEnd, _lineCache.Length );
|
||||
}
|
||||
if( index != -1 ) {
|
||||
if (index == -1)
|
||||
{
|
||||
_lineCache = string.Empty;
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
_isReadingMultiLineComment = false;
|
||||
_current += index + 2;
|
||||
}
|
||||
}
|
||||
lineOver = string.IsNullOrEmpty(_lineCache) || _current >=
|
||||
_lineCache.Length;
|
||||
}
|
||||
return _lineCache.AsSpan().Slice( _index );
|
||||
return _lineCache.AsSpan().Slice( _current );
|
||||
}
|
||||
|
||||
public unsafe ReadOnlySpan<char> ReadToken( out string operand )
|
||||
public unsafe ReadOnlySpan<char> ReadToken()
|
||||
{
|
||||
int tokenBegin,tokenEnd,mark;
|
||||
int begin, end, rest;
|
||||
readLine:
|
||||
ReadOnlySpan<char> source = ReadLine();
|
||||
char c = source[ 0 ];
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
public string ReadNoCommentLine()
|
||||
{
|
||||
string line = string.Empty;
|
||||
while( TryReadNoCommentLine( ref line ) ) { }
|
||||
return line;
|
||||
}
|
||||
|
||||
//Returns true when no usable string returns
|
||||
private bool TryReadNoCommentLine( ref string line )
|
||||
{
|
||||
int _index;
|
||||
string str = _provider.ReadLine();
|
||||
if( string.IsNullOrEmpty( str ) ) {
|
||||
return true;
|
||||
}
|
||||
if( _isReadingMultiLineComment ) {
|
||||
_index = str.LastIndexOf( @"*)" ) + 2; //calc by beginning of code
|
||||
if( _index == 1 || _index >= str.Length ) {
|
||||
return true;
|
||||
if (char.IsLetter(c)) {
|
||||
Match match = _nextWordTokenPattern.Match(
|
||||
_lineCache, _current);
|
||||
if (match.Success) {
|
||||
begin = match.Groups[Token].Index;
|
||||
end = match.Groups[End].Index;
|
||||
rest = match.Groups[Rest].Index;
|
||||
_current = rest;
|
||||
return source.Slice(0, end - begin);
|
||||
} else {
|
||||
throw new ExpInvalidTokenException(_current);
|
||||
}
|
||||
line = str.SliceAndTrim( _index );
|
||||
_isReadingMultiLineComment = false;
|
||||
return string.IsNullOrEmpty(line);
|
||||
}
|
||||
_index = str.IndexOf( @"--" );
|
||||
if( _index != -1 ) {
|
||||
line = str.SliceAndTrim( 0, _index );
|
||||
return string.IsNullOrEmpty(str);
|
||||
}
|
||||
_index = str.IndexOf(@"(*");
|
||||
if (_index > 0) {
|
||||
if (TrySliceMultiLineCommentInSingleLine(ref str, _index))
|
||||
} else if (char.IsPunctuation(c) || char.IsSymbol(c)) {
|
||||
switch (c) {
|
||||
case '"':
|
||||
Match match = _nextStringLiteralPattern.Match(
|
||||
_lineCache, _current);
|
||||
if (match.Success) {
|
||||
begin = match.Groups["token"].Index;
|
||||
end = match.Groups["end"].Index;
|
||||
rest = match.Groups["rest"].Index;
|
||||
_current = rest;
|
||||
return source[..(end - begin)];
|
||||
} else {
|
||||
throw new ExpInvalidCharException('\"');
|
||||
}
|
||||
case '(':
|
||||
char cNext = source[1];
|
||||
if (cNext == '*') {
|
||||
_isReadingMultiLineComment = true;
|
||||
_current += 2;
|
||||
goto readLine;
|
||||
} else {
|
||||
return source[..1];
|
||||
}
|
||||
case '-':
|
||||
|
||||
// 1. -number
|
||||
// 2. -- comment
|
||||
// 3. - number type param
|
||||
cNext = source[1];
|
||||
if (char.IsNumber(cNext)) {
|
||||
match = _nextNumberLiteralPattern.Match(
|
||||
_lineCache, _current);
|
||||
if (match.Success) {
|
||||
begin = match.Groups[Token].Index;
|
||||
end = match.Groups[End].Index;
|
||||
rest = match.Groups[Rest].Index;
|
||||
_current = rest;
|
||||
return source[..(end - begin)];
|
||||
} else {
|
||||
throw new ExpInvalidTokenException(_current);
|
||||
}
|
||||
} else if (cNext == '-') {
|
||||
_lineCache = string.Empty;
|
||||
_current = 0;
|
||||
goto readLine;
|
||||
} else {
|
||||
_current++;
|
||||
return source[..1];
|
||||
}
|
||||
case ')' or '[' or ']' or '{' or '}' or
|
||||
'+' or '/' or '\\' or '*' or '.' or ',':
|
||||
_current++;
|
||||
return source[..1];
|
||||
default:
|
||||
throw new ExpInvalidCharException(c);
|
||||
}
|
||||
} else if (char.IsNumber(c)) {
|
||||
Match match = _nextNumberLiteralPattern.Match(
|
||||
_lineCache, _current);
|
||||
if (match.Success) {
|
||||
begin = match.Groups[Token].Index;
|
||||
end = match.Groups[End].Index;
|
||||
rest = match.Groups[Rest].Index;
|
||||
_current = rest;
|
||||
return source[..(end - begin)];
|
||||
} else {
|
||||
throw new ExpInvalidTokenException(_current);
|
||||
}
|
||||
} else if (char.IsWhiteSpace(c)) {
|
||||
Match match = _skipBlankPattern.Match(_lineCache, _current);
|
||||
if (match.Success)
|
||||
{
|
||||
line = str;
|
||||
return string.IsNullOrEmpty(str);
|
||||
rest = match.Groups[Rest].Index;
|
||||
_current = rest;
|
||||
goto readLine;
|
||||
}
|
||||
line = str.SliceAndTrim(0,_index);
|
||||
_isReadingMultiLineComment = true;
|
||||
return string.IsNullOrEmpty(str);
|
||||
else
|
||||
{
|
||||
throw new ExpInvalidTokenException(_current);
|
||||
}
|
||||
}else {
|
||||
throw new ExpInvalidCharException( c );
|
||||
}
|
||||
line = str;
|
||||
return string.IsNullOrEmpty(str);
|
||||
}
|
||||
|
||||
//returns true when a usable string returns
|
||||
private unsafe bool TrySliceMultiLineCommentInSingleLine(ref string source, int commentBegin)
|
||||
{
|
||||
string origin = source;
|
||||
int _index = origin.IndexOf("*)",commentBegin);
|
||||
if (_index == -1)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
_index += 2;
|
||||
int begin = commentBegin - 1;
|
||||
ReadOnlySpan<char> sourceSpan = origin.AsSpan();
|
||||
while (begin >= 0 && char.IsWhiteSpace(sourceSpan[begin]))
|
||||
{
|
||||
begin--;
|
||||
}
|
||||
while (_index < origin.Length && char.IsWhiteSpace(sourceSpan[_index]))
|
||||
{
|
||||
_index++;
|
||||
}
|
||||
if (begin == 0 && _index == origin.Length - 1)
|
||||
{
|
||||
source = string.Empty;
|
||||
return true;
|
||||
}
|
||||
fixed (char* cPtr = sourceSpan)
|
||||
{
|
||||
int secLength = origin.Length - _index;
|
||||
Span<char> o = new Span<char>(cPtr, origin.Length);
|
||||
Span<char> s = o.Slice(0, begin + 1 + secLength);
|
||||
Span<char> sec = o.Slice(_index), dest = o.Slice(begin+1, secLength);
|
||||
sec.CopyTo(dest);
|
||||
source = new string(s);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
/**/
|
||||
[GeneratedRegex(
|
||||
@"(?<token>[a-zA-Z][a-zA-Z0-9_]*)(?<end>\s*?)(?<rest>\S|$)",
|
||||
RegexOptions.Singleline )]
|
||||
private static partial Regex GetNextWordTokenPattern();
|
||||
|
||||
[GeneratedRegex(
|
||||
@"(?<token>-?\s*?[0-9]+(?:\.[0-9]*)?)(?<end>\s*?)(?<rest>\S|$)",
|
||||
RegexOptions.Singleline )]
|
||||
private static partial Regex GetNumberLiteralPattern();
|
||||
|
||||
[GeneratedRegex( @"(?:\s+?)(?<rest>\S|$)", RegexOptions.Singleline )]
|
||||
private static partial Regex GetSkipBlankPattern();
|
||||
|
||||
[GeneratedRegex(
|
||||
@"(?<token>"")(?:.*?[^""\\](\\\\)*)?(?<end>"")\s*?(?<next>\S|$)",
|
||||
RegexOptions.Singleline )]
|
||||
private static partial Regex GetStringLiteralPattern();
|
||||
|
||||
private Regex _nextWordTokenPattern = GetNextWordTokenPattern(),
|
||||
_nextStringLiteralPattern = GetStringLiteralPattern(),
|
||||
_nextNumberLiteralPattern = GetNumberLiteralPattern(),
|
||||
_skipBlankPattern = GetSkipBlankPattern();
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user