youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

daily update on 20240206 by nitload youmu, token parser change to use regex generator

Browse Source
This commit is contained in:
Nitload
2025-02-06 07:45:06 +08:00
parent 4e5a1b0dfb
commit 7d0470d50b
104 changed files with 2752 additions and 3134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
内核源代码/StgSharp.Script/Express/ExpSourceReader.cs
+70 -29
View File
@@ -28,37 +28,77 @@
//
//-----------------------------------------------------------------------
//-----------------------------------------------------------------------
using CommunityToolkit.HighPerformance;
using StgSharp.Intrinsic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.RegularExpressions;
namespace StgSharp.Script.Express
{
public class ExpSourceReader
public partial class ExpSourceReader
{
private static Regex
_commentSplitter = new Regex(
@"(?<code>\S*)\s*--.*", RegexOptions.Compiled ),
_multiLineCommentBegin = new Regex(
@"(?<code>\S*)\s*\(\*.*", RegexOptions.Compiled ),
_multiLineCommentEnd = new Regex(
@".*\*\)\s*(?<code>\S*)", RegexOptions.Compiled );
private const int SingleComment = 0x2D2D, // --
MultiLineBegin = 0x282A, // (*
MultiLineEnd = 0x2A29; // *)
private Regex _nextWordTokenPattern = GetNextWordTokenPattern(),
_;
private bool _isReadingMultiLineComment;
private int _index;
private IScriptSourceProvider _provider;
private string _lineCache;
public ExpSourceReader( IScriptSourceProvider provider )
{
_provider = provider;
}
public string ReadLine()
[GeneratedRegex(@"(?<token>[a-zA-Z][a-zA-Z0-9_]*)(?<end>\s|\W)(?<rest>.*)")]
internal static partial Regex GetNextWordTokenPattern();
/// <summary>
/// <para> Read code until end of a line. Lines in multi-line comment will be ignored. Line
/// begins with part of multi-line comment will automatically remove the comment part, but
/// beginning with a full multi-line comment will not causing removing. </para> <para>(*Just
/// Like The Comment Here*) Will Not Be Removed</para>
/// </summary>
/// <returns></returns>
public unsafe ReadOnlySpan<char> ReadLine()
{
return _provider.ReadLine();
while( string.IsNullOrEmpty( _lineCache ) || _index >=
_lineCache.Length || _isReadingMultiLineComment ) {
_lineCache = _provider.ReadLine();
if( _isReadingMultiLineComment ) {
int index;
fixed( char* cPtr = _lineCache.AsSpan() ) {
index = UnsafeCompute.IndexOfCharPair(
cPtr, MultiLineEnd, _lineCache.Length );
}
if( index != -1 ) {
_lineCache = string.Empty;
continue;
}
}
}
return _lineCache.AsSpan().Slice( _index );
}
public unsafe ReadOnlySpan<char> ReadToken( out string operand )
{
int tokenBegin,tokenEnd,mark;
readLine:
}
/*
public string ReadNoCommentLine()
{
string line = string.Empty;
@@ -69,33 +109,33 @@ namespace StgSharp.Script.Express
//Returns true when no usable string returns
private bool TryReadNoCommentLine( ref string line )
{
int index;
int _index;
string str = _provider.ReadLine();
if( string.IsNullOrEmpty( str ) ) {
return true;
}
if( _isReadingMultiLineComment ) {
index = str.LastIndexOf( @"*)" ) + 2; //calc by beginning of code
if( index == 1 || index >= str.Length ) {
_index = str.LastIndexOf( @"*)" ) + 2; //calc by beginning of code
if( _index == 1 || _index >= str.Length ) {
return true;
}
line = str.SliceAndTrim( index );
line = str.SliceAndTrim( _index );
_isReadingMultiLineComment = false;
return string.IsNullOrEmpty(line);
}
index = str.IndexOf( @"--" );
if( index != -1 ) {
line = str.SliceAndTrim( 0, index );
_index = str.IndexOf( @"--" );
if( _index != -1 ) {
line = str.SliceAndTrim( 0, _index );
return string.IsNullOrEmpty(str);
}
index = str.IndexOf(@"(*");
if (index > 0) {
if (TrySliceMultiLineCommentInSingleLine(ref str, index))
_index = str.IndexOf(@"(*");
if (_index > 0) {
if (TrySliceMultiLineCommentInSingleLine(ref str, _index))
{
line = str;
return string.IsNullOrEmpty(str);
}
line = str.SliceAndTrim(0,index);
line = str.SliceAndTrim(0,_index);
_isReadingMultiLineComment = true;
return string.IsNullOrEmpty(str);
}
@@ -107,37 +147,38 @@ namespace StgSharp.Script.Express
private unsafe bool TrySliceMultiLineCommentInSingleLine(ref string source, int commentBegin)
{
string origin = source;
int index = origin.IndexOf("*)",commentBegin);
if (index == -1)
int _index = origin.IndexOf("*)",commentBegin);
if (_index == -1)
{
return false;
}
index += 2;
_index += 2;
int begin = commentBegin - 1;
ReadOnlySpan<char> sourceSpan = origin.AsSpan();
while (begin >= 0 && char.IsWhiteSpace(sourceSpan[begin]))
{
begin--;
}
while (index < origin.Length && char.IsWhiteSpace(sourceSpan[index]))
while (_index < origin.Length && char.IsWhiteSpace(sourceSpan[_index]))
{
index++;
_index++;
}
if (begin == 0 && index == origin.Length - 1)
if (begin == 0 && _index == origin.Length - 1)
{
source = string.Empty;
return true;
}
fixed (char* cPtr = sourceSpan)
{
int secLength = origin.Length - index;
int secLength = origin.Length - _index;
Span<char> o = new Span<char>(cPtr, origin.Length);
Span<char> s = o.Slice(0, begin + 1 + secLength);
Span<char> sec = o.Slice(index), dest = o.Slice(begin+1, secLength);
Span<char> sec = o.Slice(_index), dest = o.Slice(begin+1, secLength);
sec.CopyTo(dest);
source = new string(s);
}
return true;
}
/**/
}
}