ngra string regex use ir as jit engine as well

This commit is contained in:
Youmu Liu
2026-06-15 18:08:14 +08:00
parent 2741b34583
commit 861b50575a
23 changed files with 352 additions and 795 deletions
@@ -66,13 +66,14 @@ namespace StgSharp.Mathematics.Numeric
mat_mk[f32.IntrinsicNode]. mat_mk[f32.IntrinsicNode].
Get(MatrixIntrinsicHandle.Add); Get(MatrixIntrinsicHandle.Add);
MatrixParallelTask* t = stackalloc MatrixParallelTask[1]; MatrixParallelTask* t = stackalloc MatrixParallelTask[1];
/*
t->Mat1 = (MatrixKernel*)leftPtr; t->Mat1 = (MatrixKernel*)leftPtr;
t->Mat2 = (MatrixKernel*)rightPtr; t->Mat2 = (MatrixKernel*)rightPtr;
t->Mat3 = (MatrixKernel*)ansPtr; t->Mat3 = (MatrixKernel*)ansPtr;
t->ElementType = f32; t->ElementType = f32;
t->ComputeHandle = MatrixIntrinsicHandle.Add; t->ComputeHandle = MatrixIntrinsicHandle.Add;
t->Scalar.Data<long>(0) = count; t->Scalar.Data<long>(0) = count;
/**/
SequentialOperation(t, count); SequentialOperation(t, count);
} }
@@ -62,7 +62,7 @@ namespace StgSharp.Mathematics.Numeric
internal readonly Tlsf _allocator = allocator; internal readonly Tlsf _allocator = allocator;
protected internal override unsafe T* BufferPointer => (T*)_handle.Address; protected internal override unsafe T* BufferPointer => (T*)(ulong)_handle.Address;
public override void Dispose() public override void Dispose()
{ {
@@ -51,7 +51,8 @@ namespace StgSharp.Mathematics.Numeric
internal static unsafe MatrixParallelWrap ScheduleTask<TTaskQueue>( internal static unsafe MatrixParallelWrap ScheduleTask<TTaskQueue>(
MatrixParallelTask* package MatrixParallelTask* package
) where TTaskQueue : IMatrixParallelQueue<TTaskQueue> )
{ {
/* /*
int primCount = package->PrimCount; int primCount = package->PrimCount;
@@ -66,10 +66,12 @@ namespace StgSharp.Mathematics.Numeric
} }
public bool TryGetTask( public bool TryGetTask(
in MatrixParallelTask task out MatrixParallelTask task
) )
{ {
if (_cursor >= _currentCapacity) { if (_cursor >= _currentCapacity)
{
task = default;
return false; return false;
} }
task = _currentPack[_cursor++]; task = _currentPack[_cursor++];
@@ -42,42 +42,36 @@ namespace StgSharp.RegularAnalysis.Abstraction
where TType : unmanaged where TType : unmanaged
{ {
private HashSet<TNode> _allToken = new(); public HashSet<TNode> AllNodes => AllToken;
public HashSet<TNode> AllNodes public int Count => AllToken.Count;
{
get { return _allToken; }
}
public int Count
{
get { return _allToken.Count; }
}
public TNode Root public TNode Root
{ {
get; get;
internal set internal set
{ {
if (_allToken.Contains(value)) { if (AllToken.Contains(value)) {
field = value; field = value;
} }
} }
} }
private HashSet<TNode> AllToken { get; } = new();
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool AddNode( public bool AddNode(
TNode node TNode node
) )
{ {
return _allToken.Add(node); return AllToken.Add(node);
} }
public bool Contains( public bool Contains(
TNode node TNode node
) )
{ {
return _allToken.Contains(node); return AllToken.Contains(node);
} }
public IEnumerator<TNode> GetEnumerator() public IEnumerator<TNode> GetEnumerator()
@@ -92,11 +86,11 @@ namespace StgSharp.RegularAnalysis.Abstraction
) : IEnumerator<TNode> where TNode : ISyntaxNode<TNode, TType> where TType : unmanaged ) : IEnumerator<TNode> where TNode : ISyntaxNode<TNode, TType> where TType : unmanaged
{ {
private bool _completed = false; private bool _completed;
private int _state, _target; // 0: not started, 1: on right, 2: ended private int _state, _target; // 0: not started, 1: on right, 2: ended
private Stack<Record> _recycle = []; private readonly Stack<Record> _recycle = [];
private Stack<Record> _stack = []; private readonly Stack<Record> _stack = [];
// private TNode _lRoot; // private TNode _lRoot;
private TNode _root = root; private TNode _root = root;
@@ -38,8 +38,6 @@ namespace StgSharp.RegularAnalysis.Abstraction
public readonly struct Token<TLabel> where TLabel : unmanaged public readonly struct Token<TLabel> where TLabel : unmanaged
{ {
private static readonly Token<TLabel> EmptyToken = new Token<TLabel>();
public readonly int Column; public readonly int Column;
public readonly int Line; public readonly int Line;
public readonly string Value; public readonly string Value;
@@ -66,7 +64,7 @@ namespace StgSharp.RegularAnalysis.Abstraction
Flag = flag; Flag = flag;
} }
public static Token<TLabel> Empty => EmptyToken; public static Token<TLabel> Empty { get; } = new Token<TLabel>();
public override bool Equals( public override bool Equals(
[NotNullWhen(true)] object? obj [NotNullWhen(true)] object? obj
@@ -1,97 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="Node.Boarder"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis
{
internal class RegularExpressionLeftBoarderNode : RegularStateNode
{
public RegularExpressionLeftBoarderNode(
int index,
string name
)
{
Boarder = new GroupIndex
{
Index = index,
Name = name
};
}
public event EventHandler<GroupIndex> OnBoarderEntered;
public GroupIndex Boarder { get; private set; }
public override NodeType Type => NodeType.LEFT_BOARDER;
public override bool IsValidateFrom(
RegularStateNode former,
RegularContext context
)
{
return true;
}
}
internal class RegularExpressionRightBoarderNode : RegularStateNode
{
public RegularExpressionRightBoarderNode(
int index,
string name
)
{
Boarder = new GroupIndex
{
Index = index,
Name = name
};
}
public event EventHandler<GroupIndex> OnBoarderEntered;
public GroupIndex Boarder { get; private set; }
public override NodeType Type => NodeType.RIGHT_BOARDER;
public override bool IsValidateFrom(
RegularStateNode former,
RegularContext context
)
{
return true;
}
}
}
@@ -1,107 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="Node.Branch"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis
{
internal sealed class RegularBranchNode : RegularStateNode
{
private int _afterBranch;
private int _lastBranch;
private List<BranchMeasureResult> _result;
public override NodeType Type => NodeType.BRANCH;
public override bool IsValidateFrom(
RegularStateNode former,
RegularContext context
)
{
return true;
}
public override bool TryTransaction(
ReadOnlySpan<RegularStateNode> nodeSpan,
RegularContext context,
out int label
)
{
if (context.StateCode == 0)
{
// branch match
label = _afterBranch;
_ = context.LastBranch.Pop();
_lastBranch = 0;
return true;
}
// restore sequence position
int pos = context.LastBranch.Peek();
context.Position = pos;
if (AvailableNodes.Length > _lastBranch)
{
label = FailNodeIndex;
return false;
}
_lastBranch++;
label = AvailableNodes[_lastBranch];
return true;
}
}
internal struct BranchMeasureResult
{
internal BranchMeasureResult(
bool match,
int size
)
{
IsMatch = match;
Length = size;
}
public bool IsMatch { get; init; }
public static BranchMeasureResult Fail { get; } = new BranchMeasureResult();
public int Length { get; init; }
}
}
@@ -1,196 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="Node.Char"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.State;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis
{
internal enum EpsilonKind
{
Always,
Begin,
End,
LineBegin,
LineEnd,
WordBoundary,
NonWordBoundary
}
internal class RegularExpressionEpsilonNode : RegularStateNode
{
public RegularExpressionEpsilonNode(
EpsilonKind kind = EpsilonKind.Always
)
{
Kind = kind;
}
public EpsilonKind Kind { get; }
public override NodeType Type => NodeType.EPSILON;
public override bool IsValidateFrom(
RegularStateNode former,
RegularContext context
)
{
int pos = context.Position;
int len = context.Source?.Length ?? 0;
bool PrevIsWord()
{
if (pos - 1 < 0 || pos - 1 >= len) {
return false;
}
char c = context.Source[pos - 1];
return char.IsLetterOrDigit(c) || c == '_';
}
bool NextIsWord()
{
if (pos < 0 || pos >= len) {
return false;
}
char c = context.Source[pos];
return char.IsLetterOrDigit(c) || c == '_';
}
switch (Kind)
{
case EpsilonKind.Always:
return true;
case EpsilonKind.Begin:
return pos == context.Begin;
case EpsilonKind.End:
return pos == len;
case EpsilonKind.LineBegin:
if (pos == 0) {
return true;
}
if (pos - 1 < 0 || pos - 1 >= len) {
return false;
}
return context.Source[pos - 1] == '\n' || context.Source[pos - 1] == '\r';
case EpsilonKind.LineEnd:
if (pos == len) {
return true;
}
if (pos < 0 || pos >= len) {
return false;
}
return context.Source[pos] == '\n' || context.Source[pos] == '\r';
case EpsilonKind.WordBoundary:
return PrevIsWord() != NextIsWord();
case EpsilonKind.NonWordBoundary:
return PrevIsWord() == NextIsWord();
default:
return false;
}
}
}
internal class RegularExpressionCharNode : RegularStateNode
{
private char _char;
public RegularExpressionCharNode(
char c
)
{
_char = c;
}
public override NodeType Type => NodeType.CHAR;
public override bool IsValidateFrom(
RegularStateNode former,
RegularContext context
)
{
int pos = context.Position;
char c = context.CurrentCharSpan[pos + 1];
return c == _char;
}
}
internal class RegularExpressionCharsetNode : RegularStateNode
{
private bool _isNegated;
private HashSet<char> _charset;
public RegularExpressionCharsetNode(
IEnumerable<char> charset,
bool isNegated = false
)
{
_charset = new(charset);
_isNegated = isNegated;
}
public RegularExpressionCharsetNode(
ReadOnlySpan<char> charset,
bool isNegated = false
)
{
_charset = new(charset.Length);
_isNegated = isNegated;
}
public override NodeType Type => NodeType.CHARSET;
public override bool IsValidateFrom(
RegularStateNode former,
RegularContext context
)
{
int pos = context.Position;
char c = context.CurrentCharSpan[pos + 1];
bool contains = _charset.Contains(c);
return _isNegated ? !contains : contains;
}
}
}
@@ -1,27 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection.Emit;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis
{
internal sealed class RegularSequenceNode : RegularStateNode
{
private string _sequence;
public override NodeType Type => NodeType.SEQUENCE;
public override bool IsValidateFrom(
RegularStateNode former,
RegularContext context
)
{
int pos = context.Position + 1;
return context.CurrentCharSpan[pos.._sequence.Length].SequenceEqual(_sequence);
}
}
}
@@ -1,70 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="Node"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.State;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis
{
internal abstract class RegularStateNode : SimpleStateMachineNode<RegularStateNode, int, RegularContext>
{
public int FailNodeIndex { get; set; }
public abstract NodeType Type { get; }
public override bool TryTransaction(
ReadOnlySpan<RegularStateNode> nodeSpan,
RegularContext context,
out int label
)
{
int pos = context.Position;
if (pos < 0)
{
label = FailNodeIndex;
return false;
}
char c = context.CurrentCharSpan[pos];
foreach (int index in AvailableNodes)
{
if (nodeSpan[index].IsValidateFrom(this, context))
{
label = index;
return true;
}
}
label = FailNodeIndex;
return false;
}
}
}
@@ -1,65 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegularExpression.NodeState"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis
{
internal enum NodeType
{
// single unicode character: a、B、1、@、汉字 etc.
CHAR,
// collections of finate characters: [abc]、[^abc]、\d、\w、\s、\D、\W、\S etc.
CHARSET,
// increase count of a counter, half function of *+? quantifiers
COUNT,
// left boarder of a sub sequnce, usually begin of match group or lookaround
LEFT_BOARDER,
// right boarder of a sub sequnce, usually end of match group or lookaround
RIGHT_BOARDER,
// zero length char or structure
EPSILON,
// more than one cases to match, like alter|nation
BRANCH,
// a sequence of nodes to match in order, each char should be fully defined but not charset
SEQUENCE
}
}
@@ -1,138 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="StateMachine"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Internal;
using StgSharp.State;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis
{
internal class RegularExpressionState : ISimpleStateMachine<RegularStateNode, int, RegularContext>
{
private List<RegularStateNode> _nodes;
private RegularStateNode _currentState;
public RegularStateNode this[
int label
] => _nodes[label];
public ref readonly RegularStateNode GetCurrentStateNode()
{
throw new NotImplementedException();
}
public void Reset()
{
throw new NotImplementedException();
}
public bool TryMoveNext(
RegularContext ctx
)
{
throw new NotImplementedException();
}
#region node factory
public RegularExpressionCharNode AddChar(
char c
)
{
RegularExpressionCharNode node = new RegularExpressionCharNode(c);
int index = _nodes.Count;
_nodes.Add(node);
node.FailNodeIndex = index; // Point to self by default
return node;
}
public RegularExpressionCharsetNode AddCharset(
ReadOnlySpan<char> chars,
bool isNegate
)
{
RegularExpressionCharsetNode node = new RegularExpressionCharsetNode(chars, isNegate);
int index = _nodes.Count;
_nodes.Add(node);
node.FailNodeIndex = index; // Point to self by default
return node;
}
public (RegularExpressionLeftBoarderNode left,RegularExpressionRightBoarderNode right) AddBoarder(
string groupName,
int groupIndex
)
{
RegularExpressionLeftBoarderNode leftNode = new RegularExpressionLeftBoarderNode(groupIndex, groupName);
int leftIndex = _nodes.Count;
_nodes.Add(leftNode);
leftNode.FailNodeIndex = leftIndex; // Point to self by default
RegularExpressionRightBoarderNode rightNode = new RegularExpressionRightBoarderNode(groupIndex, groupName);
int rightIndex = _nodes.Count;
_nodes.Add(rightNode);
rightNode.FailNodeIndex = rightIndex; // Point to self by default
return (left: leftNode,right:rightNode);
}
public RegularExpressionCounterNode AddCounter(
int minCount,
bool isEager
)
{
RegularExpressionCounterNode node = new RegularExpressionCounterNode(minCount, isEager);
int index = _nodes.Count;
_nodes.Add(node);
node.FailNodeIndex = index; // Point to self by default
return node;
}
#endregion
}
internal class RegularContext : StateContext
{
public int Begin { get; set; }
public int Position { get; set; } = -1;
public int StateCode { get; set; } = 0;
public ReadOnlySpan<char> CurrentCharSpan => Source.AsSpan()[Begin..Position];
public string Source { get; init; }
internal Stack<int> LastBranch { get; } = new();
}
}
@@ -26,14 +26,7 @@
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction; using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Linq;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using System.Xml.Linq;
namespace StgSharp.RegularAnalysis.TextRegex namespace StgSharp.RegularAnalysis.TextRegex
{ {
@@ -0,0 +1,242 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexIR"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.TextRegex
{
internal enum RegexIRCommand
{
Search = 0,
Jump = 1,
Compare = 2,
Check = 3,
Uncheck = 4,
Span = 5,
Define = 6,
Assign = 7,
Back = 8,
}
internal abstract class RegexIR
{
internal abstract RegexIRCommand Command { get; }
internal abstract void Execute();
}
/// <summary>
/// Searches for a certain string pattern or charset from current position.
/// </summary>
internal sealed class RegexSearchStringIR(
string pattern
) : RegexIR
{
public string Pattern { get; } = pattern;
internal override RegexIRCommand Command => RegexIRCommand.Search;
internal override void Execute()
{
throw new NotImplementedException();
}
}
internal sealed class RegexSearchCharsetIR(
string charset
) : RegexIR
{
public string Charset { get; } = charset;
internal override RegexIRCommand Command => RegexIRCommand.Search;
internal override void Execute()
{
throw new NotImplementedException();
}
}
internal sealed class RegexSearchCharIR(
char c
) : RegexIR
{
public char target { get; } = c;
internal override RegexIRCommand Command => RegexIRCommand.Search;
internal override void Execute()
{
throw new NotImplementedException();
}
}
/// <summary>
/// JUMP to a certain instruction index.
/// </summary>
/// <param name="target">
///
/// </param>
internal sealed class RegexJumpIR(
int target
) : RegexIR
{
public int TargetIndex { get; } = target;
internal override RegexIRCommand Command => RegexIRCommand.Jump;
internal override void Execute()
{
throw new NotImplementedException();
}
}
internal sealed class RegexBackIR(
int index
) : RegexIR
{
public int Target { get; } = index;
internal override RegexIRCommand Command => RegexIRCommand.Back;
internal override void Execute()
{
throw new NotImplementedException();
}
}
internal sealed class RegexCompareIR(
int success,
int fail
) : RegexIR
{
public int SuccessCase { get; } = success;
public int FailCase { get; } = fail;
internal override RegexIRCommand Command => RegexIRCommand.Compare;
internal override void Execute()
{
throw new NotImplementedException();
}
}
internal sealed class RegexCheckIR : RegexIR
{
internal override RegexIRCommand Command => RegexIRCommand.Check;
internal override void Execute()
{
throw new NotImplementedException();
}
}
internal sealed class RegexUncheckIR : RegexIR
{
internal override RegexIRCommand Command => RegexIRCommand.Check;
internal override void Execute()
{
throw new NotImplementedException();
}
}
internal sealed class RegexSpanIR(
string name
) : RegexIR
{
public string Name { get; } = name;
internal override RegexIRCommand Command => RegexIRCommand.Span;
internal override void Execute()
{
throw new NotImplementedException();
}
}
internal sealed class RegexDefineCommand(
string name
) : RegexIR
{
public string Name { get; } = name;
internal override RegexIRCommand Command => RegexIRCommand.Define;
internal override void Execute()
{
throw new NotImplementedException();
}
}
internal sealed class RegexAssignCommand<T>(
string name,
T value
) : RegexIR
{
public string Name { get; } = name;
public T Value { get; } = value;
internal override RegexIRCommand Command => RegexIRCommand.Assign;
internal override void Execute()
{
throw new NotImplementedException();
}
}
}
@@ -1,6 +1,6 @@
//----------------------------------------------------------------------- //-----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="Node.Count" // file="RegexInterpreter.IRGenerate"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload // AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved. // Copyright (c) Nitload. All rights reserved.
@@ -25,48 +25,29 @@
// //
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis namespace StgSharp.RegularAnalysis.TextRegex
{ {
internal class RegularExpressionCounterNode : RegularStateNode public partial class RegexInterpreter
{ {
private readonly int _maxCount; internal List<RegexIR> GenerateIR(
private readonly int _minCount; AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree
public RegularExpressionCounterNode(
int minCount,
bool isEager
) )
{ {
_minCount = minCount; using TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(tree);
IsEager = isEager; while (enumerator.MoveNext())
} {
if (true) { }
}
public bool IsEager { get; init; } return [];
public int Count { get; private set; }
public override NodeType Type => NodeType.COUNT;
public RegularStateNode LoopBack { get; set; }
public override bool IsValidateFrom(
RegularStateNode former,
RegularContext context
)
{
return true;
}
public void Reset()
{
Count = 0;
} }
} }
@@ -106,8 +106,8 @@ namespace StgSharp.RegularAnalysis.TextRegex
int length = 0; int length = 0;
RegexAstNode right = root.Right; RegexAstNode right = root.Right;
RegexAstNode left = root.Left; RegexAstNode left = root.Left;
if (left.EqualityTypeConvert != RegexElementLabel.UNIT_SPAN && if (left.EqualityTypeConvert is not RegexElementLabel.UNIT_SPAN and
left.EqualityTypeConvert != RegexElementLabel.UNIT) { not RegexElementLabel.UNIT) {
return false; return false;
} }
string str = left.Value; string str = left.Value;
@@ -117,8 +117,8 @@ namespace StgSharp.RegularAnalysis.TextRegex
right.EqualityTypeConvert == RegexElementLabel.CONCAT) right.EqualityTypeConvert == RegexElementLabel.CONCAT)
{ {
left = right.Left; left = right.Left;
if (left.EqualityTypeConvert != RegexElementLabel.UNIT_SPAN && if (left.EqualityTypeConvert is not RegexElementLabel.UNIT_SPAN and
left.EqualityTypeConvert != RegexElementLabel.UNIT) not RegexElementLabel.UNIT)
{ {
break; break;
} }
@@ -130,8 +130,8 @@ namespace StgSharp.RegularAnalysis.TextRegex
right = root.Right; right = root.Right;
right.Parent = root; right.Parent = root;
} }
if (right.EqualityTypeConvert == RegexElementLabel.UNIT_SPAN || if (right.EqualityTypeConvert is RegexElementLabel.UNIT_SPAN or
right.EqualityTypeConvert == RegexElementLabel.UNIT) RegexElementLabel.UNIT)
{ {
root.Right = RegexAstNode.Empty; root.Right = RegexAstNode.Empty;
buffer.Add(right.Value); buffer.Add(right.Value);
@@ -367,8 +367,6 @@ namespace StgSharp.RegularAnalysis.TextRegex
Cases[i].Parent = UnionEnd; Cases[i].Parent = UnionEnd;
} }
UnionEnd.Right = Cases[0]; UnionEnd.Right = Cases[0];
TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new TreeEnumerator<RegexAstNode, RegexElementLabel>(UnionBegin);
return UnionBegin; return UnionBegin;
} }
@@ -62,7 +62,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
string regex string regex
) )
{ {
RegexInterpreter interpreter = new RegexInterpreter(); RegexInterpreter interpreter = new();
interpreter._source = regex; interpreter._source = regex;
bool isSuccess; bool isSuccess;
try try
@@ -81,11 +81,12 @@ namespace StgSharp.RegularAnalysis.TextRegex
private void Analyze() private void Analyze()
{ {
RegexTokenReader reader = new RegexTokenReader(_source); RegexTokenReader reader = new(_source);
TokenParser<RegexElementLabel, RegexElementLabel> lexer = reader.Pipe(() => new RegexTokenParser()); TokenParser<RegexElementLabel, RegexElementLabel> lexer = reader.Pipe(() => new RegexTokenParser());
while (lexer.TryReadToken(out Token<RegexElementLabel> token)) while (lexer.TryReadToken(out Token<RegexElementLabel> token))
{ {
// Console.WriteLine(token.Value);
if ((token.Flag & RegexElementLabel.SEQUENCE) != 0) if ((token.Flag & RegexElementLabel.SEQUENCE) != 0)
{ {
// char or charset // char or charset
@@ -68,6 +68,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
} }
} else } else
{ {
Console.WriteLine("returns empty");
return Token<RegexElementLabel>.Empty; return Token<RegexElementLabel>.Empty;
} }
} }
@@ -82,7 +83,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
return false; return false;
} }
t = ReadToken(); t = ReadToken();
return t == Token<RegexElementLabel>.Empty; return t != Token<RegexElementLabel>.Empty;
} }
} }
@@ -2,8 +2,8 @@
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="RegexTokenReader" // file="RegexTokenReader"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload Space // AuthorGroup: Nitload
// Copyright (c) Nitload Space. All rights reserved. // Copyright (c) Nitload. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -45,7 +45,9 @@ namespace StgSharp.RegularAnalysis.TextRegex
public RegexTokenReader() { } public RegexTokenReader() { }
public RegexTokenReader(string source) public RegexTokenReader(
string source
)
{ {
_source = source; _source = source;
} }
@@ -65,9 +67,11 @@ namespace StgSharp.RegularAnalysis.TextRegex
{ {
string value = _source[(pos + 1)..(pos + 3)]; string value = _source[(pos + 1)..(pos + 3)];
pos += 4; pos += 4;
if (uint.TryParse(value, out uint code) && Rune.TryCreate(code, out Rune result)) if (uint.TryParse(value, out uint code) &&
Rune.TryCreate(code, out Rune result))
{ {
return new Token<RegexElementLabel>(result.ToString(), 0, pos, RegexElementLabel.UNIT); return new Token<RegexElementLabel>(result.ToString(), 0, pos,
RegexElementLabel.UNIT);
} else } else
{ {
throw new InvalidCastException($"Failed to convert {value} to a valid ascii code point."); throw new InvalidCastException($"Failed to convert {value} to a valid ascii code point.");
@@ -76,9 +80,11 @@ namespace StgSharp.RegularAnalysis.TextRegex
{ {
string value = _source[(pos + 1)..(pos + 5)]; string value = _source[(pos + 1)..(pos + 5)];
pos += 6; pos += 6;
if (uint.TryParse(value, out uint code) && Rune.TryCreate(code, out Rune result)) if (uint.TryParse(value, out uint code) &&
Rune.TryCreate(code, out Rune result))
{ {
return new Token<RegexElementLabel>(result.ToString(), 0, pos, RegexElementLabel.UNIT); return new Token<RegexElementLabel>(result.ToString(), 0, pos,
RegexElementLabel.UNIT);
} else } else
{ {
throw new InvalidCastException($"Failed to convert {value} to a valid unicode code point."); throw new InvalidCastException($"Failed to convert {value} to a valid unicode code point.");
@@ -112,21 +118,24 @@ namespace StgSharp.RegularAnalysis.TextRegex
{ {
int length = (i - pos - 1); int length = (i - pos - 1);
position = i + 1; position = i + 1;
return new Token<RegexElementLabel>(_source[(pos)..(i + 1)], 0, position, return new Token<RegexElementLabel>(_source[(pos)..(i + 1)], 0,
position,
RegexElementLabel.UNIT_SET); RegexElementLabel.UNIT_SET);
} }
} }
throw new InvalidOperationException("Unclosed character set"); throw new InvalidOperationException("Unclosed character set");
case '|': case '|':
position = pos + 1; position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.ALT); return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
RegexElementLabel.ALT);
case '(': case '(':
// very comlex here // very comlex here
return ReadGroupBeginToken(); return ReadGroupBeginToken();
case ')': case ')':
position = pos + 1; position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.GROUP_END); return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
RegexElementLabel.GROUP_END);
case '{': case '{':
for (int i = pos + 1; i < _source.Length; i++) for (int i = pos + 1; i < _source.Length; i++)
{ {
@@ -141,19 +150,25 @@ namespace StgSharp.RegularAnalysis.TextRegex
case '*' or '+': case '*' or '+':
int count = _source[pos + 1] == '?' ? 2 : 1; int count = _source[pos + 1] == '?' ? 2 : 1;
position = pos + count; position = pos + count;
return new Token<RegexElementLabel>(_source[(pos)..(pos + count)], 0, pos, RegexElementLabel.COUNT); return new Token<RegexElementLabel>(_source[(pos)..(pos + count)], 0, pos,
RegexElementLabel.COUNT);
case '?': case '?':
position = pos + 1; position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.COUNT); return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
RegexElementLabel.COUNT);
case '.': case '.':
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.UNIT); return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
RegexElementLabel.UNIT);
default: default:
position = pos + 1; position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.UNIT); return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
RegexElementLabel.UNIT);
} }
} }
public bool TryReadToken(out Token<RegexElementLabel> t) public bool TryReadToken(
out Token<RegexElementLabel> t
)
{ {
if (IsEmpty) if (IsEmpty)
{ {
@@ -161,7 +176,7 @@ namespace StgSharp.RegularAnalysis.TextRegex
return false; return false;
} }
t = ReadToken(); t = ReadToken();
return Token<RegexElementLabel>.Empty == t; return Token<RegexElementLabel>.Empty != t;
} }
private Token<RegexElementLabel> ReadGroupBeginToken() private Token<RegexElementLabel> ReadGroupBeginToken()
+3 -2
View File
@@ -52,10 +52,11 @@ namespace StgSharp
// .UseModule<GraphicModule>() // .UseModule<GraphicModule>()
.InitializeModule(); .InitializeModule();
// TestRegexAnalyse.Run(); TestRegexAnalyse.Run();
// TestHlsf(); // TestHlsf();
TlsfStressTest.Run(); // TlsfStressTest.Run();
// TestSwiss(); // TestSwiss();
// TestL4Prefetch(); // TestL4Prefetch();
+6 -4
View File
@@ -2,8 +2,8 @@
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// file="TestRegexAnalyse" // file="TestRegexAnalyse"
// Project: StgSharp // Project: StgSharp
// AuthorGroup: Nitload Space // AuthorGroup: Nitload
// Copyright (c) Nitload Space. All rights reserved. // Copyright (c) Nitload. All rights reserved.
// //
// Permission is hereby granted, free of charge, to any person obtaining a copy // Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal // of this software and associated documentation files (the "Software"), to deal
@@ -42,7 +42,9 @@ namespace StgSharp
@"http|https|ftp|ftps", @"http|https|ftp|ftps",
@"abc[0-9]x|abc[0-9]y|abd[0-9]x|abd[0-9]y|xyz", @"abc[0-9]x|abc[0-9]y|abd[0-9]x|abd[0-9]y|xyz",
@"get_name|get_value|get_type|set_name|set_value|set_type", @"get_name|get_value|get_type|set_name|set_value|set_type",
@"helix|hello|help|hero|hers|her" ]; @"helix|hello|help|hero|hers|her",
@"http|https|ftp|ftps"
];
public static void Run() public static void Run()
{ {
@@ -51,7 +53,7 @@ namespace StgSharp
RegexInterpreter interpreter = RegexInterpreter.Analyze(regex); RegexInterpreter interpreter = RegexInterpreter.Analyze(regex);
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree = interpreter.Tree; AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree = interpreter.Tree;
Console.WriteLine($"result of \"{regex}\""); Console.WriteLine($"result of \"{regex}\"");
TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new TreeEnumerator<RegexAstNode, RegexElementLabel>(tree); TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(tree);
while (enumerator.MoveNext()) while (enumerator.MoveNext())
{ {
RegexAstNode node = enumerator.Current; RegexAstNode node = enumerator.Current;
+27
View File
@@ -0,0 +1,27 @@
# 适用于NGRA字符串正则的IR形式
## 两个指针
1. 活动指针,记录当前的搜索位置,下文“不更新活动指针”的意思是算计结束后自行对活动指针进行复位
2. 锚点指针,记录一个区间的开始位置
## 检查点
检查点是锚点指针的一种形式,以栈形态存储,创建检查点时,上一个检查点自动压栈,并用当前检查点替换为当前的锚点指针
## 符号区
使用DEFINE指令定义的符号定义在符号区中,符号的类型通过第一次赋值确定,源生成模式下,可以通过第一个赋值指令推导其类型
## 算子
1. 搜索算子(SEARCH %str; SEARCH %unicode;SERCH %set_expression),向后搜索确定长度序列或者满足规则的字符集,更新活动指针,不更新锚点指针,搜索结果保存在全局上下文中
2. 跳转算子(JMP %cmd_idx),跳转到指定的IR指令之前,不变化任何指针,
3. 回到(BACK %cmd_idx %count)跳转到一个跳自身位置之前的位置,并更新跳转次数,不变化任何的指针,%count应当是使用DEFINE指令定义好的符号
4. 比较指令(CONDITION %cmd_true %cmd_fail),比较前一条SEARCH指令是否成功,并指定跳转到指定的位置,如果某种条件不需要跳转,请使用0进行填充
5. 检查点指令(CHECK),将锚点指针强制拉取到活动指针的位置
6. 弹出检查点(UNCHECK),将一个检查点撤销并销毁
7. 划分区间指令(SPAN %name),将锚点指针和活动指针之间的部分划分为一个Span<>,并弹出对应的检查点
8. 定义指令(DEFINE %symbol)定义一个符号
9. 赋值指令(ASSIGN %symbol %value)将符号设定为值