youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Using new file structure. Use new regexir display method. Fix line dependency error in prefix analysing.

Browse Source 
Signed-off-by: Nitload <YoumuLiu_NSI@whv0t.onmicrosoft.com>
This commit is contained in:
Nitload
2026-07-12 02:04:57 +08:00
parent ae8a09efc2
commit b6cd2fc256
476 changed files with 16440 additions and 502 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/StgSharp.RegularAnalysis/Abstraction/AbstractSyntaxTree.cs
+238
View File
@@ -0,0 +1,238 @@
// -----------------------------------------------------------------------------
// file="AbstractSyntaxTree"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.Collections;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Xml.Linq;
using System.Xml.Schema;
namespace StgSharp.RegularAnalysis.Abstraction
{
public sealed class AbstractSyntaxTree<TNode, TType> where TNode : ISyntaxNode<TNode, TType>
where TType : unmanaged
{
public HashSet<TNode> AllNodes => AllToken;
public int Count => AllToken.Count;
public TNode Root
{
get;
internal set
{
if (AllToken.Contains(value)) {
field = value;
}
}
}
private HashSet<TNode> AllToken { get; } = new();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool AddNode(
TNode node
)
{
return AllToken.Add(node);
}
public bool Contains(
TNode node
)
{
return AllToken.Contains(node);
}
public IEnumerator<TNode> GetEnumerator()
{
return new TreeEnumerator<TNode, TType>(this);
}
}
internal class TreeEnumerator<TNode, TType>(
TNode root
) : IEnumerator<TNode> where TNode : ISyntaxNode<TNode, TType> where TType : unmanaged
{
private bool _completed;
private int _state, _target; // 0: not started, 1: on right, 2: ended
private readonly Stack<Record> _recycle = [];
private readonly Stack<Record> _stack = [];
// private TNode _lRoot;
private TNode _root = root;
internal TreeEnumerator(
AbstractSyntaxTree<TNode, TType> tree
)
: this(tree.Root) { }
public int Depth => _stack.Count;
public TNode Current
{
get
{
if (_stack.TryPeek(out Record? r)) {
return r.Node;
}
return TNode.Empty;
}
}
public void Dispose()
{
return;
}
public bool MoveNext()
{
if (_completed) {
return false;
}
while (_stack.TryPeek(out Record? cur))
{
switch (cur.State)
{
case 0:
cur.State++;
if (TNode.IsNullOrEmpty(cur.Node.Left))
{
goto case 1;
} else
{
_stack.Push(GetNewRecord(cur.Node.Left));
return true;
}
case 1:
cur.State++;
if (TNode.IsNullOrEmpty(cur.Node.Right))
{
goto case 2;
} else
{
_stack.Push(GetNewRecord(cur.Node.Right));
return true;
}
case 2:
_recycle.Push(_stack.Pop());
if (TNode.IsNullOrEmpty(cur.Node.Next))
{
if (_stack.Count == 0)
{
_completed = true;
return false;
}
} else
{
_stack.Push(GetNewRecord(cur.Node.Next));
return true;
}
break;
default:
break;
}
}
_stack.Push(GetNewRecord(_root));
return true;
}
public void Reset()
{
_completed = false;
_stack.Clear();
_state = 0;
}
/// <summary>
/// Skips the node and its subtree if the current node is the given node. Otherwise, do
/// nothing.
/// </summary>
/// <param name="node">
/// The node to skip.
/// </param>
public void SkipNode(
TNode node
)
{
if (_stack.TryPeek(out Record? r) && ReferenceEquals(r.Node, node))
{
_ = _stack.Pop();
_recycle.Push(r);
}
}
public void SkipOnce()
{
if (_stack.TryPeek(out Record? cur))
{
switch (cur.State)
{
case 0:
cur.State++;
break;
case 1:
cur.State++;
break;
case 2:
if (TNode.IsNullOrEmpty(cur.Node.Next))
{
_recycle.Push(_stack.Pop());
if (_stack.Count == 0)
{
_completed = true;
return;
}
} else
{
_stack.Push(GetNewRecord(cur.Node.Next));
}
break;
default:
break;
}
}
}
private Record GetNewRecord(
TNode node
)
{
if (_recycle.TryPop(out Record? r))
{
r.State = 0;
r.Node = node;
return r;
}
return new Record(node, 0);
}
object IEnumerator.Current => Current;
private class Record(
TNode node,
int state
)
{
public int State { get; set; } = state;
public TNode Node { get; set; } = node;
}
}
}
src/StgSharp.RegularAnalysis/Abstraction/CompileDepthState.cs
+62
View File
@@ -0,0 +1,62 @@
// -----------------------------------------------------------------------------
// file="CompileDepthState"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Abstraction
{
public interface ICompileDepthState
{
string CurrentState { get; set; }
}
public class CompileDepthMark
{
internal CompileDepthMark(
int operatorBegin,
int operandBegin,
int usage
)
{
OperatorBegin = operatorBegin;
OperandBegin = operandBegin;
Usage = usage;
}
internal CompileDepthMark(
int operatorBegin,
int operandBegin,
int usage,
ICompileDepthState stateCache
)
{
OperatorBegin = operatorBegin;
OperandBegin = operandBegin;
Usage = usage;
StateCache = stateCache;
}
public bool IsBeginOfBlock { get; internal set; }
public ICompileDepthState StateCache { get; internal set; }
public int OperandBegin { get; internal set; }
public int OperatorBegin { get; internal set; }
public int Usage { get; internal set; }
}
}
src/StgSharp.RegularAnalysis/Abstraction/CompileStack.cs
+167
View File
@@ -0,0 +1,167 @@
// -----------------------------------------------------------------------------
// file="CompileStack"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.Collections;
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Abstraction
{
public partial class CompileStack<TNode, TLabel> where TNode : ISyntaxNode<TNode, TLabel>
where TLabel : unmanaged
{
private readonly RandomAccessibleStack<DepthMark> _depthMarks = [];
private readonly RandomAccessibleStack<TNode> _operands = [];
private readonly RandomAccessibleStack<TNode> _operators = [];
public CompileStack()
{
_depthMarks.Push(new DepthMark(0, 0, 0));
}
public int OperandInDepthCount => _operands.Count - _depthMarks.Peek().OperandPos;
public int OperatorInDepthCount => _operators.Count - _depthMarks.Peek().OperatorPos;
public int Depth => _depthMarks.Count;
private record struct DepthMark(
int OperandPos,
int OperatorPos,
int Mark
);
#region operator
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public TNode PopOperator(
)
{
if (OperatorInDepthCount > 0) {
return _operators.Pop();
}
return TNode.Empty;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void PushOperator(
TNode operatorNode
)
{
_operators.Push(operatorNode);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPopOperator(
out TNode op
)
{
if (OperatorInDepthCount > 0)
{
op = _operators.Pop();
return true;
}
op = TNode.Empty;
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPeekOperator(
out TNode op
)
{
if (OperatorInDepthCount > 0)
{
op = _operators.Peek();
return true;
}
op = TNode.Empty;
return false;
}
#endregion
#region operand
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public TNode PopOperand(
)
{
if (OperandInDepthCount > 0) {
return _operands.Pop();
}
return TNode.Empty;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void PushOperand(
TNode operatorNode
)
{
_operands.Push(operatorNode);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPopOperand(
out TNode op
)
{
if (OperandInDepthCount > 0)
{
op = _operands.Pop();
return true;
}
op = TNode.Empty;
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPeekOperand(
out TNode op
)
{
if (OperandInDepthCount > 0)
{
op = _operands.Peek();
return true;
}
op = TNode.Empty;
return false;
}
#endregion
#region depth
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void DecreaseDepth(
)
{
if (Depth > 1)
{
DepthMark mark = _depthMarks.Pop();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void IncreaseDepth(
int mark
)
{
_depthMarks.Push(new DepthMark(_operands.Count, _operators.Count, mark));
}
#endregion
}
}
src/StgSharp.RegularAnalysis/Abstraction/ISyntaxNode.cs
+69
View File
@@ -0,0 +1,69 @@
// -----------------------------------------------------------------------------
// file="ISyntaxNode"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.Collections;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Text;
namespace StgSharp.RegularAnalysis.Abstraction
{
public interface ITrinitySyntaxNode<TNode, TLabel> : ISyntaxNode<TNode, TLabel>
where TNode : ISyntaxNode<TNode, TLabel>
where TLabel : unmanaged
{
TNode Third { get; set; }
}
public interface ISyntaxNode<TNode, TLabel> where TNode : ISyntaxNode<TNode, TLabel>
where TLabel : unmanaged
{
long NodeFlag { get; }
TLabel EqualityTypeConvert { get; }
TNode Previous { get; set; }
TNode Next { get; set; }
TNode Left { get; set; }
TNode Right { get; set; }
TNode Parent { get; set; }
static abstract TNode Empty { get; }
Token<TLabel> Source { get; }
protected internal int EnumState { get; set; }
void AppendNode(
TNode nextToken
);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static virtual bool IsNullOrEmpty(
TNode node
)
{
return node is null || node.Equals(TNode.Empty);
}
void PrependNode(
TNode previousNode
);
}
}
src/StgSharp.RegularAnalysis/Abstraction/ITokenParser.cs
+31
View File
@@ -0,0 +1,31 @@
// -----------------------------------------------------------------------------
// file="ITokenParser"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Abstraction
{
public abstract class TokenParser<TIn, TOut> : ITokenReader<TOut> where TIn : unmanaged
where TOut : unmanaged
{
public abstract bool IsEmpty { get; }
protected internal ITokenReader<TIn> Source { get; internal set; }
public abstract Token<TOut> ReadToken();
public abstract bool TryReadToken(
out Token<TOut> t
);
}
}
src/StgSharp.RegularAnalysis/Abstraction/ITokenReader.cs
+39
View File
@@ -0,0 +1,39 @@
// -----------------------------------------------------------------------------
// file="ITokenReader"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
namespace StgSharp.RegularAnalysis.Abstraction
{
public static class TokenReader
{
public static TokenParser<TIn, TOut> Pipe<TIn, TOut>(
this ITokenReader<TIn> source,
Func<TokenParser<TIn, TOut>> builder
) where TIn : unmanaged where TOut : unmanaged
{
TokenParser<TIn, TOut> parser = builder();
parser.Source = source;
return parser;
}
}
public interface ITokenReader<TLabel> where TLabel : unmanaged
{
bool IsEmpty { get; }
Token<TLabel> ReadToken();
bool TryReadToken(
out Token<TLabel> t
);
}
}
src/StgSharp.RegularAnalysis/Abstraction/SequenceEmitter.cs
+201
View File
@@ -0,0 +1,201 @@
// -----------------------------------------------------------------------------
// file="SequenceEmitter"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.RegularAnalysis.Text;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Abstraction
{
public static class SequenceEmitterExtensions
{
public static SequenceEmitter<string> AppendLine(
this SequenceEmitter<string> emitter
)
{
return emitter is null ? null! : emitter.Append(string.Empty);
}
public static SequenceEmitter<string> AppendLine(
this SequenceEmitter<string> emitter,
string line
)
{
return emitter is null ?
null! :
emitter.Append(string.IsNullOrEmpty(line) ? string.Empty : line);
}
}
public sealed class SequenceEmitter<T>
{
private bool _isRoot;
private readonly Node _head;
private Node _tail;
public SequenceEmitter()
{
_isRoot = true;
_head = new();
_tail = _head;
}
public int Count { get; private set; }
private static bool IsString { get; } = typeof(T) == typeof(string);
public SequenceEmitter<T> Append(
T unit
)
{
if (!_isRoot) {
throw new InvalidOperationException("This emitter has been appended to another emitter. It is now readonly.");
}
_tail.Add(unit);
Count++;
return this;
}
public SequenceEmitter<T> Append(
SequenceEmitter<T> emitter
)
{
if (emitter == null || emitter.Count == 0) {
return this;
}
if (ReferenceEquals(this, emitter)) {
throw new InvalidOperationException("Cannot append emitter to itself.");
}
if (!emitter._isRoot) {
throw new InvalidOperationException("The emitter has been appended to another emitter.");
}
_tail.Next = emitter._head;
_tail = emitter._tail;
Count += emitter.Count;
emitter._isRoot = false;
return this;
}
public Enumerator GetEnumerator()
{
return _isRoot ?
new Enumerator(this) :
throw new InvalidOperationException("This emitter has been appended to another emitter. You should not get its enumerator.");
}
public List<T> ToList()
{
List<T> result = new(Count);
Node current = _head;
while (current != null)
{
result.AddRange(current.IRList);
current = current.Next!;
}
return result;
}
#pragma warning disable CA1063
#pragma warning disable CA1816
public class Enumerator(
[NotNull] SequenceEmitter<T> emitter
) : IEnumerator<T>
{
private int _index = -1;
private Node _current = null!;
private readonly Node _root = emitter._head;
public T Current
{
get
{
if (_current is null || _index < 0 || _index >= _current.Count) {
throw new InvalidOperationException();
}
return _current[_index];
}
}
public void Dispose() { }
public bool MoveNext()
{
_current ??= _root;
int index = _index + 1;
if (index < _current.Count)
{
_index = index;
return true;
}
Node current = _current.Next!;
if (current is null || ReferenceEquals(current, _root))
{
_index = -1;
return false;
}
_current = current;
_index = -1;
return true;
}
public void Reset()
{
_current = _root;
_index = -1;
}
object IEnumerator.Current => Current!;
}
#pragma warning restore CA1063
#pragma warning restore CA1816
private sealed class Node
{
public T this[
int index
]
{
get => IRList[index];
set => IRList[index] = value;
}
public Node? Next { get; set; }
public int Count => IRList.Count;
public List<T> IRList { get; } = [];
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Add(
T ir
)
{
IRList.Add(ir);
}
}
}
}
src/StgSharp.RegularAnalysis/Abstraction/Token.cs
+96
View File
@@ -0,0 +1,96 @@
// -----------------------------------------------------------------------------
// file="Token"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Abstraction
{
public readonly struct Token<TLabel> where TLabel : unmanaged
{
public readonly int Column;
public readonly int Line;
public readonly string Value;
public readonly TLabel Flag;
public Token()
{
Column = 0;
Line = 0;
Value = string.Empty;
Flag = default;
}
public Token(
string chars,
int lineNumber,
int columnNumber,
TLabel flag
)
{
Value = chars;
Line = lineNumber;
Column = columnNumber;
Flag = flag;
}
public static Token<TLabel> Empty { get; } = new Token<TLabel>();
public override bool Equals(
[NotNullWhen(true)] object? obj
)
{
return base.Equals(obj);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Token<TLabel> ReLocate(
int line,
int column
)
{
return new Token<TLabel>(Value, line, column, Flag);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Token<TNewLabel> Remark<TNewLabel>(
TNewLabel label
) where TNewLabel : unmanaged
{
return new Token<TNewLabel>(Value, Line, Column, label);
}
public static bool operator !=(
Token<TLabel> left,
Token<TLabel> right
)
{
return !left.Value.SequenceEqual(right.Value) ||
left.Line != right.Line ||
left.Column != right.Column ||
!left.Flag.Equals(right.Flag);
}
public static bool operator ==(
Token<TLabel> left,
Token<TLabel> right
)
{
return left.Value.SequenceEqual(right.Value) &&
left.Line == right.Line &&
left.Column == right.Column &&
left.Flag.Equals(right.Flag);
}
}
}
src/StgSharp.RegularAnalysis/Global.cs
+15
View File
@@ -0,0 +1,15 @@
// -----------------------------------------------------------------------------
// file="Global"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
[assembly: InternalsVisibleTo("StgSharpDebug")]
src/StgSharp.RegularAnalysis/GroupIndex.cs
+30
View File
@@ -0,0 +1,30 @@
// -----------------------------------------------------------------------------
// file="GroupIndex"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System.Runtime.InteropServices;
namespace StgSharp.RegularAnalysis
{
public record struct GroupIndex
{
public GroupIndex(
int index,
string name
)
{
Index = index;
Name = name;
}
public int Index { get; internal set; }
public string Name { get; internal set; }
}
}
src/StgSharp.RegularAnalysis/NGRA_pattern_file.md
+1110
View File
File diff suppressed because it is too large Load Diff
src/StgSharp.RegularAnalysis/SourceGenerate.cs
+17
View File
@@ -0,0 +1,17 @@
// -----------------------------------------------------------------------------
// file="SourceGenerate"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis
{
internal class SourceGenerate { }
}
src/StgSharp.RegularAnalysis/StgSharp.RegularAnalysis.csproj
+39
View File
@@ -0,0 +1,39 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<IsRoslynComponent>true</IsRoslynComponent>
<EnforceExtendedAnalyzerRules>true</EnforceExtendedAnalyzerRules>
<EmitCompilerGeneratedFiles>true</EmitCompilerGeneratedFiles>
<CompilerGeneratedFilesOutputPath>$(BaseIntermediateOutputPath)Generated</CompilerGeneratedFilesOutputPath>
</PropertyGroup>
<ItemGroup>
<Compile Remove="bin\**" />
<EmbeddedResource Remove="bin\**" />
<None Remove="bin\**" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StgSharp.Common\StgSharp.Common.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.CodeAnalysis.Analyzers" Version="3.11.0" PrivateAssets="all" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.13.0" PrivateAssets="all" />
</ItemGroup>
<ItemGroup>
<None Remove="bin\x64\Debug\net8.0\StgSharp.RegularAnalysis.dll" />
</ItemGroup>
<ItemGroup>
<None Include="$(OutputPath)$(AssemblyName).dll" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false" />
<None Remove="bin\x64\Debug\net8.0\StgSharp.RegularAnalysis.pdb" />
<None Include="$(OutputPath)$(AssemblyName).pdb" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false" Condition="Exists('$(OutputPath)$(AssemblyName).pdb')" />
<None Remove="bin\x64\Debug\net8.0\StgSharp.Common.dll" />
<None Include="$(OutputPath)StgSharp.Common.dll" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false" Condition="Exists('$(OutputPath)StgSharp.Common.dll')" />
<None Remove="bin\x64\Debug\net8.0\StgSharp.Common.pdb" />
<None Include="$(OutputPath)StgSharp.Common.pdb" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false" Condition="Exists('$(OutputPath)StgSharp.Common.pdb')" />
</ItemGroup>
</Project>
src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs
+159
View File
@@ -0,0 +1,159 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer.ASTGenerate"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexAnalyzer
{
private static AbstractSyntaxTree<RegexAstNode, RegexElementLabel> GenerateAST(
string _source
)
{
CompileStack<RegexAstNode, RegexElementLabel> _stack = new();
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> _tree = new();
RegexTokenReader reader = new(_source);
TokenParser<RegexElementLabel, RegexElementLabel> lexer = reader.Pipe(() => new RegexTokenParser());
while (lexer.TryReadToken(out Token<RegexElementLabel> token))
{
// Console.WriteLine(token.Value);
if ((token.Flag & RegexElementLabel.SEQUENCE) != 0)
{
// char or charset
_stack.PushOperand(new RegexAstNode(token));
} else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0)
{
_stack.PushOperator(new RegexAstNode(token));
_stack.IncreaseDepth(0);
} else if ((token.Flag & RegexElementLabel.GROUP_END) != 0)
{
// close all nodes here
RegexAstNode op;
while (_stack.TryPopOperator(out op)) {
ProcessOperator(op);
}
if (_stack.OperandInDepthCount != 1) {
throw new InvalidOperationException();
}
_stack.DecreaseDepth();
op = _stack.PopOperator();
if (op.Source.Flag != RegexElementLabel.GROUP_BEGIN) {
throw new InvalidOperationException("Unmatched group begin and end symbol");
}
if (op.Source.Value.Length > 1)
{
op.Right = _stack.PopOperand();
_stack.PushOperand(op);
}
} else if ((token.Flag & RegexElementLabel.OPERATOR) != 0)
{
if (_stack.OperatorInDepthCount == 0)
{
// the first operator in stack
_stack.PushOperator(new RegexAstNode(token));
} else
{
// process operator by precedence
while (_stack.TryPeekOperator(out RegexAstNode topOp))
{
(bool Valid, int Value) cmp = RegexAstNode.ComparePrecedence(topOp.Source.Flag, token.Flag);
if (!cmp.Valid) {
throw new InvalidOperationException("Operators with same precedence are not supported.");
}
if (cmp.Value < 0)
{
_stack.PopOperator();
ProcessOperator(topOp);
continue;
} else
{
break;
}
}
_stack.PushOperator(new RegexAstNode(token));
}
}
}
if (_stack.Depth == 1)
{
if (_stack.OperandInDepthCount != 1 || _stack.OperatorInDepthCount != 0)
{
while (_stack.TryPopOperator(out RegexAstNode? op)) {
ProcessOperator(op);
}
}
if (_stack.OperandInDepthCount == 1 && _stack.OperatorInDepthCount == 0)
{
_tree.Root = _stack.PopOperand();
return _tree;
}
}
throw new InvalidOperationException("Invalid regular expression syntax.");
void ProcessOperator(
RegexAstNode op
)
{
if (op.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1) &&
_stack.TryPopOperand(out RegexAstNode? _2))
{
op.Left = _2;
op.Right = _1;
_stack.PushOperand(op);
_ = _tree.AddNode(op);
_ = _tree.AddNode(_1);
_ = _tree.AddNode(_2);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
} else if (op.EqualityTypeConvert == RegexElementLabel.COUNT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1))
{
op.Right = _1;
_stack.PushOperand(op);
_ = _tree.AddNode(_1);
_ = _tree.AddNode(op);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
} else if (op.EqualityTypeConvert == RegexElementLabel.ALT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1) &&
_stack.TryPopOperand(out RegexAstNode? _2))
{
op.Left = _2;
op.Right = _1;
_stack.PushOperand(op);
_ = _tree.AddNode(op);
_ = _tree.AddNode(_1);
_ = _tree.AddNode(_2);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
}
}
}
}
}
src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs
+454
View File
@@ -0,0 +1,454 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer.ASTOptimize"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Security;
using System.Runtime.CompilerServices;
using System.Security.Permissions;
using System.Text;
using System.Threading.Tasks;
using System.Xml.Linq;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexAnalyzer
{
private static RegexAstNode FlattenAlt(
RegexAstNode root
)
{
TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(root);
List<RegexAstNode> cases = [];
RegexAstNode current;
_ = enumerator.MoveNext(); // skip root
while (enumerator.MoveNext())
{
current = enumerator.Current;
if (current.EqualityTypeConvert != RegexElementLabel.ALT)
{
cases.Add(current);
enumerator.SkipNode(current);
}
current.Parent.RemoveChild(current);
}
for (int i = 0; i < cases.Count - 1; i++)
{
cases[i].AppendNode(cases[i + 1]);
cases[i].Parent = root;
}
cases[^1].Parent = root;
return cases[0];
}
private static bool MergeConcat(
RegexAstNode root
)
{
if (root.EqualityTypeConvert != RegexElementLabel.CONCAT) {
return false;
}
/*
* concat → 'a'
* ╱
* 'a'
*/
if ((root.Left.EqualityTypeConvert & RegexElementLabel.SEQUENCE) != 0 &&
RegexAstNode.IsNullOrEmpty(root.Right))
{
root.RemoveAndAsChild(true);
return true;
}
if ((root.Right.EqualityTypeConvert & RegexElementLabel.SEQUENCE) != 0 &&
RegexAstNode.IsNullOrEmpty(root.Left))
{
root.RemoveAndAsChild(false);
return true;
}
/*
* concat → concat
* ╱ ╲ /
* 'a' concat "abc"
* / \
* 'b' 'c'
*/
List<string> buffer = [];
int length = 0;
RegexAstNode right = root.Right;
RegexAstNode left = root.Left;
if (left.EqualityTypeConvert is not RegexElementLabel.UNIT_SPAN and
not RegexElementLabel.UNIT) {
return false;
}
string str = left.Value;
buffer.Add(str);
length += str.Length;
while (!RegexAstNode.IsNullOrEmpty(right) &&
right.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
left = right.Left;
if (left.EqualityTypeConvert is not RegexElementLabel.UNIT_SPAN and
not RegexElementLabel.UNIT)
{
break;
}
str = left.Value;
buffer.Add(str);
length += str.Length;
root.Right = right.Right;
right.Parent = RegexAstNode.Empty;
right = root.Right;
right.Parent = root;
}
if (right.EqualityTypeConvert is RegexElementLabel.UNIT_SPAN or
RegexElementLabel.UNIT)
{
root.Right = RegexAstNode.Empty;
buffer.Add(right.Value);
length += right.Value.Length;
}
str = string.Create(length, buffer, static(
span,
buffer
) => {
int offset = 0;
for (int i = 0; i < buffer.Count; i++)
{
buffer[i].AsSpan().CopyTo(span[offset..]);
offset += buffer[i].Length;
}
});
root.Left = new RegexAstNode(new Token<RegexElementLabel>(str, left.Source.Line,
left.Source.Column,
str.Length == 1 ?
RegexElementLabel.UNIT :
RegexElementLabel.UNIT_SPAN));
return true;
}
private static void OptimizeTree(
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree
)
{
// phase0: scan container and remove no parent nodes
_ = tree.AllNodes.RemoveWhere(node => node.Parent == null && node != tree.Root);
TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(tree.Root);
RegexAstNode current;
// phase1: ALT flatten
// phase2: CONCAT normalize
int i = 0;
List<RegexAstNode> altNodes = [];
while (enumerator.MoveNext())
{
current = enumerator.Current;
if (current.EqualityTypeConvert == RegexElementLabel.ALT)
{
RegexAstNode union = FlattenAlt(current);
altNodes.Add(current);
current.Right = union;
current.Left = RegexAstNode.Empty;
} else if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) {
RotateConcat(current);
}
}
// phase3: then ALT union
foreach (RegexAstNode item in altNodes)
{
List<RegexAstNode> cases = [];
RegexAstNode _case = item.Right;
do
{
cases.Add(_case);
_case = _case.Next;
} while (!RegexAstNode.IsNullOrEmpty(_case));
RegexAstNode newAlt = MergeAlt(cases);
item.ReplaceBy(newAlt);
}
// return;
// phase4: CONCAT merge
enumerator.Reset();
while (enumerator.MoveNext())
{
// suppose all concat is now in normalized form,
// we can merge adjacent unit nodes into a single unit node with string value
current = enumerator.Current;
if (current.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
if (MergeConcat(current)) {
enumerator.SkipOnce();
}
}
}
}
private static void RotateConcat(
RegexAstNode root
)
{
RegexAstNode left = root.Left;
while (!RegexAstNode.IsNullOrEmpty(left) &&
left.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
/*
* root → root
* ╱ ╲ ╱ ╲
* left c a left
* ╱ ╲ ╱ ╲
* a b b c
*/
RegexAstNode a = left.Left;
RegexAstNode b = left.Right;
RegexAstNode c = root.Right;
root.Left = a;
if (RegexAstNode.IsNullOrEmpty(a)) {
a.Parent = root;
}
root.Right = left;
left.Left = b;
left.Right = c;
if (RegexAstNode.IsNullOrEmpty(b)) {
b.Parent = left;
}
if (RegexAstNode.IsNullOrEmpty(c)) {
c.Parent = left;
}
left = root.Left;
}
}
#region merge alt
/// <summary>
/// Get union beginning part of cases
/// </summary>
/// <param name="cases">
///
/// </param>
/// <returns>
///
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static RegexAstNode MergeAlt(
List<RegexAstNode> cases
)
{
UnionAlt union = new(0, cases);
RegexAstNode root = union.AsNormal();
return root;
}
private class UnionAlt : RegexAstNode
{
private const int MaxLevel = 8;
private UnionAlt(
int level,
RegexAstNode node
)
: base(node.Source)
{
Level = level;
Cases = [];
RegexAstNode right = IsNullOrEmpty(node.Right) ?
new RegexAstNode(
new Token<RegexElementLabel>(node.Value, 0, node.Source.Column,
RegexElementLabel.UNIT)
) :
node.Right;
Cases.Add(right);
node.PrependNode(Empty);
node.AppendNode(Empty);
UnionBegin = node;
UnionEnd = node;
}
public UnionAlt(
int level,
List<RegexAstNode> c
)
: base(c[0].Source)
{
Cases = c;
Level = level;
}
public List<RegexAstNode> Cases { get; set; }
public RegexAstNode UnionBegin { get; set; }
public RegexAstNode UnionEnd { get; set; }
private int Level { get; set; }
public RegexAstNode AsNormal()
{
if (Level >= MaxLevel)
{
goto build;
}
merge:
List<RegexAstNode> buffer = [];
for (int i = 0; i < Cases.Count; i++)
{
if ((Cases[i].EqualityTypeConvert & RegexElementLabel.SEQUENCE) != 0)
{
RegexAstNode concat = new(new Token<RegexElementLabel>(string.Empty, 0,
Cases[i].Source.Column,
RegexElementLabel.CONCAT))
{
Left = Cases[i]
};
Cases[i].Parent = concat;
concat.Right = Empty;
Cases[i] = concat;
}
RegexAstNode current = Cases[i];
int j = 0;
for (; j < buffer.Count; j++)
{
if (CanMerge(current, buffer[j]))
{
// find a case in buffer that can merge with current case,
// then merge them and break
RegexAstNode b = buffer[j];
if (b is not UnionAlt alt)
{
alt = new UnionAlt(Level + 1, b);
alt.AddCase(current);
buffer[j] = alt;
} else
{
alt.AddCase(current);
}
break;
}
}
if (j >= buffer.Count) {
buffer.Add(current);
}
}
if (Cases.Count != buffer.Count) {
Cases = buffer;
}
if (buffer.Count == 1)
{
// if there is only one case,
// then we suppose that its contained cases has a same beginning,
// so we attach it to UnionEnd and stay level unchanged
UnionAlt alt = (Cases[0] as UnionAlt)!;
if (IsNullOrEmpty(UnionBegin))
{
UnionBegin = alt.UnionBegin;
UnionEnd = alt.UnionEnd;
} else
{
alt.UnionBegin.Parent = UnionEnd;
UnionEnd.Right = alt.UnionBegin;
UnionEnd = alt.UnionBegin;
}
Cases = alt.Cases;
goto merge;
}
build:
// suppose that from UnionBegin to UnionEnd is a normal form without union
for (int i = 0; i < Cases.Count; i++)
{
if (Cases[i] is UnionAlt union) {
Cases[i] = union.AsNormal();
}
}
RegexAstNode branch;
if (IsNullOrEmpty(UnionEnd))
{
branch = new RegexAstNode(new Token<RegexElementLabel>(string.Empty, 0,
Source.Column,
RegexElementLabel.ALT));
UnionBegin = branch;
UnionEnd = branch;
} else
{
branch = new RegexAstNode(new Token<RegexElementLabel>(string.Empty, 0,
UnionEnd.Source.Column,
RegexElementLabel.ALT));
UnionEnd.Right = branch;
branch.Parent = UnionEnd;
UnionEnd = branch;
}
for (int i = 0; i < Cases.Count - 1; i++)
{
Cases[i] ??= NewEmptyNode();
Cases[i].AppendNode(Cases[i + 1]);
Cases[i].Parent = UnionEnd;
}
UnionEnd.Right = Cases[0];
return UnionBegin;
}
private void AddCase(
RegexAstNode _case
)
{
RegexAstNode right = IsNullOrEmpty(_case.Right) ?
new RegexAstNode(
new Token<RegexElementLabel>(_case.Value, 0, _case.Source.Column,
RegexElementLabel.UNIT)
) :
_case.Right;
Cases.Add(right);
}
private static bool CanMerge(
RegexAstNode origin,
RegexAstNode target
)
{
// suppose that origin is not a union,
if (target is UnionAlt alt)
{
// then we suppose that alt has only one are same
return origin .EqualityTypeConvert == RegexElementLabel.CONCAT &&
(origin.Left.EqualityTypeConvert & RegexElementLabel.SINGLE) != 0 &&
origin.Left.EqualityTypeConvert == alt.UnionEnd.Left.EqualityTypeConvert &&
origin.Left.Value == alt.UnionEnd.Left.Value;
} else
{
return origin.EqualityTypeConvert == RegexElementLabel.CONCAT &&
target.EqualityTypeConvert == RegexElementLabel.CONCAT &&
(origin.Left.EqualityTypeConvert & RegexElementLabel.SINGLE) != 0 &&
origin.Left.EqualityTypeConvert == target.Left.EqualityTypeConvert &&
origin.Left.Value == target.Left.Value;
}
}
}
#endregion
/**/
}
}
src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.IRGenerate.cs
+319
View File
@@ -0,0 +1,319 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer.IRGenerate"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.ComponentModel.Design;
using System.Data;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Reflection.Emit;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using System.Xml;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexAnalyzer
{
internal static List<RegexIR> GenerateIR(
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree,
ref RegexInfo info
)
{
List<RegexIR> list = GenerateIRRecursion(tree.Root, 0, out int length).ToList();
info.MinPredictLength = length;
return list;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void EmitCount(
int curDepth,
RegexIRGenerator ir,
int min,
int max,
bool isGreedy,
RegexAstNode right,
out int minLength
)
{
bool is_operator = !RegexAstNode.IsNullOrEmpty(right) &&
(right.Source.Flag & RegexElementLabel.OPERATOR) != 0;
if (is_operator) // seq to be count is complex
{
RegexIRGenerator right_ir = GenerateIRRecursion(right, curDepth + 1, out minLength);
_ = ir.EmitCountComplex(min, max, isGreedy, right_ir.Count);
_ = ir.EmitIRStream(right_ir);
} else
{
RegexElementLabel count_source_mask = right.Source.Flag;
switch (count_source_mask)
{
case RegexElementLabel.UNIT:
minLength = 1;
_ = ir.EmitCountSeq(min, max, isGreedy, right.Value);
break;
case RegexElementLabel.UNIT_SET:
minLength = 1;
_ = ir.EmitCountSet(min, max, isGreedy, right.Value);
break;
case RegexElementLabel.UNIT_SPAN:
minLength = 1;
_ = ir.EmitCountSeq(min, max, isGreedy, right.Value);
break;
default:
throw new InvalidOperationException($"Unexpected regex element type for count operator: {count_source_mask}");
}
}
}
private static RegexIRGenerator GenerateIRRecursion(
RegexAstNode node,
int depth,
out int minLength
)
{
#pragma warning disable IDE0010
if (depth > 1024)
{
throw new OverflowException("Regex parse exceeds maximum recursion depth");
}
int curDepth = depth;
RegexAstNode left = node.Left;
RegexAstNode right = node.Right;
RegexIRGenerator leftIR;
RegexIRGenerator rightIR;
Token<RegexElementLabel> token = node.Source;
switch (node.Source.Flag)
{
case RegexElementLabel.GROUP_BEGIN:
string group_source = token.Value;
// process group type
bool is_group = group_source.Length == 1; // simple group: (pattern)
string group_name =
group_source.StartsWith(@"(?<", StringComparison.Ordinal) ?
group_source[3..^1] : /* named group (?<name>) */
string.Empty; /* unnamed group (?:) */
RegexIRGenerator groupIR = new();
rightIR = GenerateIRRecursion(right, curDepth + 1, out minLength);
_ = groupIR.EmitTry(rightIR.Count)
.EmitIRStream(rightIR)
.EmitCondition(2, 1)
.EmitPop(false)
.EmitPack(true, group_name);
return groupIR;
case RegexElementLabel.COUNT:
string count_source = token.Value;
bool isGreedy = !(count_source.Length > 1 && count_source.EndsWith('?'));
RegexAstNode seq = right;
ReadOnlySpan<char> count_source_span = count_source;
RegexIRGenerator count_ir = new();
int length_once;
if (count_source_span[0] == '*') // infinite count
{
EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq, out _);
minLength = 0;
} else if (count_source_span[0] == '+') // at least one
{
EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq, out length_once);
minLength = length_once;
} else if (count_source_span[0] == '?') // zero or one
{
EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq, out _);
minLength = 0;
} else if (count_source_span[0] == '{') // zero or one, non-greedy
{
int comma_pos = count_source_span.IndexOf(',');
if (comma_pos == -1)
{
if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count))
{
EmitCount(curDepth, count_ir, count, count, isGreedy, seq,
out length_once);
minLength = length_once * count;
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
}
} else
{
if (int.TryParse(count_source[1..comma_pos], out int min) &&
int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max))
{
EmitCount(curDepth, count_ir, min, max, isGreedy, seq,
out length_once);
minLength = length_once * min;
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
}
}
} else
{
throw new InvalidOperationException();
} // unknown count format
return count_ir;
case RegexElementLabel.CONCAT:
RegexIRGenerator concat = new();
// process left node
int left_length = 0;
if (RegexAstNode.IsNullOrEmpty(left))
{
// do nothing
} else if ((left.Source.Flag & RegexElementLabel.SEQUENCE) != 0)
{
switch (left.Source.Flag)
{
case RegexElementLabel.UNIT_SPAN:
left_length = left.Value.Length;
_ = concat.EmitMatchSeq(left.Value);
break;
case RegexElementLabel.UNIT:
left_length = 1;
_ = concat.EmitMatchSeq(left.Value);
break;
case RegexElementLabel.UNIT_SET:
left_length = 1;
_ = concat.EmitMatchSet(left.Value);
break;
default:
break;
}
} else if ((left.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{
leftIR = GenerateIRRecursion(left, curDepth + 1, out left_length);
_ = concat.EmitIRStream(leftIR);
}
// process right node
int right_length = 0;
if (RegexAstNode.IsNullOrEmpty(right))
{
// do nothing
} else if ((right.Source.Flag & RegexElementLabel.SEQUENCE) != 0)
{
switch (right.Source.Flag)
{
case RegexElementLabel.UNIT_SPAN:
right_length = right.Value.Length;
_ = concat.EmitMatchSeq(right.Value);
break;
case RegexElementLabel.UNIT:
right_length = 1;
_ = concat.EmitMatchSeq(right.Value);
break;
case RegexElementLabel.UNIT_SET:
right_length = 1;
_ = concat.EmitMatchSet(right.Value);
break;
default:
break;
}
} else if ((right.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{
rightIR = GenerateIRRecursion(right, curDepth + 1, out right_length);
_ = concat.EmitIRStream(rightIR);
}
minLength = left_length + right_length;
return concat;
case RegexElementLabel.ALT:
RegexIRGenerator alt = new();
List<RegexIRGenerator> casesIRList = [];
RegexAstNode cur_node = right;
int ir_count = 0;
int min_case_length = int.MaxValue;
while (!RegexAstNode.IsNullOrEmpty(cur_node))
{
RegexIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1, out int case_length);
min_case_length = int.Min(case_length, min_case_length);
casesIRList.Add(_case);
ir_count += _case.Count;
cur_node = cur_node.Next;
}
int case_count = casesIRList.Count;
while (case_count != 0)
{
RegexIRGenerator cur_ir = casesIRList[^case_count];
ir_count -= cur_ir.Count;
case_count--;
_ = alt.EmitTry(cur_ir.Count)
.EmitIRStream(cur_ir)
.EmitCondition(1/* need to calc here */, 2)
.EmitPop(true);
}
_ = alt.EmitPop(false);
minLength = min_case_length;
return alt;
case RegexElementLabel.UNIT_SPAN:
RegexIRGenerator unit = new();
_ = unit.EmitMatchSeq(node.Value);
minLength = node.Value.Length;
return unit;
case RegexElementLabel.UNIT:
RegexIRGenerator unit_seq = new();
_ = unit_seq.EmitMatchSeq(node.Value);
minLength = 1;
return unit_seq;
case RegexElementLabel.UNIT_SET:
RegexIRGenerator unit_set = new();
_ = unit_set.EmitMatchSet(node.Value);
minLength = 1;
return unit_set;
default:
minLength = 0;
return null!;
}
#pragma warning restore IDE0010
// info.MinPredictLength = ComputeMinLength(tree.Root);
static int ComputeMinLength(
RegexAstNode node
)
{
if (node.EqualityTypeConvert is RegexElementLabel.UNIT_SPAN or
RegexElementLabel.UNIT)
{
return node.Value.Length;
} else if (node.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
return ComputeMinLength(node.Left) + ComputeMinLength(node.Right);
} else if (node.EqualityTypeConvert == RegexElementLabel.ALT)
{
RegexAstNode case_0 = node.Right;
int length = int.MaxValue;
do
{
length = int.Min(ComputeMinLength(case_0), length);
case_0 = case_0.Next;
} while (!RegexAstNode.IsNullOrEmpty(case_0));
return length;
} else if (node.EqualityTypeConvert == RegexElementLabel.COUNT)
{
string source = node.Source.Value;
int single_count = ComputeMinLength(node.Right);
return 0;
} else
{
return 0;
}
}
}
}
}
src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.IROptimize.cs
+493
View File
@@ -0,0 +1,493 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer.IROptimize"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexAnalyzer
{
internal static void ScanAndOptimizeIR(
List<RegexIR> baseIR,
out List<RegexIR> optIR
)
{
int opt_count;
List<RegexIR> origin_ir = baseIR;
List<RegexIR> opt_ir = new(origin_ir.Count);
RegexIR[] opt_window = new RegexIR[8];
Span<RegexIR> opt_win_sp = opt_window;
int result;
// phase 0: common opt
phase_zero:
opt_count = 0;
int index = 0;
Span<RegexIR> origin_span = CollectionsMarshal.AsSpan(origin_ir);
do
{
int process_count = 0;
int last = int.Min(origin_span.Length, index + 8);
opt_win_sp.Clear();
origin_span[index..last].CopyTo(opt_win_sp);
if ((result = TryExpandPlus(opt_win_sp, opt_ir)) >= 0)
{
process_count++;
if (result > 0)
{
RegexCountComplexIR cc = Unsafe.As<RegexCountComplexIR>(opt_win_sp[0]);
Span<RegexIR> sp = origin_span.Slice(index + 1, result);
TryExpandComplexPlus(cc.IsGreedy, sp, opt_ir);
process_count += result;
}
opt_count++;
} else if (TryOptDotCountCount(opt_win_sp, opt_ir) >= 0)
{
process_count += 2;
opt_count++;
} else if (TryFlattenCount(opt_win_sp, opt_ir) >= 0)
{
process_count++;
opt_count++;
} else if ((result = TryExpandOnce(opt_win_sp, opt_ir)) >= 0)
{
process_count++;
if (result > 0)
{
RegexCountComplexIR cc = Unsafe.As<RegexCountComplexIR>(opt_win_sp[0]);
Span<RegexIR> sp = origin_span.Slice(index + 1, result);
TryExpandComplex(sp, opt_ir, true);
process_count += result;
}
opt_count++;
} else if (opt_win_sp[0] is IRegexComplex { IsFullyOptimized: true } skip)
{
int count = skip.CommandCount;
process_count += count + 1;
Span<RegexIR> opt_block = origin_span.Slice(index, count + 1);
opt_ir.AddRange(opt_block);
} else if (opt_win_sp[0] is RegexTryIR { IsFullyOptimized: false } try_ir)
{
process_count++;
int cmd_cnt = try_ir.CommandCount;
Span<RegexIR> try_sp = origin_span.Slice(index + 1, cmd_cnt);
ScanAndOptimizeIR(try_sp, out List<RegexIR> opt_try);
RegexTryIR new_try = new(opt_try.Count)
{
IsFullyOptimized = true
};
opt_ir.Add(new_try);
opt_ir.AddRange(opt_try);
process_count += cmd_cnt;
opt_count++;
} else
{
opt_ir.Add(opt_win_sp[0]);
process_count++;
}
index += process_count;
} while (index < origin_ir.Count);
if (opt_count > 0)
{
// need to swap the origin_ir and opt_ir, and clear the opt_ir for next round
if (ReferenceEquals(origin_ir, baseIR))
{
origin_ir = new(opt_ir.Count);
}
(origin_ir, opt_ir) = (opt_ir, origin_ir);
opt_ir.Clear();
goto phase_zero;
}
// phase 1: dot star replace
(origin_ir, opt_ir) = (opt_ir, origin_ir);
opt_ir.Clear();
index = 0;
opt_count = 0;
origin_span = CollectionsMarshal.AsSpan(origin_ir);
do
{
int process_count = 0;
int last = int.Min(origin_span.Length, index + 8);
opt_win_sp.Clear();
origin_span[index..last].CopyTo(opt_win_sp);
if (opt_win_sp[0] is IRegexComplex { IsFullyOptimized: true } skip)
{
int count = skip.CommandCount;
process_count += count + 1;
Span<RegexIR> opt_block = origin_span.Slice(index, count + 1);
opt_ir.AddRange(opt_block);
} else
if (TryReplaceDotStar(opt_win_sp, opt_ir) >= 0)
{
opt_count++;
process_count++;
} else
{
opt_ir.Add(opt_win_sp[0]);
process_count++;
}
index += process_count;
} while (index < origin_ir.Count);
if (opt_count == 0)
{
optIR = opt_ir;
return;
}
// phase 2: find-1 merge and try-find-1 rotate
do
{
(origin_ir, opt_ir) = (opt_ir, origin_ir);
opt_ir.Clear();
opt_count = 0;
index = 0;
origin_span = CollectionsMarshal.AsSpan(origin_ir);
do
{
int process_count = 0;
int last = int.Min(origin_span.Length, index + 8);
opt_win_sp.Clear();
origin_span[index..last].CopyTo(opt_win_sp);
if (opt_win_sp[0] is IRegexComplex { IsFullyOptimized: true, CommandCount: > 0 } skip)
{
int count = skip.CommandCount;
process_count += count + 1;
Span<RegexIR> opt_block = origin_span.Slice(index, count + 1);
opt_ir.AddRange(opt_block);
} else if (TyrMergeFindRestTwice(opt_win_sp, opt_ir) == 0)
{
process_count += 2;
opt_count++;
} else if ((result = TyrMergeFindRestTry(opt_win_sp, opt_ir)) >= 0)
{
Span<RegexIR> opt_block = origin_span.Slice(index + 2, result);
opt_ir.AddRange(opt_block);
process_count += 2;
process_count += result;
opt_count++;
} else
{
opt_ir.Add(opt_win_sp[0]);
process_count++;
}
index += process_count;
} while (index < origin_ir.Count);
} while (opt_count > 0);
optIR = opt_ir;
}
private static void ScanAndOptimizeIR(
Span<RegexIR> baseIR,
out List<RegexIR> opt_ir
)
{
List<RegexIR> originList = new(baseIR.Length);
CollectionsMarshal.SetCount(originList, baseIR.Length);
baseIR.CopyTo(CollectionsMarshal.AsSpan(originList));
ScanAndOptimizeIR(originList, out opt_ir);
}
private static int TryFlattenCount(
Span<RegexIR> window,
List<RegexIR> opt
)
{
const int flatten_size = 32;
RegexIR ir = window[0];
if (ir is not RegexCountSeqIR seq) {
return -1;
}
int min = seq.Min;
int max = seq.Max;
if (min == 0) {
return -1;
}
string pattern = seq.Pattern;
int pattern_size = pattern.Length;
if (pattern_size > flatten_size) {
opt.Add(ir);
}
int min_expand = flatten_size / pattern_size;
min_expand = int.Min(min_expand, min);
string expand = string.Concat(Enumerable.Repeat(pattern, min_expand));
int new_min = min / min_expand;
int new_rest = min % min_expand;
if (new_min == 1)
{
opt.Add(new RegexMatchSeqIR(expand));
} else
{
opt.Add(new RegexCountSeqIR(new_min, new_min, seq.IsGreedy, expand));
}
if (new_rest > 0)
{
string expand_tail = string.Concat(Enumerable.Repeat(pattern, new_rest));
opt.Add(new RegexMatchSeqIR(expand_tail));
}
int rest_count = max - min;
rest_count = int.Max(rest_count, -1);
opt.Add(new RegexCountSeqIR(0, rest_count, seq.IsGreedy, pattern));
return 0;
}
#region dot count count
private static int TryOptDotCountCount(
Span<RegexIR> window,
List<RegexIR> opt
)
{
/* .* <any>*
* convert to
* .*
*/
RegexIR cur_ir = window[0];
if (cur_ir is not RegexCountSetIR { IsDotStar: true } dot) {
return -1;
}
if (window[1] is not IRegexCount { IsStar: true } after) {
return -1;
}
if (dot.IsGreedy)
{
opt.Add(window[0]);
return 0;
}
return -1;
}
#endregion
private static int TryReplaceDotStar(
Span<RegexIR> window,
List<RegexIR> opt
)
{
if (window[0] is not RegexCountSetIR { IsDotStar: true } dot) {
return -1;
}
if (dot.IsGreedy)
{
RegexFindComplexIR find = new(false, -1)
{
IsFullyOptimized = true
};
opt.Add(find);
return 0;
} else
{
RegexFindComplexIR find = new(true, -1)
{
IsFullyOptimized = true
};
opt.Add(find);
return 0;
}
}
#region merge find and try-find
private static int TyrMergeFindRestTwice(
Span<RegexIR> window,
List<RegexIR> opt
)
{
if (window[0] is not RegexFindComplexIR { CommandCount: -1 } find_1 ||
window[1] is not RegexFindComplexIR find_2) {
return -1;
}
RegexFindComplexIR find_try = new(find_1.IsNearest && find_2.IsNearest, find_2.CommandCount)
{
IsFullyOptimized = true
};
opt.Add(find_try);
return 0;
}
private static int TyrMergeFindRestTry(
Span<RegexIR> window,
List<RegexIR> opt
)
{
if (window[0] is not RegexFindComplexIR { CommandCount: -1 } _find ||
window[1] is not RegexTryIR _try) {
return -1;
}
RegexFindComplexIR find_try = new(_find.IsNearest, _try.CommandCount)
{
IsFullyOptimized = true
};
opt.Add(find_try);
return find_try.CommandCount;
}
#endregion
#region expand once
private static void TryExpandComplex(
Span<RegexIR> complex,
List<RegexIR> opt,
bool is_once
)
{
if (!is_once) {
opt.Add(new RegexTryIR(complex.Length));
}
opt.AddRange(complex);
}
private static int TryExpandOnce(
Span<RegexIR> window,
List<RegexIR> opt
)
{
RegexIR ir = window[0];
if (ir is IRegexCount { Min: 1, Max: 1 })
{
string pattern;
switch (ir.CommandMask & RegexIRCommand.Count)
{
case RegexIRCommand.Count_Complex:
RegexCountComplexIR complex = Unsafe.As<RegexCountComplexIR>(ir);
return complex.CommandCount;
case RegexIRCommand.Count_Seq:
RegexCountSeqIR seq = Unsafe.As<RegexCountSeqIR>(ir);
pattern = seq.Pattern;
opt.Add(new RegexMatchSeqIR(pattern));
return 0;
case RegexIRCommand.Count_Set:
RegexCountSetIR set = Unsafe.As<RegexCountSetIR>(ir);
pattern = set.CharSet;
opt.Add(new RegexMatchSetIR(pattern));
return 0;
default:
return -1;
}
} else if (ir is IRegexCount { Min: 0, Max: 1 })
{
string pattern;
switch (ir.CommandMask & RegexIRCommand.Count)
{
case RegexIRCommand.Count_Complex:
RegexCountComplexIR complex = Unsafe.As<RegexCountComplexIR>(ir);
return complex.CommandCount;
case RegexIRCommand.Count_Seq:
RegexCountSeqIR seq = Unsafe.As<RegexCountSeqIR>(ir);
pattern = seq.Pattern;
opt.Add(new RegexTryIR(1));
opt.Add(new RegexMatchSeqIR(pattern));
return 0;
case RegexIRCommand.Count_Set:
opt.Add(new RegexTryIR(1));
RegexCountSetIR set = Unsafe.As<RegexCountSetIR>(ir);
pattern = set.CharSet;
opt.Add(new RegexMatchSetIR(pattern));
return 0;
default:
return -1;
}
} else
{
return -1;
}
}
#endregion
#region expand plus
private static void TryExpandComplexPlus(
bool is_greedy,
Span<RegexIR> complex,
List<RegexIR> opt
)
{
ScanAndOptimizeIR(complex, out List<RegexIR>? part_opt);
opt.AddRange(part_opt);
RegexCountComplexIR c_ir = new(0, -1, is_greedy, part_opt.Count)
{
IsFullyOptimized = true
};
opt.Add(c_ir);
opt.AddRange(part_opt);
}
/// <summary>
///
/// </summary>
/// <param name="window">
///
/// </param>
/// <param name="opt">
///
/// </param>
/// <returns>
/// Negative if cannot opt, 0 if match successfully, positive if meets an IRegexComplex
/// interface, need to call TryExpandComplexPlus
/// </returns>
private static int TryExpandPlus(
Span<RegexIR> window,
List<RegexIR> opt
)
{
/* a+
* convert to
* aa*
*/
RegexIR ir = window[0];
if (ir is not IRegexCount { IsPlus: true } count) {
return -1;
}
bool is_greedy = count.IsGreedy;
string pattern;
switch (ir.CommandMask & RegexIRCommand.Count)
{
case RegexIRCommand.Count_Complex:
RegexCountComplexIR complex = Unsafe.As<RegexCountComplexIR>(ir);
return complex.CommandCount;
case RegexIRCommand.Count_Seq:
RegexCountSeqIR seq = Unsafe.As<RegexCountSeqIR>(ir);
pattern = seq.Pattern;
opt.Add(new RegexMatchSeqIR(pattern));
opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern));
return 0;
case RegexIRCommand.Count_Set:
RegexCountSeqIR set = Unsafe.As<RegexCountSeqIR>(ir);
pattern = set.Pattern;
opt.Add(new RegexMatchSeqIR(pattern));
opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern));
return 0;
default:
return -1;
}
}
#endregion
}
}
src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.PrefixEmit.cs
+531
View File
@@ -0,0 +1,531 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer.PrefixEmit"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public static partial class RegexAnalyzer
{
internal static List<RegexIR> EmitPrefix(
List<RegexIR> irList,
ref RegexInfo info
)
{
List<RegexIR> origin = irList;
List<RegexIR> result = new(origin.Count);
Span<RegexIR> origin_span = CollectionsMarshal.AsSpan(origin);
int region_name = 1;
EmitPrefix(origin_span, result, 0, 0);
int single_line_count = FixLineObservation(result);
info.RegionCount = region_name - 1;
info.SingleLineResultCount = single_line_count;
return result;
void EmitPrefix(
ReadOnlySpan<RegexIR> region,
List<RegexIR> result,
int currentRegion,
int lineBase
)
{
ReadOnlySpan<RegexIR> emit_window;
int index = 0;
while (index < region.Length)
{
emit_window = region[index..];
int emitResult;
if ((emitResult =
TryEmitGroupFindPack(emit_window, result, currentRegion, lineBase)) > 0)
{
index += emitResult;
} else if ((emitResult =
TryEmitTryGroupTail(emit_window, result, currentRegion,
lineBase)) > 0)
{
index += emitResult;
} else if ((emitResult =
TryEmitAltChain(emit_window, result, currentRegion, lineBase)) > 0)
{
index += emitResult;
} else if ((emitResult =
TryEmitAltSuccessTail(emit_window, result, currentRegion,
lineBase)) > 0)
{
index += emitResult;
} else if ((emitResult =
TryEmitAltFailTail(emit_window, result, currentRegion,
lineBase)) > 0)
{
index += emitResult;
} else if ((emitResult =
TryEmitComplexIR(emit_window, result, currentRegion, lineBase)) > 0)
{
index += emitResult;
} else if ((emitResult =
TryEmitTryIR(emit_window, result, currentRegion, lineBase)) > 0)
{
index += emitResult;
} else
{
result.Add(CloneIR(emit_window[0], currentRegion));
index++;
}
}
}
int TryEmitTryGroupTail(
ReadOnlySpan<RegexIR> origin,
List<RegexIR> result,
int currentRegion,
int lineBase
)
{
if (origin.Length < 5 || origin[0] is not RegexTryIR _try) {
return -1;
}
int tailOffset = _try.CommandCount + 1;
if (origin.Length < tailOffset + 3 ||
origin[tailOffset] is not RegexConditionIR { SuccessCase: 2, FailCase: 1 } condition ||
origin[tailOffset + 1] is not RegexPopIR { IsSuccess: false } pop ||
origin[tailOffset + 2] is not RegexPackIR { Condition: true } pack) {
return -1;
}
int tryOutputIndex = result.Count;
int emitCount = TryEmitTryIR(origin, result, currentRegion, lineBase,
foldSingleInstructionTry:false);
if (emitCount <= 0 || result[tryOutputIndex] is not RegexTryIR emittedTry) {
return -1;
}
_ = condition;
int regionId = emittedTry.Prefix.WorkingRegion;
RegexIR prefixedPop = CloneIR(pop, regionId);
ApplyPrefix(prefixedPop, regionId,
RegexIRPrefixMask.Prev_Fail | RegexIRPrefixMask.Observe_Region,
regionId);
RegexIR prefixedPack = CloneIR(pack, regionId);
ApplyPrefix(prefixedPack, regionId,
RegexIRPrefixMask.Prev_Success | RegexIRPrefixMask.Observe_Region,
regionId);
result.Add(prefixedPop);
result.Add(prefixedPack);
return emitCount + 3;
}
int TryEmitAltChain(
ReadOnlySpan<RegexIR> origin,
List<RegexIR> result,
int currentRegion,
int lineBase
)
{
if (origin.Length < 3 ||
origin[0] is not RegexConditionIR { SuccessCase: 1, FailCase: 2 } ||
origin[1] is not RegexPopIR { IsSuccess: true } ||
origin[2] is not RegexTryIR) {
return -1;
}
int observeRegion = currentRegion;
RegexIRPrefixMask observeMask = RegexIRPrefixMask.Observe_Region;
int observeSource = currentRegion;
if (result.Count > 0)
{
RegexIR previous = result[^1];
observeRegion = GetObservedRegion(previous, currentRegion);
GetObservation(previous, result, lineBase, currentRegion, out observeMask,
out observeSource);
MarkPopResult(previous, RegexIRPrefixMask.Pop_Success);
}
RegexIRPrefixMask mask = RegexIRPrefixMask.Prev_Fail | observeMask;
int emit_count = TryEmitTryIR(origin[2..], result, currentRegion, lineBase, mask,
observeSource, observeRegion);
return emit_count < 0 ? -1 : emit_count + 2;
}
int TryEmitAltSuccessTail(
ReadOnlySpan<RegexIR> origin,
List<RegexIR> result,
int currentRegion,
int lineBase
)
{
if (origin.Length < 2 ||
origin[0] is not RegexConditionIR { SuccessCase: 1, FailCase: 2 } ||
origin[1] is not RegexPopIR { IsSuccess: true } ||
result.Count == 0) {
return -1;
}
RegexIR previous = result[^1];
MarkPopResult(previous, RegexIRPrefixMask.Pop_Success);
return 2;
}
int TryEmitAltFailTail(
ReadOnlySpan<RegexIR> origin,
List<RegexIR> result,
int currentRegion,
int lineBase
)
{
if (origin.Length < 1 ||
origin[0] is not RegexPopIR { IsSuccess: false } pop ||
result.Count == 0) {
return -1;
}
RegexIR previous = result[^1];
if ((previous.Prefix.Mask & RegexIRPrefixMask.Pop_Success) == 0) {
return -1;
}
GetObservation(previous, result, lineBase, currentRegion,
out RegexIRPrefixMask observeMask, out int observeSource);
RegexIR prefixedPop = CloneIR(pop, currentRegion);
ApplyPrefix(prefixedPop, currentRegion, RegexIRPrefixMask.Prev_Fail | observeMask,
observeSource);
result.Add(prefixedPop);
return 1;
}
int TryEmitComplexIR(
ReadOnlySpan<RegexIR> origin,
List<RegexIR> result,
int currentRegion,
int lineBase
)
{
if (origin.Length == 0) {
return -1;
}
if (origin[0] is RegexCountComplexIR count)
{
int commandCount = count.CommandCount;
if (origin.Length < commandCount + 1) {
return -1;
}
List<RegexIR> sub_ir = new(commandCount);
EmitPrefix(origin.Slice(1, commandCount), sub_ir, currentRegion,
lineBase + result.Count + 1);
RegexCountComplexIR newCount = new(count.Min, count.Max, count.IsGreedy,
sub_ir.Count)
{
IsFullyOptimized = count.IsFullyOptimized,
Prefix =
new RegexIRPrefix(
count.Prefix.WorkingRegion == 0 ?
currentRegion :
count.Prefix.WorkingRegion,
count.Prefix.Mask,
count.Prefix.ObserveSource)
};
result.Add(newCount);
result.AddRange(sub_ir);
return commandCount + 1;
}
if (origin[0] is RegexFindComplexIR { CommandCount: > 0 } find)
{
int commandCount = find.CommandCount;
if (origin.Length < commandCount + 1) {
return -1;
}
List<RegexIR> sub_ir = new(commandCount);
EmitPrefix(origin.Slice(1, commandCount), sub_ir, currentRegion,
lineBase + result.Count + 1);
RegexFindComplexIR newFind = new(find.IsNearest, sub_ir.Count)
{
IsFullyOptimized = find.IsFullyOptimized,
Prefix =
new RegexIRPrefix(
find.Prefix.WorkingRegion == 0 ?
currentRegion :
find.Prefix.WorkingRegion,
find.Prefix.Mask,
find.Prefix.ObserveSource)
};
result.Add(newFind);
result.AddRange(sub_ir);
return commandCount + 1;
}
return -1;
}
// convert TRY into region
int TryEmitTryIR(
ReadOnlySpan<RegexIR> origin,
List<RegexIR> result,
int currentRegion,
int lineBase,
RegexIRPrefixMask mask = 0,
int observeSource = 0,
int reuseRegion = 0,
bool foldSingleInstructionTry = true
)
{
if (origin.Length == 0 || origin[0] is not RegexTryIR _try) {
return -1;
}
int commandCount = _try.CommandCount;
if (origin.Length < commandCount + 1) {
return -1;
}
int regionId = reuseRegion == 0 ? region_name : reuseRegion;
if (reuseRegion == 0) {
region_name++;
}
if (foldSingleInstructionTry &&
commandCount == 1 &&
IsFoldableSingleTryInstruction(origin[1]))
{
RegexIR folded = CloneIR(origin[1], regionId);
int foldedObserveSource =
(mask & RegexIRPrefixMask.ObserveMethod) != 0 ?
observeSource :
_try.Prefix.ObserveSource;
folded.Prefix = new RegexIRPrefix(
regionId,
folded.Prefix.Mask | _try.Prefix.Mask | mask,
foldedObserveSource);
result.Add(folded);
return commandCount + 1;
}
List<RegexIR> sub_ir = new(commandCount);
ReadOnlySpan<RegexIR> try_region = origin.Slice(1, commandCount);
EmitPrefix(try_region, sub_ir, regionId, lineBase + result.Count + 1);
RegexTryIR newTry = new(sub_ir.Count)
{
IsFullyOptimized = _try.IsFullyOptimized,
Prefix = new RegexIRPrefix(regionId, _try.Prefix.Mask | mask, observeSource)
};
result.Add(newTry);
result.AddRange(sub_ir);
return commandCount + 1;
}
int TryEmitGroupFindPack(
ReadOnlySpan<RegexIR> origin,
List<RegexIR> result,
int currentRegion,
int lineBase
)
{
if (origin.Length < 5 ||
origin[0] is not RegexTryIR { CommandCount: 1 } _try ||
origin[1] is not RegexFindComplexIR find ||
origin[2] is not RegexConditionIR { SuccessCase: 2, FailCase: 1 } ||
origin[3] is not RegexPopIR { IsSuccess: false } ||
origin[4] is not RegexPackIR { Condition: true } pack) {
return -1;
}
int regionId = region_name;
region_name++;
RegexIRPrefixMask findMask = find.Prefix.Mask | _try.Prefix.Mask;
int findObserveSource =
(find.Prefix.Mask & RegexIRPrefixMask.ObserveMethod) != 0 ?
find.Prefix.ObserveSource :
_try.Prefix.ObserveSource;
RegexFindComplexIR newFind = new(find.IsNearest, find.CommandCount)
{
IsFullyOptimized = find.IsFullyOptimized,
Prefix = new RegexIRPrefix(regionId, findMask, findObserveSource)
};
RegexPackIR newPack = new(pack.Condition, pack.GroupName)
{
Prefix =
new RegexIRPrefix(
regionId,
pack.Prefix.Mask | RegexIRPrefixMask.Prev_Success | RegexIRPrefixMask.Observe_Line,
-1)
};
result.Add(newFind);
result.Add(newPack);
return 5;
}
static int GetObservedRegion(
RegexIR ir,
int currentRegion
)
{
return ir.Prefix.WorkingRegion == 0 ? currentRegion : ir.Prefix.WorkingRegion;
}
static void GetObservation(
RegexIR ir,
List<RegexIR> result,
int lineBase,
int currentRegion,
out RegexIRPrefixMask observeMask,
out int observeSource
)
{
if (IsLineObservable(ir))
{
observeMask = RegexIRPrefixMask.Observe_Line;
observeSource = -1;
} else
{
observeMask = RegexIRPrefixMask.Observe_Region;
observeSource = GetObservedRegion(ir, currentRegion);
}
}
static int FixLineObservation(
List<RegexIR> result
)
{
// Observe_Line uses relative offsets while nested regions are emitted.
// Positive offsets point to larger line numbers; this pass resolves them
// into absolute IR line indexes after the final instruction order is known.
HashSet<int> single_required = [];
for (int i = 0; i < result.Count; i++)
{
RegexIRPrefix prefix = result[i].Prefix;
if ((prefix.Mask & RegexIRPrefixMask.Observe_Line) == 0)
{
continue;
}
int observeSource = i + prefix.ObserveSource;
RegexIRPrefix prefix_new = new RegexIRPrefix(
prefix.WorkingRegion,
prefix.Mask,
observeSource);
result[i].Prefix = prefix_new;
_ = single_required.Add(prefix_new.ObserveSource);
}
return single_required.Count;
}
static bool IsFoldableSingleTryInstruction(
RegexIR ir
)
{
return IsLineObservable(ir) &&
(ir.Prefix.Mask & RegexIRPrefixMask.ExeCondition) == 0 &&
(ir.Prefix.Mask & RegexIRPrefixMask.ObserveMethod) == 0;
}
static bool IsLineObservable(
RegexIR ir
)
{
return ir is
RegexMatchSeqIR or
RegexMatchSetIR or
RegexCountSeqIR or
RegexCountSetIR or
RegexFindSeqIR or
//RegexFindSetIR or
RegexFindComplexIR;
}
static void ApplyPrefix(
RegexIR ir,
int workingRegion,
RegexIRPrefixMask mask,
int observeSource
)
{
ir.Prefix = new RegexIRPrefix(
workingRegion,
ir.Prefix.Mask | mask,
observeSource);
}
// Only annotate the pop result of an already emitted instruction.
// The working region and observe source belong to the execution guard
// and must stay untouched, otherwise a region id leaks into ObserveSource.
static void MarkPopResult(
RegexIR ir,
RegexIRPrefixMask popMask
)
{
ir.Prefix = new RegexIRPrefix(
ir.Prefix.WorkingRegion,
ir.Prefix.Mask | popMask,
ir.Prefix.ObserveSource);
}
static RegexIR CloneIR(
RegexIR ir,
int currentRegion
)
{
RegexIR clone = ir switch
{
RegexFindComplexIR find => new RegexFindComplexIR(find.IsNearest, find.CommandCount)
{
IsFullyOptimized = find.IsFullyOptimized
},
RegexFindSeqIR find => new RegexFindSeqIR(find.Pattern, find.IsFirst),
RegexMatchSeqIR match => new RegexMatchSeqIR(match.Pattern),
RegexMatchSetIR match => new RegexMatchSetIR(match.Charset),
RegexConditionIR condition => new RegexConditionIR(condition.SuccessCase, condition.FailCase),
RegexPackIR pack => new RegexPackIR(pack.Condition, pack.GroupName),
RegexCountComplexIR count => new RegexCountComplexIR(count.Min, count.Max,
count.IsGreedy,
count.CommandCount)
{
IsFullyOptimized = count.IsFullyOptimized
},
RegexCountSeqIR count => new RegexCountSeqIR(count.Min, count.Max,
count.IsGreedy, count.Pattern),
RegexCountSetIR count => new RegexCountSetIR(count.Min, count.Max,
count.IsGreedy, count.CharSet),
RegexTryIR tryIR => new RegexTryIR(tryIR.CommandCount)
{
IsFullyOptimized = tryIR.IsFullyOptimized
},
RegexPopIR pop => new RegexPopIR(pop.IsSuccess),
_ => throw new NotSupportedException($"Unsupported IR type: {ir.GetType().Name}")
};
clone.Prefix = new RegexIRPrefix(
ir.Prefix.WorkingRegion == 0 ? currentRegion : ir.Prefix.WorkingRegion,
ir.Prefix.Mask,
ir.Prefix.ObserveSource);
return clone;
}
}
}
}
src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.SourceGen.CharSetEmitter.cs
+105
View File
@@ -0,0 +1,105 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer.SourceGen.CharSetEmitter"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal enum CharSetType
{
None,
Single,
Range,
Set
}
internal record struct RegexCharSet(
string value,
CharSetType t,
bool accept
)
{
public bool IsAcceptable { get; set; } = accept;
public CharSetType Type { get; set; } = t;
public string Value { get; set; } = value;
public override readonly string ToString()
{
return $"{{value:{value}, type:{Type}, accept:{IsAcceptable}}}";
}
}
internal static partial class RegexAnalyzerBackEnd
{
public static RegexCharSet[] FigureCharSetType(
ReadOnlySpan<char> source
)
{
// range or single
bool accept;
if (source[0] == '[')
{
int idx;
accept = source[1] != '^';
idx = accept ? 1 : 2;
ReadOnlySpan<char> seq = source.Slice(idx, source.Length - idx - 1);
if (seq.IndexOf('-') == -1)
{
return [new RegexCharSet(seq.ToString(), CharSetType.Single, accept)];
} else
{
List<RegexCharSet> list = [];
for (; seq.Length > 0; )
{
int pos = seq.IndexOf('-');
if (pos == -1)
{
RegexCharSet single = new(seq.ToString(), CharSetType.Single, accept);
list.Add(single);
break;
} else
if (pos > 1) {
list.Add(new(seq[0..(pos - 1)].ToString(), CharSetType.Single, accept));
}
list.Add(new($"{seq[pos - 1]}{seq[pos + 1]}", CharSetType.Range, accept));
seq = seq[(pos + 2)..];
}
return [.. list];
}
} else if (source[0] == '\\')
{
// set
accept = char.IsAsciiLetterUpper(source[0]);
return char.ToLower(source[0], CultureInfo.InvariantCulture)
switch
{
's' => [new("s", CharSetType.Set, accept)],
'w' => [new("w", CharSetType.Set, accept)],
'd' => [new("d", CharSetType.Set, accept)],
_ => throw new NotSupportedException()
};
} else
{
// unknown, throw
throw new InvalidCastException($"Unknown char set: {source}");
}
}
}
}
src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.SourceGen.cs
+79
View File
@@ -0,0 +1,79 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer.SourceGen"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Microsoft.CodeAnalysis;
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexAnalyzer
{
internal static SequenceEmitter<string> GenerateSource(
TextRegexSource source
)
{
SequenceEmitter<string> se = new();
List<SequenceEmitter<string>> sub_match_list = [];
RegexInfo info = source.Info;
List<RegexIR> ir = source.IRs;
ReadOnlySpan<RegexIR> ir_sp = CollectionsMarshal.AsSpan(ir);
RegexIR[] window = new RegexIR[8];
Span<RegexIR> scan_win = window;
_ = se.AppendLine(@"public override MatchResult Match(ReadOnlySpan<char> text)")//
.AppendLine(@"{");
int region_count = info.RegionCount;
for (int i = 0; i < region_count; i++)
{
_ = se.AppendLine($@"int region_{i}_start = -1;")//
.AppendLine($@"int region_{i}_end = -1;")
.AppendLine($@"bool region_{i}_success;");
}
int index = 0;
do
{
int count = int.Min(8, ir_sp.Length - index);
ir_sp.Slice(index, count).CopyTo(scan_win);
} while (index < ir.Count);
foreach (SequenceEmitter<string> _sub in sub_match_list)
{
_ = se.AppendLine() //
.Append(_sub);
}
_ = se.Append(@"}");
return se;
/**/
int TryGenerateMatch(
Span<RegexIR> scan_win,
SequenceEmitter<string> sb
)
{
if (scan_win[0] is not RegexMatchSeqIR seq) {
return -1;
}
_ = sb.Append(string.Empty);
return 1;
}
/**/
}
}
}
src/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs
+111
View File
@@ -0,0 +1,111 @@
// -----------------------------------------------------------------------------
// file="RegexAnalyzer"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection.Emit;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public record TextRegexSource
{
internal TextRegexSource(
string pattern,
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> ast,
List<RegexIR> base_ir,
List<RegexIR> ir,
RegexInfo info,
Exception? analyseException = null
)
{
Pattern = pattern;
Ast = ast;
IRs = ir;
BaseIR = base_ir;
AnalyseException = analyseException;
Info = info;
}
public Exception? AnalyseException { get; set; }
public string Pattern { get; }
internal AbstractSyntaxTree<RegexAstNode, RegexElementLabel> Ast { get; }
internal List<RegexIR> BaseIR { get; }
internal List<RegexIR> IRs { get; }
internal RegexInfo Info { get; }
}
public static partial class RegexAnalyzer
{
public static TextRegexSource Analyze(
string regex
)
{
#pragma warning disable CA1031
// RegexAnalyzer interpreter = new();
// interpreter._source = regex;
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree;
RegexInfo info = new();
try
{
tree = GenerateAST(regex);
OptimizeTree(tree);
}
catch (Exception ex)
{
return new TextRegexSource(regex, null, null, null, info, ex);
}
List<RegexIR> base_ir;
try
{
base_ir = GenerateIR(tree, ref info);
}
catch (Exception ex)
{
return new TextRegexSource(regex, tree, null, null, info, ex);
}
try
{
ScanAndOptimizeIR(base_ir, out List<RegexIR> opt_ir);
List<RegexIR> prefix_ir = EmitPrefix(opt_ir, ref info);
return new TextRegexSource(regex, tree, base_ir, prefix_ir, info);
}
catch (Exception ex)
{
return new TextRegexSource(regex, tree, base_ir, null, info, ex);
}
#pragma warning restore CA1031
}
}
internal struct RegexInfo
{
public int RegionCount { get; set; }
public int MinPredictLength { get; set; }
public int SingleLineResultCount { get; set; }
}
}
src/StgSharp.RegularAnalysis/Text/RegexAstNode.cs
+192
View File
@@ -0,0 +1,192 @@
// -----------------------------------------------------------------------------
// file="RegexAstNode"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System.Runtime.CompilerServices;
namespace StgSharp.RegularAnalysis.Text
{
internal class RegexAstNode : ISyntaxNode<RegexAstNode, RegexElementLabel>
{
// private RegexAstNode _left, _right;
public RegexAstNode(
Token<RegexElementLabel> source
)
{
Source = source;
}
public int EnumState { get; set; }
public long NodeFlag => (long)Source.Flag;
public static RegexAstNode Empty { get; }
= new EmptyRegexAstNode();
public RegexAstNode Previous { get; set; }
public RegexAstNode Next { get; set; }
public RegexAstNode Parent { get; set; }
public RegexAstNode Left
{
get;
set
{
_ = (field?.Parent = Empty);
field = value;
if (!IsNullOrEmpty(value)) {
value.Parent = this;
}
}
}
public RegexAstNode Right
{
get;
set
{
_ = (field?.Parent = Empty);
field = value;
if (!IsNullOrEmpty(value)) {
value.Parent = this;
}
}
}
public RegexElementLabel EqualityTypeConvert => Source.Flag;
public string Value => Source.Value;
public Token<RegexElementLabel> Source { get; private protected set; }
public void AppendNode(
RegexAstNode nextToken
)
{
Next = nextToken;
nextToken.Previous = this;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static (bool Valid, int Value) ComparePrecedence(
RegexElementLabel left,
RegexElementLabel right
)
{
left &= RegexElementLabel.OPERATOR;
right &= RegexElementLabel.OPERATOR;
int v_left = (int)left, v_right = (int)right;
if (v_left == 0 || v_right == 0) {
return (false,0);
}
return (true,v_left - v_right);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(
RegexAstNode node
)
{
return node is null || ReferenceEquals(node, Empty);
}
/**/
public void PrependNode(
RegexAstNode previousNode
)
{
Previous = previousNode;
previousNode.Next = this;
}
public void RemoveChild(
RegexAstNode node
)
{
if (Left == node)
{
Left = Empty;
} else if (Right == node) {
Right = Empty;
}
node.Parent = Empty;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static RegexAstNode NewEmptyNode(
)
{
return new EmptyRegexAstNode();
}
internal void RemoveAndAsChild(
bool useLeft
)
{
if (useLeft)
{
Source = Left.Source;
Left.Parent = Empty;
Left = Empty;
} else
{
Source = Right.Source;
Right.Parent = Empty;
Right = Empty;
}
}
/**/
/// <summary>
/// Replace the current node with another node, and update the parent and sibling
/// references accordingly. This is implemented by copying metadata and nearby
/// environment of the given node to the current node, and then update the nearby nodes to
/// point to the current node.
/// </summary>
/// <param name="node">
/// The node to replace the current node with.
/// </param>
internal void ReplaceBy(
RegexAstNode node
)
{
Source = node.Source;
Left = node.Left;
Right = node.Right;
if (!IsNullOrEmpty(Left)) {
Left.Parent = this;
}
if (!IsNullOrEmpty(Right)) {
Right.Parent = this;
}
Previous = node.Previous;
Next = node.Next;
RegexAstNode prev = Previous;
RegexAstNode next = Next;
if (!IsNullOrEmpty(prev)) {
prev.Next = this;
}
if (!IsNullOrEmpty(next)) {
next.Previous = this;
}
}
private sealed class EmptyRegexAstNode : RegexAstNode
{
public EmptyRegexAstNode() : base(Token<RegexElementLabel>.Empty) { }
}
}
}
src/StgSharp.RegularAnalysis/Text/RegexElementLabel.cs
+37
View File
@@ -0,0 +1,37 @@
// -----------------------------------------------------------------------------
// file="RegexElementLabel"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
[Flags]
public enum RegexElementLabel
{
NONE = 0,
UNIT = 1,
UNIT_SET = 2,
UNIT_SPAN = 4,
GROUP_BEGIN = 8,
GROUP_END = 16,
COUNT = 32,
CONCAT = 64,
ALT = 128,
SEQUENCE = UNIT | UNIT_SPAN | UNIT_SET,
SINGLE = UNIT | UNIT_SET,
OPERATOR = COUNT | ALT | CONCAT,
VAST_OPERATOR = OPERATOR | GROUP_BEGIN,
ATOM_BEGIN = SEQUENCE | GROUP_BEGIN ,
ATOM_END = SEQUENCE | GROUP_END | COUNT,
}
}
src/StgSharp.RegularAnalysis/Text/RegexIR.Abstraction.cs
+71
View File
@@ -0,0 +1,71 @@
// -----------------------------------------------------------------------------
// file="RegexIR.Abstraction"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal interface IRegexComplex
{
bool IsFullyOptimized { get; set; }
int CommandCount { get; set; }
}
internal interface IRegexSequence
{
string Pattern { get; }
}
internal interface IRegexCount
{
bool IsGreedy { get; }
bool IsStar { get; }
bool IsPlus { get; }
int Min { get; }
int Max { get; }
}
internal abstract class RegexIR
{
protected RegexIR(RegexIRCommand command)
{
CommandMask = command;
}
public bool SingleLineResultRequired { get; set; }
public int SingleLineResultIndex { get; set; }
public RegexIRPrefix Prefix { get; set; }
internal RegexIRCommand CommandMask { get; init; }
internal string AsCommand(int lineNum)
{
return $"[{lineNum:D3}] {Prefix}, {AsCommand()}";
}
protected abstract string AsCommand();
}
}
src/StgSharp.RegularAnalysis/Text/RegexIR.Prefix.cs
+71
View File
@@ -0,0 +1,71 @@
// -----------------------------------------------------------------------------
// file="RegexIR.Prefix"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal enum RegexIRPrefixMask
{
Prev_Success = 1 << 0,
Prev_Fail = 1 << 1,
Always = 1 << 2,
ExeCondition = Prev_Fail | Prev_Success | Always,
Pop_Fail = 1 << 3,
Pop_Success = 1 << 4,
PopCondition = Pop_Success | Pop_Fail,
Observe_Line = 1 << 5,
Observe_Region = 1 << 6,
ObserveMethod = Observe_Line | Observe_Region
}
internal record struct RegexIRPrefix(int WorkingRegion, RegexIRPrefixMask Mask, int ObserveSource)
{
public override string ToString()
{
return $"{WorkingRegion:D4}, {DisplayMask(Mask , RegexIRPrefixMask.ExeCondition)}, {DisplayMask(Mask, RegexIRPrefixMask.PopCondition)}, {DisplayMask(Mask,RegexIRPrefixMask.ObserveMethod)}, {ObserveSource:D4}";
}
private static string DisplayMask(RegexIRPrefixMask mask, RegexIRPrefixMask range)
{
return range switch
{
RegexIRPrefixMask.ExeCondition => ((int)(mask & RegexIRPrefixMask.ExeCondition) >> 0) switch
{
0 => "DEF ",
(1 << 0) => "Sccs",
(1 << 1) => "Fail",
(1 << 2) => "Alws",
_ => "????"
},
RegexIRPrefixMask.PopCondition => ((int)(mask & RegexIRPrefixMask.PopCondition) >> 3) switch
{
0 => "DEF ",
(1 << 0) => "Fail",
(1 << 1) => "Sccs",
_ => "????"
},
RegexIRPrefixMask.ObserveMethod => ((int)(mask & RegexIRPrefixMask.ObserveMethod) >> 5) switch
{
0 => "NONE",
(1 << 0) => "LINE",
(1 << 1) => "REGI",
_ => "????"
},
_ => throw new ArgumentOutOfRangeException(nameof(range), range, null)
};
}
}
}
src/StgSharp.RegularAnalysis/Text/RegexIR.cs
+220
View File
@@ -0,0 +1,220 @@
// -----------------------------------------------------------------------------
// file="RegexIR"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Microsoft.CodeAnalysis.CSharp.Syntax;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal enum RegexIRCommand
{
Match_Seq = 1 << 0,
Match_Set = 1 << 1,
Condition = 1 << 2,
Try = 1 << 3,
Count_Complex = 1 << 4,
Count_Seq = 1 << 5,
Count_Set = 1 << 6,
Pop = 1 << 7,
Pack = 1 << 8,
Find_Seq = 1 << 9,
Find_Set = 1 << 10,
Find_Complex = 1 << 11,
Match = Match_Seq | Match_Set,
Count = Count_Seq | Count_Set | Count_Complex,
Find = Find_Seq | Find_Set | Find_Complex,
}
internal sealed class RegexFindComplexIR(bool nearest, int count) : RegexIR(RegexIRCommand.Find_Complex), IRegexComplex
{
public bool IsFullyOptimized { get; set; }
public bool IsNearest { get; } = nearest;
public int CommandCount { get; set; } = count;
protected override string AsCommand()
{
return $"FIND_COMPLEX {(IsNearest ? "NEAREST" : "FURTHEST")} COUNT: {CommandCount}";
}
}
internal sealed class RegexFindSeqIR(string pattern, bool isFirst) : RegexIR(RegexIRCommand.Find_Seq), IRegexSequence
{
public bool IsFirst { get; } = isFirst;
public string Pattern { get; } = pattern;
protected override string AsCommand()
{
return $"FIND_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "<EPSILON>" : Pattern)}\"";
}
}
/// <summary>
/// Searches for a certain string pattern or charset from current position.
/// </summary>
internal sealed class RegexMatchSeqIR(string pattern) : RegexIR(RegexIRCommand.Match_Seq), IRegexSequence
{
public string Pattern { get; } = pattern;
protected override string AsCommand()
{
return $"MATCH_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "<EPSILON>" : Pattern)}\"";
}
}
internal sealed class RegexMatchSetIR(string charset) : RegexIR(RegexIRCommand.Match_Set)
{
public string Charset { get; } = charset;
protected override string AsCommand()
{
return $"MATCH_SET \"{Charset}\"";
}
}
internal sealed class RegexConditionIR(int success, int fail) : RegexIR(RegexIRCommand.Condition)
{
public int SuccessCase { get; } = success;
public int FailCase { get; } = fail;
protected override string AsCommand()
{
return $"CONDITION SUCCESS: {SuccessCase} FAIL: {FailCase}";
}
}
internal sealed class RegexPackIR(bool condition, string name = "") : RegexIR(RegexIRCommand.Pack)
{
public bool Condition { get; } = condition;
public string GroupName { get; } = name;
protected override string AsCommand()
{
return $"PACK {(Condition ? "SUCCESS" : "FAIL")} GROUP: \"{GroupName}\"";
}
}
internal sealed class RegexCountComplexIR(int min, int max, bool isGreedy, int operation) : RegexIR(RegexIRCommand.Count_Complex), IRegexCount, IRegexComplex
{
public bool IsGreedy { get; } = isGreedy;
public bool IsStar => Min == 0 && Max < 0;
public bool IsPlus => Min == 1 && Max < 0;
public bool IsFullyOptimized { get; set; }
public int Min { get; } = min;
public int Max { get; } = max;
public int CommandCount { get; set; } = operation;
protected override string AsCommand()
{
return $"COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {CommandCount}";
}
}
internal sealed class RegexCountSeqIR(int min, int max, bool isGreedy, string seq) : RegexIR(RegexIRCommand.Count_Seq), IRegexCount, IRegexSequence
{
public bool IsGreedy { get; } = isGreedy;
public bool IsStar => Min == 0 && Max < 0;
public bool IsPlus => Min == 1 && Max < 0;
public int Min { get; } = min;
public int Max { get; } = max;
public string Pattern { get; } = seq.Length == 2 && seq[0] == '\\' ? seq[1..2] : seq;
protected override string AsCommand()
{
return $"COUNT_SEQ {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} SEQ: \"{Pattern}\"";
}
}
internal sealed class RegexCountSetIR(int min, int max, bool isGreedy, string set) : RegexIR(RegexIRCommand.Count_Set), IRegexCount
{
public bool IsGreedy { get; } = isGreedy;
public bool IsDotStar => CharSet == "." && Min == 0 && Max < 0;
public bool IsStar => Min == 0 && Max < 0;
public bool IsPlus => Min == 1 && Max < 0;
public int Min { get; } = min;
public int Max { get; } = max;
public string CharSet { get; } = set;
protected override string AsCommand()
{
return $"COUNT_SET {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} SET: \"{CharSet}\"";
}
}
internal sealed class RegexTryIR(int count) : RegexIR(RegexIRCommand.Try), IRegexComplex
{
public bool IsFullyOptimized { get; set; }
public int CommandCount { get; set; } = count;
protected override string AsCommand()
{
return $"TRY {CommandCount}";
}
}
internal sealed class RegexPopIR(bool isSuccess) : RegexIR(RegexIRCommand.Pop)
{
public bool IsSuccess { get; } = isSuccess;
protected override string AsCommand()
{
return $"POP {(IsSuccess ? "SUCCESS" : "FAIL")}";
}
}
}
src/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs
+128
View File
@@ -0,0 +1,128 @@
// -----------------------------------------------------------------------------
// file="RegexIRGenerator"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal sealed class RegexIRGenerator
{
private readonly SequenceEmitter<RegexIR> _stream = new();
public int Count => _stream.Count;
public RegexIRGenerator EmitCondition(
int success,
int fail
)
{
return Emit(new RegexConditionIR(success, fail));
}
public RegexIRGenerator EmitCountComplex(
int min,
int max,
bool isGreedy,
int operation
)
{
return Emit(new RegexCountComplexIR(min, max, isGreedy, operation));
}
public RegexIRGenerator EmitCountSeq(
int min,
int max,
bool isGreedy,
string seq
)
{
return Emit(new RegexCountSeqIR(min, max, isGreedy, seq));
}
public RegexIRGenerator EmitCountSet(
int min,
int max,
bool isGreedy,
string set
)
{
return Emit(new RegexCountSetIR(min, max, isGreedy, set));
}
public RegexIRGenerator EmitIRStream(
RegexIRGenerator generator
)
{
ArgumentNullException.ThrowIfNull(generator);
_ = _stream.Append(generator._stream);
return this;
}
public RegexIRGenerator EmitMatchSeq(
string unit
)
{
return Emit(new RegexMatchSeqIR(unit));
}
public RegexIRGenerator EmitMatchSet(
string unit
)
{
return Emit(new RegexMatchSetIR(unit));
}
public RegexIRGenerator EmitPack(
bool condition,
string name = ""
)
{
return Emit(new RegexPackIR(condition, name));
}
public RegexIRGenerator EmitPop(
bool isSuccess
)
{
return Emit(new RegexPopIR(isSuccess));
}
public RegexIRGenerator EmitTry(
int count
)
{
return Emit(new RegexTryIR(count));
}
public SequenceEmitter<RegexIR>.Enumerator GetEnumerator()
{
return _stream.GetEnumerator();
}
public List<RegexIR> ToList()
{
return _stream.ToList();
}
private RegexIRGenerator Emit(
RegexIR ir
)
{
_ = _stream.Append(ir);
return this;
}
}
}
src/StgSharp.RegularAnalysis/Text/RegexTokenParser.cs
+69
View File
@@ -0,0 +1,69 @@
// -----------------------------------------------------------------------------
// file="RegexTokenParser"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Diagnostics.Tracing;
using System.IO.Pipes;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal class RegexTokenParser : TokenParser<RegexElementLabel, RegexElementLabel>
{
private readonly Queue<Token<RegexElementLabel>> _buffer = [];
private RegexElementLabel _prev;
public override bool IsEmpty => Source.IsEmpty && _buffer.Count == 0;
public override Token<RegexElementLabel> ReadToken()
{
if (_buffer.TryDequeue(out Token<RegexElementLabel> token)) {
return token;
}
if (Source.TryReadToken(out Token<RegexElementLabel> orig))
{
if (((int)(orig.Flag & RegexElementLabel.ATOM_BEGIN) != 0) && ((int)_prev != 0))
{
Token<RegexElementLabel> ret = new Token<RegexElementLabel>(string.Empty,
orig.Line,
orig.Column,
RegexElementLabel.CONCAT);
_buffer.Enqueue(orig);
_prev = orig.Flag & RegexElementLabel.ATOM_END;
return ret;
} else
{
_prev = orig.Flag & RegexElementLabel.ATOM_END;
return orig;
}
} else
{
return Token<RegexElementLabel>.Empty;
}
}
public override bool TryReadToken(
out Token<RegexElementLabel> t
)
{
if (IsEmpty)
{
t = Token<RegexElementLabel>.Empty;
return false;
}
t = ReadToken();
return t != Token<RegexElementLabel>.Empty;
}
}
}
src/StgSharp.RegularAnalysis/Text/RegexTokenReader.cs
+291
View File
@@ -0,0 +1,291 @@
// -----------------------------------------------------------------------------
// file="RegexTokenReader"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Security.Permissions;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal class RegexTokenReader : ITokenReader<RegexElementLabel>
{
private static string _source;
private int position = 0;
public RegexTokenReader() { }
public RegexTokenReader(string source)
{
_source = source;
}
public bool IsEmpty => string.IsNullOrEmpty(_source) || position >= _source.Length;
public Token<RegexElementLabel> ReadToken()
{
int pos = position;
switch (_source[pos])
{
case '\\':
if (pos + 1 == _source.Length) {
throw new InvalidOperationException("Unclosed char");
}
if (_source[pos + 1] == 'x')
{
string value = _source[(pos + 1)..(pos + 3)];
pos += 4;
if (uint.TryParse(value, out uint code) && Rune.TryCreate(code, out Rune result))
{
return new Token<RegexElementLabel>(result.ToString(), 0, pos, RegexElementLabel.UNIT);
} else
{
throw new InvalidCastException($"Failed to convert {value} to a valid ascii code point.");
}
} else if (_source[pos + 1] == 'u')
{
string value = _source[(pos + 1)..(pos + 5)];
pos += 6;
if (uint.TryParse(value, out uint code) && Rune.TryCreate(code, out Rune result))
{
return new Token<RegexElementLabel>(result.ToString(), 0, pos, RegexElementLabel.UNIT);
} else
{
throw new InvalidCastException($"Failed to convert {value} to a valid unicode code point.");
}
} else if (_source[pos + 1] == 'p')
{
int i = pos + 2;
if (_source[i] != '{') {
throw new InvalidOperationException("Invalid Unicode property escape sequence.");
}
for (; i < _source.Length; i++)
{
if (_source[i] == '}')
{
position = i + 1;
return new Token<RegexElementLabel>(_source[(pos)..(i + 1)], 0, pos,
RegexElementLabel.UNIT_SET);
}
}
throw new InvalidOperationException("Unclosed Unicode property escape sequence.");
} else if (_source[pos + 1] == '.')
{
position = pos + 2;
return new Token<RegexElementLabel>(".", 0, pos, RegexElementLabel.UNIT);
} else
{
position = pos + 2;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 2)], 0, pos,
RegexElementLabel.UNIT_SET);
}
case '[':
for (int i = pos + 1; i < _source.Length; i++)
{
if (_source[i] == ']' && _source[i - 1] != '\\')
{
int length = (i - pos - 1);
position = i + 1;
return new Token<RegexElementLabel>(_source[(pos)..(i + 1)], 0, position,
RegexElementLabel.UNIT_SET);
}
}
throw new InvalidOperationException("Unclosed character set");
case '|':
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.ALT);
case '(':
// very comlex here
return ReadGroupBeginToken();
case ')':
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.GROUP_END);
case '{':
for (int i = pos + 1; i < _source.Length; i++)
{
if (_source[i] == '}')
{
position = i + 1;
return new Token<RegexElementLabel>(_source[(pos)..(i + 1)], 0, pos,
RegexElementLabel.COUNT);
}
}
throw new InvalidOperationException("Unclosed count specifier");
case '*' or '+':
int count = pos + 1 >= _source.Length ? 1 : (_source[pos + 1] == '?' ? 2 : 1);
position = pos + count;
return new Token<RegexElementLabel>(_source[(pos)..(pos + count)], 0, pos, RegexElementLabel.COUNT);
case '?':
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.COUNT);
case '.':
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.UNIT_SET);
default:
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos, RegexElementLabel.UNIT);
}
}
public bool TryReadToken(out Token<RegexElementLabel> t)
{
if (IsEmpty)
{
t = Token<RegexElementLabel>.Empty;
return false;
}
t = ReadToken();
return Token<RegexElementLabel>.Empty != t;
}
private Token<RegexElementLabel> ReadGroupBeginToken()
{
if (IsEmpty) {
throw new InvalidOperationException("Source is null.");
}
int start = position;
if (start >= _source.Length || _source[start] != '(') {
throw new InvalidOperationException("ReadGroupBeginToken called when current char is not '('.");
}
// Case 1: plain capturing group "("
if (start + 1 >= _source.Length || _source[start + 1] != '?')
{
position = start + 1;
return new Token<RegexElementLabel>("(", 0, start, RegexElementLabel.GROUP_BEGIN);
}
// From here we have "(?"
if (start + 2 >= _source.Length)
{
throw new InvalidOperationException("Unfinished group prefix '(?'.");
}
char c2 = _source[start + 2];
// Case 2: non-capturing group "(?:"
if (c2 == ':')
{
position = start + 3;
return new Token<RegexElementLabel>("(?:", 0, start, RegexElementLabel.GROUP_BEGIN);
}
// Assertions (write the first-level cases, but not supported yet)
if (c2 == '=')
{
// "(?="
position = start + 3;
throw new NotSupportedException("Positive lookahead '(?=...)' is not supported yet.");
}
if (c2 == '!')
{
// "(?!"
position = start + 3;
throw new NotSupportedException("Negative lookahead '(?!...)' is not supported yet.");
}
// Named capturing group or lookbehind starts with "(?<"
if (c2 == '<')
{
if (start + 3 >= _source.Length) {
throw new InvalidOperationException("Unfinished group prefix '(?<'.");
}
char c3 = _source[start + 3];
// Look behind prefixes: "(?<=" and "(?<!"
if (c3 == '=')
{
position = start + 4;
throw new NotSupportedException("Positive look behind '(?<=...)' is not supported yet.");
}
if (c3 == '!')
{
position = start + 4;
throw new NotSupportedException("Negative look behind '(?<!...)' is not supported yet.");
}
// Otherwise: "(?<name>"
int nameStart = start + 3;
// Read name until '>'
int i = nameStart;
// Name must start with [A-Za-z_]
if (i >= _source.Length)
{
throw new InvalidOperationException("Unfinished named group prefix.");
}
char first = _source[i];
if (!(first == '_' || char.IsLetter(first))) {
throw new InvalidOperationException($"Invalid named group name start '{first}' at position {i}.");
}
i++;
while (i < _source.Length)
{
char ch = _source[i];
if (ch == '>')
{
break;
}
if (!(ch == '_' || char.IsLetterOrDigit(ch))) {
throw new InvalidOperationException($"Invalid character '{ch}' in group name at position {i}.");
}
i++;
}
if (i >= _source.Length || _source[i] != '>') {
throw new InvalidOperationException("Unclosed named group prefix; missing '>'.");
}
// Consume "(?<name>"
int consumedLen = i - start + 1; // include '>'
string text = _source.Substring(start, consumedLen);
position = start + consumedLen;
return new Token<RegexElementLabel>(text, 0, start, RegexElementLabel.GROUP_BEGIN);
}
// Optional: comments "(?#...)" or inline options "(?i:...)" etc.
// First-level cases listed but not supported yet:
if (c2 == '#')
{
position = start + 3;
throw new NotSupportedException("Comment group '(?#...)' is not supported yet.");
}
// Inline options: (?i:...), (?m:...), (?-i:...), (?im), etc.
// Too many dialect differences; reserve the prefix but mark unsupported.
if (char.IsLetter(c2) || c2 == '-')
{
// We don't know how long the option spec is here; do NOT advance position too much.
// Keep position at '(' so caller can decide how to handle, or just throw.
position = start;
throw new NotSupportedException("Inline option groups like '(?i:...)' are not supported yet.");
}
// Unknown "(?X" prefix
position = start;
throw new InvalidOperationException($"Unknown or unsupported group prefix starting at position {start}: '{_source.Substring(start, Math.Min(4, _source.Length - start))}'");
}
}
}
src/StgSharp.RegularAnalysis/Text/RegexTokenUtil.cs
+29
View File
@@ -0,0 +1,29 @@
// -----------------------------------------------------------------------------
// file="RegexTokenUtil"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal static class RegexTokenUtil
{
public static bool IsAccurateChar(
this Token<RegexElementLabel> token
)
{
return ((token.Flag & RegexElementLabel.UNIT) != 0 && token.Value.Length == 1) ||
(token.Flag & RegexElementLabel.UNIT_SPAN) != 0;
}
}
}
src/StgSharp.RegularAnalysis/Text/TextRegex.cs
+26
View File
@@ -0,0 +1,26 @@
// -----------------------------------------------------------------------------
// file="TextRegex"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public abstract class TextRegex
{
public abstract MatchResult Match(
ReadOnlySpan<char> text
);
}
public ref struct MatchResult { }
}
src/StgSharp.RegularAnalysis/regex_node_transaction.md
+24
View File
@@ -0,0 +1,24 @@
### basic
char/charset->char/charset
### count
char/charset-match->count increment
count increment-loop back->char/charset
char/charset-miss match->count end
count end-exit->count increment-judge->next node
### match group/lookaround
left boarder-> match group start //push stack, add group index
-> match logic // recored index
-> match group end //pop stack, set group end, build sub sequence