youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

discuss regex ir opt

Browse Source
This commit is contained in:
Youmu Liu
2026-06-22 18:18:16 +08:00
parent 062ffacf88
commit 7e5e4d1f7a
18 changed files with 1499 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.RegularAnalysis/Text/RegexAstNode.cs
+214
View File
@@ -0,0 +1,214 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexAstNode"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System.Runtime.CompilerServices;
namespace StgSharp.RegularAnalysis.Text
{
internal class RegexAstNode : ISyntaxNode<RegexAstNode, RegexElementLabel>
{
// private RegexAstNode _left, _right;
public RegexAstNode(
Token<RegexElementLabel> source
)
{
Source = source;
}
public int EnumState { get; set; }
public long NodeFlag => (long)Source.Flag;
public static RegexAstNode Empty { get; }
= new EmptyRegexAstNode();
public RegexAstNode Previous { get; set; }
public RegexAstNode Next { get; set; }
public RegexAstNode Parent { get; set; }
public RegexAstNode Left
{
get;
set
{
_ = (field?.Parent = Empty);
field = value;
if (!IsNullOrEmpty(value)) {
value.Parent = this;
}
}
}
public RegexAstNode Right
{
get;
set
{
_ = (field?.Parent = Empty);
field = value;
if (!IsNullOrEmpty(value)) {
value.Parent = this;
}
}
}
public RegexElementLabel EqualityTypeConvert => Source.Flag;
public string Value => Source.Value;
public string CodeConvertTemplate { get; }
public Token<RegexElementLabel> Source { get; private protected set; }
public void AppendNode(
RegexAstNode nextToken
)
{
Next = nextToken;
nextToken.Previous = this;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static (bool Valid, int Value) ComparePrecedence(
RegexElementLabel left,
RegexElementLabel right
)
{
left &= RegexElementLabel.OPERATOR;
right &= RegexElementLabel.OPERATOR;
int v_left = (int)left, v_right = (int)right;
if (v_left == 0 || v_right == 0) {
return (false,0);
}
return (true,v_left - v_right);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(
RegexAstNode node
)
{
return node is null || ReferenceEquals(node, Empty);
}
/**/
public void PrependNode(
RegexAstNode previousNode
)
{
Previous = previousNode;
previousNode.Next = this;
}
public void RemoveChild(
RegexAstNode node
)
{
if (Left == node)
{
Left = Empty;
} else if (Right == node) {
Right = Empty;
}
node.Parent = Empty;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static RegexAstNode NewEmptyNode(
)
{
return new EmptyRegexAstNode();
}
internal void RemoveAndAsChild(
bool useLeft
)
{
if (useLeft)
{
Source = Left.Source;
Left.Parent = Empty;
Left = Empty;
} else
{
Source = Right.Source;
Right.Parent = Empty;
Right = Empty;
}
}
/**/
/// <summary>
/// Replace the current node with another node, and update the parent and sibling
/// references accordingly. This is implemented by copying metadata and nearby
/// environment of the given node to the current node, and then update the nearby nodes to
/// point to the current node.
/// </summary>
/// <param name="node">
/// The node to replace the current node with.
/// </param>
internal void ReplaceBy(
RegexAstNode node
)
{
Source = node.Source;
Left = node.Left;
Right = node.Right;
if (!IsNullOrEmpty(Left)) {
Left.Parent = this;
}
if (!IsNullOrEmpty(Right)) {
Right.Parent = this;
}
Previous = node.Previous;
Next = node.Next;
RegexAstNode prev = Previous;
RegexAstNode next = Next;
if (!IsNullOrEmpty(prev)) {
prev.Next = this;
}
if (!IsNullOrEmpty(next)) {
next.Previous = this;
}
}
private sealed class EmptyRegexAstNode : RegexAstNode
{
public EmptyRegexAstNode() : base(Token<RegexElementLabel>.Empty) { }
}
}
}
StgSharp.RegularAnalysis/Text/RegexElementLabel.cs
+57
View File
@@ -0,0 +1,57 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexElementLabel"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
[Flags]
public enum RegexElementLabel
{
NONE = 0,
UNIT = 1,
UNIT_SET = 2,
UNIT_SPAN = 4,
GROUP_BEGIN = 8,
GROUP_END = 16,
COUNT = 32,
CONCAT = 64,
ALT = 128,
SEQUENCE = UNIT | UNIT_SPAN | UNIT_SET,
SINGLE = UNIT | UNIT_SET,
OPERATOR = COUNT | ALT | CONCAT,
VAST_OPERATOR = OPERATOR | GROUP_BEGIN,
ATOM_BEGIN = SEQUENCE | GROUP_BEGIN ,
ATOM_END = SEQUENCE | GROUP_END | COUNT,
}
}
StgSharp.RegularAnalysis/Text/RegexHIR.cs
+260
View File
@@ -0,0 +1,260 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexIR"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal enum RegexIRCommand
{
Match_Seq = 0,
Match_Set = 1,
Condition = 3,
Try = 4,
Count = 5,
Count_Seq = 6,
Count_Set = 7,
Pop = 8,
Pack = 9,
}
internal abstract class RegexHIR
{
internal abstract RegexIRCommand Command { get; }
internal abstract string AsCommand(
int lineNum
);
}
/// <summary>
/// Searches for a certain string pattern or charset from current position.
/// </summary>
internal sealed class RegexMatchSeqHIR(
string pattern
) : RegexHIR
{
public string Pattern { get; } = pattern;
internal override RegexIRCommand Command => RegexIRCommand.Match_Seq;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t MATCH_SEQ \"{(string.IsNullOrEmpty(Pattern) ? "<EPSILON>" : Pattern)}\"";
}
}
internal sealed class RegexMatchSetHIR(
string charset
) : RegexHIR
{
public string Charset { get; } = charset;
internal override RegexIRCommand Command => RegexIRCommand.Match_Set;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t MATCH_SET \"{Charset}\"";
}
}
internal sealed class RegexConditionHIR(
int success,
int fail
) : RegexHIR
{
public int SuccessCase { get; } = success;
public int FailCase { get; } = fail;
internal override RegexIRCommand Command => RegexIRCommand.Condition;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t CONDITION SUCCESS: {SuccessCase} FAIL: {FailCase}";
}
}
internal sealed class RegexPackHIR(
bool condition,
string name = ""
) : RegexHIR
{
public bool Condition { get; } = condition;
public string GroupName { get; } = name;
internal override RegexIRCommand Command => RegexIRCommand.Pack;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t PACK {(Condition ? "SUCCESS" : "FAIL")} GROUP: \"{GroupName}\"";
}
}
internal sealed class RegexCountHIR(
int min,
int max,
bool isGreedy,
int operation
) : RegexHIR
{
public bool IsGreedy { get; } = isGreedy;
public int Min { get; } = min;
public int Max { get; } = max;
public int OperationIndex { get; } = operation;
internal override RegexIRCommand Command { get; } = RegexIRCommand.Count;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {OperationIndex}";
}
}
internal sealed class RegexCountSeqHIR(
int min,
int max,
bool isGreedy,
string seq
) : RegexHIR
{
public bool IsGreedy { get; } = isGreedy;
public int Min { get; } = min;
public int Max { get; } = max;
public string Sequence { get; } = seq;
internal override RegexIRCommand Command { get; } = RegexIRCommand.Count_Seq;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t COUNT_SEQ {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} SEQ: \"{Sequence}\"";
}
}
internal sealed class RegexCountSetHIR(
int min,
int max,
bool isGreedy,
string set
) : RegexHIR
{
public bool IsGreedy { get; } = isGreedy;
public int Min { get; } = min;
public int Max { get; } = max;
public string CharSet { get; } = set;
internal override RegexIRCommand Command => RegexIRCommand.Count_Set;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t COUNT_SET {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} SET: \"{CharSet}\"";
}
}
internal sealed class RegexTryHIR(
int count
) : RegexHIR
{
public int CountOfCommand { get; } = count;
internal override RegexIRCommand Command => RegexIRCommand.Try;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t TRY {CountOfCommand}";
}
}
internal sealed class RegexPopHIR(
bool isSuccess
) : RegexHIR
{
public bool IsSuccess { get; } = isSuccess;
internal override RegexIRCommand Command => RegexIRCommand.Pop;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t POP {(IsSuccess ? "SUCCESS" : "FAIL")}";
}
}
}
StgSharp.RegularAnalysis/Text/RegexHIRGenerator.cs
+177
View File
@@ -0,0 +1,177 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexHIRGenerator"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal class RegexHIRGenerator
{
private Node _head = new();
private Node _tail;
public RegexHIRGenerator()
{
_tail = _head;
}
public int Count { get; private set; }
public void EmitCondition(
int success,
int fail
)
{
_tail.Add(new RegexConditionHIR(success, fail));
Count++;
}
public void EmitCount(
int min,
int max,
bool isGreedy,
int operation
)
{
_tail.Add(new RegexCountHIR(min, max, isGreedy, operation));
Count++;
}
public void EmitCountSeq(
int min,
int max,
bool isGreedy,
string seq
)
{
_tail.Add(new RegexCountSeqHIR(min, max, isGreedy, seq));
Count++;
}
public void EmitCountSet(
int min,
int max,
bool isGreedy,
string set
)
{
_tail.Add(new RegexCountSetHIR(min, max, isGreedy, set));
Count++;
}
public void EmitIRStream(
RegexHIRGenerator stream
)
{
if (stream is null || stream.Count == 0) {
return;
}
_tail.Next = stream._head;
_tail = stream._tail;
Count += stream.Count;
}
public void EmitMatchSeq(
string unit
)
{
_tail.Add(new RegexMatchSeqHIR(unit));
Count++;
}
public void EmitMatchSet(
string unit
)
{
_tail.Add(new RegexMatchSetHIR(unit));
Count++;
}
public void EmitPack(
bool condition,
string name = ""
)
{
_tail.Add(new RegexPackHIR(condition, name));
Count++;
}
public void EmitPop(
bool isSuccess
)
{
_tail.Add(new RegexPopHIR(isSuccess));
Count++;
}
public void EmitTry(
int count
)
{
_tail.Add(new RegexTryHIR(count));
Count++;
}
public List<RegexHIR> ToList()
{
List<RegexHIR> result = new(Count);
Node current = _head;
while (current != null)
{
result.AddRange(current.IRList);
current = current.Next!;
}
return result;
}
private class Node
{
public Node? Next { get; set; }
public int Count => IRList.Count;
public List<RegexHIR> IRList { get; } = [];
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Add(
RegexHIR ir
)
{
IRList.Add(ir);
}
}
}
}
StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs
+415
View File
@@ -0,0 +1,415 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexInterpreter.Optimize"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Security;
using System.Runtime.CompilerServices;
using System.Security.Permissions;
using System.Text;
using System.Threading.Tasks;
using System.Xml.Linq;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexInterpreter
{
private static RegexAstNode FlattenAlt(
RegexAstNode root
)
{
TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(root);
List<RegexAstNode> cases = [];
RegexAstNode current;
_ = enumerator.MoveNext(); // skip root
while (enumerator.MoveNext())
{
current = enumerator.Current;
if (current.EqualityTypeConvert != RegexElementLabel.ALT)
{
cases.Add(current);
enumerator.SkipNode(current);
}
current.Parent.RemoveChild(current);
}
for (int i = 0; i < cases.Count - 1; i++)
{
cases[i].AppendNode(cases[i + 1]);
cases[i].Parent = root;
}
cases[^1].Parent = root;
return cases[0];
}
private static bool MergeConcat(
RegexAstNode root
)
{
if (root.EqualityTypeConvert != RegexElementLabel.CONCAT) {
return false;
}
/*
* concat → 'a'
* ╱
* 'a'
*/
if ((root.Left.EqualityTypeConvert & RegexElementLabel.SEQUENCE) != 0 &&
RegexAstNode.IsNullOrEmpty(root.Right))
{
root.RemoveAndAsChild(true);
return true;
}
if ((root.Right.EqualityTypeConvert & RegexElementLabel.SEQUENCE) != 0 &&
RegexAstNode.IsNullOrEmpty(root.Left))
{
root.RemoveAndAsChild(false);
return true;
}
/*
* concat → concat
* ╱ ╲ /
* 'a' concat "abc"
* / \
* 'b' 'c'
*/
List<string> buffer = [];
int length = 0;
RegexAstNode right = root.Right;
RegexAstNode left = root.Left;
if (left.EqualityTypeConvert is not RegexElementLabel.UNIT_SPAN and
not RegexElementLabel.UNIT) {
return false;
}
string str = left.Value;
buffer.Add(str);
length += str.Length;
while (!RegexAstNode.IsNullOrEmpty(right) &&
right.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
left = right.Left;
if (left.EqualityTypeConvert is not RegexElementLabel.UNIT_SPAN and
not RegexElementLabel.UNIT)
{
break;
}
str = left.Value;
buffer.Add(str);
length += str.Length;
root.Right = right.Right;
right.Parent = RegexAstNode.Empty;
right = root.Right;
right.Parent = root;
}
if (right.EqualityTypeConvert is RegexElementLabel.UNIT_SPAN or
RegexElementLabel.UNIT)
{
root.Right = RegexAstNode.Empty;
buffer.Add(right.Value);
length += right.Value.Length;
}
str = string.Create(length, buffer, static(
span,
buffer
) => {
int offset = 0;
for (int i = 0; i < buffer.Count; i++)
{
buffer[i].AsSpan().CopyTo(span[offset..]);
offset += buffer[i].Length;
}
});
root.Left = new RegexAstNode(new Token<RegexElementLabel>(str, left.Source.Line,
left.Source.Column,
str.Length == 1 ?
RegexElementLabel.UNIT :
RegexElementLabel.UNIT_SPAN));
return true;
}
private static void RotateConcat(
RegexAstNode root
)
{
RegexAstNode left = root.Left;
while (!RegexAstNode.IsNullOrEmpty(left) &&
left.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
/*
* root → root
* ╱ ╲ ╱ ╲
* left c a left
* ╱ ╲ ╱ ╲
* a b b c
*/
RegexAstNode a = left.Left;
RegexAstNode b = left.Right;
RegexAstNode c = root.Right;
root.Left = a;
if (RegexAstNode.IsNullOrEmpty(a)) {
a.Parent = root;
}
root.Right = left;
left.Left = b;
left.Right = c;
if (RegexAstNode.IsNullOrEmpty(b)) {
b.Parent = left;
}
if (RegexAstNode.IsNullOrEmpty(c)) {
c.Parent = left;
}
left = root.Left;
}
}
#region merge alt
/// <summary>
/// Get union beginning part of cases
/// </summary>
/// <param name="cases">
///
/// </param>
/// <returns>
///
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static RegexAstNode MergeAlt(
List<RegexAstNode> cases
)
{
UnionAlt union = new(0, cases);
RegexAstNode root = union.AsNormal();
return root;
}
private class UnionAlt : RegexAstNode
{
private const int MaxLevel = 8;
private UnionAlt(
int level,
RegexAstNode node
)
: base(node.Source)
{
Level = level;
Cases = [];
RegexAstNode right = IsNullOrEmpty(node.Right) ?
new RegexAstNode(
new Token<RegexElementLabel>(node.Value, 0, node.Source.Column,
RegexElementLabel.UNIT)
) :
node.Right;
Cases.Add(right);
node.PrependNode(Empty);
node.AppendNode(Empty);
UnionBegin = node;
UnionEnd = node;
}
public UnionAlt(
int level,
List<RegexAstNode> c
)
: base(c[0].Source)
{
Cases = c;
Level = level;
}
public List<RegexAstNode> Cases { get; set; }
public RegexAstNode UnionBegin { get; set; }
public RegexAstNode UnionEnd { get; set; }
private int Level { get; set; }
public RegexAstNode AsNormal()
{
if (Level >= MaxLevel)
{
goto build;
}
merge:
List<RegexAstNode> buffer = [];
for (int i = 0; i < Cases.Count; i++)
{
if ((Cases[i].EqualityTypeConvert & RegexElementLabel.SEQUENCE) != 0)
{
RegexAstNode concat = new(new Token<RegexElementLabel>(string.Empty, 0,
Cases[i].Source.Column,
RegexElementLabel.CONCAT))
{
Left = Cases[i]
};
Cases[i].Parent = concat;
concat.Right = Empty;
Cases[i] = concat;
}
RegexAstNode current = Cases[i];
int j = 0;
for (; j < buffer.Count; j++)
{
if (CanMerge(current, buffer[j]))
{
// find a case in buffer that can merge with current case,
// then merge them and break
RegexAstNode b = buffer[j];
if (b is not UnionAlt alt)
{
alt = new UnionAlt(Level + 1, b);
alt.AddCase(current);
buffer[j] = alt;
} else
{
alt.AddCase(current);
}
break;
}
}
if (j >= buffer.Count) {
buffer.Add(current);
}
}
if (Cases.Count != buffer.Count) {
Cases = buffer;
}
if (buffer.Count == 1)
{
// if there is only one case,
// then we suppose that its contained cases has a same beginning,
// so we attach it to UnionEnd and stay level unchanged
UnionAlt alt = (Cases[0] as UnionAlt)!;
if (IsNullOrEmpty(UnionBegin))
{
UnionBegin = alt.UnionBegin;
UnionEnd = alt.UnionEnd;
} else
{
alt.UnionBegin.Parent = UnionEnd;
UnionEnd.Right = alt.UnionBegin;
UnionEnd = alt.UnionBegin;
}
Cases = alt.Cases;
goto merge;
}
build:
// suppose that from UnionBegin to UnionEnd is a normal form without union
for (int i = 0; i < Cases.Count; i++)
{
if (Cases[i] is UnionAlt union) {
Cases[i] = union.AsNormal();
}
}
RegexAstNode branch;
if (IsNullOrEmpty(UnionEnd))
{
branch = new RegexAstNode(new Token<RegexElementLabel>(string.Empty, 0,
Source.Column,
RegexElementLabel.ALT));
UnionBegin = branch;
UnionEnd = branch;
} else
{
branch = new RegexAstNode(new Token<RegexElementLabel>(string.Empty, 0,
UnionEnd.Source.Column,
RegexElementLabel.ALT));
UnionEnd.Right = branch;
branch.Parent = UnionEnd;
UnionEnd = branch;
}
for (int i = 0; i < Cases.Count - 1; i++)
{
Cases[i] ??= NewEmptyNode();
Cases[i].AppendNode(Cases[i + 1]);
Cases[i].Parent = UnionEnd;
}
UnionEnd.Right = Cases[0];
return UnionBegin;
}
private void AddCase(
RegexAstNode _case
)
{
RegexAstNode right = IsNullOrEmpty(_case.Right) ?
new RegexAstNode(
new Token<RegexElementLabel>(_case.Value, 0, _case.Source.Column,
RegexElementLabel.UNIT)
) :
_case.Right;
Cases.Add(right);
}
private static bool CanMerge(
RegexAstNode origin,
RegexAstNode target
)
{
// suppose that origin is not a union,
if (target is UnionAlt alt)
{
// then we suppose that alt has only one are same
return origin .EqualityTypeConvert == RegexElementLabel.CONCAT &&
(origin.Left.EqualityTypeConvert & RegexElementLabel.SINGLE) != 0 &&
origin.Left.EqualityTypeConvert == alt.UnionEnd.Left.EqualityTypeConvert &&
origin.Left.Value == alt.UnionEnd.Left.Value;
} else
{
return origin.EqualityTypeConvert == RegexElementLabel.CONCAT &&
target.EqualityTypeConvert == RegexElementLabel.CONCAT &&
(origin.Left.EqualityTypeConvert & RegexElementLabel.SINGLE) != 0 &&
origin.Left.EqualityTypeConvert == target.Left.EqualityTypeConvert &&
origin.Left.Value == target.Left.Value;
}
}
}
#endregion
/**/
}
}
StgSharp.RegularAnalysis/Text/RegexInterpreter.HIRGenerate.cs
+282
View File
@@ -0,0 +1,282 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexInterpreter.HIRGenerate"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.ComponentModel.Design;
using System.Data;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Reflection.Emit;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using System.Xml;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexInterpreter
{
internal static List<RegexHIR> GenerateIR(
AbstractSyntaxTree<RegexAstNode, RegexElementLabel> tree
)
{
return GenerateIRRecursion(tree.Root, 0).ToList();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void EmitCount(
int curDepth,
RegexHIRGenerator ir,
int min,
int max,
bool isGreedy,
RegexAstNode right
)
{
bool isRightOperator = !RegexAstNode.IsNullOrEmpty(right) &&
(right.Source.Flag & RegexElementLabel.OPERATOR) != 0;
if (isRightOperator) // seq to be count is complex
{
ir.EmitCount(min, max, isGreedy, 1);
RegexHIRGenerator rightIR = GenerateIRRecursion(right, curDepth + 1);
ir.EmitTry(rightIR.Count);
ir.EmitIRStream(rightIR);
} else
{
RegexElementLabel count_source_mask = right.Source.Flag;
switch (count_source_mask)
{
case RegexElementLabel.UNIT:
ir.EmitCountSeq(min, max, isGreedy, right.Source.Value);
break;
case RegexElementLabel.UNIT_SET:
ir.EmitCountSet(min, max, isGreedy, right.Source.Value);
break;
case RegexElementLabel.UNIT_SPAN:
ir.EmitCountSeq(min, max, isGreedy, right.Source.Value);
break;
default:
throw new InvalidOperationException($"Unexpected regex element type for count operator: {count_source_mask}");
}
}
}
private static RegexHIRGenerator GenerateIRRecursion(
RegexAstNode node,
int depth
)
{
#pragma warning disable IDE0010
if (depth > 1024)
{
throw new OverflowException("Regex parse exceeds maximum recursion depth");
}
int curDepth = depth;
RegexAstNode left = node.Left;
RegexAstNode right = node.Right;
RegexHIRGenerator leftIR;
RegexHIRGenerator rightIR;
Token<RegexElementLabel> token = node.Source;
switch (node.Source.Flag)
{
case RegexElementLabel.GROUP_BEGIN:
string group_source = token.Value;
// process group type
bool is_group = group_source.Length == 1; // simple group: (pattern)
string group_name =
group_source.StartsWith(@"(?<", StringComparison.Ordinal) ?
group_source[3..^1] : /* named group (?<name>) */
string.Empty; /* unnamed group (?:) */
RegexHIRGenerator groupIR = new();
rightIR = GenerateIRRecursion(right, curDepth + 1);
groupIR.EmitTry(rightIR.Count);
groupIR.EmitIRStream(rightIR);
groupIR.EmitCondition(2, 1);
groupIR.EmitPop(false);
groupIR.EmitPack(true, group_name);
return groupIR;
case RegexElementLabel.COUNT:
string count_source = token.Value;
bool isGreedy = !(count_source.Length > 1 && count_source.EndsWith('?'));
int seq_mask = (RegexAstNode.IsNullOrEmpty(left) ? 0 : 2) + (RegexAstNode.IsNullOrEmpty(right) ?
0 :
1);
RegexAstNode seq = seq_mask switch
{
// no seq, invalid count operator
0 => throw new InvalidOperationException("Count operator must have a sequence to operate on"),
1 => right,// right seq
2 => left,// left seq
// both sides, invalid count operator
3 => throw new InvalidOperationException("Count operator cannot have two sequences to operate on"),
_ => throw new InvalidOperationException("Unexpected sequence mask for count operator"),
};
ReadOnlySpan<char> count_source_span = count_source;
RegexHIRGenerator countIR = new();
if (count_source_span[0] == '*') // infinite count
{
EmitCount(curDepth, countIR, 0, -1, isGreedy, seq);
} else if (count_source_span[0] == '+') // at least one
{
EmitCount(curDepth, countIR, 1, -1, isGreedy, seq);
} else if (count_source_span[0] == '?') // zero or one
{
EmitCount(curDepth, countIR, 0, 1, isGreedy, seq);
} else if (count_source_span[0] == '{') // zero or one, non-greedy
{
int comma_pos = count_source_span.IndexOf(',');
if (comma_pos == -1)
{
if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count))
{
EmitCount(curDepth, countIR, count, count, isGreedy, seq);
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
}
} else
{
if (int.TryParse(count_source[1..comma_pos], out int min) &&
int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max))
{
EmitCount(curDepth, countIR, min, max, isGreedy, seq);
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
}
}
} else
{
throw new InvalidOperationException();
} // unknown count format
return countIR;
case RegexElementLabel.CONCAT:
RegexHIRGenerator concat = new();
// process left node
if (RegexAstNode.IsNullOrEmpty(left))
{
// do nothing
} else if ((left.Source.Flag & RegexElementLabel.SEQUENCE) != 0)
{
switch (left.Source.Flag)
{
case RegexElementLabel.UNIT_SPAN:
goto case RegexElementLabel.UNIT;
case RegexElementLabel.UNIT:
concat.EmitMatchSeq(left.Source.Value);
break;
case RegexElementLabel.UNIT_SET:
concat.EmitMatchSet(left.Source.Value);
break;
default:
break;
}
} else if ((left.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{
leftIR = GenerateIRRecursion(left, curDepth + 1);
concat.EmitIRStream(leftIR);
}
// process right node
if (RegexAstNode.IsNullOrEmpty(right))
{
// do nothing
} else if ((right.Source.Flag & RegexElementLabel.SEQUENCE) != 0)
{
switch (right.Source.Flag)
{
case RegexElementLabel.UNIT_SPAN:
goto case RegexElementLabel.UNIT;
case RegexElementLabel.UNIT:
concat.EmitMatchSeq(right.Source.Value);
break;
case RegexElementLabel.UNIT_SET:
concat.EmitMatchSet(right.Source.Value);
break;
default:
break;
}
} else if ((right.Source.Flag & RegexElementLabel.VAST_OPERATOR) != 0)
{
rightIR = GenerateIRRecursion(right, curDepth + 1);
concat.EmitIRStream(rightIR);
}
return concat;
case RegexElementLabel.ALT:
RegexHIRGenerator alt = new();
List<RegexHIRGenerator> casesIRList = [];
RegexAstNode cur_node = right;
int ir_count = 0;
while (!RegexAstNode.IsNullOrEmpty(cur_node))
{
RegexHIRGenerator _case = GenerateIRRecursion(cur_node, curDepth + 1);
casesIRList.Add(_case);
ir_count += _case.Count;
cur_node = cur_node.Next;
}
int case_count = casesIRList.Count;
while (case_count != 0)
{
RegexHIRGenerator cur_ir = casesIRList[^case_count];
ir_count -= cur_ir.Count;
case_count--;
alt.EmitTry(cur_ir.Count);
alt.EmitIRStream(cur_ir);
alt.EmitCondition(1/* need to calc here */, 2);
alt.EmitPop(true);
}
alt.EmitPop(false);
return alt;
case RegexElementLabel.UNIT_SPAN:
goto case RegexElementLabel.UNIT;
case RegexElementLabel.UNIT:
RegexHIRGenerator unit_seq = new();
unit_seq.EmitMatchSeq(node.Source.Value);
return unit_seq;
case RegexElementLabel.UNIT_SET:
RegexHIRGenerator unit_set = new();
unit_set.EmitMatchSet(node.Source.Value);
return unit_set;
default:
return null!;
}
#pragma warning restore IDE0010
}
}
}
StgSharp.RegularAnalysis/Text/RegexInterpreter.IRScan.cs
+69
View File
@@ -0,0 +1,69 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexInterpreter.IRScan"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexInterpreter
{
internal static void ScanAndMarkIROptimization(
RegexHIR ir
)
{
switch (ir.Command)
{
case RegexIRCommand.Match_Seq:
break;
case RegexIRCommand.Match_Set:
break;
case RegexIRCommand.Pack:
break;
case RegexIRCommand.Condition:
break;
case RegexIRCommand.Try:
break;
case RegexIRCommand.Count:
break;
case RegexIRCommand.Count_Seq:
break;
case RegexIRCommand.Count_Set:
break;
case RegexIRCommand.Pop:
break;
default:
break;
}
}
}
}
StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs
+51
View File
@@ -0,0 +1,51 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexInterpreter.SourceGenerate"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexInterpreter
{
internal static string GetSourceCode(
IEnumerable<RegexHIR> irs
)
{
StringBuilder sb = new StringBuilder();
foreach (RegexHIR ir in irs) {
ScanAndMarkIROptimization(ir);
}
return sb.ToString();
}
}
}
StgSharp.RegularAnalysis/Text/RegexInterpreter.cs
+275
View File
@@ -0,0 +1,275 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexInterpreter"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection.Emit;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public partial class RegexInterpreter
{
private AbstractSyntaxTree<RegexAstNode, RegexElementLabel> _tree = new();
private CompileStack<RegexAstNode, RegexElementLabel> _stack = new();
private Exception _interpreteException;
private string _source;
private RegexInterpreter() { }
internal AbstractSyntaxTree<RegexAstNode, RegexElementLabel> Tree
{
get
{
if (_interpreteException != null) {
throw new InvalidOperationException("The regex interpretation failed.", _interpreteException);
}
return _tree;
}
}
public static RegexInterpreter Analyze(
string regex
)
{
RegexInterpreter interpreter = new();
interpreter._source = regex;
bool isSuccess;
try
{
interpreter.Analyze();
interpreter.OptimizeTree();
isSuccess = true;
}
catch (Exception ex)
{
interpreter._interpreteException = ex;
isSuccess = false;
}
return interpreter;
}
private void Analyze()
{
RegexTokenReader reader = new(_source);
TokenParser<RegexElementLabel, RegexElementLabel> lexer = reader.Pipe(() => new RegexTokenParser());
while (lexer.TryReadToken(out Token<RegexElementLabel> token))
{
// Console.WriteLine(token.Value);
if ((token.Flag & RegexElementLabel.SEQUENCE) != 0)
{
// char or charset
_stack.PushOperand(new RegexAstNode(token));
} else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0)
{
_stack.PushOperator(new RegexAstNode(token));
_stack.IncreaseDepth(0);
} else if ((token.Flag & RegexElementLabel.GROUP_END) != 0)
{
// close all nodes here
RegexAstNode op;
while (_stack.TryPopOperator(out op)) {
ProcessOperator(op);
}
if (_stack.OperandInDepthCount != 1) {
throw new InvalidOperationException();
}
_stack.DecreaseDepth();
op = _stack.PopOperator();
if (op.Source.Flag != RegexElementLabel.GROUP_BEGIN) {
throw new InvalidOperationException("Unmatched group begin and end symbol");
}
if (op.Source.Value.Length > 1)
{
op.Right = _stack.PopOperand();
_stack.PushOperand(op);
}
} else if ((token.Flag & RegexElementLabel.OPERATOR) != 0)
{
if (_stack.OperatorInDepthCount == 0)
{
// the first operator in stack
_stack.PushOperator(new RegexAstNode(token));
} else
{
// process operator by precedence
while (_stack.TryPeekOperator(out RegexAstNode topOp))
{
(bool Valid, int Value) cmp = RegexAstNode.ComparePrecedence(topOp.Source.Flag, token.Flag);
if (!cmp.Valid) {
throw new InvalidOperationException("Operators with same precedence are not supported.");
}
if (cmp.Value < 0)
{
_stack.PopOperator();
ProcessOperator(topOp);
continue;
} else
{
break;
}
}
_stack.PushOperator(new RegexAstNode(token));
}
}
}
if (_stack.Depth == 1)
{
if (_stack.OperandInDepthCount != 1 || _stack.OperatorInDepthCount != 0)
{
while (_stack.TryPopOperator(out RegexAstNode? op)) {
ProcessOperator(op);
}
}
if (_stack.OperandInDepthCount == 1 && _stack.OperatorInDepthCount == 0)
{
_tree.Root = _stack.PopOperand();
return;
}
}
throw new InvalidOperationException("Invalid regular expression syntax.");
}
private void OptimizeTree()
{
// phase0: scan container and remove no parent nodes
_ = _tree.AllNodes.RemoveWhere(node => node.Parent == null && node != _tree.Root);
TreeEnumerator<RegexAstNode, RegexElementLabel> enumerator = new(_tree.Root);
RegexAstNode current;
// phase1: ALT flatten
// phase2: CONCAT normalize
int i = 0;
List<RegexAstNode> altNodes = [];
while (enumerator.MoveNext())
{
current = enumerator.Current;
if (current.EqualityTypeConvert == RegexElementLabel.ALT)
{
RegexAstNode union = FlattenAlt(current);
altNodes.Add(current);
current.Right = union;
current.Left = RegexAstNode.Empty;
} else if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) {
RotateConcat(current);
}
}
List<RegexAstNode> cases = [];
// phase3: then ALT union
foreach (RegexAstNode item in altNodes)
{
RegexAstNode _case = item.Right;
do
{
cases.Add(_case);
_case = _case.Next;
} while (!RegexAstNode.IsNullOrEmpty(_case));
RegexAstNode newAlt = MergeAlt(cases);
item.ReplaceBy(newAlt);
}
// return;
// phase4: CONCAT merge
enumerator.Reset();
while (enumerator.MoveNext())
{
// suppose all concat is now in normalized form,
// we can merge adjacent unit nodes into a single unit node with string value
current = enumerator.Current;
if (current.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
if (MergeConcat(current)) {
enumerator.SkipOnce();
}
}
}
// TODO enum AST again , do ALT tail merge and simple ALT front merge
}
private void ProcessOperator(
RegexAstNode op
)
{
if (op.EqualityTypeConvert == RegexElementLabel.CONCAT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1) &&
_stack.TryPopOperand(out RegexAstNode? _2))
{
op.Left = _2;
op.Right = _1;
_stack.PushOperand(op);
_tree.AddNode(op);
_tree.AddNode(_1);
_tree.AddNode(_2);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
} else if (op.EqualityTypeConvert == RegexElementLabel.COUNT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1))
{
op.Left = _1;
_stack.PushOperand(op);
_tree.AddNode(_1);
_tree.AddNode(op);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
} else if (op.EqualityTypeConvert == RegexElementLabel.ALT)
{
if (_stack.TryPopOperand(out RegexAstNode? _1) &&
_stack.TryPopOperand(out RegexAstNode? _2))
{
op.Left = _2;
op.Right = _1;
_stack.PushOperand(op);
_tree.AddNode(op);
_tree.AddNode(_1);
_tree.AddNode(_2);
} else
{
throw new InvalidOperationException("Insufficient operands for CONCAT operator.");
}
}
}
}
}
StgSharp.RegularAnalysis/Text/RegexLIR.cs
+60
View File
@@ -0,0 +1,60 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexLIR"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal enum LIRCommand
{
Search = 0,
Count = 1,
Try = 2,
}
internal abstract class RegexLIR
{
public abstract LIRCommand Command { get; }
}
internal enum RegexSearchMethod
{
Match,
Find,
Assert,
}
}
StgSharp.RegularAnalysis/Text/RegexTokenParser.cs
+90
View File
@@ -0,0 +1,90 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexTokenParser"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Diagnostics.Tracing;
using System.IO.Pipes;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal class RegexTokenParser : TokenParser<RegexElementLabel, RegexElementLabel>
{
private readonly Queue<Token<RegexElementLabel>> _buffer = [];
private RegexElementLabel _prev;
public override bool IsEmpty => Source.IsEmpty && _buffer.Count == 0;
public override Token<RegexElementLabel> ReadToken()
{
if (_buffer.TryDequeue(out Token<RegexElementLabel> token)) {
return token;
}
if (Source.TryReadToken(out Token<RegexElementLabel> orig))
{
if (((int)(orig.Flag & RegexElementLabel.ATOM_BEGIN) != 0) && ((int)_prev != 0))
{
Token<RegexElementLabel> ret = new Token<RegexElementLabel>(string.Empty,
orig.Line,
orig.Column,
RegexElementLabel.CONCAT);
_buffer.Enqueue(orig);
_prev = orig.Flag & RegexElementLabel.ATOM_END;
return ret;
} else
{
_prev = orig.Flag & RegexElementLabel.ATOM_END;
return orig;
}
} else
{
Console.WriteLine("returns empty");
return Token<RegexElementLabel>.Empty;
}
}
public override bool TryReadToken(
out Token<RegexElementLabel> t
)
{
if (IsEmpty)
{
t = Token<RegexElementLabel>.Empty;
return false;
}
t = ReadToken();
return t != Token<RegexElementLabel>.Empty;
}
}
}
StgSharp.RegularAnalysis/Text/RegexTokenReader.cs
+322
View File
@@ -0,0 +1,322 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexTokenReader"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Security.Permissions;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal class RegexTokenReader : ITokenReader<RegexElementLabel>
{
private static string _source;
private int position = 0;
public RegexTokenReader() { }
public RegexTokenReader(
string source
)
{
_source = source;
}
public bool IsEmpty => string.IsNullOrEmpty(_source) || position >= _source.Length;
public Token<RegexElementLabel> ReadToken()
{
int pos = position;
switch (_source[pos])
{
case '\\':
if (pos + 1 == _source.Length) {
throw new InvalidOperationException("Unclosed char");
}
if (_source[pos + 1] == 'x')
{
string value = _source[(pos + 1)..(pos + 3)];
pos += 4;
if (uint.TryParse(value, out uint code) &&
Rune.TryCreate(code, out Rune result))
{
return new Token<RegexElementLabel>(result.ToString(), 0, pos,
RegexElementLabel.UNIT);
} else
{
throw new InvalidCastException($"Failed to convert {value} to a valid ascii code point.");
}
} else if (_source[pos + 1] == 'u')
{
string value = _source[(pos + 1)..(pos + 5)];
pos += 6;
if (uint.TryParse(value, out uint code) &&
Rune.TryCreate(code, out Rune result))
{
return new Token<RegexElementLabel>(result.ToString(), 0, pos,
RegexElementLabel.UNIT);
} else
{
throw new InvalidCastException($"Failed to convert {value} to a valid unicode code point.");
}
} else if (_source[pos + 1] == 'p')
{
int i = pos + 2;
if (_source[i] != '{') {
throw new InvalidOperationException("Invalid Unicode property escape sequence.");
}
for (; i < _source.Length; i++)
{
if (_source[i] == '}')
{
position = i + 1;
return new Token<RegexElementLabel>(_source[(pos)..(i + 1)], 0, pos,
RegexElementLabel.UNIT_SET);
}
}
throw new InvalidOperationException("Unclosed Unicode property escape sequence.");
} else
{
position = pos + 2;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 2)], 0, pos,
RegexElementLabel.UNIT_SET);
}
case '[':
for (int i = pos + 1; i < _source.Length; i++)
{
if (_source[i] == ']' && _source[i - 1] != '\\')
{
int length = (i - pos - 1);
position = i + 1;
return new Token<RegexElementLabel>(_source[(pos)..(i + 1)], 0,
position,
RegexElementLabel.UNIT_SET);
}
}
throw new InvalidOperationException("Unclosed character set");
case '|':
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
RegexElementLabel.ALT);
case '(':
// very comlex here
return ReadGroupBeginToken();
case ')':
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
RegexElementLabel.GROUP_END);
case '{':
for (int i = pos + 1; i < _source.Length; i++)
{
if (_source[i] == '}')
{
position = i + 1;
return new Token<RegexElementLabel>(_source[(pos)..(i + 1)], 0, pos,
RegexElementLabel.COUNT);
}
}
throw new InvalidOperationException("Unclosed count specifier");
case '*' or '+':
int count = pos + 1 >= _source.Length ? 1 : (_source[pos + 1] == '?' ? 2 : 1);
position = pos + count;
return new Token<RegexElementLabel>(_source[(pos)..(pos + count)], 0, pos,
RegexElementLabel.COUNT);
case '?':
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
RegexElementLabel.COUNT);
case '.':
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
RegexElementLabel.UNIT);
default:
position = pos + 1;
return new Token<RegexElementLabel>(_source[(pos)..(pos + 1)], 0, pos,
RegexElementLabel.UNIT);
}
}
public bool TryReadToken(
out Token<RegexElementLabel> t
)
{
if (IsEmpty)
{
t = Token<RegexElementLabel>.Empty;
return false;
}
t = ReadToken();
return Token<RegexElementLabel>.Empty != t;
}
private Token<RegexElementLabel> ReadGroupBeginToken()
{
if (IsEmpty) {
throw new InvalidOperationException("Source is null.");
}
int start = position;
if (start >= _source.Length || _source[start] != '(') {
throw new InvalidOperationException("ReadGroupBeginToken called when current char is not '('.");
}
// Case 1: plain capturing group "("
if (start + 1 >= _source.Length || _source[start + 1] != '?')
{
position = start + 1;
return new Token<RegexElementLabel>("(", 0, start, RegexElementLabel.GROUP_BEGIN);
}
// From here we have "(?"
if (start + 2 >= _source.Length)
{
throw new InvalidOperationException("Unfinished group prefix '(?'.");
}
char c2 = _source[start + 2];
// Case 2: non-capturing group "(?:"
if (c2 == ':')
{
position = start + 3;
return new Token<RegexElementLabel>("(?:", 0, start, RegexElementLabel.GROUP_BEGIN);
}
// Assertions (write the first-level cases, but not supported yet)
if (c2 == '=')
{
// "(?="
position = start + 3;
throw new NotSupportedException("Positive lookahead '(?=...)' is not supported yet.");
}
if (c2 == '!')
{
// "(?!"
position = start + 3;
throw new NotSupportedException("Negative lookahead '(?!...)' is not supported yet.");
}
// Named capturing group or lookbehind starts with "(?<"
if (c2 == '<')
{
if (start + 3 >= _source.Length) {
throw new InvalidOperationException("Unfinished group prefix '(?<'.");
}
char c3 = _source[start + 3];
// Lookbehind prefixes: "(?<=" and "(?<!"
if (c3 == '=')
{
position = start + 4;
throw new NotSupportedException("Positive lookbehind '(?<=...)' is not supported yet.");
}
if (c3 == '!')
{
position = start + 4;
throw new NotSupportedException("Negative lookbehind '(?<!...)' is not supported yet.");
}
// Otherwise: "(?<name>"
int nameStart = start + 3;
// Read name until '>'
int i = nameStart;
// Name must start with [A-Za-z_]
if (i >= _source.Length)
{
throw new InvalidOperationException("Unfinished named group prefix.");
}
char first = _source[i];
if (!(first == '_' || char.IsLetter(first))) {
throw new InvalidOperationException($"Invalid named group name start '{first}' at position {i}.");
}
i++;
while (i < _source.Length)
{
char ch = _source[i];
if (ch == '>')
{
break;
}
if (!(ch == '_' || char.IsLetterOrDigit(ch))) {
throw new InvalidOperationException($"Invalid character '{ch}' in group name at position {i}.");
}
i++;
}
if (i >= _source.Length || _source[i] != '>') {
throw new InvalidOperationException("Unclosed named group prefix; missing '>'.");
}
// Consume "(?<name>"
int consumedLen = i - start + 1; // include '>'
string text = _source.Substring(start, consumedLen);
position = start + consumedLen;
return new Token<RegexElementLabel>(text, 0, start, RegexElementLabel.GROUP_BEGIN);
}
// Optional: comments "(?#...)" or inline options "(?i:...)" etc.
// First-level cases listed but not supported yet:
if (c2 == '#')
{
position = start + 3;
throw new NotSupportedException("Comment group '(?#...)' is not supported yet.");
}
// Inline options: (?i:...), (?m:...), (?-i:...), (?im), etc.
// Too many dialect differences; reserve the prefix but mark unsupported.
if (char.IsLetter(c2) || c2 == '-')
{
// We don't know how long the option spec is here; do NOT advance position too much.
// Keep position at '(' so caller can decide how to handle, or just throw.
position = start;
throw new NotSupportedException("Inline option groups like '(?i:...)' are not supported yet.");
}
// Unknown "(?X" prefix
position = start;
throw new InvalidOperationException($"Unknown or unsupported group prefix starting at position {start}: '{_source.Substring(start, Math.Min(4, _source.Length - start))}'");
}
}
}
StgSharp.RegularAnalysis/Text/RegexTokenUtil.cs
+49
View File
@@ -0,0 +1,49 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexTokenUtil"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.RegularAnalysis.Abstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
internal static class RegexTokenUtil
{
public static bool IsAccurateChar(
this Token<RegexElementLabel> token
)
{
return ((token.Flag & RegexElementLabel.UNIT) != 0 && token.Value.Length == 1) ||
(token.Flag & RegexElementLabel.UNIT_SPAN) != 0;
}
}
}
StgSharp.RegularAnalysis/Text/TextHelper.cs
+50
View File
@@ -0,0 +1,50 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="TextHelper"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public static class TextHelper
{
public static bool TryLastIndexOf(
this ReadOnlySpan<char> span,
ReadOnlySpan<char> value,
out int index
)
{
index = span.LastIndexOf(value);
return index != -1;
}
}
}
StgSharp.RegularAnalysis/Text/TextRegex.cs
+47
View File
@@ -0,0 +1,47 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="TextRegex"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.RegularAnalysis.Text
{
public abstract class TextRegex
{
public abstract bool IsMatch(
ReadOnlySpan<char> text
);
}
public ref struct MatchResult { }
}