youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

regex ir opt code on rewrite

Browse Source
This commit is contained in:
Youmu Liu
2026-06-25 19:52:11 +08:00
parent 45f9b04b6d
commit 5cd2b8b4ba
7 changed files with 281 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/Collections/CapacityFixedList.cs
+66 -19
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="CapacityFixedList"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -35,7 +35,9 @@ namespace StgSharp.Collections
public static class CapacityFixedListBuilder
{
internal static CapacityFixedList<T> Create<T>(ReadOnlySpan<T> values)
internal static CapacityFixedList<T> Create<T>(
ReadOnlySpan<T> values
)
{
return new CapacityFixedList<T>(values);
}
@@ -50,28 +52,39 @@ namespace StgSharp.Collections
private readonly int _capacity;
private int _index;
public CapacityFixedList(int size)
public CapacityFixedList(
int size
)
{
_values = new T[size];
_capacity = size;
}
public CapacityFixedList(ReadOnlySpan<T> values)
public CapacityFixedList(
ReadOnlySpan<T> values
)
{
_values = values.ToArray();
}
public T this[int index]
public T this[
int index
]
{
get => index < _index ? _values[index] : throw new ArgumentOutOfRangeException(nameof(index));
set => _values[index] = index < _index ? value : throw new ArgumentOutOfRangeException(nameof(index));
get => index < _index ?
_values[index] :
throw new ArgumentOutOfRangeException(nameof(index));
set => _values[index] =
index < _index ? value : throw new ArgumentOutOfRangeException(nameof(index));
}
public bool IsReadOnly => false;
public int Count => _index + 1;
public void Add(T item)
public void Add(
T item
)
{
if (_index < _capacity - 1)
{
@@ -82,12 +95,19 @@ namespace StgSharp.Collections
public void Clear()
{
_values = new T[_capacity];
if (RuntimeHelpers.IsReferenceOrContainsReferences<T>()) {
Array.Fill(_values, default);
}
_index = 0;
}
public bool Contains(T item)
public bool Contains(
T item
)
{
if (item is null) {
return false;
}
for (int i = 0; i <= _index; i++)
{
if (item.Equals(_values[i])) {
@@ -97,7 +117,10 @@ namespace StgSharp.Collections
return false;
}
public void CopyTo(T[] array, int arrayIndex)
public void CopyTo(
T[] array,
int arrayIndex
)
{
_values.CopyTo(array, arrayIndex);
}
@@ -109,7 +132,9 @@ namespace StgSharp.Collections
}
}
public int IndexOf(T item)
public int IndexOf(
T item
)
{
T v;
for (int i = 0; i <= _index; i++)
@@ -121,7 +146,9 @@ namespace StgSharp.Collections
return -1;
}
public bool Remove(T item)
public bool Remove(
T item
)
{
T v;
for (int i = 0; i <= _index; i++)
@@ -136,8 +163,13 @@ namespace StgSharp.Collections
return false;
}
public void RemoveAll(Predicate<T> predicate)
public void RemoveAll(
Predicate<T> predicate
)
{
if (predicate is null) {
return;
}
for (int i = _index; i > 0; i--)
{
if (predicate(_values[i]))
@@ -148,26 +180,41 @@ namespace StgSharp.Collections
}
}
public void RemoveAt(int index)
public void RemoveAt(
int index
)
{
if (index <= _index)
{
_values[index] = _values[_index];
Array.Copy(_values, index + 1, _values, index, _index - index);
if (RuntimeHelpers.IsReferenceOrContainsReferences<T>()) {
_values[_index] = default!;
}
_index--;
}
}
public void Resize(int newSize)
public void Resize(
int newSize
)
{
Array.Resize(ref _values, newSize);
}
internal Span<T> ToSpan()
{
return _values;
}
IEnumerator IEnumerable.GetEnumerator()
{
yield return GetEnumerator();
}
void IList<T>.Insert(int index, T item)
void IList<T>.Insert(
int index,
T item
)
{
throw new NotSupportedException();
}
StgSharp.RegularAnalysis/Text/RegexIR.cs
+49 -17
View File
@@ -37,16 +37,34 @@ namespace StgSharp.RegularAnalysis.Text
internal enum RegexIRCommand
{
Match_Seq = 0,
Match_Set = 1,
Condition = 3,
Try = 4,
Count = 5,
Count_Seq = 6,
Count_Set = 7,
Pop = 8,
Pack = 9,
Find_Seq = 10,
Match_Seq = 1 << 0,
Match_Set = 1 << 1,
Condition = 1 << 2,
Try = 1 << 3,
Count_Complex = 1 << 4,
Count_Seq = 1 << 5,
Count_Set = 1 << 6,
Pop = 1 << 7,
Pack = 1 << 8,
Find_Seq = 1 << 9,
Match = Match_Seq | Match_Set,
Count = Count_Seq | Count_Set | Count_Complex,
}
internal interface IRegexCount
{
bool IsGreedy { get; }
bool IsStar { get; }
bool IsPlus { get; }
int Min { get; }
int Max { get; }
}
@@ -163,29 +181,33 @@ namespace StgSharp.RegularAnalysis.Text
}
internal sealed class RegexCountIR(
internal sealed class RegexCountComplexIR(
int min,
int max,
bool isGreedy,
int operation
) : RegexIR
) : RegexIR, IRegexCount
{
public bool IsGreedy { get; } = isGreedy;
public bool IsStar => Min == 0 && Max < 0;
public bool IsPlus => Min == 1 && Max < 0;
public int Min { get; } = min;
public int Max { get; } = max;
public int OperationIndex { get; } = operation;
public int OperationCount { get; } = operation;
internal override RegexIRCommand Command { get; } = RegexIRCommand.Count;
internal override RegexIRCommand Command { get; } = RegexIRCommand.Count_Complex;
internal override string AsCommand(
int lineNum
)
{
return $"[{lineNum}]\t COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {OperationIndex}";
return $"[{lineNum}]\t COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {OperationCount}";
}
}
@@ -195,11 +217,15 @@ namespace StgSharp.RegularAnalysis.Text
int max,
bool isGreedy,
string seq
) : RegexIR
) : RegexIR, IRegexCount
{
public bool IsGreedy { get; } = isGreedy;
public bool IsStar => Min == 0 && Max < 0;
public bool IsPlus => Min == 1 && Max < 0;
public int Min { get; } = min;
public int Max { get; } = max;
@@ -222,11 +248,17 @@ namespace StgSharp.RegularAnalysis.Text
int max,
bool isGreedy,
string set
) : RegexIR
) : RegexIR, IRegexCount
{
public bool IsGreedy { get; } = isGreedy;
public bool IsDotStar => CharSet == "." && Min == 0 && Max < 0;
public bool IsStar => Min == 0 && Max < 0;
public bool IsPlus => Min == 1 && Max < 0;
public int Min { get; } = min;
public int Max { get; } = max;
StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs
+3 -3
View File
@@ -1,6 +1,6 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexHIRGenerator"
// file="RegexIRGenerator"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
@@ -56,14 +56,14 @@ namespace StgSharp.RegularAnalysis.Text
Count++;
}
public void EmitCount(
public void EmitCountComplex(
int min,
int max,
bool isGreedy,
int operation
)
{
_tail.Add(new RegexCountIR(min, max, isGreedy, operation));
_tail.Add(new RegexCountComplexIR(min, max, isGreedy, operation));
Count++;
}
StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs
+19 -20
View File
@@ -1,6 +1,6 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="RegexInterpreter.HIRGenerate"
// file="RegexInterpreter.IRGenerate"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
@@ -57,31 +57,30 @@ namespace StgSharp.RegularAnalysis.Text
int min,
int max,
bool isGreedy,
RegexAstNode right
RegexAstNode body
)
{
bool isRightOperator = !RegexAstNode.IsNullOrEmpty(right) &&
(right.Source.Flag & RegexElementLabel.OPERATOR) != 0;
bool is_operator = !RegexAstNode.IsNullOrEmpty(body) &&
(body.Source.Flag & RegexElementLabel.OPERATOR) != 0;
if (isRightOperator) // seq to be count is complex
if (is_operator) // seq to be count is complex
{
ir.EmitCount(min, max, isGreedy, 1);
RegexIRGenerator rightIR = GenerateIRRecursion(right, curDepth + 1);
ir.EmitTry(rightIR.Count);
ir.EmitIRStream(rightIR);
RegexIRGenerator body_ir = GenerateIRRecursion(body, curDepth + 1);
ir.EmitCountComplex(min, max, isGreedy, body_ir.Count);
ir.EmitIRStream(body_ir);
} else
{
RegexElementLabel count_source_mask = right.Source.Flag;
RegexElementLabel count_source_mask = body.Source.Flag;
switch (count_source_mask)
{
case RegexElementLabel.UNIT:
ir.EmitCountSeq(min, max, isGreedy, right.Source.Value);
ir.EmitCountSeq(min, max, isGreedy, body.Source.Value);
break;
case RegexElementLabel.UNIT_SET:
ir.EmitCountSet(min, max, isGreedy, right.Source.Value);
ir.EmitCountSet(min, max, isGreedy, body.Source.Value);
break;
case RegexElementLabel.UNIT_SPAN:
ir.EmitCountSeq(min, max, isGreedy, right.Source.Value);
ir.EmitCountSeq(min, max, isGreedy, body.Source.Value);
break;
default:
throw new InvalidOperationException($"Unexpected regex element type for count operator: {count_source_mask}");
@@ -144,17 +143,17 @@ namespace StgSharp.RegularAnalysis.Text
_ => throw new InvalidOperationException("Unexpected sequence mask for count operator"),
};
ReadOnlySpan<char> count_source_span = count_source;
RegexIRGenerator countIR = new();
RegexIRGenerator count_ir = new();
if (count_source_span[0] == '*') // infinite count
{
EmitCount(curDepth, countIR, 0, -1, isGreedy, seq);
EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq);
} else if (count_source_span[0] == '+') // at least one
{
EmitCount(curDepth, countIR, 1, -1, isGreedy, seq);
EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq);
} else if (count_source_span[0] == '?') // zero or one
{
EmitCount(curDepth, countIR, 0, 1, isGreedy, seq);
EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq);
} else if (count_source_span[0] == '{') // zero or one, non-greedy
{
int comma_pos = count_source_span.IndexOf(',');
@@ -162,7 +161,7 @@ namespace StgSharp.RegularAnalysis.Text
{
if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count))
{
EmitCount(curDepth, countIR, count, count, isGreedy, seq);
EmitCount(curDepth, count_ir, count, count, isGreedy, seq);
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
@@ -172,7 +171,7 @@ namespace StgSharp.RegularAnalysis.Text
if (int.TryParse(count_source[1..comma_pos], out int min) &&
int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max))
{
EmitCount(curDepth, countIR, min, max, isGreedy, seq);
EmitCount(curDepth, count_ir, min, max, isGreedy, seq);
} else
{
throw new InvalidCastException($"Invalid count format: {count_source}");
@@ -183,7 +182,7 @@ namespace StgSharp.RegularAnalysis.Text
throw new InvalidOperationException();
} // unknown count format
return countIR;
return count_ir;
case RegexElementLabel.CONCAT:
RegexIRGenerator concat = new();
StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs
+137 -124
View File
@@ -25,9 +25,12 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
@@ -37,154 +40,164 @@ namespace StgSharp.RegularAnalysis.Text
public partial class RegexAnalyzer
{
// this array is not thread safe so opt process must be done in single thread.
private RegexIR[] _opt_buffer = new RegexIR[16];
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal RegexInfo ScanAndOptimizeIR(
List<RegexIR> baseIR,
out List<RegexIR> optIR
internal static RegexInfo ScanAndOptimizeIR(
List<RegexIR> baseIR,
out List<RegexIR> optIR
)
{
int index = 0;
Span<RegexIR> origin_span = CollectionsMarshal.AsSpan(baseIR);
List<RegexIR> optimizedIR = new(baseIR.Count);
List<RegexIR> segment = [];
List<RegexIR> opt_ir = new(baseIR.Count);
CapacityFixedList<RegexIR> opt_window = new(8);
Span<RegexIR> opt_win_sp = [.. opt_window];
RegexInfo info = new();
do
{
OptimizeIR(origin_span, ref index, segment, info);
optimizedIR.AddRange(segment);
index++;
int process_count = 0;
int last = int.Min(origin_span.Length, index + 8);
origin_span[index..last].CopyTo(opt_win_sp);
do
{
// plus_expand:
int expand_result = TryExpandPlus(opt_win_sp, opt_ir);
if (expand_result == 0)
{
opt_win_sp[0] = opt_ir[^1];
process_count++;
// goto dot_count_count:
} else if (expand_result > 0)
{
// meets a count-complex
RegexCountComplexIR cnt_cmp_ir = Unsafe.As<RegexCountComplexIR>(opt_win_sp[0]);
if (TryExpandComplexPlus(cnt_cmp_ir.IsGreedy, origin_span[(index + 1)..(index + expand_result)], opt_ir))
{
process_count += expand_result + 1;
break;
}
}
// dot_count_count:
if (TryOptDotCountCount(opt_win_sp, opt_ir))
{
process_count += expand_result == 0 ? 1 : 2;
break;
}
if (TryFlatternCount(opt_win_sp, opt_ir)) { }
} while (false);
index += process_count;
} while (index < baseIR.Count);
optIR = optimizedIR;
optIR = opt_ir;
return info;
}
private int ExpandCount(
string pattern,
int min,
int max,
bool is_greedy,
in Span<RegexIR> span
private static bool TryExpandComplexPlus(
bool is_greedy,
Span<RegexIR> complex,
List<RegexIR> opt
)
{
// expand count operation to less cycles.
int count = complex.Length;
opt.AddRange(complex);
opt.Add(new RegexCountComplexIR(0, -1, is_greedy, count));
return true;
}
const int fast_seq_length = 32;
#region expand plus
int fast_count = fast_seq_length / pattern.Length;
fast_count = int.Min(fast_count, min);
if (fast_count > 0)
private static int TryExpandPlus(
Span<RegexIR> window,
List<RegexIR> opt
)
{
/* a+
* convert to
* aa*
*/
RegexIR ir = window[0];
if (ir is not IRegexCount { IsPlus: true } count) {
return -1;
}
bool is_greedy = count.IsGreedy;
string pattern;
switch (ir.Command & RegexIRCommand.Count)
{
string fast_pattern = string.Concat(Enumerable.Repeat(pattern, fast_count));
case RegexIRCommand.Count_Complex:
RegexCountComplexIR complex = Unsafe.As<RegexCountComplexIR>(ir);
return complex.OperationCount;
case RegexIRCommand.Count_Seq:
RegexCountSeqIR seq = Unsafe.As<RegexCountSeqIR>(ir);
pattern = seq.Sequence;
opt.Add(new RegexMatchSeqIR(pattern));
opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern));
return 0;
case RegexIRCommand.Count_Set:
RegexCountSeqIR set = Unsafe.As<RegexCountSeqIR>(ir);
pattern = set.Sequence;
opt.Add(new RegexMatchSeqIR(pattern));
opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern));
return 0;
default:
return -1;
}
}
int fast_min = min / fast_count;
int consumed_count = fast_min * fast_count;
int remain_min = min - consumed_count;
int remain_max = max - consumed_count;
remain_max = int.Max(-1, remain_max); //for negative stands for infinitive
if (remain_max == 0)
{
// a{2}
RegexCountSeqIR fast = new(fast_min, fast_min, is_greedy, fast_pattern);
span[0] = fast;
return 1;
} else
{
// a{2,5}
RegexCountSeqIR fast = new(fast_min, fast_min, is_greedy, fast_pattern);
RegexCountSeqIR tail = new(remain_min, remain_max, is_greedy, pattern);
span[0] = fast;
span[1] = tail;
return 2;
}
} else if (fast_count == 0)
#endregion
private static bool TryFlatternCount(
Span<RegexIR> window,
List<RegexIR> opt
)
{
RegexIR ir = window[0];
if (ir is not RegexCountSeqIR seq) {
return false;
}
return true;
}
#region dot count count
private static int TryOptDotCountCount(
Span<RegexIR> window,
List<RegexIR> opt
)
{
/* .* <any>*
* convert to
* .*
*/
RegexIR cur_ir = window[0];
if (cur_ir is not RegexCountSetIR { IsDotStar: true } dot) {
return -1;
}
if (window[1] is not IRegexCount { IsStar: true } after) {
return -1;
}
if (dot.IsGreedy)
{
// least count is 0
span[0] = new RegexCountSeqIR(0, max, is_greedy, pattern);
return 1;
opt.Add(window[0]);
return 0;
} else
{
// least count is infinitive
span[0] = new RegexCountSeqIR(-1, -1, is_greedy, pattern);
return 1;
/*
* .* ?a* is different
* it is something like
* FIND(a)?a* :.*
*/
RegexIRGenerator g = new();
g.EmitTry(1);
}
return -1;
}
private void OptimizeDot(
RegexCountSetIR root,
Span<RegexIR> ir,
ref int index,
List<RegexIR> optimizedIR,
RegexInfo info
)
{
int next = index + 1;
optimizedIR.Clear();
if (next >= ir.Length)
{
optimizedIR.Add(root);
return;
}
RegexIR nextIR = ir[next];
if (nextIR is RegexMatchSeqIR match_seq)
{
// opt .*abc style
RegexFindSeqIR find = new(match_seq.Pattern);
optimizedIR.Add(find);
info.MinLength += find.Pattern.Length;
return;
} else if (nextIR is RegexCountSeqIR count_seq)
{
// opt .* (abc)* or .*(abc){2,5} style
string pattern = count_seq.Sequence;
int min = count_seq.Min;
int max = count_seq.Max;
switch ((min, max))
{
case (0, _):
break;
default:
RegexFindSeqIR find = new(pattern);
// RegexCountSeqIR
break;
}
}
}
private void OptimizeIR(
Span<RegexIR> ir,
ref int index,
List<RegexIR> optimizedIR,
RegexInfo info
)
{
optimizedIR.Clear();
// TODO optimize here
RegexIR root = ir[index];
switch (root.Command)
{
case RegexIRCommand.Count_Set:
RegexCountSetIR count = (root as RegexCountSetIR)!;
if (count.CharSet == ".")
{
// opt for .+ / .* / .{} pattern
OptimizeDot(count, ir, ref index, optimizedIR, info);
}
break;
case RegexIRCommand.Match_Seq:
info.MinLength += (root as RegexMatchSeqIR)!.Pattern.Length;
optimizedIR.Add(root);
break;
default:
optimizedIR.Add(root);
break;
}
}
#endregion
internal class RegexInfo
{
StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs
+3 -2
View File
@@ -41,8 +41,9 @@ namespace StgSharp.RegularAnalysis.Text
)
{
StringBuilder sb = new StringBuilder();
foreach (RegexIR ir in irs) {
ScanAndMarkIROptimization(ir);
foreach (RegexIR ir in irs)
{
// ScanAndMarkIROptimization(ir);
}
return sb.ToString();
}
future/ngra-regex-ir.md
+4
View File
@@ -55,10 +55,14 @@ FIND_SEQ(a, first)
MATCH_SEQ(a)
COUNT_SEQ(a,0,+inf,greedy)
完成
3. a{1} -> a
4. .\*a\*特征，优化为.*
完成，最优先
5. a{m,n}特征，展开为
MATCH_SEQ(aa...m个a....aa)
COUNT_SEQ(a,0,n,greedy)