regex ir opt code on rewrite

StgSharp.Common/Collections/CapacityFixedList.cs

@@ -2,8 +2,8 @@
 // -----------------------------------------------------------------------
 // file="CapacityFixedList"
 // Project: StgSharp
-// AuthorGroup: Nitload Space
+// AuthorGroup: Nitload
-// Copyright (c) Nitload Space. All rights reserved.
+// Copyright (c) Nitload. All rights reserved.
 //     
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to deal
@@ -35,7 +35,9 @@ namespace StgSharp.Collections
     public static class CapacityFixedListBuilder
     {
 
-        internal static CapacityFixedList<T> Create<T>(ReadOnlySpan<T> values)
+        internal static CapacityFixedList<T> Create<T>(
+                                             ReadOnlySpan<T> values
+        )
         {
             return new CapacityFixedList<T>(values);
         }
@@ -50,28 +52,39 @@ namespace StgSharp.Collections
         private readonly int _capacity;
         private int _index;
 
-        public CapacityFixedList(int size)
+        public CapacityFixedList(
+               int size
+        )
         {
             _values = new T[size];
             _capacity = size;
         }
 
-        public CapacityFixedList(ReadOnlySpan<T> values)
+        public CapacityFixedList(
+               ReadOnlySpan<T> values
+        )
         {
             _values = values.ToArray();
         }
 
-        public T this[int index]
+        public T this[
+                 int index
+        ]
         {
-            get => index < _index ? _values[index] : throw new ArgumentOutOfRangeException(nameof(index));
+            get => index < _index ?
-            set => _values[index] = index < _index ? value : throw new ArgumentOutOfRangeException(nameof(index));
+                   _values[index] :
+                   throw new ArgumentOutOfRangeException(nameof(index));
+            set => _values[index] =
+                   index < _index ? value : throw new ArgumentOutOfRangeException(nameof(index));
         }
 
         public bool IsReadOnly => false;
 
         public int Count => _index + 1;
 
-        public void Add(T item)
+        public void Add(
+                    T item
+        )
         {
             if (_index < _capacity - 1)
             {
@@ -82,12 +95,19 @@ namespace StgSharp.Collections
 
         public void Clear()
         {
-            _values = new T[_capacity];
+            if (RuntimeHelpers.IsReferenceOrContainsReferences<T>()) {
+                Array.Fill(_values, default);
+            }
             _index = 0;
         }
 
-        public bool Contains(T item)
+        public bool Contains(
+                    T item
+        )
         {
+            if (item is null) {
+                return false;
+            }
             for (int i = 0; i <= _index; i++)
             {
                 if (item.Equals(_values[i])) {
@@ -97,7 +117,10 @@ namespace StgSharp.Collections
             return false;
         }
 
-        public void CopyTo(T[] array, int arrayIndex)
+        public void CopyTo(
+                    T[] array,
+                    int arrayIndex
+        )
         {
             _values.CopyTo(array, arrayIndex);
         }
@@ -109,7 +132,9 @@ namespace StgSharp.Collections
             }
         }
 
-        public int IndexOf(T item)
+        public int IndexOf(
+                   T item
+        )
         {
             T v;
             for (int i = 0; i <= _index; i++)
@@ -121,7 +146,9 @@ namespace StgSharp.Collections
             return -1;
         }
 
-        public bool Remove(T item)
+        public bool Remove(
+                    T item
+        )
         {
             T v;
             for (int i = 0; i <= _index; i++)
@@ -136,8 +163,13 @@ namespace StgSharp.Collections
             return false;
         }
 
-        public void RemoveAll(Predicate<T> predicate)
+        public void RemoveAll(
+                    Predicate<T> predicate
+        )
         {
+            if (predicate is null) {
+                return;
+            }
             for (int i = _index; i > 0; i--)
             {
                 if (predicate(_values[i]))
@@ -148,26 +180,41 @@ namespace StgSharp.Collections
             }
         }
 
-        public void RemoveAt(int index)
+        public void RemoveAt(
+                    int index
+        )
         {
             if (index <= _index)
             {
-                _values[index] = _values[_index];
+                Array.Copy(_values, index + 1, _values, index, _index - index);
+                if (RuntimeHelpers.IsReferenceOrContainsReferences<T>()) {
+                    _values[_index] = default!;
+                }
                 _index--;
             }
         }
 
-        public void Resize(int newSize)
+        public void Resize(
+                    int newSize
+        )
         {
             Array.Resize(ref _values, newSize);
         }
 
+        internal Span<T> ToSpan()
+        {
+            return _values;
+        }
+
         IEnumerator IEnumerable.GetEnumerator()
         {
             yield return GetEnumerator();
         }
 
-        void IList<T>.Insert(int index, T item)
+        void IList<T>.Insert(
+                      int index,
+                      T item
+        )
         {
             throw new NotSupportedException();
         }

StgSharp.RegularAnalysis/Text/RegexIR.cs

@@ -37,16 +37,34 @@ namespace StgSharp.RegularAnalysis.Text
     internal enum RegexIRCommand
     {
 
-        Match_Seq = 0,
+        Match_Seq = 1 << 0,
-        Match_Set = 1,
+        Match_Set = 1 << 1,
-        Condition = 3,
+        Condition = 1 << 2,
-        Try = 4,
+        Try = 1 << 3,
-        Count = 5,
+        Count_Complex = 1 << 4,
-        Count_Seq = 6,
+        Count_Seq = 1 << 5,
-        Count_Set = 7,
+        Count_Set = 1 << 6,
-        Pop = 8,
+        Pop = 1 << 7,
-        Pack = 9,
+        Pack = 1 << 8,
-        Find_Seq = 10,
+        Find_Seq = 1 << 9,
+
+        Match = Match_Seq | Match_Set,
+        Count = Count_Seq | Count_Set | Count_Complex,
+
+    }
+
+    internal interface IRegexCount
+    {
+
+        bool IsGreedy { get; }
+
+        bool IsStar { get; }
+
+        bool IsPlus { get; }
+
+        int Min { get; }
+
+        int Max { get; }
 
     }
 
@@ -163,29 +181,33 @@ namespace StgSharp.RegularAnalysis.Text
 
     }
 
-    internal sealed class RegexCountIR(
+    internal sealed class RegexCountComplexIR(
                           int min,
                           int max,
                           bool isGreedy,
                           int operation
-    ) : RegexIR
+    ) : RegexIR, IRegexCount
     {
 
         public bool IsGreedy { get; } = isGreedy;
 
+        public bool IsStar => Min == 0 && Max < 0;
+
+        public bool IsPlus => Min == 1 && Max < 0;
+
         public int Min { get; } = min;
 
         public int Max { get; } = max;
 
-        public int OperationIndex { get; } = operation;
+        public int OperationCount { get; } = operation;
 
-        internal override RegexIRCommand Command { get; } = RegexIRCommand.Count;
+        internal override RegexIRCommand Command { get; } = RegexIRCommand.Count_Complex;
 
         internal override string AsCommand(
                                  int lineNum
         )
         {
-            return $"[{lineNum}]\t COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {OperationIndex}";
+            return $"[{lineNum}]\t COUNT {(IsGreedy ? "GREEDY" : "NON_GREEDY")} MIN: {Min} MAX: {Max} OPERATION: {OperationCount}";
         }
 
     }
@@ -195,11 +217,15 @@ namespace StgSharp.RegularAnalysis.Text
                           int max,
                           bool isGreedy,
                           string seq
-    ) : RegexIR
+    ) : RegexIR, IRegexCount
     {
 
         public bool IsGreedy { get; } = isGreedy;
 
+        public bool IsStar => Min == 0 && Max < 0;
+
+        public bool IsPlus => Min == 1 && Max < 0;
+
         public int Min { get; } = min;
 
         public int Max { get; } = max;
@@ -222,11 +248,17 @@ namespace StgSharp.RegularAnalysis.Text
                           int max,
                           bool isGreedy,
                           string set
-    ) : RegexIR
+    ) : RegexIR, IRegexCount
     {
 
         public bool IsGreedy { get; } = isGreedy;
 
+        public bool IsDotStar => CharSet == "." && Min == 0 && Max < 0;
+
+        public bool IsStar => Min == 0 && Max < 0;
+
+        public bool IsPlus => Min == 1 && Max < 0;
+
         public int Min { get; } = min;
 
         public int Max { get; } = max;

StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs

@@ -1,6 +1,6 @@
 //-----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-// file="RegexHIRGenerator"
+// file="RegexIRGenerator"
 // Project: StgSharp
 // AuthorGroup: Nitload
 // Copyright (c) Nitload. All rights reserved.
@@ -56,14 +56,14 @@ namespace StgSharp.RegularAnalysis.Text
             Count++;
         }
 
-        public void EmitCount(
+        public void EmitCountComplex(
                     int min,
                     int max,
                     bool isGreedy,
                     int operation
         )
         {
-            _tail.Add(new RegexCountIR(min, max, isGreedy, operation));
+            _tail.Add(new RegexCountComplexIR(min, max, isGreedy, operation));
             Count++;
         }
 

StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs

@@ -1,6 +1,6 @@
 //-----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-// file="RegexInterpreter.HIRGenerate"
+// file="RegexInterpreter.IRGenerate"
 // Project: StgSharp
 // AuthorGroup: Nitload
 // Copyright (c) Nitload. All rights reserved.
@@ -57,31 +57,30 @@ namespace StgSharp.RegularAnalysis.Text
                             int min,
                             int max,
                             bool isGreedy,
-                            RegexAstNode right
+                            RegexAstNode body
         )
         {
-            bool isRightOperator = !RegexAstNode.IsNullOrEmpty(right) &&
+            bool is_operator = !RegexAstNode.IsNullOrEmpty(body) &&
-                                   (right.Source.Flag & RegexElementLabel.OPERATOR) != 0;
+                               (body.Source.Flag & RegexElementLabel.OPERATOR) != 0;
 
-            if (isRightOperator)        // seq to be count is complex
+            if (is_operator)        // seq to be count is complex
             {
-                ir.EmitCount(min, max, isGreedy, 1);
+                RegexIRGenerator body_ir = GenerateIRRecursion(body, curDepth + 1);
-                RegexIRGenerator rightIR = GenerateIRRecursion(right, curDepth + 1);
+                ir.EmitCountComplex(min, max, isGreedy, body_ir.Count);
-                ir.EmitTry(rightIR.Count);
+                ir.EmitIRStream(body_ir);
-                ir.EmitIRStream(rightIR);
             } else
             {
-                RegexElementLabel count_source_mask = right.Source.Flag;
+                RegexElementLabel count_source_mask = body.Source.Flag;
                 switch (count_source_mask)
                 {
                     case RegexElementLabel.UNIT:
-                        ir.EmitCountSeq(min, max, isGreedy, right.Source.Value);
+                        ir.EmitCountSeq(min, max, isGreedy, body.Source.Value);
                         break;
                     case RegexElementLabel.UNIT_SET:
-                        ir.EmitCountSet(min, max, isGreedy, right.Source.Value);
+                        ir.EmitCountSet(min, max, isGreedy, body.Source.Value);
                         break;
                     case RegexElementLabel.UNIT_SPAN:
-                        ir.EmitCountSeq(min, max, isGreedy, right.Source.Value);
+                        ir.EmitCountSeq(min, max, isGreedy, body.Source.Value);
                         break;
                     default:
                         throw new InvalidOperationException($"Unexpected regex element type for count operator: {count_source_mask}");
@@ -144,17 +143,17 @@ namespace StgSharp.RegularAnalysis.Text
                         _ => throw new InvalidOperationException("Unexpected sequence mask for count operator"),
                     };
                     ReadOnlySpan<char> count_source_span = count_source;
-                    RegexIRGenerator countIR = new();
+                    RegexIRGenerator count_ir = new();
 
                     if (count_source_span[0] == '*')              // infinite count
                     {
-                        EmitCount(curDepth, countIR, 0, -1, isGreedy, seq);
+                        EmitCount(curDepth, count_ir, 0, -1, isGreedy, seq);
                     } else if (count_source_span[0] == '+')       // at least one
                     {
-                        EmitCount(curDepth, countIR, 1, -1, isGreedy, seq);
+                        EmitCount(curDepth, count_ir, 1, -1, isGreedy, seq);
                     } else if (count_source_span[0] == '?')       // zero or one
                     {
-                        EmitCount(curDepth, countIR, 0, 1, isGreedy, seq);
+                        EmitCount(curDepth, count_ir, 0, 1, isGreedy, seq);
                     } else if (count_source_span[0] == '{')        // zero or one, non-greedy
                     {
                         int comma_pos = count_source_span.IndexOf(',');
@@ -162,7 +161,7 @@ namespace StgSharp.RegularAnalysis.Text
                         {
                             if (int.TryParse(count_source[1..(isGreedy ? ^2 : ^1)], out int count))
                             {
-                                EmitCount(curDepth, countIR, count, count, isGreedy, seq);
+                                EmitCount(curDepth, count_ir, count, count, isGreedy, seq);
                             } else
                             {
                                 throw new InvalidCastException($"Invalid count format: {count_source}");
@@ -172,7 +171,7 @@ namespace StgSharp.RegularAnalysis.Text
                             if (int.TryParse(count_source[1..comma_pos], out int min) &&
                                 int.TryParse(count_source[(comma_pos + 1)..(isGreedy ? ^1 : ^2)], out int max))
                             {
-                                EmitCount(curDepth, countIR, min, max, isGreedy, seq);
+                                EmitCount(curDepth, count_ir, min, max, isGreedy, seq);
                             } else
                             {
                                 throw new InvalidCastException($"Invalid count format: {count_source}");
@@ -183,7 +182,7 @@ namespace StgSharp.RegularAnalysis.Text
                         throw new InvalidOperationException();
                     }   // unknown count format
 
-                    return countIR;
+                    return count_ir;
                 case RegexElementLabel.CONCAT:
                     RegexIRGenerator concat = new();
 

StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs

@@ -25,9 +25,12 @@
 //     
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
+using StgSharp.Collections;
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Numerics;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Text;
 using System.Threading.Tasks;
@@ -37,154 +40,164 @@ namespace StgSharp.RegularAnalysis.Text
     public partial class RegexAnalyzer
     {
 
-        // this array is not thread safe so opt process must be done in single thread.
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private RegexIR[] _opt_buffer = new RegexIR[16];
 
-        internal RegexInfo ScanAndOptimizeIR(
+        internal static RegexInfo ScanAndOptimizeIR(
-                           List<RegexIR> baseIR,
+                                  List<RegexIR> baseIR,
-                           out List<RegexIR> optIR
+                                  out List<RegexIR> optIR
         )
         {
             int index = 0;
             Span<RegexIR> origin_span = CollectionsMarshal.AsSpan(baseIR);
-            List<RegexIR> optimizedIR = new(baseIR.Count);
+            List<RegexIR> opt_ir = new(baseIR.Count);
-            List<RegexIR> segment = [];
+            CapacityFixedList<RegexIR> opt_window = new(8);
+            Span<RegexIR> opt_win_sp = [.. opt_window];
             RegexInfo info = new();
             do
             {
-                OptimizeIR(origin_span, ref index, segment, info);
+                int process_count = 0;
-                optimizedIR.AddRange(segment);
+                int last = int.Min(origin_span.Length, index + 8);
-                index++;
+                origin_span[index..last].CopyTo(opt_win_sp);
+
+                do
+                {
+                    // plus_expand:
+                    int expand_result = TryExpandPlus(opt_win_sp, opt_ir);
+                    if (expand_result == 0)
+                    {
+                        opt_win_sp[0] = opt_ir[^1];
+                        process_count++;
+
+                        // goto dot_count_count:
+                    } else if (expand_result > 0)
+                    {
+                        // meets a count-complex
+                        RegexCountComplexIR cnt_cmp_ir = Unsafe.As<RegexCountComplexIR>(opt_win_sp[0]);
+                        if (TryExpandComplexPlus(cnt_cmp_ir.IsGreedy, origin_span[(index + 1)..(index + expand_result)], opt_ir))
+                        {
+                            process_count += expand_result + 1;
+                            break;
+                        }
+                    }
+
+                    // dot_count_count:
+                    if (TryOptDotCountCount(opt_win_sp, opt_ir))
+                    {
+                        process_count += expand_result == 0 ? 1 : 2;
+                        break;
+                    }
+
+                    if (TryFlatternCount(opt_win_sp, opt_ir)) { }
+                } while (false);
+
+                index += process_count;
             } while (index < baseIR.Count);
-            optIR = optimizedIR;
+            optIR = opt_ir;
             return info;
         }
 
-        private int ExpandCount(
+        private static bool TryExpandComplexPlus(
-                    string pattern,
+                            bool is_greedy,
-                    int min,
+                            Span<RegexIR> complex,
-                    int max,
+                            List<RegexIR> opt
-                    bool is_greedy,
-                    in Span<RegexIR> span
         )
         {
-            // expand count operation to less cycles.
+            int count = complex.Length;
+            opt.AddRange(complex);
+            opt.Add(new RegexCountComplexIR(0, -1, is_greedy, count));
+            return true;
+        }
 
-            const int fast_seq_length = 32;
+        #region expand plus
 
-            int fast_count = fast_seq_length / pattern.Length;
+        private static int TryExpandPlus(
-            fast_count = int.Min(fast_count, min);
+                           Span<RegexIR> window,
-            if (fast_count > 0)
+                           List<RegexIR> opt
+        )
+        {
+            /* a+
+             * convert to
+             * aa*
+             */
+
+            RegexIR ir = window[0];
+            if (ir is not IRegexCount { IsPlus: true } count) {
+                return -1;
+            }
+            bool is_greedy = count.IsGreedy;
+            string pattern;
+            switch (ir.Command & RegexIRCommand.Count)
             {
-                string fast_pattern = string.Concat(Enumerable.Repeat(pattern, fast_count));
+                case RegexIRCommand.Count_Complex:
+                    RegexCountComplexIR complex = Unsafe.As<RegexCountComplexIR>(ir);
+                    return complex.OperationCount;
+                case RegexIRCommand.Count_Seq:
+                    RegexCountSeqIR seq = Unsafe.As<RegexCountSeqIR>(ir);
+                    pattern = seq.Sequence;
+                    opt.Add(new RegexMatchSeqIR(pattern));
+                    opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern));
+                    return 0;
+                case RegexIRCommand.Count_Set:
+                    RegexCountSeqIR set = Unsafe.As<RegexCountSeqIR>(ir);
+                    pattern = set.Sequence;
+                    opt.Add(new RegexMatchSeqIR(pattern));
+                    opt.Add(new RegexCountSeqIR(0, -1, is_greedy, pattern));
+                    return 0;
+                default:
+                    return -1;
+            }
+        }
 
-                int fast_min = min / fast_count;
+            #endregion
-                int consumed_count = fast_min * fast_count;
+
-                int remain_min = min - consumed_count;
+        private static bool TryFlatternCount(
-                int remain_max = max - consumed_count;
+                            Span<RegexIR> window,
-                remain_max = int.Max(-1, remain_max);    //for negative stands for infinitive
+                            List<RegexIR> opt
-                if (remain_max == 0)
+        )
-                {
+        {
-                    // a{2}
+            RegexIR ir = window[0];
-                    RegexCountSeqIR fast = new(fast_min, fast_min, is_greedy, fast_pattern);
+            if (ir is not RegexCountSeqIR seq) {
-                    span[0] = fast;
+                return false;
-                    return 1;
+            }
-                } else
+            return true;
-                {
+        }
-                    // a{2,5}
+
-                    RegexCountSeqIR fast = new(fast_min, fast_min, is_greedy, fast_pattern);
+        #region dot count count
-                    RegexCountSeqIR tail = new(remain_min, remain_max, is_greedy, pattern);
+
-                    span[0] = fast;
+        private static int TryOptDotCountCount(
-                    span[1] = tail;
+                           Span<RegexIR> window,
-                    return 2;
+                           List<RegexIR> opt
-                }
+        )
-            } else if (fast_count == 0)
+        {
+            /* .* <any>*
+             * convert to
+             * .*
+             */
+            RegexIR cur_ir = window[0];
+            if (cur_ir is not RegexCountSetIR { IsDotStar: true } dot) {
+                return -1;
+            }
+            if (window[1] is not IRegexCount { IsStar: true } after) {
+                return -1;
+            }
+            if (dot.IsGreedy)
             {
-                // least count is 0
+                opt.Add(window[0]);
-                span[0] = new RegexCountSeqIR(0, max, is_greedy, pattern);
+                return 0;
-                return 1;
             } else
             {
-                // least count is infinitive
+                /*
-                span[0] = new RegexCountSeqIR(-1, -1, is_greedy, pattern);
+                 * .* ?a* is different
-                return 1;
+                 * it is something like
+                 * FIND(a)?a* :.*
+                 */
+                RegexIRGenerator g = new();
+                g.EmitTry(1);
             }
+            return -1;
         }
 
-        private void OptimizeDot(
+            #endregion
-                     RegexCountSetIR root,
-                     Span<RegexIR> ir,
-                     ref int index,
-                     List<RegexIR> optimizedIR,
-                     RegexInfo info
-        )
-        {
-            int next = index + 1;
-            optimizedIR.Clear();
-            if (next >= ir.Length)
-            {
-                optimizedIR.Add(root);
-                return;
-            }
-            RegexIR nextIR = ir[next];
-            if (nextIR is RegexMatchSeqIR match_seq)
-            {
-                // opt .*abc style
-                RegexFindSeqIR find = new(match_seq.Pattern);
-                optimizedIR.Add(find);
-                info.MinLength += find.Pattern.Length;
-                return;
-            } else if (nextIR is RegexCountSeqIR count_seq)
-            {
-                // opt .* (abc)* or .*(abc){2,5} style
-                string pattern = count_seq.Sequence;
-                int min = count_seq.Min;
-                int max = count_seq.Max;
-                switch ((min, max))
-                {
-                    case (0, _):
-
-                        break;
-                    default:
-                        RegexFindSeqIR find = new(pattern);
-
-                        // RegexCountSeqIR 
-
-                        break;
-                }
-            }
-        }
-
-        private void OptimizeIR(
-                     Span<RegexIR> ir,
-                     ref int index,
-                     List<RegexIR> optimizedIR,
-                     RegexInfo info
-        )
-        {
-            optimizedIR.Clear();
-
-            // TODO optimize here
-            RegexIR root = ir[index];
-            switch (root.Command)
-            {
-                case RegexIRCommand.Count_Set:
-                    RegexCountSetIR count = (root as RegexCountSetIR)!;
-                    if (count.CharSet == ".")
-                    {
-                        // opt for .+ / .* / .{}    pattern
-                        OptimizeDot(count, ir, ref index, optimizedIR, info);
-                    }
-                    break;
-                case RegexIRCommand.Match_Seq:
-                    info.MinLength += (root as RegexMatchSeqIR)!.Pattern.Length;
-                    optimizedIR.Add(root);
-                    break;
-                default:
-                    optimizedIR.Add(root);
-                    break;
-            }
-        }
 
         internal class RegexInfo
         {

StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs

@@ -41,8 +41,9 @@ namespace StgSharp.RegularAnalysis.Text
         )
         {
             StringBuilder sb = new StringBuilder();
-            foreach (RegexIR ir in irs) {
+            foreach (RegexIR ir in irs)
-                ScanAndMarkIROptimization(ir);
+            {
+                // ScanAndMarkIROptimization(ir);
             }
             return sb.ToString();
         }

future/ngra-regex-ir.md

@@ -55,10 +55,14 @@ FIND_SEQ(a, first)
 MATCH_SEQ(a)
 COUNT_SEQ(a,0,+inf,greedy)
 
+完成
+
 3. a{1} -> a
 
 4. .\*a\*特征，优化为.*
 
+完成，最优先
+
 5. a{m,n}特征，展开为
 MATCH_SEQ(aa...m个a....aa)
 COUNT_SEQ(a,0,n,greedy)