diff --git a/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs new file mode 100644 index 00000000..cd0730b4 --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs @@ -0,0 +1,179 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="RegexAnalyzer.ASTGenerate" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using StgSharp.RegularAnalysis.Abstraction; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + public partial class RegexAnalyzer + { + + private static AbstractSyntaxTree GenerateAST( + string _source + ) + { + CompileStack _stack = new(); + AbstractSyntaxTree _tree = new(); + RegexTokenReader reader = new(_source); + TokenParser lexer = reader.Pipe(() => new RegexTokenParser()); + + while (lexer.TryReadToken(out Token token)) + { + // Console.WriteLine(token.Value); + if ((token.Flag & RegexElementLabel.SEQUENCE) != 0) + { + // char or charset + _stack.PushOperand(new RegexAstNode(token)); + } else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0) + { + _stack.PushOperator(new RegexAstNode(token)); + _stack.IncreaseDepth(0); + } else if ((token.Flag & RegexElementLabel.GROUP_END) != 0) + { + // close all nodes here + RegexAstNode op; + while (_stack.TryPopOperator(out op)) { + ProcessOperator(op); + } + if (_stack.OperandInDepthCount != 1) { + throw new InvalidOperationException(); + } + _stack.DecreaseDepth(); + op = _stack.PopOperator(); + if (op.Source.Flag != RegexElementLabel.GROUP_BEGIN) { + throw new InvalidOperationException("Unmatched group begin and end symbol"); + } + if (op.Source.Value.Length > 1) + { + op.Right = _stack.PopOperand(); + _stack.PushOperand(op); + } + } else if ((token.Flag & RegexElementLabel.OPERATOR) != 0) + { + if (_stack.OperatorInDepthCount == 0) + { + // the first operator in stack + _stack.PushOperator(new RegexAstNode(token)); + } else + { + // process operator by precedence + while (_stack.TryPeekOperator(out RegexAstNode topOp)) + { + (bool Valid, int Value) cmp = RegexAstNode.ComparePrecedence(topOp.Source.Flag, token.Flag); + + if (!cmp.Valid) { + throw new InvalidOperationException("Operators with same precedence are not supported."); + } + if (cmp.Value < 0) + { + _stack.PopOperator(); + ProcessOperator(topOp); + continue; + } else + { + break; + } + } + _stack.PushOperator(new RegexAstNode(token)); + } + } + } + if (_stack.Depth == 1) + { + if (_stack.OperandInDepthCount != 1 || _stack.OperatorInDepthCount != 0) + { + while (_stack.TryPopOperator(out RegexAstNode? op)) { + ProcessOperator(op); + } + } + if (_stack.OperandInDepthCount == 1 && _stack.OperatorInDepthCount == 0) + { + _tree.Root = _stack.PopOperand(); + return _tree; + } + } + throw new InvalidOperationException("Invalid regular expression syntax."); + + + void ProcessOperator( + RegexAstNode op + ) + { + if (op.EqualityTypeConvert == RegexElementLabel.CONCAT) + { + if (_stack.TryPopOperand(out RegexAstNode? _1) && + _stack.TryPopOperand(out RegexAstNode? _2)) + { + op.Left = _2; + op.Right = _1; + _stack.PushOperand(op); + _ = _tree.AddNode(op); + _ = _tree.AddNode(_1); + _ = _tree.AddNode(_2); + } else + { + throw new InvalidOperationException("Insufficient operands for CONCAT operator."); + } + } else if (op.EqualityTypeConvert == RegexElementLabel.COUNT) + { + if (_stack.TryPopOperand(out RegexAstNode? _1)) + { + op.Left = _1; + _stack.PushOperand(op); + _ = _tree.AddNode(_1); + _ = _tree.AddNode(op); + } else + { + throw new InvalidOperationException("Insufficient operands for CONCAT operator."); + } + } else if (op.EqualityTypeConvert == RegexElementLabel.ALT) + { + if (_stack.TryPopOperand(out RegexAstNode? _1) && + _stack.TryPopOperand(out RegexAstNode? _2)) + { + op.Left = _2; + op.Right = _1; + _stack.PushOperand(op); + _ = _tree.AddNode(op); + _ = _tree.AddNode(_1); + _ = _tree.AddNode(_2); + } else + { + throw new InvalidOperationException("Insufficient operands for CONCAT operator."); + } + } + } + } + + } +} diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs b/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs similarity index 87% rename from StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs rename to StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs index 038417b8..0197a4f5 100644 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.ASTOptimize.cs +++ b/StgSharp.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexInterpreter.ASTOptimize" +// file="RegexAnalyzer.ASTOptimize" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -156,6 +156,67 @@ namespace StgSharp.RegularAnalysis.Text return true; } + private static void OptimizeTree( + AbstractSyntaxTree tree + ) + { + // phase0: scan container and remove no parent nodes + _ = tree.AllNodes.RemoveWhere(node => node.Parent == null && node != tree.Root); + TreeEnumerator enumerator = new(tree.Root); + RegexAstNode current; + + // phase1: ALT flatten + // phase2: CONCAT normalize + int i = 0; + List altNodes = []; + while (enumerator.MoveNext()) + { + current = enumerator.Current; + if (current.EqualityTypeConvert == RegexElementLabel.ALT) + { + RegexAstNode union = FlattenAlt(current); + altNodes.Add(current); + current.Right = union; + current.Left = RegexAstNode.Empty; + } else if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) { + RotateConcat(current); + } + } + + List cases = []; + + // phase3: then ALT union + foreach (RegexAstNode item in altNodes) + { + RegexAstNode _case = item.Right; + do + { + cases.Add(_case); + _case = _case.Next; + } while (!RegexAstNode.IsNullOrEmpty(_case)); + RegexAstNode newAlt = MergeAlt(cases); + item.ReplaceBy(newAlt); + } + + // return; + + // phase4: CONCAT merge + enumerator.Reset(); + while (enumerator.MoveNext()) + { + // suppose all concat is now in normalized form, + // we can merge adjacent unit nodes into a single unit node with string value + current = enumerator.Current; + if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) + { + if (MergeConcat(current)) { + enumerator.SkipOnce(); + } + } + } + + } + private static void RotateConcat( RegexAstNode root ) diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexAnalyzer.IRGenerate.cs similarity index 87% rename from StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs rename to StgSharp.RegularAnalysis/Text/RegexAnalyzer.IRGenerate.cs index 1491e9b4..7080e962 100644 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IRGenerate.cs +++ b/StgSharp.RegularAnalysis/Text/RegexAnalyzer.IRGenerate.cs @@ -1,30 +1,27 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexInterpreter.IRGenerate.cs" +// file="RegexInterpreter.IRGenerate" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. // -// Permission is hereby granted, free of charge, to any person -// obtaining a copy of this software and associated documentation -// files (the “Software”), to deal in the Software without restriction, -// including without limitation the rights to use, copy, modify, merge, -// publish, distribute, sublicense, and/or sell copies of the Software, -// and to permit persons to whom the Software is furnished to do so, -// subject to the following conditions: -// -// The above copyright notice and -// this permission notice shall be included in all copies -// or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED “AS IS”, -// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, -// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH -// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- @@ -46,7 +43,9 @@ namespace StgSharp.RegularAnalysis.Text public partial class RegexAnalyzer { - internal static List GenerateIR(AbstractSyntaxTree tree) + internal static List GenerateIR( + AbstractSyntaxTree tree + ) { return GenerateIRRecursion(tree.Root, 0).ToList(); } @@ -58,7 +57,8 @@ namespace StgSharp.RegularAnalysis.Text int min, int max, bool isGreedy, - RegexAstNode right) + RegexAstNode right + ) { bool is_operator = !RegexAstNode.IsNullOrEmpty(right) && (right.Source.Flag & RegexElementLabel.OPERATOR) != 0; @@ -88,7 +88,10 @@ namespace StgSharp.RegularAnalysis.Text } } - private static RegexIRGenerator GenerateIRRecursion(RegexAstNode node, int depth) + private static RegexIRGenerator GenerateIRRecursion( + RegexAstNode node, + int depth + ) { #pragma warning disable IDE0010 if (depth > 1024) @@ -117,7 +120,7 @@ namespace StgSharp.RegularAnalysis.Text RegexIRGenerator groupIR = new(); rightIR = GenerateIRRecursion(right, curDepth + 1); - groupIR.EmitMatchComplex(rightIR.Count); + groupIR.EmitTry(rightIR.Count); groupIR.EmitIRStream(rightIR); groupIR.EmitCondition(2, 1); groupIR.EmitPop(false); @@ -251,7 +254,7 @@ namespace StgSharp.RegularAnalysis.Text RegexIRGenerator cur_ir = casesIRList[^case_count]; ir_count -= cur_ir.Count; case_count--; - alt.EmitMatchComplex(cur_ir.Count); + alt.EmitTry(cur_ir.Count); alt.EmitIRStream(cur_ir); alt.EmitCondition(1/* need to calc here */, 2); alt.EmitPop(true); diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs b/StgSharp.RegularAnalysis/Text/RegexAnalyzer.IROptimize.cs similarity index 51% rename from StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs rename to StgSharp.RegularAnalysis/Text/RegexAnalyzer.IROptimize.cs index 8d548280..16c9ebaf 100644 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.IROptimize.cs +++ b/StgSharp.RegularAnalysis/Text/RegexAnalyzer.IROptimize.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="RegexInterpreter.IROptimize" +// file="RegexAnalyzer.IROptimize" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -40,80 +40,125 @@ namespace StgSharp.RegularAnalysis.Text public partial class RegexAnalyzer { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static RegexInfo ScanAndOptimizeIR( - List baseIR, - out List optIR + internal static void ScanAndOptimizeIR( + List baseIR, + out List optIR ) { - int index = 0; + int opt_count; List origin_ir = baseIR; - Span origin_span = CollectionsMarshal.AsSpan(baseIR); - List opt_ir = new(baseIR.Count); - CapacityFixedList opt_window = new(8); - Span opt_win_sp = [.. opt_window]; - RegexInfo info = new(); + List opt_ir = new(origin_ir.Count); + RegexIR[] opt_window = new RegexIR[8]; + Span opt_win_sp = opt_window; + opt_cycle: + opt_count = 0; + int index = 0; + Span origin_span = CollectionsMarshal.AsSpan(origin_ir); + do { int process_count = 0; int last = int.Min(origin_span.Length, index + 8); origin_span[index..last].CopyTo(opt_win_sp); - do + int result; + if ((result = TryExpandPlus(opt_win_sp, opt_ir)) >= 0) { - int result = 0; - if ((result = TryExpandPlus(opt_win_sp, opt_ir)) >= 0) + process_count++; + if (result > 0) { - process_count++; - if (result > 0) - { - RegexCountComplexIR cc = Unsafe.As(opt_win_sp[0]); - Span sp = origin_span.Slice(index + 1, result); - TryExpandComplexPlus(cc.IsGreedy, sp, opt_ir); - process_count += result; - } - } else if ((result = TryOptDotCountCount(opt_win_sp, opt_ir)) >= 0) - { - process_count++; - } else if (TryFlattenCount(opt_win_sp, opt_ir)) { - process_count++; + RegexCountComplexIR cc = Unsafe.As(opt_win_sp[0]); + Span sp = origin_span.Slice(index + 1, result); + TryExpandComplexPlus(cc.IsGreedy, sp, opt_ir); + process_count += result; } - } while (false); + opt_count++; + } else if (TryOptDotCountCount(opt_win_sp, opt_ir) >= 0) + { + process_count += 2; + opt_count++; + } else if (TryFlattenCount(opt_win_sp, opt_ir) >= 0) + { + process_count++; + opt_count++; + } else if ((result = TryExpandOnce(opt_win_sp, opt_ir)) >= 0) + { + process_count++; + if (result > 0) + { + RegexCountComplexIR cc = Unsafe.As(opt_win_sp[0]); + Span sp = origin_span.Slice(index + 1, result); + TryExpandComplex(sp, opt_ir, true); + process_count += result; + } + opt_count++; + } else if (opt_win_sp[0] is RegexTryIR{ IsFullyOptimized:false} try_ir) + { + process_count++; + int cmd_cnt = try_ir.CommandCount; + Span try_sp = origin_span.Slice(index + 1, cmd_cnt); + ScanAndOptimizeIR(try_sp, out List opt_try); + RegexTryIR new_try = new(opt_try.Count) + { + IsFullyOptimized = true + }; + opt_ir.Add(new_try); + opt_ir.AddRange(opt_try); + process_count += cmd_cnt; + opt_count++; + } else + { + opt_ir.Add(opt_win_sp[0]); + process_count++; + } index += process_count; - } while (index < baseIR.Count); + } while (index < origin_ir.Count); + if (opt_count > 0) + { + // need to swap the origin_ir and opt_ir, and clear the opt_ir for next round + if (ReferenceEquals(origin_ir, baseIR)) + { + origin_ir = new(opt_ir.Count); + } + (origin_ir, opt_ir) = (opt_ir, origin_ir); + opt_ir.Clear(); + goto opt_cycle; + } optIR = opt_ir; - return info; } - private static RegexInfo ScanAndOptimizeIR( - Span baseIR, - List opt_ir + private static void ScanAndOptimizeIR( + Span baseIR, + out List opt_ir ) { List originList = new(baseIR.Length); CollectionsMarshal.SetCount(originList, baseIR.Length); baseIR.CopyTo(CollectionsMarshal.AsSpan(originList)); - return ScanAndOptimizeIR(originList, out opt_ir); + ScanAndOptimizeIR(originList, out opt_ir); } - private static bool TryFlattenCount( - Span window, - List opt + private static int TryFlattenCount( + Span window, + List opt ) { const int falttern_size = 32; RegexIR ir = window[0]; if (ir is not RegexCountSeqIR seq) { - return false; + return -1; + } + int min = seq.Min; + int max = seq.Max; + if (min == 0) { + return -1; } string pattern = seq.Sequence; int pattern_size = pattern.Length; if (pattern_size > falttern_size) { opt.Add(ir); } - int min = seq.Min; - int max = seq.Max; int min_expand = falttern_size / pattern_size; min_expand = int.Min(min_expand, min); string expand = string.Concat(Enumerable.Repeat(pattern, min_expand)); @@ -121,10 +166,10 @@ namespace StgSharp.RegularAnalysis.Text int new_rest = min % min_expand; if (new_min == 1) { - opt.Add(new RegexMatchSeqIR(pattern)); + opt.Add(new RegexMatchSeqIR(expand)); } else { - opt.Add(new RegexCountSeqIR(new_min, new_min, seq.IsGreedy, pattern)); + opt.Add(new RegexCountSeqIR(new_min, new_min, seq.IsGreedy, expand)); } if (new_rest > 0) { @@ -134,7 +179,7 @@ namespace StgSharp.RegularAnalysis.Text int rest_count = max - min; rest_count = int.Max(rest_count, -1); opt.Add(new RegexCountSeqIR(0, rest_count, seq.IsGreedy, pattern)); - return true; + return 0; } #region dot count count @@ -176,17 +221,89 @@ namespace StgSharp.RegularAnalysis.Text } + #region expand once + + private static void TryExpandComplex( + Span complex, + List opt, + bool is_once + ) + { + if (!is_once) { + opt.Add(new RegexTryIR(complex.Length)); + } + + opt.AddRange(complex); + } + + private static int TryExpandOnce( + Span window, + List opt + ) + { + RegexIR ir = window[0]; + if (ir is IRegexCount { Min: 1, Max: 1 }) + { + string pattern; + switch (ir.Command & RegexIRCommand.Count) + { + case RegexIRCommand.Count_Complex: + RegexCountComplexIR complex = Unsafe.As(ir); + return complex.CommandCount; + case RegexIRCommand.Count_Seq: + RegexCountSeqIR seq = Unsafe.As(ir); + pattern = seq.Sequence; + opt.Add(new RegexMatchSeqIR(pattern)); + return 0; + case RegexIRCommand.Count_Set: + RegexCountSetIR set = Unsafe.As(ir); + pattern = set.CharSet; + opt.Add(new RegexMatchSetIR(pattern)); + return 0; + default: + return -1; + } + } else if (ir is IRegexCount { Min: 0, Max: 1 }) + { + string pattern; + switch (ir.Command & RegexIRCommand.Count) + { + case RegexIRCommand.Count_Complex: + RegexCountComplexIR complex = Unsafe.As(ir); + return complex.CommandCount; + case RegexIRCommand.Count_Seq: + RegexCountSeqIR seq = Unsafe.As(ir); + pattern = seq.Sequence; + opt.Add(new RegexTryIR(1)); + opt.Add(new RegexMatchSeqIR(pattern)); + return 0; + case RegexIRCommand.Count_Set: + opt.Add(new RegexTryIR(1)); + RegexCountSetIR set = Unsafe.As(ir); + pattern = set.CharSet; + opt.Add(new RegexMatchSetIR(pattern)); + return 0; + default: + return -1; + } + } else + { + return -1; + } + } + + #endregion + + #region expand plus - private static bool TryExpandComplexPlus( + private static void TryExpandComplexPlus( bool is_greedy, Span complex, List opt ) { - int count = complex.Length; - List part_opt = new(count); - RegexInfo info = ScanAndOptimizeIR(complex, part_opt); + ScanAndOptimizeIR(complex, out List? part_opt); opt.AddRange(part_opt); RegexCountComplexIR c_ir = new(0, -1, is_greedy, part_opt.Count) { @@ -194,7 +311,6 @@ namespace StgSharp.RegularAnalysis.Text }; opt.Add(c_ir); opt.AddRange(part_opt); - return true; } /// diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs b/StgSharp.RegularAnalysis/Text/RegexAnalyzer.SourceGenerate.cs similarity index 100% rename from StgSharp.RegularAnalysis/Text/RegexInterpreter.SourceGenerate.cs rename to StgSharp.RegularAnalysis/Text/RegexAnalyzer.SourceGenerate.cs diff --git a/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs b/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs new file mode 100644 index 00000000..63eba341 --- /dev/null +++ b/StgSharp.RegularAnalysis/Text/RegexAnalyzer.cs @@ -0,0 +1,94 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="RegexAnalyzer" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using StgSharp.RegularAnalysis.Abstraction; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection.Emit; +using System.Runtime.CompilerServices; +using System.Text; + +using System.Threading.Tasks; + +namespace StgSharp.RegularAnalysis.Text +{ + public record TextRegexSource + { + + internal TextRegexSource( + string pattern, + AbstractSyntaxTree ast, + List base_ir, + List ir, + Exception? analyseException = null + ) + { + Pattern = pattern; + Ast = ast; + IRs = ir; + BaseIR = base_ir; + AnalyseException = analyseException; + } + + public Exception? AnalyseException { get; set; } + + public string Pattern { get; } + + internal AbstractSyntaxTree Ast { get; } + + internal List BaseIR { get; } + + internal List IRs { get; } + + } + + public static partial class RegexAnalyzer + { + + public static TextRegexSource Analyze( + string regex + ) + { + // RegexAnalyzer interpreter = new(); + // interpreter._source = regex; + try + { + AbstractSyntaxTree tree = GenerateAST(regex); + OptimizeTree(tree); + List base_ir = GenerateIR(tree); + ScanAndOptimizeIR(base_ir, out List opt_ir); + return new TextRegexSource(regex, tree, base_ir, opt_ir); + } + catch (Exception ex) + { + return new TextRegexSource(regex, null, null, null, ex); + } + } + + } +} diff --git a/StgSharp.RegularAnalysis/Text/RegexIR.cs b/StgSharp.RegularAnalysis/Text/RegexIR.cs index dfc12c8c..ede82aab 100644 --- a/StgSharp.RegularAnalysis/Text/RegexIR.cs +++ b/StgSharp.RegularAnalysis/Text/RegexIR.cs @@ -312,16 +312,14 @@ namespace StgSharp.RegularAnalysis.Text } - internal sealed class RegexMatchComplexIR( + internal sealed class RegexTryIR( int count ) : RegexIR, IRegexComplex { public bool IsFullyOptimized { get; set; } - public int CountOfCommand { get; } = count; - - public int CommandCount { get; set; } + public int CommandCount { get; set; } = count; internal override RegexIRCommand Command => RegexIRCommand.Try; @@ -329,7 +327,7 @@ namespace StgSharp.RegularAnalysis.Text int lineNum ) { - return $"[{lineNum}]\t TRY {CountOfCommand}"; + return $"[{lineNum}]\t TRY {CommandCount}"; } } diff --git a/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs b/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs index 4c05aa47..190bd505 100644 --- a/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs +++ b/StgSharp.RegularAnalysis/Text/RegexIRGenerator.cs @@ -134,11 +134,11 @@ namespace StgSharp.RegularAnalysis.Text Count++; } - public void EmitMatchComplex( + public void EmitTry( int count ) { - _tail.Add(new RegexMatchComplexIR(count)); + _tail.Add(new RegexTryIR(count)); Count++; } diff --git a/StgSharp.RegularAnalysis/Text/RegexInterpreter.cs b/StgSharp.RegularAnalysis/Text/RegexInterpreter.cs deleted file mode 100644 index 141df63b..00000000 --- a/StgSharp.RegularAnalysis/Text/RegexInterpreter.cs +++ /dev/null @@ -1,275 +0,0 @@ -//----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -// file="RegexInterpreter" -// Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. -// -// ----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -using StgSharp.RegularAnalysis.Abstraction; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection.Emit; -using System.Runtime.CompilerServices; -using System.Text; - -using System.Threading.Tasks; - -namespace StgSharp.RegularAnalysis.Text -{ - public partial class RegexAnalyzer - { - - private AbstractSyntaxTree _tree = new(); - private CompileStack _stack = new(); - private Exception _interpreteException; - private string _source; - - private RegexAnalyzer() { } - - internal AbstractSyntaxTree Tree - { - get - { - if (_interpreteException != null) { - throw new InvalidOperationException("The regex interpretation failed.", _interpreteException); - } - return _tree; - } - } - -<<<<<<< HEAD:StgSharp.RegularAnalysis/TextRegex/RegexInterpreter.cs - public static RegexInterpreter Analyze(string regex) -======= - public static RegexAnalyzer Analyze( - string regex - ) ->>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/RegexInterpreter.cs - { - RegexAnalyzer interpreter = new(); - interpreter._source = regex; - bool isSuccess; - try - { - interpreter.Analyze(); - interpreter.OptimizeTree(); - isSuccess = true; - } - catch (Exception ex) - { - interpreter._interpreteException = ex; - isSuccess = false; - } - return interpreter; - } - - private void Analyze() - { - RegexTokenReader reader = new(_source); - TokenParser lexer = reader.Pipe(() => new RegexTokenParser()); - - while (lexer.TryReadToken(out Token token)) - { - // Console.WriteLine(token.Value); - if ((token.Flag & RegexElementLabel.SEQUENCE) != 0) - { - // char or charset - _stack.PushOperand(new RegexAstNode(token)); - } else if ((token.Flag & RegexElementLabel.GROUP_BEGIN) != 0) - { - _stack.PushOperator(new RegexAstNode(token)); - _stack.IncreaseDepth(0); - } else if ((token.Flag & RegexElementLabel.GROUP_END) != 0) - { - // close all nodes here - RegexAstNode op; - while (_stack.TryPopOperator(out op)) { - ProcessOperator(op); - } - if (_stack.OperandInDepthCount != 1) { - throw new InvalidOperationException(); - } - _stack.DecreaseDepth(); - op = _stack.PopOperator(); - if (op.Source.Flag != RegexElementLabel.GROUP_BEGIN) { - throw new InvalidOperationException("Unmatched group begin and end symbol"); - } - if (op.Source.Value.Length > 1) - { - op.Right = _stack.PopOperand(); - _stack.PushOperand(op); - } - } else if ((token.Flag & RegexElementLabel.OPERATOR) != 0) - { - if (_stack.OperatorInDepthCount == 0) - { - // the first operator in stack - _stack.PushOperator(new RegexAstNode(token)); - } else - { - // process operator by precedence - while (_stack.TryPeekOperator(out RegexAstNode topOp)) - { - (bool Valid, int Value) cmp = RegexAstNode.ComparePrecedence(topOp.Source.Flag, token.Flag); - - if (!cmp.Valid) { - throw new InvalidOperationException("Operators with same precedence are not supported."); - } - if (cmp.Value < 0) - { - _stack.PopOperator(); - ProcessOperator(topOp); - continue; - } else - { - break; - } - } - _stack.PushOperator(new RegexAstNode(token)); - } - } - } - if (_stack.Depth == 1) - { - if (_stack.OperandInDepthCount != 1 || _stack.OperatorInDepthCount != 0) - { - while (_stack.TryPopOperator(out RegexAstNode? op)) { - ProcessOperator(op); - } - } - if (_stack.OperandInDepthCount == 1 && _stack.OperatorInDepthCount == 0) - { - _tree.Root = _stack.PopOperand(); - return; - } - } - throw new InvalidOperationException("Invalid regular expression syntax."); - } - - private void OptimizeTree() - { - // phase0: scan container and remove no parent nodes - _ = _tree.AllNodes.RemoveWhere(node => node.Parent == null && node != _tree.Root); - TreeEnumerator enumerator = new(_tree.Root); - RegexAstNode current; - - // phase1: ALT flatten - // phase2: CONCAT normalize - int i = 0; - List altNodes = []; - while (enumerator.MoveNext()) - { - current = enumerator.Current; - if (current.EqualityTypeConvert == RegexElementLabel.ALT) - { - RegexAstNode union = FlattenAlt(current); - altNodes.Add(current); - current.Right = union; - current.Left = RegexAstNode.Empty; - } else if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) { - RotateConcat(current); - } - } - - List cases = []; - - // phase3: then ALT union - foreach (RegexAstNode item in altNodes) - { - RegexAstNode _case = item.Right; - do - { - cases.Add(_case); - _case = _case.Next; - } while (!RegexAstNode.IsNullOrEmpty(_case)); - RegexAstNode newAlt = MergeAlt(cases); - item.ReplaceBy(newAlt); - } - - // return; - - // phase4: CONCAT merge - enumerator.Reset(); - while (enumerator.MoveNext()) - { - // suppose all concat is now in normalized form, - // we can merge adjacent unit nodes into a single unit node with string value - current = enumerator.Current; - if (current.EqualityTypeConvert == RegexElementLabel.CONCAT) - { - if (MergeConcat(current)) { - enumerator.SkipOnce(); - } - } - } - - // TODO enum AST again , do ALT tail merge and simple ALT front merge - } - - private void ProcessOperator(RegexAstNode op) - { - if (op.EqualityTypeConvert == RegexElementLabel.CONCAT) - { - if (_stack.TryPopOperand(out RegexAstNode? _1) && _stack.TryPopOperand(out RegexAstNode? _2)) - { - op.Left = _2; - op.Right = _1; - _stack.PushOperand(op); - _tree.AddNode(op); - _tree.AddNode(_1); - _tree.AddNode(_2); - } else - { - throw new InvalidOperationException("Insufficient operands for CONCAT operator."); - } - } else if (op.EqualityTypeConvert == RegexElementLabel.COUNT) - { - if (_stack.TryPopOperand(out RegexAstNode? _1)) - { - op.Left = _1; - _stack.PushOperand(op); - _tree.AddNode(_1); - _tree.AddNode(op); - } else - { - throw new InvalidOperationException("Insufficient operands for CONCAT operator."); - } - } else if (op.EqualityTypeConvert == RegexElementLabel.ALT) - { - if (_stack.TryPopOperand(out RegexAstNode? _1) && _stack.TryPopOperand(out RegexAstNode? _2)) - { - op.Left = _2; - op.Right = _1; - _stack.PushOperand(op); - _tree.AddNode(op); - _tree.AddNode(_1); - _tree.AddNode(_2); - } else - { - throw new InvalidOperationException("Insufficient operands for CONCAT operator."); - } - } - } - - } -} diff --git a/StgSharp.RegularAnalysis/Text/TextRegex.cs b/StgSharp.RegularAnalysis/Text/TextRegex.cs index 04194dd4..ba92d6d0 100644 --- a/StgSharp.RegularAnalysis/Text/TextRegex.cs +++ b/StgSharp.RegularAnalysis/Text/TextRegex.cs @@ -28,10 +28,6 @@ using System; using System.Collections.Generic; using System.Linq; -<<<<<<< HEAD:StgSharp.RegularAnalysis/TextRegex/TextRegex.cs -======= -using System.Runtime.CompilerServices; ->>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/TextRegex.cs using System.Text; using System.Threading.Tasks; @@ -39,22 +35,19 @@ namespace StgSharp.RegularAnalysis.Text { public abstract class TextRegex { -<<<<<<< HEAD:StgSharp.RegularAnalysis/TextRegex/TextRegex.cs - public ref struct Match - { - private ReadOnlySpan _sequence; - - - } - } -======= public abstract bool IsMatch( ReadOnlySpan text ); + public ref struct Match + { + + private ReadOnlySpan _sequence; + + } + } public ref struct MatchResult { } ->>>>>>> 359dd6728716c1c0014d64b72ba70c6c65bb8cf8:StgSharp.RegularAnalysis/Text/TextRegex.cs } diff --git a/StgSharpDebug/TestRegexAnalyse.cs b/StgSharpDebug/TestRegexAnalyse.cs index 907efc05..83d26af1 100644 --- a/StgSharpDebug/TestRegexAnalyse.cs +++ b/StgSharpDebug/TestRegexAnalyse.cs @@ -40,6 +40,7 @@ namespace StgSharp { private static string[] regexSource = [ + @"http|https|ftp|ftps", @"abc[0-9]x|abc[0-9]y|abd[0-9]x|abd[0-9]y|xyz", @"get_name|get_value|get_type|set_name|set_value|set_type", @@ -51,8 +52,8 @@ namespace StgSharp { foreach (string regex in regexSource) { - RegexAnalyzer interpreter = RegexAnalyzer.Analyze(regex); - AbstractSyntaxTree tree = interpreter.Tree; + TextRegexSource interpreterSource = RegexAnalyzer.Analyze(regex); + AbstractSyntaxTree tree = interpreterSource.Ast; Console.WriteLine($"result of \"{regex}\""); TreeEnumerator enumerator = new(tree); while (enumerator.MoveNext()) @@ -61,11 +62,16 @@ namespace StgSharp Console.Write(new string('|', enumerator.Depth)); Console.WriteLine($"\t{node.Source.Value}\t{node.Source.Flag}"); } - List ir = RegexAnalyzer.GenerateIR(tree); Console.WriteLine("-----IR-----"); + List ir = interpreterSource.BaseIR; for (int i = 0; i < ir.Count; i++) { Console.WriteLine(ir[i].AsCommand(i)); } + Console.WriteLine("-----Optimized once-----"); + List opt_ir = interpreterSource.IRs; + for (int i = 0; i < opt_ir.Count; i++) { + Console.WriteLine(opt_ir[i].AsCommand(i)); + } Console.WriteLine(); Console.WriteLine(); Console.WriteLine(); diff --git a/future/ngra-regex-ir.md b/future/ngra-regex-ir.md index de81e805..b4c7d390 100644 --- a/future/ngra-regex-ir.md +++ b/future/ngra-regex-ir.md @@ -59,6 +59,8 @@ COUNT_SEQ(a,0,+inf,greedy) 3. a{1} -> a +完成 + 4. .\*a\*特征，优化为.* 完成，最优先 @@ -67,6 +69,4 @@ COUNT_SEQ(a,0,+inf,greedy) MATCH_SEQ(aa...m个a....aa) COUNT_SEQ(a,0,n,greedy) -6. .*?a*$ - -7. 对于一般的.*?，应该将后续的所有东西全都放进一个巨大的try，code gen时使用最慢的一位一位offset直到直到第一个匹配rest的东西 \ No newline at end of file +完成 \ No newline at end of file