From f372f83171151bc155bd78879eef3d2517088e4c Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:17:09 +0800 Subject: [PATCH 1/7] process cases of ALT as special rules --- Directory.Build.targets | 34 +++- ReadMe.md | 2 +- .../OpenGL/Generated/OpenGLFunction.Base.g.cs | 6 +- .../OpenGL/Generated/OpenGLFunction.v4_1.g.cs | 6 +- src/Nitload.Graphics/OpenGL/glRenderStream.cs | 21 ++- src/Nitload.Graphics/PlainGeometryMesh.cs | 12 +- src/Nitload.Graphics/RenderStream.cs | 10 +- .../Numeric/Graphics/GMatrix44_T.cs | 76 ++++++++- src/Nitload.Native/CMakeLists.txt | 26 ++- src/Nitload.Native/Nitload.Native.vcxproj | 7 +- src/Nitload.Native/README.md | 26 +++ .../Text/TextRegexSourceGen.GenClass.cs | 136 ++++----------- .../Text/TextRegexSourceGen.GenSpecial.cs | 159 +++++++++++++++++- src/README.md | 2 +- test/README.md | 32 ++++ test/StgSharpDebug/Program.cs | 22 +-- test/StgSharpDebug/StgSharpDebug.csproj | 10 +- test/StgSharpDebug/TestRegexGenerate.cs | 65 +++++++ 18 files changed, 486 insertions(+), 166 deletions(-) create mode 100644 test/StgSharpDebug/TestRegexGenerate.cs diff --git a/Directory.Build.targets b/Directory.Build.targets index 749f92a9..3bd7f0ac 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -1,19 +1,37 @@ + + + + clang-debug + clang-release + $(SolutionRoot)cmake_build/$(NativeCMakePreset) + <_NativeCMakeOutputDir>$([System.String]::Copy('$(NativeBuildDir)').Replace('\', '/')) + + + + + + + Condition="'$(DisablePostBuildCopy)'!='true' And '$(MSBuildProjectExtension)'=='.csproj' And '$(DesignTimeBuild)'!='true'"> - <_OutputFiles Include="$(OutputPath)**\*.*" - Exclude="$(OutputPath)**\*.user" /> + + <_OutputFiles Include="$(TargetDir)**\*.*" + Exclude="$(TargetDir)**\*.user;$(TargetDir)**\$(NativeLibName).*;$(TargetDir)**\lib$(NativeLibName).*;$(TargetDir)**\StgSharp.Native.*;$(TargetDir)**\libStgSharp.Native.*" /> + BeforeTargets="CopyOutputToBuildDir" + Condition="'$(DisableNativeCopy)'!='true' And '$(MSBuildProjectExtension)'=='.csproj' And '$(DesignTimeBuild)'!='true'"> + + @@ -45,12 +67,12 @@ value) { - fixed (GMatrix44* __value = &value) - { - __context->glUniformMatrix4fv(location, 1, transpose ? (byte)1 : (byte)0, (float*)__value); - } + __context->glUniformMatrix4fv(location, 1, transpose ? (byte)1 : (byte)0, (float*)&value); } + /// /// Forwards to the native OpenGL entry point. diff --git a/src/Nitload.Graphics/OpenGL/Generated/OpenGLFunction.v4_1.g.cs b/src/Nitload.Graphics/OpenGL/Generated/OpenGLFunction.v4_1.g.cs index 512d07be..997c6b9d 100644 --- a/src/Nitload.Graphics/OpenGL/Generated/OpenGLFunction.v4_1.g.cs +++ b/src/Nitload.Graphics/OpenGL/Generated/OpenGLFunction.v4_1.g.cs @@ -633,11 +633,9 @@ namespace Nitload.Graphics.OpenGL [MethodImpl(MethodImplOptions.AggressiveInlining)] public void ProgramUniform(GlHandle program, int location, bool transpose, GMatrix44 value) { - fixed (GMatrix44* __value = &value) - { - __context->glProgramUniformMatrix4fv(program.Value, location, 1, transpose ? (byte)1 : (byte)0, (float*)__value); + __context->glProgramUniformMatrix4fv(program.Value, location, 1, transpose ? (byte)1 : (byte)0, (float*)&value); } - } + /// /// Forwards to the native OpenGL entry point. diff --git a/src/Nitload.Graphics/OpenGL/glRenderStream.cs b/src/Nitload.Graphics/OpenGL/glRenderStream.cs index c2ab1c90..0f505671 100644 --- a/src/Nitload.Graphics/OpenGL/glRenderStream.cs +++ b/src/Nitload.Graphics/OpenGL/glRenderStream.cs @@ -42,10 +42,11 @@ namespace Nitload.Graphics.OpenGL { ContextHandle = Marshal.AllocHGlobal(Marshal.SizeOf()); OpenglContext* contextID = (OpenglContext*)ContextHandle; - contextID->glGetString = (delegate*)IntPtr.Zero; + contextID->glGetString = (delegate* unmanaged[Cdecl])IntPtr.Zero; } - if (GL == null) { - _context = OpenGLFunction.BuildGlFunctionPackage(ContextHandle); + if (GL == null) + { + // _context = OpenGLFunction.BuildGlFunctionPackage(ContextHandle); } } @@ -60,7 +61,8 @@ namespace Nitload.Graphics.OpenGL public unsafe void MakeAsCurrentContext() { GraphicFramework.glfwMakeContextCurrent(CanvasHandle); - OpenGLFunction.CurrentGL = this.GL; + + // OpenGLFunction.CurrentGL = this.GL; } /// @@ -73,7 +75,8 @@ namespace Nitload.Graphics.OpenGL SwapBuffers(); renderToThread[this] = ThreadHelper.EmptyThread; GraphicFramework.glfwMakeContextCurrent(IntPtr.Zero); - OpenGLFunction.CurrentGL = null; + + // OpenGLFunction.CurrentGL = null; } /// @@ -121,7 +124,7 @@ namespace Nitload.Graphics.OpenGL [MethodImpl(MethodImplOptions.AggressiveInlining)] protected sealed override void NativeCameraViewRange( Radius fieldOfRange, - Vec2 offset, + Vec2 offset, (float frontDepth, float backDepth) viewDepth ) { @@ -130,9 +133,9 @@ namespace Nitload.Graphics.OpenGL [MethodImpl(MethodImplOptions.AggressiveInlining)] protected sealed override void NativeCameraViewTarget( - Vec3 position, - Vec3 direction, - Vec3 up + Vec3 position, + Vec3 direction, + Vec3 up ) { NativeCamera.SetViewDirection(position, direction, up); diff --git a/src/Nitload.Graphics/PlainGeometryMesh.cs b/src/Nitload.Graphics/PlainGeometryMesh.cs index c2ebf751..213c9974 100644 --- a/src/Nitload.Graphics/PlainGeometryMesh.cs +++ b/src/Nitload.Graphics/PlainGeometryMesh.cs @@ -16,14 +16,15 @@ namespace Nitload.Graphics public class PlainGeometryMesh { - private readonly PlainGeometry _shape; - private readonly Vec2[] _textureCoordinates; + private readonly Vec2[] _textureCoordinates; private readonly Image _textureImage; + private readonly PlainGeometry _shape; + public PlainGeometryMesh( PlainGeometry shape, Image texture, - Vec2[] texCoord + Vec2[] texCoord ) { ArgumentNullException.ThrowIfNull(shape); @@ -40,14 +41,13 @@ namespace Nitload.Graphics _textureCoordinates = texCoord; } - public Vec4[] VertexArray => _shape.VertexStream; + public Vec3[] VertexArray => _shape.VertexStream; public Image TextureImage => _textureImage; public ReadOnlySpan MeshIndices => _shape.VertexIndices; - public ReadOnlySpan TextureCoord => MemoryMarshal.Cast( - _textureCoordinates); + public ReadOnlySpan> TextureCoord => _textureCoordinates; public PlainGeometryMesh Resize( PlainGeometry newShape diff --git a/src/Nitload.Graphics/RenderStream.cs b/src/Nitload.Graphics/RenderStream.cs index 5aec9383..ae515e43 100644 --- a/src/Nitload.Graphics/RenderStream.cs +++ b/src/Nitload.Graphics/RenderStream.cs @@ -5,10 +5,10 @@ // SPDX-License-Identifier: MIT // ----------------------------------------------------------------------------- +using Nitload.Common.Timing; using Nitload.Mathematics; using Nitload.Mathematics.Numeric.Graphics; using Nitload.Mathematics.Numeric.Graphics; -using Nitload.Common.Timing; using System; using System.Collections.Generic; @@ -123,14 +123,14 @@ namespace Nitload.Graphics protected abstract void NativeCameraViewRange( Radius fieldOfRange, - Vec2 offset, + Vec2 offset, (float frontDepth, float backDepth) viewDepth ); protected abstract void NativeCameraViewTarget( - Vec3 position, - Vec3 direction, - Vec3 up + Vec3 position, + Vec3 direction, + Vec3 up ); #endregion diff --git a/src/Nitload.Mathematics/Numeric/Graphics/GMatrix44_T.cs b/src/Nitload.Mathematics/Numeric/Graphics/GMatrix44_T.cs index 4824e1e2..9db36e62 100644 --- a/src/Nitload.Mathematics/Numeric/Graphics/GMatrix44_T.cs +++ b/src/Nitload.Mathematics/Numeric/Graphics/GMatrix44_T.cs @@ -24,12 +24,11 @@ namespace Nitload.Mathematics.Numeric.Graphics private Buffer _buffer; - public Vec4 this[ - int x + internal unsafe ref Vec4 this[ + int x ] { - readonly get => _buffer[x]; - set => _buffer[x] = value; + get => ref Unsafe.Add(ref Unsafe.As, Vec4>(ref this), x); } public T this[ @@ -178,6 +177,75 @@ namespace Nitload.Mathematics.Numeric.Graphics } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static GMatrix44 operator -( + GMatrix44 left, + GMatrix44 right + ) + { + Unsafe.SkipInit(out GMatrix44 result); + result._buffer[0] = left._buffer[0] - right._buffer[0]; + result._buffer[1] = left._buffer[1] - right._buffer[1]; + result._buffer[2] = left._buffer[2] - right._buffer[2]; + result._buffer[3] = left._buffer[3] - right._buffer[3]; + + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static GMatrix44 operator *( + GMatrix44 left, + T right + ) + { + Unsafe.SkipInit(out GMatrix44 result); + result._buffer[0] = left._buffer[0] * right; + result._buffer[1] = left._buffer[1] * right; + result._buffer[2] = left._buffer[2] * right; + result._buffer[3] = left._buffer[3] * right; + + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static GMatrix44 operator *( + T left, + GMatrix44 right + ) + { + return right * left; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static GMatrix44 operator *( + GMatrix44 left, + GMatrix44 right + ) + { + Unsafe.SkipInit(out GMatrix44 result); + result._buffer[0] = left._buffer[0] + right._buffer[0]; + result._buffer[1] = left._buffer[1] + right._buffer[1]; + result._buffer[2] = left._buffer[2] + right._buffer[2]; + result._buffer[3] = left._buffer[3] + right._buffer[3]; + + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static GMatrix44 operator +( + GMatrix44 left, + GMatrix44 right + ) + { + Unsafe.SkipInit(out GMatrix44 result); + result._buffer[0] = left._buffer[0] + right._buffer[0]; + result._buffer[1] = left._buffer[1] + right._buffer[1]; + result._buffer[2] = left._buffer[2] + right._buffer[2]; + result._buffer[3] = left._buffer[3] + right._buffer[3]; + + return result; + } + [InlineArray(4)] private struct Buffer { diff --git a/src/Nitload.Native/CMakeLists.txt b/src/Nitload.Native/CMakeLists.txt index 223b0059..aeea2cc5 100644 --- a/src/Nitload.Native/CMakeLists.txt +++ b/src/Nitload.Native/CMakeLists.txt @@ -70,7 +70,13 @@ else() set(NIF_ARCH_TAG ${NIF_RESOLVED_ARCH_NAME}) endif() -set(NIF_BUILD_OUTPUT_DIR ${NIF_SOLUTION_ROOT}/bin/${NIF_OS_TAG}-${NIF_ARCH_TAG}) +set(NIF_EXPECTED_RID "" CACHE STRING "Optional managed target RID to validate before deployment") +if(NIF_EXPECTED_RID AND NOT NIF_EXPECTED_RID STREQUAL "${NIF_OS_TAG}-${NIF_ARCH_TAG}") + message(FATAL_ERROR "Native target ${NIF_OS_TAG}-${NIF_ARCH_TAG} does not match managed target ${NIF_EXPECTED_RID}. Use a matching compiler/toolchain and a separate build directory.") +endif() + +set(NIF_BUILD_OUTPUT_DIR "${NIF_SOLUTION_ROOT}/bin/${NIF_OS_TAG}-${NIF_ARCH_TAG}" + CACHE PATH "Native artifact directory shared with managed build deployment") # Compile-time intermediates (native PDB / .obj side files) live inside the CMake # build tree (cmake_build/) so they get cleaned up with it. set(NIF_OBJECT_ROOT_DIR ${CMAKE_CURRENT_BINARY_DIR}/obj/${PROJECT_NAME}) @@ -86,9 +92,21 @@ source_group(TREE ${CMAKE_CURRENT_LIST_DIR} FILES ${NIF_NATIVE_PROJECT_FILES}) set_target_properties(${PROJECT_NAME} PROPERTIES C_STANDARD 11 C_STANDARD_REQUIRED ON - RUNTIME_OUTPUT_DIRECTORY ${NIF_BUILD_OUTPUT_DIR} - LIBRARY_OUTPUT_DIRECTORY ${NIF_BUILD_OUTPUT_DIR} - ARCHIVE_OUTPUT_DIRECTORY ${NIF_BUILD_OUTPUT_DIR}) + OUTPUT_NAME "Nitload.Native" + PDB_NAME "Nitload.Native" + DEBUG_POSTFIX "" + RELEASE_POSTFIX "" + RELWITHDEBINFO_POSTFIX "" + MINSIZEREL_POSTFIX "" + RUNTIME_OUTPUT_DIRECTORY "${NIF_BUILD_OUTPUT_DIR}" + LIBRARY_OUTPUT_DIRECTORY "${NIF_BUILD_OUTPUT_DIR}" + ARCHIVE_OUTPUT_DIRECTORY "${NIF_BUILD_OUTPUT_DIR}" + PDB_OUTPUT_DIRECTORY "${NIF_BUILD_OUTPUT_DIR}") + +# P/Invoke loads Nitload.Native.dll on Windows, including GNU-style Clang toolchains. +if(WIN32) + set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "") +endif() foreach(_config Debug Release RelWithDebInfo MinSizeRel) string(TOUPPER ${_config} _config_upper) diff --git a/src/Nitload.Native/Nitload.Native.vcxproj b/src/Nitload.Native/Nitload.Native.vcxproj index ec4f2119..abe5b157 100644 --- a/src/Nitload.Native/Nitload.Native.vcxproj +++ b/src/Nitload.Native/Nitload.Native.vcxproj @@ -85,9 +85,10 @@ <_ArchTag Condition="'$(Platform)'=='x64'">x64 <_ArchTag Condition="'$(Platform)'=='ARM64'">arm64 - - $(SolutionDir)cmake_build\win_$(_ArchTag)\ - $(SolutionDir)obj\$(ProjectName)\$(Configuration)\$(Platform)\ + + Nitload.Native + $(NativeBuildDir) + $(SolutionRoot)obj\$(ProjectName)\$(Configuration)\$(Platform)\ true diff --git a/src/Nitload.Native/README.md b/src/Nitload.Native/README.md index 1b19e6fe..cd6e5b1f 100644 --- a/src/Nitload.Native/README.md +++ b/src/Nitload.Native/README.md @@ -6,6 +6,32 @@ Though this project do has a .vcxproj file, you can only edit but not compile in visul studio. Currently you can only compile this lib via Clang and cmake. +## Build output and managed deployment + +From the repository root: + +```powershell +cmake --preset clang-debug -S src/Nitload.Native +cmake --build cmake_build/clang-debug --target Nitload.Native +``` + +CMake writes `Nitload.Native.dll` and its linker PDB to `bin/win-x64` on Windows x64 +(Linux uses `bin/linux-x64/libNitload.Native.so`). Debug and release presets currently +share this platform directory; the most recent native build is the deployed version. +`NIF_BUILD_OUTPUT_DIR` can override the destination. Managed builds pass their expected +RID as `NIF_EXPECTED_RID` so an incompatible native target is rejected before deployment. + +`StgSharpDebug.csproj` enables `BuildNativeBeforeManaged` and `RequireNativeLibrary`. +Its build runs the incremental CMake build (`clang-debug` for Debug, `clang-release` +otherwise), then deploys the native library to the executable's actual `TargetDir`. +Other managed projects copy an existing native artifact without invoking CMake. +Managed-output staging never copies deployed native files back over the CMake artifact. + +To use a prebuilt native artifact, pass `-p:BuildNativeBeforeManaged=false`. A matching +Clang/Ninja environment and the local GLFW static library are required for native builds. +Cross-target builds require a matching toolchain/preset; changing a managed platform alone +does not cross-compile the C library. + ## Code Layout Conventions: diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs index 26da0a18..39047e19 100644 --- a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs +++ b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs @@ -39,13 +39,13 @@ namespace Nitload.RegularAnalysis.Text GenerateConcat(root, sc, func_list, context, exit_label); break; case RegexElementLabel.COUNT: - GenerateCount(root, sc, func_list, context); + GenerateCount(root, sc, func_list, context, exit_label); break; case RegexElementLabel.GROUP_BEGIN: - GenerateGroup(root, sc, func_list, context); + GenerateGroup(root, sc, func_list, context, exit_label); break; case RegexElementLabel.ALT: - GenerateAlt(root, sc, func_list, context); + GenerateAlt(root, sc, func_list, context, exit_label); break; default: @@ -167,17 +167,20 @@ namespace Nitload.RegularAnalysis.Text ) { RegexCharSetPayload payload = value.PayloadAs(); - string[] rules = new string[payload.Set.Count]; - for (int i = 0; i < rules.Length; i++) { - rules[i] = GenerateCharSetExpression(payload.Set[i]); - } - // Combine rule-level complements before applying the enclosing charset complement. // An empty union accepts nothing; its complement accepts any available character. - string predicate = rules.Length == 0 ? "false" : string.Join(" || ", rules); - string mismatch = payload.Accept ? $"!({predicate})" : $"({predicate})"; string remain_span = context.RemainSpan; - _ = sc.AppendLine($"if ({remain_span}.IsEmpty || {mismatch})") + _ = sc.AppendLine($"if ({remain_span}.IsEmpty ||") + .AppendLine(payload.Accept ? " !(" : " ("); + if (payload.Set.Count == 0) { + _ = sc.AppendLine(" false"); + } + for (int i = 0; i < payload.Set.Count; i++) + { + string suffix = i + 1 < payload.Set.Count ? " ||" : string.Empty; + _ = sc.AppendLine($" {GenerateCharSetExpression(payload.Set[i])}{suffix}"); + } + _ = sc.AppendLine(" ))") .AppendLine("{") .AppendLine($"goto {exit.GetLabelName()};") .AppendLine("}") @@ -244,9 +247,6 @@ namespace Nitload.RegularAnalysis.Text string required_chunk_index = $"__required_chunk_index_{scratch_scope_index}"; string additional_count = $"__additional_count_{scratch_scope_index}"; - // TODO(NGRA): Replace this local COUNT exit with the inherited failure ExitLabel. - // The temporary goto preserves the former local exit but still lets siblings execute. - if (min <= flattened_unit_count_threshold) { if (min > 0) @@ -428,9 +428,6 @@ namespace Nitload.RegularAnalysis.Text string additional_count = $"__additional_count_{scratch_scope_index}"; string starts_with = $"{nameof(MemoryExtensions.StartsWith)}"; - // TODO(NGRA): Replace this local COUNT exit with the inherited failure ExitLabel. - // The temporary goto preserves the former local exit but still lets siblings execute. - if (literal.Length > flattened_unit_count_threshold) { // One repetition is already larger than the flattening threshold. Match the @@ -551,93 +548,28 @@ namespace Nitload.RegularAnalysis.Text RegexAstNode node, [NotNull] in SequenceEmitter sc, in List> func_list, - in SourceGenContext context + in SourceGenContext context, + in ExitLabel exit_label ) { + ExitLabel success = context.RequestExitLabel(ExitMode.Goto); RegexAstNode current = node.Right; - - // TODO(NGRA-NAMING): Reserve exclusive names for future ALT cursor/capture snapshots; - // their lifetime extends through retry labels, not just the candidate's emitted block. - ExitLabel exit_label = context.RequestExitLabel(ExitMode.Goto); - - // TODO(NGRA): Walk every normalized ALT candidate exactly once. The final traversal - // shape and failure continuation are not wired yet. - while (RegexAstNode.IsNullOrEmpty(current)) + while (!RegexAstNode.IsNullOrEmpty(current)) { - RegexElementLabel op = current.Label & RegexElementLabel.SEQUENCE; - if (op != 0) - { - string code = current.SourceCode; - switch (op) - { - case RegexElementLabel.UNIT: - - // TODO(NGRA): Guard against an empty remaining span before indexing it. - _ = sc.AppendLine(@$"if({context.RemainSpan}[0] == {FormatCharLiteral(code[0])})") - .AppendLine("{") - .AppendLine($@"{context.RemainSpan} = {context.RemainSpan}.Slice(1);") - .AppendLine($@"goto {exit_label};") - .AppendLine("}"); - break; - case RegexElementLabel.UNIT_SET: - - // TODO(NGRA): Guard the remaining span, handle empty/Any sets, and - // validate negated-set composition before emitting this predicate. - RegexCharSetPayload payload = current.PayloadAs(); - IReadOnlyList split = payload.Set; - bool accept = payload.Accept; - RegexCharSet _set = split[0]; - _ = sc.AppendLine($"if({(accept ? string.Empty : "!(")} {GenerateCharSetExpression(_set)} ||"); - for (int i = 1; i < split.Count - 1; i++) - { - _set = split[i]; - _ = sc.AppendLine($" {GenerateCharSetExpression(_set)} ||"); - } - _set = split[^1]; - _ = sc.AppendLine($" {GenerateCharSetExpression(_set)} ){(accept ? string.Empty : ')')}") - .AppendLine("{") - .AppendLine($@"{context.RemainSpan} = {context.RemainSpan}.Slice(1);") - .AppendLine($@"goto {exit_label};") - .AppendLine("}"); - break; - case RegexElementLabel.UNIT_SPAN: - - // TODO(NGRA): Advance by code.Length rather than one character and - // preserve the ALT rollback cursor on failure. - string literal = FormatStringLiteral(code); - _ = sc.AppendLine(@$"if({context.RemainSpan}.{MemoryExtensions.StartsWith}({literal}))") - .AppendLine("{") - .AppendLine($@"{context.RemainSpan} = {context.RemainSpan}.Slice({code.Length});") - .AppendLine($@"goto {exit_label};") - .AppendLine("}"); - - break; - default: - - // TODO(NGRA): Route unsupported ALT candidates through the recursive - // matcher or report a source-generation diagnostic. - break; - } - - // TODO(NGRA): Advance to the next ALT candidate after a scalar candidate - // misses; the current sequence branch does not update current. - } else - { - GenerateMethodSource(current, sc, func_list, context); - current = current.Next; - } + GenerateAltCandidate(current, sc, func_list, context, success); + current = current.Next; } - // TODO(NGRA): Emit the all-alternatives-failed continuation before the shared success - // label, including restoration of the entry cursor and capture state. - _ = sc.AppendLine($@"{exit_label}: {{ }}"); + _ = sc.AppendLine($"goto {exit_label.GetLabelName()};") + .AppendLine($"{success.GetLabelName()}: ;"); } private static void GenerateGroup( RegexAstNode node, [NotNull] in SequenceEmitter sc, in List> func_list, - in SourceGenContext context + in SourceGenContext context, + in ExitLabel exit_label ) { RegexAstNode left = node.Left; @@ -648,14 +580,13 @@ namespace Nitload.RegularAnalysis.Text // TODO(NGRA-NAMING): Reserve exclusive names for generated group snapshots and capture // temporaries. A capture's user-provided name is not a unique generated local name. - // TODO(NGRA): Select the owned group body using Empty-sentinel checks, snapshot its - // start cursor, and restore it when the child matcher fails. - if (left is not null) + // TODO(NGRA): Snapshot the group start cursor and restore it when the child fails. + if (!RegexAstNode.IsNullOrEmpty(left)) { - GenerateMethodSource(left, sc, func_list, context); + GenerateMethodSource(left, sc, func_list, context, exit_label); } else - if (right is not null) { - GenerateMethodSource(right, sc, func_list, context); + if (!RegexAstNode.IsNullOrEmpty(right)) { + GenerateMethodSource(right, sc, func_list, context, exit_label); } string group_name = config.Name; @@ -685,7 +616,8 @@ namespace Nitload.RegularAnalysis.Text RegexAstNode node, [NotNull] in SequenceEmitter sc, in List> func_list, - in SourceGenContext context + in SourceGenContext context, + in ExitLabel exit_label ) { RegexAstNode value = RegexAstNode.IsNullOrEmpty(node.Left) ? node.Right : node.Left; @@ -699,12 +631,11 @@ namespace Nitload.RegularAnalysis.Text // TODO(NGRA-NAMING): Reserve exclusive names for future repetition/rollback state // across complex COUNT retries; recursive children must use the same method pool. - ExitLabel exit_label = context.RequestExitLabel(ExitMode.Goto); if ((value.Label & RegexElementLabel.SEQUENCE) == 0) { // TODO(NGRA): Emit min/max repetition, zero-width progress protection, greedy vs. // lazy ordering, and per-attempt rollback for a complex COUNT body. - GenerateMethodSource(value, sc, func_list, context); + GenerateMethodSource(value, sc, func_list, context, exit_label); } else { switch (value.Label & RegexElementLabel.SEQUENCE) @@ -726,9 +657,6 @@ namespace Nitload.RegularAnalysis.Text } } - // TODO(NGRA): Connect COUNT success/failure continuations to the surrounding rollback - // scope; the label is currently only a placeholder. - _ = sc.AppendLine($@"{exit_label}: {{ }}"); } #endregion diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenSpecial.cs b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenSpecial.cs index 1fbeb591..9deefd61 100644 --- a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenSpecial.cs +++ b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenSpecial.cs @@ -1,6 +1,6 @@ // ----------------------------------------------------------------------------- // file="TextRegexSourceGen.GenSpecial" -// Project: Nitload +// Project: StgSharp // Copyright (c) Nitload. // SPDX-License-Identifier: MIT // ----------------------------------------------------------------------------- @@ -24,11 +24,162 @@ namespace Nitload.RegularAnalysis.Text // to the general path; returning true prevents that subtree from being emitted twice. return node.Label switch { - RegexElementLabel.CONCAT => TryGenerateFindConcat(node, sc, func_list, context, exit_label), + RegexElementLabel.CONCAT => TryGenerateFindConcat(node, sc, func_list, context, + exit_label), _ => false, }; } + #region ALT candidate rules + + private static bool TryGetSimpleAltNode( + RegexAstNode node, + out RegexAstNode simple + ) + { + simple = node; + while (simple.Label == RegexElementLabel.CONCAT) + { + bool left_empty = RegexAstNode.IsNullOrEmpty(simple.Left); + bool right_empty = RegexAstNode.IsNullOrEmpty(simple.Right); + if (left_empty == right_empty) { + return false; + } + simple = left_empty ? simple.Right : simple.Left; + } + + // Inspect transparent wrappers without changing ownership or the candidate Next link. + // COUNT and GROUP are never peeled: repetition and capture semantics must survive. + return simple.Label is RegexElementLabel.UNIT or RegexElementLabel.UNIT_SET or + RegexElementLabel.UNIT_SPAN; + } + + private static void GenerateAltCandidate( + RegexAstNode node, + in SequenceEmitter sc, + in List> func_list, + in SourceGenContext context, + in ExitLabel success + ) + { + if (TryGetSimpleAltNode(node, out RegexAstNode simple)) { + node = simple; + } + + switch (node.Label) + { + case RegexElementLabel.UNIT: + GenerateAltUnit(node, sc, context, success); + break; + case RegexElementLabel.UNIT_SET: + GenerateAltUnitSet(node, sc, context, success); + break; + case RegexElementLabel.UNIT_SPAN: + GenerateAltUnitSpan(node, sc, context, success); + break; + case RegexElementLabel.CONCAT when RegexAstNode.IsNullOrEmpty(node.Left) && + RegexAstNode.IsNullOrEmpty(node.Right): + case RegexElementLabel.EPSILON: + + // An empty expression succeeds without consuming input. + _ = sc.AppendLine($"goto {success.GetLabelName()};"); + break; + case RegexElementLabel.CONCAT: + case RegexElementLabel.COUNT: + case RegexElementLabel.GROUP_BEGIN: + case RegexElementLabel.ALT: + GenerateAltOperator(node, sc, func_list, context, success); + break; + default: + throw new NotSupportedException($"Unsupported ALT candidate: {node.Label}."); + } + } + + private static void GenerateAltUnit( + RegexAstNode node, + in SequenceEmitter sc, + in SourceGenContext context, + in ExitLabel success + ) + { + string literal = node.Value; + if (literal.Length != 1) { + throw new InvalidOperationException("ALT UNIT requires one decoded character."); + } + + _ = sc.AppendLine($"if (!{context.RemainSpan}.IsEmpty && {context.RemainSpan}[0] == {FormatCharLiteral(literal[0])})") + .AppendLine("{") + .AppendLine($"{context.RemainSpan} = {context.RemainSpan}.Slice(1);") + .AppendLine($"goto {success.GetLabelName()};") + .AppendLine("}"); + } + + private static void GenerateAltUnitSet( + RegexAstNode node, + in SequenceEmitter sc, + in SourceGenContext context, + in ExitLabel success + ) + { + RegexCharSetPayload payload = node.PayloadAs(); + _ = sc.AppendLine($"if (!{context.RemainSpan}.IsEmpty &&") + .AppendLine(payload.Accept ? " (" : " !("); + if (payload.Set.Count == 0) { + _ = sc.AppendLine(" false"); + } + for (int i = 0; i < payload.Set.Count; i++) + { + string suffix = i + 1 < payload.Set.Count ? " ||" : string.Empty; + _ = sc.AppendLine($" {GenerateCharSetExpression(payload.Set[i])}{suffix}"); + } + _ = sc.AppendLine(" ))") + .AppendLine("{") + .AppendLine($"{context.RemainSpan} = {context.RemainSpan}.Slice(1);") + .AppendLine($"goto {success.GetLabelName()};") + .AppendLine("}"); + } + + private static void GenerateAltUnitSpan( + RegexAstNode node, + in SequenceEmitter sc, + in SourceGenContext context, + in ExitLabel success + ) + { + string literal = node.Value; + if (literal.Length == 0) { + throw new InvalidOperationException("ALT UNIT_SPAN requires a non-empty literal."); + } + string predicate = $"{context.RemainSpan}.{nameof(MemoryExtensions.StartsWith)}({FormatStringLiteral(literal)})"; + _ = sc.AppendLine($"if ({predicate})") + .AppendLine("{") + .AppendLine($"{context.RemainSpan} = {context.RemainSpan}.Slice({literal.Length});") + .AppendLine($"goto {success.GetLabelName()};") + .AppendLine("}"); + } + + private static void GenerateAltOperator( + RegexAstNode node, + in SequenceEmitter sc, + in List> func_list, + in SourceGenContext context, + in ExitLabel success + ) + { + ExitLabel failure = context.RequestExitLabel(ExitMode.Goto); + string entry = $"__alt_entry_{context.RequestScratchScopeIndex()}"; + _ = sc.AppendLine($"{ROS_char} {entry} = {context.RemainSpan};"); + GenerateMethodSource(node, sc, func_list, context, failure); + _ = sc.AppendLine($"goto {success.GetLabelName()};") + .AppendLine($"{failure.GetLabelName()}:") + .AppendLine($"{context.RemainSpan} = {entry};"); + + // TODO(NGRA): Restore capture state once GROUP emission supports captures. + // Retrying an already successful candidate after a later continuation fails is separate. + } + + #endregion + #region FIND concatenation private static bool TryGenerateFindConcat( @@ -78,7 +229,8 @@ namespace Nitload.RegularAnalysis.Text { // (?.*)abc - if (RegexAstNode.IsNullOrEmpty(node) || node.Label != RegexElementLabel.CONCAT) { + if (RegexAstNode.IsNullOrEmpty(node) || node.Label != RegexElementLabel.CONCAT) + { return false; } RegexAstNode group = node.Left; @@ -106,6 +258,5 @@ namespace Nitload.RegularAnalysis.Text // An escaped star is a literal and must not be classified as dot-star repetition. #endregion - } } diff --git a/src/README.md b/src/README.md index c18375f8..2960c206 100644 --- a/src/README.md +++ b/src/README.md @@ -28,4 +28,4 @@ cmake --build cmake_build/clang-release dotnet build Nitload.sln ``` -Managed builds copy an existing native binary when one is available; they do not invoke CMake automatically. Generated OpenGL files under `Nitload.Graphics/OpenGL/Generated` should be updated through the generator in `tools/Nitload.GenerateGL`, not edited by hand. +Managed library builds copy an existing native binary when one is available; they do not invoke CMake automatically. The `test/StgSharpDebug` executable explicitly enables the CMake build before managed compilation. Generated OpenGL files under `Nitload.Graphics/OpenGL/Generated` should be updated through the generator in `tools/Nitload.GenerateGL`, not edited by hand. diff --git a/test/README.md b/test/README.md index fa1aa32c..167dcd3e 100644 --- a/test/README.md +++ b/test/README.md @@ -16,4 +16,36 @@ Run the active debug harness directly when investigating a subsystem: dotnet run --project test/StgSharpDebug/StgSharpDebug.csproj ``` +## NGRA generation preview + +`StgSharpDebug/TestRegexGenerate.cs` contains a standalone debugging routine that calls the +existing NGRA emitter without Roslyn attribute binding. Select it from the debug `Main`: + +```powershell +dotnet run --project test/StgSharpDebug/StgSharpDebug.csproj -- --regex-preview +``` + +This branch returns before `World` or native-memory initialization. Without the switch, the +existing debug flow is unchanged. You can also call `TestRegexGenerate.Run()` directly from +`Main` while debugging; pass a pattern to `Run("abc")` for a single preview. +The switch only selects runtime behavior; building the debug project still builds its normal +project references, including Graphics and Mathematics, and prepares the native library with +CMake. `BuildNativeBeforeManaged` defaults to true; Debug selects `clang-debug`, and other +configurations select `clang-release`. `RequireNativeLibrary` reports a missing DLL at build +time instead of allowing the default runtime path to fail later. + +For managed-only preview work without a native toolchain or DLL, add +`-p:BuildNativeBeforeManaged=false -p:RequireNativeLibrary=false` before `--` in the run command. + +Enter raw regular expressions without C# string delimiters. The routine prints only the emitted +code, without status headers or file output. Exceptions propagate directly to the caller/debugger. +Use `:exit` or end-of-input to leave the loop. Empty input is an empty regex. For one-shot mode: + +```powershell +dotnet run --project test/StgSharpDebug/StgSharpDebug.csproj -- --regex-preview 'a{2,3}' +``` + +Set breakpoints in `TestRegexGenerate.cs` or any existing `Generate*` method. The output is a +method-body fragment; this routine does not compile or execute it. + New repeatable tests should use a test project and avoid machine-specific paths, external mutable data, and dependencies on previous test execution. Keep one-off investigations in `SingleFile` or the manual debug harness until their expected behavior is stable enough for automation. diff --git a/test/StgSharpDebug/Program.cs b/test/StgSharpDebug/Program.cs index 0160e0e6..0ff354d0 100644 --- a/test/StgSharpDebug/Program.cs +++ b/test/StgSharpDebug/Program.cs @@ -5,13 +5,13 @@ // SPDX-License-Identifier: MIT // ----------------------------------------------------------------------------- -using Nitload.Graphics; using Nitload.Common.HighPerformance.Memory; using Nitload.Common.Internal; +using Nitload.Common.Timing; +using Nitload.Common.Timing; +using Nitload.Graphics; using Nitload.Mathematics; using Nitload.Mathematics.Numeric; -using Nitload.Common.Timing; -using Nitload.Common.Timing; using System; using System.Runtime.InteropServices; @@ -28,6 +28,13 @@ namespace Nitload string[] args ) { + // Select the managed-only emitter preview before initializing the normal runtime. + if (args.Length > 0 && args[0] == "--regex-preview") + { + Environment.ExitCode = TestRegexGenerate.Run(args[1..]); + return; + } + try { World.Initialize(new() @@ -39,13 +46,8 @@ namespace Nitload // .UseModule() .InitializeModule(); - _ = SlabAllocator.FromBuffer( - (nuint)NativeMemory.AlignedAlloc(65536 * 4, 4), - 65536, - buffer => NativeMemory.AlignedFree((void*)buffer) - ); - - TestRegexAnalyse.Run(); + // TestRegexAnalyse.Run(); + _ = TestRegexGenerate.Run("abc|123"); // TestHlsf(); diff --git a/test/StgSharpDebug/StgSharpDebug.csproj b/test/StgSharpDebug/StgSharpDebug.csproj index 3cd98ede..70048fe6 100644 --- a/test/StgSharpDebug/StgSharpDebug.csproj +++ b/test/StgSharpDebug/StgSharpDebug.csproj @@ -2,6 +2,10 @@ Exe + true + true + + true @@ -9,6 +13,11 @@ + + + + + @@ -18,4 +27,3 @@ - diff --git a/test/StgSharpDebug/TestRegexGenerate.cs b/test/StgSharpDebug/TestRegexGenerate.cs new file mode 100644 index 00000000..6a53a4a1 --- /dev/null +++ b/test/StgSharpDebug/TestRegexGenerate.cs @@ -0,0 +1,65 @@ +// ----------------------------------------------------------------------------- +// file="TestRegexGenerate" +// Project: Nitload +// Copyright (c) Nitload. +// SPDX-License-Identifier: MIT +// ----------------------------------------------------------------------------- + +using Nitload.RegularAnalysis.Abstraction; +using Nitload.RegularAnalysis.Text; +using System; +using System.Collections.Generic; +using System.Runtime.ExceptionServices; +using static Nitload.RegularAnalysis.Text.TextRegexSourceGen; + +namespace Nitload +{ + internal static class TestRegexGenerate + { + public static int Run(params string[] args) + { + if (args.Length > 1) { + throw new ArgumentException("Expected at most one regex pattern.", nameof(args)); + } + if (args.Length == 1) + { + Print(args[0]); + return 0; + } + + while (true) + { + Console.Write("regex> "); + string? pattern = Console.ReadLine(); + if (pattern is null or ":exit") { + return 0; + } + Print(pattern); + } + } + + private static void Print(string pattern) + { + TextRegexSource analysis = RegexAnalyzer.Analyze(pattern); + if (analysis.AnalyseException is not null) { + ExceptionDispatchInfo.Throw(analysis.AnalyseException); + } + + SequenceEmitter body = new(); + List> functions = []; + SourceGenContext context = new(); + ExitLabel exit = context.RequestExitLabel(ExitMode.Goto); + GenerateMethodSource(analysis.Ast.Root, body, functions, context, exit); + + foreach (string line in body.ToList()) { + Console.WriteLine(line); + } + foreach (SequenceEmitter function in functions) + { + foreach (string line in function.ToList()) { + Console.WriteLine(line); + } + } + } + } +} From 5a3286dc21fea416bb6ba28810bc7255314c9436 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:29:42 +0800 Subject: [PATCH 2/7] meet difficult problem on COUNT node --- .../Text/TextRegexSourceGen.GenClass.cs | 168 ++++++++++++++---- .../Text/TextRegexSourceGen.GenSpecial.cs | 44 +++++ test/StgSharpDebug/Program.cs | 13 +- test/StgSharpDebug/TestRegexGenerate.cs | 33 ++-- 4 files changed, 188 insertions(+), 70 deletions(-) diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs index 39047e19..6791bf37 100644 --- a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs +++ b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs @@ -220,8 +220,11 @@ namespace Nitload.RegularAnalysis.Text int max, bool is_greedy, [NotNull] in SequenceEmitter sc, + in List> func_list, in SourceGenContext context, - in ExitLabel exit + in ExitLabel exit, + IReadOnlyList? continuation = null, + int next = 0 ) { if (min < 0 || (max >= 0 && max < min)) { @@ -300,22 +303,60 @@ namespace Nitload.RegularAnalysis.Text } int additional_limit = max < 0 ? -1 : max - min; + bool has_continuation = continuation is not null && next < continuation.Count; + bool can_retry = has_continuation && additional_limit != 0; + string additional_entry = $"__additional_entry_{scratch_scope_index}"; + if (can_retry) { + _ = sc.AppendLine($"{ROS_char} {additional_entry} = {remain_span};"); + } + if (can_retry || (is_greedy && additional_limit > 0)) { + _ = sc.AppendLine($"int {additional_count} = 0;"); + } if (is_greedy && additional_limit != 0) { - _ = additional_limit < 0 ? - sc.AppendLine($"while (!{remain_span}.IsEmpty && {remain_span}[0] == {expected_literal})") + string limit = additional_limit < 0 ? string.Empty : + $"{additional_count} < {additional_limit} && "; + _ = sc.AppendLine($"while ({limit}!{remain_span}.IsEmpty && {remain_span}[0] == {expected_literal})") .AppendLine("{") - .AppendLine($"{remain_span} = {remain_span}.Slice(1);") - .AppendLine("}") : - sc.AppendLine($"for (int {additional_count} = 0; {additional_count} < {additional_limit} && !{remain_span}.IsEmpty && {remain_span}[0] == {expected_literal}; {additional_count}++)") - .AppendLine("{") - .AppendLine($"{remain_span} = {remain_span}.Slice(1);") - .AppendLine("}"); - } else if (!is_greedy && additional_limit != 0) + .AppendLine($"{remain_span} = {remain_span}.Slice(1);"); + if (can_retry || additional_limit > 0) { + _ = sc.AppendLine($"{additional_count}++;"); + } + _ = sc.AppendLine("}"); + } + if (can_retry) { - // Lazy COUNT deliberately stops at Min. Its optional tail is consumed only when - // a future continuation attempt fails and requests another repetition. - _ = sc.AppendLine("// TODO(NGRA): Retry the lazy COUNT tail from its continuation."); + // The same optional count drives scanning and retries. Required units stay consumed. + ExitLabel retry = context.RequestExitLabel(ExitMode.Goto); + _ = sc.AppendLine("while (true)") + .AppendLine("{") + .AppendLine($"{remain_span} = {additional_entry}.Slice({additional_count});"); + GenerateConcatTail(continuation!, next, sc, func_list, context, retry); + _ = sc.AppendLine("break;") + .AppendLine($"{retry.GetLabelName()}: {{}}"); + if (is_greedy) + { + _ = sc.AppendLine($"if ({additional_count} == 0)") + .AppendLine("{") + .AppendLine($"{remain_span} = {additional_entry};") + .AppendLine($"goto {exit_label_node};") + .AppendLine("}") + .AppendLine($"{additional_count}--;"); + } else + { + string limit = additional_limit < 0 ? string.Empty : + $"{additional_count} == {additional_limit} || "; + _ = sc.AppendLine($"{remain_span} = {additional_entry}.Slice({additional_count});") + .AppendLine($"if ({limit}{remain_span}.IsEmpty || {remain_span}[0] != {expected_literal})") + .AppendLine("{") + .AppendLine($"{remain_span} = {additional_entry};") + .AppendLine($"goto {exit_label_node};") + .AppendLine("}") + .AppendLine($"{additional_count}++;"); + } + _ = sc.AppendLine("}"); + } else if (has_continuation) { + GenerateConcatTail(continuation!, next, sc, func_list, context, exit); } } @@ -399,8 +440,11 @@ namespace Nitload.RegularAnalysis.Text int max, bool is_greedy, [NotNull] in SequenceEmitter sc, + in List> func_list, in SourceGenContext context, - in ExitLabel exit + in ExitLabel exit, + IReadOnlyList? continuation = null, + int next = 0 ) { if (min < 0 || (max >= 0 && max < min)) { @@ -510,26 +554,60 @@ namespace Nitload.RegularAnalysis.Text } int additional_limit = max < 0 ? -1 : max - min; + bool has_continuation = continuation is not null && next < continuation.Count; + bool can_retry = has_continuation && additional_limit != 0; + string additional_entry = $"__additional_entry_{scratch_scope_index}"; + if (can_retry) { + _ = sc.AppendLine($"{ROS_char} {additional_entry} = {remain_span};"); + } + if (can_retry || (is_greedy && additional_limit > 0)) { + _ = sc.AppendLine($"int {additional_count} = 0;"); + } if (is_greedy && additional_limit != 0) { - if (additional_limit < 0) + string limit = additional_limit < 0 ? string.Empty : + $"{additional_count} < {additional_limit} && "; + _ = sc.AppendLine($"while ({limit}{remain_span}.{starts_with}({expected_literal}))") + .AppendLine("{") + .AppendLine($"{remain_span} = {remain_span}.Slice({literal.Length});"); + if (can_retry || additional_limit > 0) { + _ = sc.AppendLine($"{additional_count}++;"); + } + _ = sc.AppendLine("}"); + } + if (can_retry) + { + // Each retry moves by a complete literal using the scan's own optional counter. + ExitLabel retry = context.RequestExitLabel(ExitMode.Goto); + _ = sc.AppendLine("while (true)") + .AppendLine("{") + .AppendLine($"{remain_span} = {additional_entry}.Slice({additional_count} * {literal.Length});"); + GenerateConcatTail(continuation!, next, sc, func_list, context, retry); + _ = sc.AppendLine("break;") + .AppendLine($"{retry.GetLabelName()}: {{}}"); + if (is_greedy) { - _ = sc.AppendLine($"while ({remain_span}.{starts_with}({expected_literal}))") + _ = sc.AppendLine($"if ({additional_count} == 0)") .AppendLine("{") - .AppendLine($"{remain_span} = {remain_span}.Slice({literal.Length});") - .AppendLine("}"); + .AppendLine($"{remain_span} = {additional_entry};") + .AppendLine($"goto {exit_label_node};") + .AppendLine("}") + .AppendLine($"{additional_count}--;"); } else { - _ = sc.AppendLine($"for (int {additional_count} = 0; {additional_count} < {additional_limit} && {remain_span}.{starts_with}({expected_literal}); {additional_count}++)") + string limit = additional_limit < 0 ? string.Empty : + $"{additional_count} == {additional_limit} || "; + _ = sc.AppendLine($"{remain_span} = {additional_entry}.Slice({additional_count} * {literal.Length});") + .AppendLine($"if ({limit}!{remain_span}.{starts_with}({expected_literal}))") .AppendLine("{") - .AppendLine($"{remain_span} = {remain_span}.Slice({literal.Length});") - .AppendLine("}"); + .AppendLine($"{remain_span} = {additional_entry};") + .AppendLine($"goto {exit_label_node};") + .AppendLine("}") + .AppendLine($"{additional_count}++;"); } - } else if (!is_greedy && additional_limit != 0) - { - // Lazy COUNT deliberately stops at Min. Its optional tail is consumed only when - // a future continuation attempt fails and requests another repetition. - _ = sc.AppendLine("// TODO(NGRA): Retry the lazy COUNT tail from its continuation."); + _ = sc.AppendLine("}"); + } else if (has_continuation) { + GenerateConcatTail(continuation!, next, sc, func_list, context, exit); } } @@ -604,12 +682,10 @@ namespace Nitload.RegularAnalysis.Text in ExitLabel exit_label = default ) { - if (!RegexAstNode.IsNullOrEmpty(node.Left)) { - GenerateMethodSource(node.Left, sc, func_list, context, exit_label); - } - if (!RegexAstNode.IsNullOrEmpty(node.Right)) { - GenerateMethodSource(node.Right, sc, func_list, context, exit_label); - } + // Keep the complete suffix visible even when CONCAT is left-associated. + List sequence = []; + AppendConcatNodes(node, sequence); + GenerateConcatTail(sequence, 0, sc, func_list, context, exit_label); } private static void GenerateCount( @@ -617,13 +693,25 @@ namespace Nitload.RegularAnalysis.Text [NotNull] in SequenceEmitter sc, in List> func_list, in SourceGenContext context, - in ExitLabel exit_label + in ExitLabel exit_label, + IReadOnlyList? continuation = null, + int next = 0 ) { RegexAstNode value = RegexAstNode.IsNullOrEmpty(node.Left) ? node.Right : node.Left; if (RegexAstNode.IsNullOrEmpty(value)) { throw new InvalidOperationException("COUNT requires an operand."); } + // Inspect transparent CONCAT wrappers without rewriting the repetition body. + while (value.Label == RegexElementLabel.CONCAT) + { + bool left_empty = RegexAstNode.IsNullOrEmpty(value.Left); + bool right_empty = RegexAstNode.IsNullOrEmpty(value.Right); + if (left_empty == right_empty) { + break; + } + value = left_empty ? value.Right : value.Left; + } RegexCountPayload payload = node.PayloadAs(); int min = payload.Min; int max = payload.Max; @@ -641,14 +729,16 @@ namespace Nitload.RegularAnalysis.Text switch (value.Label & RegexElementLabel.SEQUENCE) { case RegexElementLabel.UNIT: - GenerateUnitCount(value, min, max, is_greedy, sc, context, exit_label); - break; + GenerateUnitCount(value, min, max, is_greedy, sc, func_list, context, + exit_label, continuation, next); + return; case RegexElementLabel.UNIT_SET: GenerateUnitSetCount(value, min, max, is_greedy, sc, context, exit_label); break; case RegexElementLabel.UNIT_SPAN: - GenerateUnitSpanCount(value, min, max, is_greedy, sc, context, exit_label); - break; + GenerateUnitSpanCount(value, min, max, is_greedy, sc, func_list, context, + exit_label, continuation, next); + return; default: // TODO(NGRA): Diagnose malformed COUNT children instead of emitting an @@ -656,7 +746,9 @@ namespace Nitload.RegularAnalysis.Text break; } } - + if (continuation is not null) { + GenerateConcatTail(continuation, next, sc, func_list, context, exit_label); + } } #endregion diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenSpecial.cs b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenSpecial.cs index 9deefd61..d0724085 100644 --- a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenSpecial.cs +++ b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenSpecial.cs @@ -180,6 +180,50 @@ namespace Nitload.RegularAnalysis.Text #endregion + #region CONCAT continuation + + private static void AppendConcatNodes( + RegexAstNode node, + List sequence + ) + { + if (RegexAstNode.IsNullOrEmpty(node)) { + return; + } + if (node.Label == RegexElementLabel.CONCAT && + !RegexAstNode.IsDotStar(node.Left) && !IsNamedFind(node)) + { + AppendConcatNodes(node.Left, sequence); + AppendConcatNodes(node.Right, sequence); + } else { + sequence.Add(node); + } + } + + private static void GenerateConcatTail( + IReadOnlyList sequence, + int start, + in SequenceEmitter sc, + in List> func_list, + in SourceGenContext context, + in ExitLabel exit_label + ) + { + for (int i = start; i < sequence.Count; i++) + { + RegexAstNode node = sequence[i]; + if (node.Label == RegexElementLabel.COUNT) + { + // COUNT owns both its repetition and the continuation that may retry it. + GenerateCount(node, sc, func_list, context, exit_label, sequence, i + 1); + return; + } + GenerateMethodSource(node, sc, func_list, context, exit_label); + } + } + + #endregion + #region FIND concatenation private static bool TryGenerateFindConcat( diff --git a/test/StgSharpDebug/Program.cs b/test/StgSharpDebug/Program.cs index 0ff354d0..0406adfe 100644 --- a/test/StgSharpDebug/Program.cs +++ b/test/StgSharpDebug/Program.cs @@ -24,17 +24,8 @@ namespace Nitload private static string testExpPath = @"D:\Programmer\stepcode\data\ap203\ap203.exp"; - internal static unsafe void Main( - string[] args - ) + internal static unsafe void Main() { - // Select the managed-only emitter preview before initializing the normal runtime. - if (args.Length > 0 && args[0] == "--regex-preview") - { - Environment.ExitCode = TestRegexGenerate.Run(args[1..]); - return; - } - try { World.Initialize(new() @@ -47,7 +38,7 @@ namespace Nitload .InitializeModule(); // TestRegexAnalyse.Run(); - _ = TestRegexGenerate.Run("abc|123"); + _ = TestRegexGenerate.Run("abc1{1,2}23"); // TestHlsf(); diff --git a/test/StgSharpDebug/TestRegexGenerate.cs b/test/StgSharpDebug/TestRegexGenerate.cs index 6a53a4a1..4e09804b 100644 --- a/test/StgSharpDebug/TestRegexGenerate.cs +++ b/test/StgSharpDebug/TestRegexGenerate.cs @@ -1,6 +1,6 @@ // ----------------------------------------------------------------------------- // file="TestRegexGenerate" -// Project: Nitload +// Project: StgSharp // Copyright (c) Nitload. // SPDX-License-Identifier: MIT // ----------------------------------------------------------------------------- @@ -16,29 +16,19 @@ namespace Nitload { internal static class TestRegexGenerate { - public static int Run(params string[] args) - { - if (args.Length > 1) { - throw new ArgumentException("Expected at most one regex pattern.", nameof(args)); - } - if (args.Length == 1) - { - Print(args[0]); - return 0; - } - while (true) - { - Console.Write("regex> "); - string? pattern = Console.ReadLine(); - if (pattern is null or ":exit") { - return 0; - } - Print(pattern); - } + public static int Run( + string arg + ) + { + Console.WriteLine($"Text regex to be tested: {arg} :\n\n\n"); + Print(arg); + return 0; } - private static void Print(string pattern) + private static void Print( + string pattern + ) { TextRegexSource analysis = RegexAnalyzer.Analyze(pattern); if (analysis.AnalyseException is not null) { @@ -61,5 +51,6 @@ namespace Nitload } } } + } } From 9e9078ced10a3d054493a471b9f0b65f40128726 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:30:31 +0800 Subject: [PATCH 3/7] meet difficult problem on COUNT node --- test/StgSharpDebug/Program.cs | 2 +- test/StgSharpDebug/TestRegexAnalyse.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/StgSharpDebug/Program.cs b/test/StgSharpDebug/Program.cs index 0406adfe..05d101de 100644 --- a/test/StgSharpDebug/Program.cs +++ b/test/StgSharpDebug/Program.cs @@ -37,7 +37,7 @@ namespace Nitload // .UseModule() .InitializeModule(); - // TestRegexAnalyse.Run(); + TestRegexAnalyse.Run(); _ = TestRegexGenerate.Run("abc1{1,2}23"); // TestHlsf(); diff --git a/test/StgSharpDebug/TestRegexAnalyse.cs b/test/StgSharpDebug/TestRegexAnalyse.cs index 46fbbe9b..3f3ea350 100644 --- a/test/StgSharpDebug/TestRegexAnalyse.cs +++ b/test/StgSharpDebug/TestRegexAnalyse.cs @@ -52,7 +52,7 @@ namespace Nitload } } - [GeneratedRegex(@"123.*\w123", RegexOptions.ExplicitCapture)] + [GeneratedRegex(@"(a{2,4}bc)d", RegexOptions.ExplicitCapture)] internal static partial Regex GetRegex(); } From 57587bec160025805c904536f840cc0d2c36864c Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:21:23 +0800 Subject: [PATCH 4/7] use new exit lable --- .../Text/TextRegexSourceGen.ExitLabel.cs | 28 +- .../Text/TextRegexSourceGen.GenClass.cs | 246 +++++++++--------- .../Text/TextRegexSourceGen.GenContext.cs | 10 +- .../Text/TextRegexSourceGen.GenSpecial.cs | 62 +---- .../Text/TextRegexSourceGen.cs | 10 +- test/StgSharpDebug/TestRegexGenerate.cs | 10 +- 6 files changed, 169 insertions(+), 197 deletions(-) diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.ExitLabel.cs b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.ExitLabel.cs index 766bd8e7..23a9b1db 100644 --- a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.ExitLabel.cs +++ b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.ExitLabel.cs @@ -16,12 +16,15 @@ namespace Nitload.RegularAnalysis.Text internal partial class TextRegexSourceGen { - internal enum ExitMode + internal enum ExitKind { - Break, - Continue, - Goto, + MatchFailed, + MatchEnd, + CountRetry, + CountResume, + AltNext, + AltMatched, } @@ -29,25 +32,28 @@ namespace Nitload.RegularAnalysis.Text { public ExitLabel( - ExitMode mode, + ExitKind kind, int order ) { - Mode = mode; + Kind = kind; LabelOrder = order; } - public ExitMode Mode { get; init; } + public ExitKind Kind { get; init; } public int LabelOrder { get; init; } public string GetLabelName() { - return Mode switch + return Kind switch { - ExitMode.Break => $@"__break_{LabelOrder}", - ExitMode.Continue => $@"__continue_{LabelOrder}", - ExitMode.Goto => $@"__goto_{LabelOrder}", + ExitKind.MatchFailed => $@"__match_failed_{LabelOrder}", + ExitKind.MatchEnd => $@"__match_end_{LabelOrder}", + ExitKind.CountRetry => $@"__count_retry_{LabelOrder}", + ExitKind.CountResume => $@"__count_resume_{LabelOrder}", + ExitKind.AltNext => $@"__alt_next_{LabelOrder}", + ExitKind.AltMatched => $@"__alt_matched_{LabelOrder}", _ => throw new NotImplementedException(), }; } diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs index 6791bf37..87fd828b 100644 --- a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs +++ b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs @@ -14,20 +14,31 @@ namespace Nitload.RegularAnalysis.Text internal partial class TextRegexSourceGen { + /// + /// Emits a subtree and updates the retry target for the rest of the current path. + /// + /// + /// local_fail is the owning region's fallback. backtrack starts at that fallback and + /// advances when a COUNT registers a retry point. Returning from recursion does not + /// discard it. Mutually exclusive branches must use separate backtrack variables. + /// internal static void GenerateMethodSource( RegexAstNode ast, in SequenceEmitter sc, in List> func_list, in SourceGenContext context, - in ExitLabel exit_label = default, + in ExitLabel local_fail, + ref ExitLabel backtrack, bool is_find = false ) { + // local_fail belongs to this region. COUNT updates only the current path's + // backtrack target; CONCAT passes that update on to every later sibling. RegexAstNode root = ast; if (root.Label == RegexElementLabel.EPSILON) { return; } - if (TryGenerateSpecialSource(root, sc, func_list, context, exit_label)) { + if (TryGenerateSpecialSource(root, sc, func_list, context, local_fail, ref backtrack)) { return; } RegexElementLabel op = root.Label & (RegexElementLabel.VAST_OPERATOR); @@ -36,16 +47,16 @@ namespace Nitload.RegularAnalysis.Text switch (op) { case RegexElementLabel.CONCAT: - GenerateConcat(root, sc, func_list, context, exit_label); + GenerateConcat(root, sc, func_list, context, local_fail, ref backtrack); break; case RegexElementLabel.COUNT: - GenerateCount(root, sc, func_list, context, exit_label); + GenerateCount(root, sc, func_list, context, local_fail, ref backtrack); break; case RegexElementLabel.GROUP_BEGIN: - GenerateGroup(root, sc, func_list, context, exit_label); + GenerateGroup(root, sc, func_list, context, local_fail, ref backtrack); break; case RegexElementLabel.ALT: - GenerateAlt(root, sc, func_list, context, exit_label); + GenerateAlt(root, sc, func_list, context, backtrack); break; default: @@ -59,13 +70,13 @@ namespace Nitload.RegularAnalysis.Text switch (op) { case RegexElementLabel.UNIT: - GenerateUnit(root, sc, context, exit_label); + GenerateUnit(root, sc, context, backtrack); break; case RegexElementLabel.UNIT_SET: - GenerateUnitSet(root, sc, context, exit_label); + GenerateUnitSet(root, sc, context, backtrack); break; case RegexElementLabel.UNIT_SPAN: - GenerateUnitSpan(root, sc, context, exit_label); + GenerateUnitSpan(root, sc, context, backtrack); break; default: break; @@ -220,11 +231,9 @@ namespace Nitload.RegularAnalysis.Text int max, bool is_greedy, [NotNull] in SequenceEmitter sc, - in List> func_list, in SourceGenContext context, in ExitLabel exit, - IReadOnlyList? continuation = null, - int next = 0 + ref ExitLabel backtrack ) { if (min < 0 || (max >= 0 && max < min)) { @@ -303,61 +312,54 @@ namespace Nitload.RegularAnalysis.Text } int additional_limit = max < 0 ? -1 : max - min; - bool has_continuation = continuation is not null && next < continuation.Count; - bool can_retry = has_continuation && additional_limit != 0; + if (additional_limit == 0) { + return; + } + + // Save the optional portion once. Its counter is shared by scanning and retries. string additional_entry = $"__additional_entry_{scratch_scope_index}"; - if (can_retry) { - _ = sc.AppendLine($"{ROS_char} {additional_entry} = {remain_span};"); - } - if (can_retry || (is_greedy && additional_limit > 0)) { - _ = sc.AppendLine($"int {additional_count} = 0;"); - } - if (is_greedy && additional_limit != 0) + ExitLabel retry = context.RequestExitLabel(ExitKind.CountRetry); + ExitLabel resume = context.RequestExitLabel(ExitKind.CountResume); + _ = sc.AppendLine($"{ROS_char} {additional_entry} = {remain_span};") + .AppendLine($"int {additional_count} = 0;"); + if (is_greedy) { string limit = additional_limit < 0 ? string.Empty : $"{additional_count} < {additional_limit} && "; _ = sc.AppendLine($"while ({limit}!{remain_span}.IsEmpty && {remain_span}[0] == {expected_literal})") .AppendLine("{") - .AppendLine($"{remain_span} = {remain_span}.Slice(1);"); - if (can_retry || additional_limit > 0) { - _ = sc.AppendLine($"{additional_count}++;"); - } - _ = sc.AppendLine("}"); + .AppendLine($"{remain_span} = {remain_span}.Slice(1);") + .AppendLine($"{additional_count}++;") + .AppendLine("}"); } - if (can_retry) + + // Initial success skips retry handling. Later failures jump back here from anywhere + // in this path; exhaustion forwards to the target inherited before this COUNT. + _ = sc.AppendLine($"goto {resume.GetLabelName()};") + .AppendLine($"{retry.GetLabelName()}: {{}}"); + if (is_greedy) { - // The same optional count drives scanning and retries. Required units stay consumed. - ExitLabel retry = context.RequestExitLabel(ExitMode.Goto); - _ = sc.AppendLine("while (true)") + _ = sc.AppendLine($"if ({additional_count} == 0)") .AppendLine("{") - .AppendLine($"{remain_span} = {additional_entry}.Slice({additional_count});"); - GenerateConcatTail(continuation!, next, sc, func_list, context, retry); - _ = sc.AppendLine("break;") - .AppendLine($"{retry.GetLabelName()}: {{}}"); - if (is_greedy) - { - _ = sc.AppendLine($"if ({additional_count} == 0)") - .AppendLine("{") - .AppendLine($"{remain_span} = {additional_entry};") - .AppendLine($"goto {exit_label_node};") - .AppendLine("}") - .AppendLine($"{additional_count}--;"); - } else - { - string limit = additional_limit < 0 ? string.Empty : - $"{additional_count} == {additional_limit} || "; - _ = sc.AppendLine($"{remain_span} = {additional_entry}.Slice({additional_count});") - .AppendLine($"if ({limit}{remain_span}.IsEmpty || {remain_span}[0] != {expected_literal})") - .AppendLine("{") - .AppendLine($"{remain_span} = {additional_entry};") - .AppendLine($"goto {exit_label_node};") - .AppendLine("}") - .AppendLine($"{additional_count}++;"); - } - _ = sc.AppendLine("}"); - } else if (has_continuation) { - GenerateConcatTail(continuation!, next, sc, func_list, context, exit); + .AppendLine($"{remain_span} = {additional_entry};") + .AppendLine($"goto {exit_label_node};") + .AppendLine("}") + .AppendLine($"{additional_count}--;"); + } else + { + string limit = additional_limit < 0 ? string.Empty : + $"{additional_count} == {additional_limit} || "; + _ = sc.AppendLine($"{remain_span} = {additional_entry}.Slice({additional_count});") + .AppendLine($"if ({limit}{remain_span}.IsEmpty || {remain_span}[0] != {expected_literal})") + .AppendLine("{") + .AppendLine($"{remain_span} = {additional_entry};") + .AppendLine($"goto {exit_label_node};") + .AppendLine("}") + .AppendLine($"{additional_count}++;"); } + _ = sc.AppendLine($"{remain_span} = {additional_entry}.Slice({additional_count});") + .AppendLine($"{resume.GetLabelName()}: {{}}"); + backtrack = retry; } private static void GenerateUnitSetCount( @@ -440,11 +442,9 @@ namespace Nitload.RegularAnalysis.Text int max, bool is_greedy, [NotNull] in SequenceEmitter sc, - in List> func_list, in SourceGenContext context, in ExitLabel exit, - IReadOnlyList? continuation = null, - int next = 0 + ref ExitLabel backtrack ) { if (min < 0 || (max >= 0 && max < min)) { @@ -554,61 +554,54 @@ namespace Nitload.RegularAnalysis.Text } int additional_limit = max < 0 ? -1 : max - min; - bool has_continuation = continuation is not null && next < continuation.Count; - bool can_retry = has_continuation && additional_limit != 0; + if (additional_limit == 0) { + return; + } + + // Save the optional portion once. Its counter is shared by scanning and retries. string additional_entry = $"__additional_entry_{scratch_scope_index}"; - if (can_retry) { - _ = sc.AppendLine($"{ROS_char} {additional_entry} = {remain_span};"); - } - if (can_retry || (is_greedy && additional_limit > 0)) { - _ = sc.AppendLine($"int {additional_count} = 0;"); - } - if (is_greedy && additional_limit != 0) + ExitLabel retry = context.RequestExitLabel(ExitKind.CountRetry); + ExitLabel resume = context.RequestExitLabel(ExitKind.CountResume); + _ = sc.AppendLine($"{ROS_char} {additional_entry} = {remain_span};") + .AppendLine($"int {additional_count} = 0;"); + if (is_greedy) { string limit = additional_limit < 0 ? string.Empty : $"{additional_count} < {additional_limit} && "; _ = sc.AppendLine($"while ({limit}{remain_span}.{starts_with}({expected_literal}))") .AppendLine("{") - .AppendLine($"{remain_span} = {remain_span}.Slice({literal.Length});"); - if (can_retry || additional_limit > 0) { - _ = sc.AppendLine($"{additional_count}++;"); - } - _ = sc.AppendLine("}"); + .AppendLine($"{remain_span} = {remain_span}.Slice({literal.Length});") + .AppendLine($"{additional_count}++;") + .AppendLine("}"); } - if (can_retry) + + // Initial success skips retry handling. Later failures jump back here from anywhere + // in this path; exhaustion forwards to the target inherited before this COUNT. + _ = sc.AppendLine($"goto {resume.GetLabelName()};") + .AppendLine($"{retry.GetLabelName()}: {{}}"); + if (is_greedy) { - // Each retry moves by a complete literal using the scan's own optional counter. - ExitLabel retry = context.RequestExitLabel(ExitMode.Goto); - _ = sc.AppendLine("while (true)") + _ = sc.AppendLine($"if ({additional_count} == 0)") .AppendLine("{") - .AppendLine($"{remain_span} = {additional_entry}.Slice({additional_count} * {literal.Length});"); - GenerateConcatTail(continuation!, next, sc, func_list, context, retry); - _ = sc.AppendLine("break;") - .AppendLine($"{retry.GetLabelName()}: {{}}"); - if (is_greedy) - { - _ = sc.AppendLine($"if ({additional_count} == 0)") - .AppendLine("{") - .AppendLine($"{remain_span} = {additional_entry};") - .AppendLine($"goto {exit_label_node};") - .AppendLine("}") - .AppendLine($"{additional_count}--;"); - } else - { - string limit = additional_limit < 0 ? string.Empty : - $"{additional_count} == {additional_limit} || "; - _ = sc.AppendLine($"{remain_span} = {additional_entry}.Slice({additional_count} * {literal.Length});") - .AppendLine($"if ({limit}!{remain_span}.{starts_with}({expected_literal}))") - .AppendLine("{") - .AppendLine($"{remain_span} = {additional_entry};") - .AppendLine($"goto {exit_label_node};") - .AppendLine("}") - .AppendLine($"{additional_count}++;"); - } - _ = sc.AppendLine("}"); - } else if (has_continuation) { - GenerateConcatTail(continuation!, next, sc, func_list, context, exit); + .AppendLine($"{remain_span} = {additional_entry};") + .AppendLine($"goto {exit_label_node};") + .AppendLine("}") + .AppendLine($"{additional_count}--;"); + } else + { + string limit = additional_limit < 0 ? string.Empty : + $"{additional_count} == {additional_limit} || "; + _ = sc.AppendLine($"{remain_span} = {additional_entry}.Slice({additional_count} * {literal.Length});") + .AppendLine($"if ({limit}!{remain_span}.{starts_with}({expected_literal}))") + .AppendLine("{") + .AppendLine($"{remain_span} = {additional_entry};") + .AppendLine($"goto {exit_label_node};") + .AppendLine("}") + .AppendLine($"{additional_count}++;"); } + _ = sc.AppendLine($"{remain_span} = {additional_entry}.Slice({additional_count} * {literal.Length});") + .AppendLine($"{resume.GetLabelName()}: {{}}"); + backtrack = retry; } #endregion @@ -630,7 +623,7 @@ namespace Nitload.RegularAnalysis.Text in ExitLabel exit_label ) { - ExitLabel success = context.RequestExitLabel(ExitMode.Goto); + ExitLabel success = context.RequestExitLabel(ExitKind.AltMatched); RegexAstNode current = node.Right; while (!RegexAstNode.IsNullOrEmpty(current)) { @@ -639,7 +632,7 @@ namespace Nitload.RegularAnalysis.Text } _ = sc.AppendLine($"goto {exit_label.GetLabelName()};") - .AppendLine($"{success.GetLabelName()}: ;"); + .AppendLine($"{success.GetLabelName()}: {{}}"); } private static void GenerateGroup( @@ -647,7 +640,8 @@ namespace Nitload.RegularAnalysis.Text [NotNull] in SequenceEmitter sc, in List> func_list, in SourceGenContext context, - in ExitLabel exit_label + in ExitLabel local_fail, + ref ExitLabel backtrack ) { RegexAstNode left = node.Left; @@ -661,10 +655,10 @@ namespace Nitload.RegularAnalysis.Text // TODO(NGRA): Snapshot the group start cursor and restore it when the child fails. if (!RegexAstNode.IsNullOrEmpty(left)) { - GenerateMethodSource(left, sc, func_list, context, exit_label); + GenerateMethodSource(left, sc, func_list, context, local_fail, ref backtrack); } else if (!RegexAstNode.IsNullOrEmpty(right)) { - GenerateMethodSource(right, sc, func_list, context, exit_label); + GenerateMethodSource(right, sc, func_list, context, local_fail, ref backtrack); } string group_name = config.Name; @@ -679,13 +673,16 @@ namespace Nitload.RegularAnalysis.Text [NotNull] in SequenceEmitter sc, in List> func_list, in SourceGenContext context, - in ExitLabel exit_label = default + in ExitLabel local_fail, + ref ExitLabel backtrack ) { - // Keep the complete suffix visible even when CONCAT is left-associated. - List sequence = []; - AppendConcatNodes(node, sequence); - GenerateConcatTail(sequence, 0, sc, func_list, context, exit_label); + if (!RegexAstNode.IsNullOrEmpty(node.Left)) { + GenerateMethodSource(node.Left, sc, func_list, context, local_fail, ref backtrack); + } + if (!RegexAstNode.IsNullOrEmpty(node.Right)) { + GenerateMethodSource(node.Right, sc, func_list, context, local_fail, ref backtrack); + } } private static void GenerateCount( @@ -693,9 +690,8 @@ namespace Nitload.RegularAnalysis.Text [NotNull] in SequenceEmitter sc, in List> func_list, in SourceGenContext context, - in ExitLabel exit_label, - IReadOnlyList? continuation = null, - int next = 0 + in ExitLabel local_fail, + ref ExitLabel backtrack ) { RegexAstNode value = RegexAstNode.IsNullOrEmpty(node.Left) ? node.Right : node.Left; @@ -716,6 +712,7 @@ namespace Nitload.RegularAnalysis.Text int min = payload.Min; int max = payload.Max; bool is_greedy = payload.IsGreedy; + ExitLabel failure = backtrack; // TODO(NGRA-NAMING): Reserve exclusive names for future repetition/rollback state // across complex COUNT retries; recursive children must use the same method pool. @@ -723,21 +720,19 @@ namespace Nitload.RegularAnalysis.Text { // TODO(NGRA): Emit min/max repetition, zero-width progress protection, greedy vs. // lazy ordering, and per-attempt rollback for a complex COUNT body. - GenerateMethodSource(value, sc, func_list, context, exit_label); + GenerateMethodSource(value, sc, func_list, context, local_fail, ref backtrack); } else { switch (value.Label & RegexElementLabel.SEQUENCE) { case RegexElementLabel.UNIT: - GenerateUnitCount(value, min, max, is_greedy, sc, func_list, context, - exit_label, continuation, next); + GenerateUnitCount(value, min, max, is_greedy, sc, context, failure, ref backtrack); return; case RegexElementLabel.UNIT_SET: - GenerateUnitSetCount(value, min, max, is_greedy, sc, context, exit_label); + GenerateUnitSetCount(value, min, max, is_greedy, sc, context, failure); break; case RegexElementLabel.UNIT_SPAN: - GenerateUnitSpanCount(value, min, max, is_greedy, sc, func_list, context, - exit_label, continuation, next); + GenerateUnitSpanCount(value, min, max, is_greedy, sc, context, failure, ref backtrack); return; default: @@ -746,9 +741,6 @@ namespace Nitload.RegularAnalysis.Text break; } } - if (continuation is not null) { - GenerateConcatTail(continuation, next, sc, func_list, context, exit_label); - } } #endregion diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenContext.cs b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenContext.cs index b7642b11..c17f8dc4 100644 --- a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenContext.cs +++ b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenContext.cs @@ -36,7 +36,7 @@ namespace Nitload.RegularAnalysis.Text public const string _remain_span = "__remain"; - private int _max_exit_lable; + private int _exit_label_count; private int _scratch_scope_count; @@ -54,12 +54,12 @@ namespace Nitload.RegularAnalysis.Text } public ExitLabel RequestExitLabel( - ExitMode exitMode + ExitKind kind ) { - int index = _max_exit_lable; - _max_exit_lable++; - return new(exitMode, index); + int index = _exit_label_count; + _exit_label_count++; + return new(kind, index); } public int RequestScratchScopeIndex() diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenSpecial.cs b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenSpecial.cs index d0724085..ffc61839 100644 --- a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenSpecial.cs +++ b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenSpecial.cs @@ -17,7 +17,8 @@ namespace Nitload.RegularAnalysis.Text in SequenceEmitter sc, in List> func_list, in SourceGenContext context, - in ExitLabel exit_label + in ExitLabel local_fail, + ref ExitLabel backtrack ) { // A special rule owns the entire matched subtree. Returning false leaves emission @@ -25,7 +26,7 @@ namespace Nitload.RegularAnalysis.Text return node.Label switch { RegexElementLabel.CONCAT => TryGenerateFindConcat(node, sc, func_list, context, - exit_label), + local_fail, ref backtrack), _ => false, }; } @@ -166,10 +167,12 @@ namespace Nitload.RegularAnalysis.Text in ExitLabel success ) { - ExitLabel failure = context.RequestExitLabel(ExitMode.Goto); + ExitLabel failure = context.RequestExitLabel(ExitKind.AltNext); string entry = $"__alt_entry_{context.RequestScratchScopeIndex()}"; _ = sc.AppendLine($"{ROS_char} {entry} = {context.RemainSpan};"); - GenerateMethodSource(node, sc, func_list, context, failure); + // Keep branch-local retries isolated from both siblings and the caller's path. + ExitLabel branch_backtrack = failure; + GenerateMethodSource(node, sc, func_list, context, failure, ref branch_backtrack); _ = sc.AppendLine($"goto {success.GetLabelName()};") .AppendLine($"{failure.GetLabelName()}:") .AppendLine($"{context.RemainSpan} = {entry};"); @@ -180,50 +183,6 @@ namespace Nitload.RegularAnalysis.Text #endregion - #region CONCAT continuation - - private static void AppendConcatNodes( - RegexAstNode node, - List sequence - ) - { - if (RegexAstNode.IsNullOrEmpty(node)) { - return; - } - if (node.Label == RegexElementLabel.CONCAT && - !RegexAstNode.IsDotStar(node.Left) && !IsNamedFind(node)) - { - AppendConcatNodes(node.Left, sequence); - AppendConcatNodes(node.Right, sequence); - } else { - sequence.Add(node); - } - } - - private static void GenerateConcatTail( - IReadOnlyList sequence, - int start, - in SequenceEmitter sc, - in List> func_list, - in SourceGenContext context, - in ExitLabel exit_label - ) - { - for (int i = start; i < sequence.Count; i++) - { - RegexAstNode node = sequence[i]; - if (node.Label == RegexElementLabel.COUNT) - { - // COUNT owns both its repetition and the continuation that may retry it. - GenerateCount(node, sc, func_list, context, exit_label, sequence, i + 1); - return; - } - GenerateMethodSource(node, sc, func_list, context, exit_label); - } - } - - #endregion - #region FIND concatenation private static bool TryGenerateFindConcat( @@ -231,7 +190,8 @@ namespace Nitload.RegularAnalysis.Text in SequenceEmitter sc, in List> func_list, in SourceGenContext context, - in ExitLabel exit_label + in ExitLabel local_fail, + ref ExitLabel backtrack ) { RegexAstNode left = node.Left; @@ -246,7 +206,7 @@ namespace Nitload.RegularAnalysis.Text { // TODO(NGRA): Emit the unnamed FIND candidate loop and retry the right subtree // until it succeeds or no anchor candidate remains. - GenerateMethodSource(right, sc, func_list, context, exit_label, true); + GenerateMethodSource(right, sc, func_list, context, local_fail, ref backtrack, true); return true; } @@ -256,7 +216,7 @@ namespace Nitload.RegularAnalysis.Text { // TODO(NGRA): Emit named FIND retry and capture the text preceding the // successful anchor candidate. - GenerateMethodSource(right, sc, func_list, context, exit_label, true); + GenerateMethodSource(right, sc, func_list, context, local_fail, ref backtrack, true); } else { // TODO(NGRA): Capture the remaining input for a terminal named dot-star. diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.cs b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.cs index bfe9cc54..eec6d7e3 100644 --- a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.cs +++ b/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.cs @@ -172,7 +172,15 @@ namespace Nitload.RegularAnalysis.Text SequenceEmitter methodSource = new(); List> localFunctions = []; - GenerateMethodSource(root, methodSource, localFunctions, generationContext); + ExitLabel local_fail = generationContext.RequestExitLabel(ExitKind.MatchFailed); + ExitLabel backtrack = local_fail; + ExitLabel end = generationContext.RequestExitLabel(ExitKind.MatchEnd); + GenerateMethodSource(root, methodSource, localFunctions, generationContext, + local_fail, ref backtrack); + _ = methodSource.AppendLine($"goto {end.GetLabelName()};") + .AppendLine($"{local_fail.GetLabelName()}: {{}}") + .AppendLine("// TODO(NGRA): Produce the root failure result here.") + .AppendLine($"{end.GetLabelName()}: {{}}"); foreach (SequenceEmitter localFunction in localFunctions) { _ = methodSource.AppendLine() .AppendLine() diff --git a/test/StgSharpDebug/TestRegexGenerate.cs b/test/StgSharpDebug/TestRegexGenerate.cs index 4e09804b..92c6277d 100644 --- a/test/StgSharpDebug/TestRegexGenerate.cs +++ b/test/StgSharpDebug/TestRegexGenerate.cs @@ -38,8 +38,14 @@ namespace Nitload SequenceEmitter body = new(); List> functions = []; SourceGenContext context = new(); - ExitLabel exit = context.RequestExitLabel(ExitMode.Goto); - GenerateMethodSource(analysis.Ast.Root, body, functions, context, exit); + ExitLabel local_fail = context.RequestExitLabel(ExitKind.MatchFailed); + ExitLabel backtrack = local_fail; + ExitLabel end = context.RequestExitLabel(ExitKind.MatchEnd); + GenerateMethodSource(analysis.Ast.Root, body, functions, context, local_fail, ref backtrack); + _ = body.AppendLine($"goto {end.GetLabelName()};") + .AppendLine($"{local_fail.GetLabelName()}: {{}}") + .AppendLine("// TODO(NGRA): Produce the root failure result here.") + .AppendLine($"{end.GetLabelName()}: {{}}"); foreach (string line in body.ToList()) { Console.WriteLine(line); From e04adf594a6a4a3aa434855b04e00720a583c3c9 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Tue, 15 Sep 2026 18:06:52 +0800 Subject: [PATCH 5/7] stop all work on text regex, ngra text regex is now frozen --- ReadMe.md | 14 +- future/NGRA_Transformation_Plan.md | 916 ++++++++++++++++++ .../Abstraction/SequenceEmitter.cs | 1 - .../{ => Legacy}/GroupIndex.cs | 0 src/Nitload.RegularAnalysis/Legacy/README.md | 51 + .../{ => Legacy}/SourceGenerate.cs | 0 .../Text/RegexAnalyzer.ASTGenerate.cs | 0 .../Text/RegexAnalyzer.ASTNodeOptimize.cs | 0 .../Text/RegexAnalyzer.ASTOptimize.cs | 0 .../{ => Legacy}/Text/RegexAnalyzer.cs | 0 .../Text/RegexAnanlyzer.ASTNodeOptimize.cs | 0 .../Text/RegexAstNode.AdditionalPayload.cs | 0 .../{ => Legacy}/Text/RegexAstNode.cs | 0 .../{ => Legacy}/Text/RegexElementLabel.cs | 0 .../{ => Legacy}/Text/RegexTokenParser.cs | 0 .../{ => Legacy}/Text/RegexTokenReader.cs | 0 .../{ => Legacy}/Text/TextRegex.cs | 0 .../{ => Legacy}/Text/TextRegexAttribute.cs | 0 .../Text/TextRegexSourceGen.Diagnostic.cs | 0 .../Text/TextRegexSourceGen.ExitLabel.cs | 0 .../Text/TextRegexSourceGen.GenClass.cs | 0 .../Text/TextRegexSourceGen.GenContext.cs | 0 .../Text/TextRegexSourceGen.GenFile.cs | 0 .../Text/TextRegexSourceGen.GenSpecial.cs | 0 .../Text/TextRegexSourceGen.Source.cs | 0 .../{ => Legacy}/Text/TextRegexSourceGen.cs | 0 .../Nitload.RegularAnalysis.csproj | 20 +- src/README.md | 2 +- test/README.md | 44 +- .../TextRegex}/TestRegexAnalyse.cs | 0 .../TextRegex}/TestRegexGenerate.cs | 0 test/StgSharpDebug/Program.cs | 3 +- test/StgSharpDebug/StgSharpDebug.csproj | 6 +- 33 files changed, 1008 insertions(+), 49 deletions(-) create mode 100644 future/NGRA_Transformation_Plan.md rename src/Nitload.RegularAnalysis/{ => Legacy}/GroupIndex.cs (100%) create mode 100644 src/Nitload.RegularAnalysis/Legacy/README.md rename src/Nitload.RegularAnalysis/{ => Legacy}/SourceGenerate.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/RegexAnalyzer.ASTGenerate.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/RegexAnalyzer.ASTNodeOptimize.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/RegexAnalyzer.ASTOptimize.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/RegexAnalyzer.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/RegexAnanlyzer.ASTNodeOptimize.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/RegexAstNode.AdditionalPayload.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/RegexAstNode.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/RegexElementLabel.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/RegexTokenParser.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/RegexTokenReader.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/TextRegex.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/TextRegexAttribute.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/TextRegexSourceGen.Diagnostic.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/TextRegexSourceGen.ExitLabel.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/TextRegexSourceGen.GenClass.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/TextRegexSourceGen.GenContext.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/TextRegexSourceGen.GenFile.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/TextRegexSourceGen.GenSpecial.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/TextRegexSourceGen.Source.cs (100%) rename src/Nitload.RegularAnalysis/{ => Legacy}/Text/TextRegexSourceGen.cs (100%) rename test/StgSharpDebug/{ => Legacy/TextRegex}/TestRegexAnalyse.cs (100%) rename test/StgSharpDebug/{ => Legacy/TextRegex}/TestRegexGenerate.cs (100%) diff --git a/ReadMe.md b/ReadMe.md index 5b6371ba..4d4ca78e 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -34,12 +34,24 @@ Numerics and linear algebra built on top of the core library: Nitload General Regular Analysis (NGRA) infrastructure for recognizing and processing regular languages: -- **Regex/Automata Core**: Regular expression processing and automata utilities +- **NFA Direction**: Planned generic ε-NFA sequence execution, followed by weighted automata and NGRA-Tensor research - **Parsing Primitives**: Reusable tokenizer and grammar building blocks - **Analysis Pipeline**: Structured processing for language-like inputs NGRA is a foundation for higher-level language tooling. It does not replace `Nitload.Script`; the planned Script implementation depends on NGRA and has not started yet. The source currently under `Nitload.Script` is retained from the earlier EXPRESS implementation and is not part of the active core solution. +The former Text Regex parser, matcher, and source generator are [archived](src/Nitload.RegularAnalysis/Legacy/README.md) +and excluded from compilation. Generic AST and token infrastructure remain active. + +Although Text Regex development is frozen, it produced valuable reusable results: the five-link +AST model, tree normalization and merging techniques, token/analysis infrastructure, and practical +experience with COUNT continuations, group boundaries, and rollback. These results remain available +to inform the new NFA architecture. + +Future string-regex functionality will use .NET `GeneratedRegex`. NGRA will no longer pursue +custom `Span` or SIMD optimizations for string matching; its research focus is generalized +sequence analysis and, later, weighted relation networks. + ### Nitload.Graphics Managed graphics infrastructure built on Common and Mathematics: diff --git a/future/NGRA_Transformation_Plan.md b/future/NGRA_Transformation_Plan.md new file mode 100644 index 00000000..54f332ef --- /dev/null +++ b/future/NGRA_Transformation_Plan.md @@ -0,0 +1,916 @@ +# NGRA 转型计划 + +> 面向本地 Codex / 项目维护者的快速架构说明 +> 项目：Nitload General Regular Analyzer（NGRA） + +## 1. 转型结论 + +NGRA 不再以“高性能字符串正则表达式引擎”为主要目标。 + +原先围绕 `Span`、专用字符串匹配、复杂回溯、Source Generator 等方向继续投入的收益已经明显下降。后续核心工作转向： + +1. 以 **ε-NFA / NFA** 为基础执行模型； +2. 将输入从 `char` 推广为任意可比较、可判定类别的序列元素； +3. 面向 Token Stream、语法前端、半上下文模式分析等场景； +4. 支持自动机归并、分层自动机、状态/断言/前序引用； +5. 在此基础上继续研究 **Weighted Automaton / NGRA-Tensor**。 + +因此，Text Regex 从“主架构”降级为“兼容前端 / 参考实现 / 测试子系统”。 + +--- + +## 2. 当前代码处理原则 + +### 2.1 保留现有 Text Regex 半成品 + +当前 Text Regex Source Generator、COUNT/CONCAT/ALT lowering、生成式 matcher 等代码不删除。 + +原因： + +- 五角星 AST 模型仍然是有效成果； +- AST 搜索、规范化、旋转、合并等代码仍可复用； +- 当前实现记录了 COUNT continuation、group boundary、rollback 等真实工程问题； +- 可作为未来自动机 IR 与旧实现行为的对照； +- 删除会丢失大量探索成果。 + +建议： + +- 将旧生成器移动到 `Legacy` / `Experimental` / `Archive` 目录； +- 通过 `.csproj` 的 `Compile Remove` 或独立非构建项目，使其暂时不参与主编译； +- 不再优先修补完整 `.NET Regex` 兼容性。 + +### 2.2 五角星 AST 继续保留 + +现有节点关系： + +```text + Parent + ↑ +Previous ← Node → Next + ↙ ↘ + Left Right +``` + +继续作为通用语法树 / 结构树基础设施。 + +需要坚持： + +- AST 用于表达语法和结构； +- 自动机 / CFG / Relation Graph 用于表达执行与传播； +- 不再要求 AST 本身承担全部 backtracking / continuation 语义。 + +--- + +## 3. 新主线：ε-NFA + +第一阶段优先实现一个简单、正确、无性能野心的 ε-NFA 模拟器。 + +### 3.1 基础状态模型 + +```text +Current Active States + ↓ +ε-closure + ↓ +consume(input) + ↓ +Next Active States + ↓ +ε-closure +``` + +核心状态： + +```text +State +Transition +StartState +AcceptStates +ActiveStateSet +``` + +Transition 分为： + +```text +ε transition +matched transition +``` + +输入类型不能写死为 `char`。 + +建议： + +```csharp +EpsilonNfa +``` + +或： + +```csharp +EpsilonNfa +``` + +匹配条件可以抽象为： + +```csharp +bool Match(in TInput input) +``` + +或者后续升级为更明确的 Predicate / Category Matcher。 + +### 3.2 第一阶段支持的 Pattern + +第一版只要求： + +```text +literal / predicate +concat +alternation +* ++ +? +{m,n} +ordinary grouping +``` + +暂不追求： + +```text +capture rollback +复杂 greedy / lazy 行为 +lookaround +backreference +完整 .NET Regex compatibility +``` + +其中 `{m,n}` 可以优先在构造期展开为 NFA 结构，而不是运行时维护复杂 COUNT 控制流。 + +--- + +## 4. 为什么以 NFA 为主，而不是 DFA + +DFA 仍然有价值，但不再是必须经过的阶段。 + +核心路线： + +```text +Pattern + ↓ +ε-NFA + ↓ +NFA execution + ↓ +Weighted NFA + ↓ +NGRA-Tensor +``` + +DFA 只作为可选 specialization： + +```text +Boolean +Static +No dynamic relation +No weighted state +``` + +满足这些条件的局部子图，可以由编译器确定化为 DFA fast path。 + +总体原则： + +> NFA 是主语义模型，DFA 是可选优化后端。 + +原因： + +普通 NFA 的活动状态： + +```text +a_i ∈ {0,1} +``` + +加权机可自然推广为： + +```text +a_i ∈ R +``` + +Tensor 版本可以继续推广为： + +```text +a_i ∈ R^d +``` + +多状态同时活跃不再是需要“消除的非确定性”，而会成为模型本身的状态表示。 + +--- + +## 5. NGRA 的目标：广义序列分析 + +NGRA 不再局限于字符串。 + +典型输入： + +```text +char +token +syntax node +semantic node +sensor frame +event +instruction-like object +``` + +Pattern 的作用也从： + +```text +“这个字符串是不是符合 regex” +``` + +扩展为： + +```text +“这个序列在当前状态、历史、类别和上下文条件下， +是否满足某个结构化模式” +``` + +--- + +## 6. 半上下文 Token Analyzer + +NGRA 的近期目标之一，是成为一种介于普通 Regex 与完整 Parser 之间的 Token Stream Analyzer。 + +可用能力包括： + +```text +Token 类别匹配 +Token 属性匹配 +前序状态引用 +局部断言 +Pattern 状态 +符号绑定 +分层子自动机 +局部上下文 +``` + +因此它适合作为： + +```text +Lexer 后端 +Token Regex +语法前端 +编译器预分析器 +DSL Parser Frontend +结构化流分析器 +``` + +“半上下文”的含义不是严格的形式语言分类，而是： + +> Pattern 不只依赖当前 token，也可以依赖有限的已知状态、绑定、前序结果和结构信息，但不要求一开始就构造完整通用 Parser。 + +--- + +## 7. 自动机归并 + +大量 Pattern 应优先在 NFA 层归并，而不是各自独立执行。 + +例如： + +```text + ε → Pattern A + / +GlobalStart ─ε → Pattern B + \ + ε → Pattern C +``` + +Accepting State 可以附带： + +```text +PatternId +MatchKind +Payload +``` + +后续目标： + +- 合并公共路径； +- 消除冗余 ε 边； +- 状态压缩； +- bitset active state； +- pattern priority； +- 多层自动机组合。 + +--- + +## 8. 分层自动机 + +后续重点不是继续增加字符串 Regex 特性，而是建立分层自动机。 + +示意： + +```text +Scalar / Token Layer + ↓ +Local Pattern Layer + ↓ +Phrase / Region Layer + ↓ +Sentence / Structure Layer + ↓ +Semantic / Relation Layer +``` + +低层自动机可以生成高层节点，高层节点继续作为新序列输入。 + +即： + +```text +Input Stream + ↓ +Automaton L0 + ↓ +Derived Nodes + ↓ +Automaton L1 + ↓ +Higher-level Nodes +``` + +这应成为 NGRA 进入编译器前端、自然语言结构分析和复杂流分析的主路径。 + +--- + +## 9. Assertion / Context 的重新定义 + +不再围绕字符串 Regex 的 lookahead / lookbehind 设计核心架构。 + +通用定义应更接近： + +```text +Assert(Context) -> bool +``` + +或者： + +```text +RelationGate(State, Context, History) -> bool +``` + +因此： + +- 字符串前瞻/后顾只是特殊情况； +- 前序引用只是状态引用的一种； +- group/capture 不是核心抽象； +- Context / Binding / Region / History 才是可推广概念。 + +--- + +# NGRA-Tensor 构想 + +## 10. 基本方向 + +NGRA-Tensor 不是简单地“给 NFA 边加 float”。 + +它希望把： + +```text +Boolean multi-state automaton +``` + +推广为： + +```text +Weighted / Tensor Relation Network +``` + +基础演化： + +```text +NFA +a_i ∈ {0,1} + +↓ weighted + +Weighted Automaton +a_i ∈ R + +↓ tensor/state + +NGRA-Tensor +a_i ∈ R^d +``` + +--- + +## 11. Relation 成为一等对象 + +不再采用固定 `Port4` 作为核心语义。 + +推荐模型： + +```text +Source Node + ↓ +[ Relation ] + ↓ +Destination Node +``` + +Relation 包含： + +```text +Source +Destination +Type +Parameters +Weight / Weight Function +Optional State +Eligibility / Gate +``` + +语义上： + +```text +Relation: +决定一条联系是否生效， +以及它如何把 Source 信息贡献给 Destination。 +``` + +Node 负责保存状态。 + +Blend 负责汇合多条 Relation 的贡献。 + +--- + +## 12. Pull，而不是主动广播 + +GPU / 执行层优先采用 destination pull： + +```text +Destination Node + ↓ +请求自己依赖的 Relations + ↓ +读取 Source states + ↓ +Relation Eval + ↓ +Blend +``` + +而不是： + +```text +Source Node + ↓ +主动向大量 Destination 广播 +``` + +优点： + +- 减少 scatter； +- 减少原子写冲突； +- 减少 readiness 管理； +- 更适合 GPU destination-centric execution。 + +--- + +## 13. Relation 的三段语义 + +推荐拆成： + +```text +Eligibility + ↓ +Compatibility + ↓ +Transfer +``` + +即： + +1. **Eligibility**：这条 Relation 当前能不能参与； +2. **Compatibility**：参与后有多重要； +3. **Transfer**：真正传递什么信息。 + +形式上： + +```text +gate_ij = G(state, context, relation) +score_ij = F(query, key, relation) +value_ij = T(source_state, relation) +``` + +最终： + +```text +output_i = Blend(gate_ij * weight_ij * value_ij) +``` + +--- + +## 14. 与 Attention / QKV 的关系 + +QKV 可以作为 Relation 内部算子的第一种 baseline，但不应决定 NGRA-Tensor 的外层拓扑。 + +标准 Attention： + +```text +Q = “我需要什么” +K = “我适不适合被引用” +V = “真正传递的内容” +``` + +数学上： + +```text +score_ij = q_i · k_j +weight_ij = softmax(score_ij) +output_i = Σ weight_ij * v_j +``` + +NGRA-Tensor 的差异： + +- Relation 是显式、长期存在、可带类型的； +- 候选关系通常是稀疏的； +- 不一定对全部 node 做 dense attention； +- Relation 可以有自己的参数、状态、类型和 gate； +- Attention 可以只是某种 Relation Eval backend。 + +--- + +## 15. 稀疏关系，而不是全连接爆破 + +Transformer 的典型关系空间： + +```text +L × L +``` + +NGRA-Tensor 的目标： + +```text +E << N² +``` + +即只计算显式 Relation。 + +但必须承认： + +- 稀疏图带来 gather； +- 不规则访问会伤害 GPU； +- 小算子过多会降低吞吐。 + +因此不能简单逐 Relation 启动 kernel。 + +--- + +## 16. Relation Tile / 脑区式分块 + +后续 GPU backend 应研究： + +```text +Semantic Relation Graph + ↓ +Graph Partition + ↓ +Relation Tiles + ↓ +Block-sparse / Dense subgraph execution +``` + +目标： + +> 块内允许不规则访问，块间尽量转成规则张量传输。 + +示意： + +```text +Dense / semi-dense local region + ↓ +少量跨区 Relation + ↓ +another dense / semi-dense region +``` + +类似“脑区”： + +- 区内关系较密； +- 区间关系较稀； +- 每个区可以编译成中型 tensor operator； +- 跨区使用显式 Relation。 + +这可能成为 NGRA-Tensor 与传统 Transformer 之间的重要折中。 + +--- + +## 17. 语义稀疏 vs 物理稠密 + +前端语义可以是： + +```text +Sparse Relation Graph +``` + +GPU backend 可以通过： + +```text +node reorder +feature reorder +relation reorder +graph clustering +block detection +tile packing +kernel fusion +shared-memory staging +``` + +将其转为： + +```text +Dense blocks ++ +Block-sparse blocks ++ +Irregular sparse tails +``` + +原则： + +> 模型的图结构不等于 GPU 必须逐边解释。 + +--- + +## 18. Node State 可以是高维 Tensor + +NGRA-Tensor 不等于“每个 Node 只有一个 scalar”。 + +Node 可以持有： + +```text +x_i = [f0, f1, ..., fN] +``` + +但 Relation 不一定读取全部 feature。 + +因此可能存在两层稀疏： + +```text +Topology sparsity +Feature sparsity +``` + +Relation 可以只对特定 feature/subspace 感兴趣。 + +例如： + +```text +Relation A: +Q features = [1, 2, 7] +K features = [4, 8] +V features = [16, 17, 18] +``` + +语义上只取这些 feature。 + +物理执行时 GPU compiler 可以为了带宽一次读入更大的连续 block。 + +--- + +## 19. Tensor 观念 + +后续 NGRA-Tensor 设计不要把 Tensor 只理解为“算子”。 + +Tensor 可以同时是： + +```text +状态 +参数 +关系 +mask +中间结果 +数据集合 +算子 +``` + +“算子”只是某个 Tensor 在某次 contraction 中扮演的角色。 + +建议以“语义轴”理解 Tensor，例如： + +```text +Node × Feature +Destination × Source +Relation × Feature +Time × Node × Feature +``` + +优先思考： + +- 哪些轴保留； +- 哪些轴被收缩； +- 哪些轴表示关系； +- 哪些轴表示 feature； +- 哪些轴可以 sparse； +- 哪些轴可以 dense。 + +--- + +## 20. 训练 / 运行时关系发现 + +NGRA-Tensor 长期设想允许结构学习。 + +训练阶段： + +```text +短程强关系 + ↓ +词 / Token 局部关系 + ↓ +词组 / 短句 + ↓ +长句 / 语义 + ↓ +知识网络 / 长程弱关系 +``` + +运行时： + +当现有网络： + +```text +NoActivePath +LowConfidence +HighAmbiguity +Unknown +``` + +可进入较慢的 Relation Discovery： + +```text +Stable Graph + ↓ +Unknown + ↓ +Candidate Relation Search + ↓ +Candidate Graph + ↓ +验证 + ↓ +可能固化进入 Stable Graph +``` + +不建议运行时每一步都修改主图。 + +应区分： + +```text +Fast inference time +Slow structural learning time +``` + +--- + +# 近期开发优先级 + +## Phase 1 — ε-NFA Reference Runtime + +实现： + +```text +EpsilonNfa +StateId +Transition +ε-closure +Consume +Accept +``` + +目标： + +- 语义简单； +- 正确性优先； +- 不做复杂性能优化； +- 不绑定 `char`。 + +## Phase 2 — Pattern → NFA Lowering + +从现有 AST / Pattern IR 构造 NFA： + +```text +literal +predicate +concat +alt +repeat +``` + +## Phase 3 — NFA Merge + +实现： + +```text +multi-pattern merge +PatternId +shared start +accept payload +``` + +然后测试大量 Pattern 同时运行。 + +## Phase 4 — Token Stream + +将输入切换到： + +```text +Token +``` + +支持： + +```text +Token kind +Token value +Token metadata +simple binding +context predicate +``` + +目标：作为编译器前端 / Token Regex 原型。 + +## Phase 5 — Hierarchical Automata + +允许低层自动机输出高层节点，并再次进入更高层自动机。 + +## Phase 6 — Weighted Automaton + +将： + +```text +active : bool +``` + +推广为： + +```text +activation : float +``` + +先不要急于引入复杂 Tensor。 + +## Phase 7 — NGRA-Tensor Prototype + +研究： + +```text +Node state tensor +Typed Relation +Eligibility +Compatibility +Transfer +Blend +Sparse / block-sparse execution +``` + +QKV/Attention 可作为 baseline Relation operator。 + +--- + +# 非目标 + +近期不要优先投入： + +```text +完整 .NET Regex 兼容 +极端 Span 优化 +复杂 Source Generator matcher +完整 capture rollback +所有 lookaround/backreference +为了 Regex 完整性设计核心 IR +``` + +这些功能如确实需要，可以： + +```text +System.Text.RegularExpressions +``` + +作为字符串兼容层。 + +这不是架构退让，而是目标调整。 + +--- + +# 一句话定位 + +NGRA 的新定位： + +> **以 NFA 为基础的广义序列模式与分层自动机框架，并逐步推广到加权 Relation Network 与 NGRA-Tensor。** + +Text Regex： + +> 一个兼容前端 / 参考实现，不再定义 NGRA 的核心架构。 + +NGRA-Tensor： + +> 把 Boolean NFA 的多状态传播推广为显式、稀疏、可加权、可训练、可分块执行的 Relation Network。 diff --git a/src/Nitload.RegularAnalysis/Abstraction/SequenceEmitter.cs b/src/Nitload.RegularAnalysis/Abstraction/SequenceEmitter.cs index fd300036..92d29754 100644 --- a/src/Nitload.RegularAnalysis/Abstraction/SequenceEmitter.cs +++ b/src/Nitload.RegularAnalysis/Abstraction/SequenceEmitter.cs @@ -5,7 +5,6 @@ // SPDX-License-Identifier: MIT // ----------------------------------------------------------------------------- -using Nitload.RegularAnalysis.Text; using System; using System.Collections; using System.Collections.Generic; diff --git a/src/Nitload.RegularAnalysis/GroupIndex.cs b/src/Nitload.RegularAnalysis/Legacy/GroupIndex.cs similarity index 100% rename from src/Nitload.RegularAnalysis/GroupIndex.cs rename to src/Nitload.RegularAnalysis/Legacy/GroupIndex.cs diff --git a/src/Nitload.RegularAnalysis/Legacy/README.md b/src/Nitload.RegularAnalysis/Legacy/README.md new file mode 100644 index 00000000..54102f26 --- /dev/null +++ b/src/Nitload.RegularAnalysis/Legacy/README.md @@ -0,0 +1,51 @@ +# Frozen Text Regex implementation + +Text Regex is archived following the [NGRA transformation plan](../../../future/NGRA_Transformation_Plan.md). +It is no longer NGRA's execution architecture or an active development target. + +## Retained results and future string matching + +Freezing development does not diminish the value of the work already completed. The project +produced reusable AST/token infrastructure, the five-link tree model and its normalization, +rotation, and merging techniques, as well as tested fixed-width COUNT retry logic. The archived +generator also records useful engineering lessons about continuations, group boundaries, label +ownership, and rollback. These are reference material for the new architecture, not a claim that +the old matcher is feature-complete. + +Future string-regex functionality will use .NET `GeneratedRegex`. Further research into custom +`Span` and SIMD optimizations for the Text Regex matcher is discontinued. This boundary +applies to the string-regex implementation, not to SIMD work elsewhere in Nitload. + +## Contents + +- `Text/`: the text tokenizer/parser, regex-specific AST and optimizers, matcher contract, + attribute, and all source-generator files. +- `GroupIndex.cs`: the former group index/name record. +- `SourceGenerate.cs`: the unused source-generation entry stub. +- [Archived debug routines](../../../test/StgSharpDebug/Legacy/TextRegex): the original + AST inspection and generated-source preview programs. + +Source contents and namespaces are preserved. The generic infrastructure in `../Abstraction/` +remains active, including the five-link node interface, generic AST, token abstractions, +compilation stack, and sequence emitter. Text-specific tree transformations remain here +as source material for future Pattern-to-NFA lowering. + +## Build boundary + +The project excludes `Legacy/**/*.cs` from `Compile` and retains the files as `None` items. +They are not copied to build/publish output or packed. The active project no longer references +Roslyn or packages its assembly as a source generator. Debug routines have the same exclusion +in their own project and are not called from `Main`. + +To investigate this implementation again, use a separate experimental project that explicitly +includes the archived files and supplies the former Roslyn dependencies: +`Microsoft.CodeAnalysis.CSharp` 4.13.0 and `Microsoft.CodeAnalysis.Analyzers` 3.11.0. +The debug preview also required Roslyn at runtime and the `StgSharpDebug` friend assembly. +Re-enabling files is not sufficient to make the unfinished matcher complete. + +Historical notes remain in [IR_RETIRED.md](../IR_RETIRED.md) and +[regex_node_transaction.md](../regex_node_transaction.md). They describe earlier experiments, +not the new NFA execution contract. Older `.discard` material is unchanged. + +The next runtime is planned around generic sequence input and Boolean ε-NFA execution; +weighted automata and NGRA-Tensor remain later development/research stages. diff --git a/src/Nitload.RegularAnalysis/SourceGenerate.cs b/src/Nitload.RegularAnalysis/Legacy/SourceGenerate.cs similarity index 100% rename from src/Nitload.RegularAnalysis/SourceGenerate.cs rename to src/Nitload.RegularAnalysis/Legacy/SourceGenerate.cs diff --git a/src/Nitload.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs b/src/Nitload.RegularAnalysis/Legacy/Text/RegexAnalyzer.ASTGenerate.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/RegexAnalyzer.ASTGenerate.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/RegexAnalyzer.ASTGenerate.cs diff --git a/src/Nitload.RegularAnalysis/Text/RegexAnalyzer.ASTNodeOptimize.cs b/src/Nitload.RegularAnalysis/Legacy/Text/RegexAnalyzer.ASTNodeOptimize.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/RegexAnalyzer.ASTNodeOptimize.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/RegexAnalyzer.ASTNodeOptimize.cs diff --git a/src/Nitload.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs b/src/Nitload.RegularAnalysis/Legacy/Text/RegexAnalyzer.ASTOptimize.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/RegexAnalyzer.ASTOptimize.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/RegexAnalyzer.ASTOptimize.cs diff --git a/src/Nitload.RegularAnalysis/Text/RegexAnalyzer.cs b/src/Nitload.RegularAnalysis/Legacy/Text/RegexAnalyzer.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/RegexAnalyzer.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/RegexAnalyzer.cs diff --git a/src/Nitload.RegularAnalysis/Text/RegexAnanlyzer.ASTNodeOptimize.cs b/src/Nitload.RegularAnalysis/Legacy/Text/RegexAnanlyzer.ASTNodeOptimize.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/RegexAnanlyzer.ASTNodeOptimize.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/RegexAnanlyzer.ASTNodeOptimize.cs diff --git a/src/Nitload.RegularAnalysis/Text/RegexAstNode.AdditionalPayload.cs b/src/Nitload.RegularAnalysis/Legacy/Text/RegexAstNode.AdditionalPayload.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/RegexAstNode.AdditionalPayload.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/RegexAstNode.AdditionalPayload.cs diff --git a/src/Nitload.RegularAnalysis/Text/RegexAstNode.cs b/src/Nitload.RegularAnalysis/Legacy/Text/RegexAstNode.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/RegexAstNode.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/RegexAstNode.cs diff --git a/src/Nitload.RegularAnalysis/Text/RegexElementLabel.cs b/src/Nitload.RegularAnalysis/Legacy/Text/RegexElementLabel.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/RegexElementLabel.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/RegexElementLabel.cs diff --git a/src/Nitload.RegularAnalysis/Text/RegexTokenParser.cs b/src/Nitload.RegularAnalysis/Legacy/Text/RegexTokenParser.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/RegexTokenParser.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/RegexTokenParser.cs diff --git a/src/Nitload.RegularAnalysis/Text/RegexTokenReader.cs b/src/Nitload.RegularAnalysis/Legacy/Text/RegexTokenReader.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/RegexTokenReader.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/RegexTokenReader.cs diff --git a/src/Nitload.RegularAnalysis/Text/TextRegex.cs b/src/Nitload.RegularAnalysis/Legacy/Text/TextRegex.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/TextRegex.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/TextRegex.cs diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexAttribute.cs b/src/Nitload.RegularAnalysis/Legacy/Text/TextRegexAttribute.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/TextRegexAttribute.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/TextRegexAttribute.cs diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.Diagnostic.cs b/src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.Diagnostic.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.Diagnostic.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.Diagnostic.cs diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.ExitLabel.cs b/src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.ExitLabel.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.ExitLabel.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.ExitLabel.cs diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs b/src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.GenClass.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenClass.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.GenClass.cs diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenContext.cs b/src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.GenContext.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenContext.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.GenContext.cs diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenFile.cs b/src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.GenFile.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenFile.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.GenFile.cs diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenSpecial.cs b/src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.GenSpecial.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.GenSpecial.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.GenSpecial.cs diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.Source.cs b/src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.Source.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.Source.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.Source.cs diff --git a/src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.cs b/src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.cs similarity index 100% rename from src/Nitload.RegularAnalysis/Text/TextRegexSourceGen.cs rename to src/Nitload.RegularAnalysis/Legacy/Text/TextRegexSourceGen.cs diff --git a/src/Nitload.RegularAnalysis/Nitload.RegularAnalysis.csproj b/src/Nitload.RegularAnalysis/Nitload.RegularAnalysis.csproj index c410a7ce..3fca8c47 100644 --- a/src/Nitload.RegularAnalysis/Nitload.RegularAnalysis.csproj +++ b/src/Nitload.RegularAnalysis/Nitload.RegularAnalysis.csproj @@ -1,26 +1,14 @@ - - true - true - true - $(BaseIntermediateOutputPath)Generated - - - - - - - - - - - + + + + diff --git a/src/README.md b/src/README.md index 2960c206..24efcff3 100644 --- a/src/README.md +++ b/src/README.md @@ -10,7 +10,7 @@ This directory contains the production source code, native runtime, supporting a | `Nitload.Mathematics` | Numeric types, vectors, matrices, geometry, and managed access to accelerated math operations. | | `Nitload.Graphics` | OpenGL bindings, rendering infrastructure, shaders, textures, windows, and input integration. | | `Nitload.Native` | C native runtime, GLFW/STBI integration, allocators, platform code, and optimized SIMD/matrix kernels. See its own `README.md` for native build details. | -| `Nitload.RegularAnalysis` | NGRA regular-language analysis, regex processing, intermediate representations, and source-generation infrastructure. | +| `Nitload.RegularAnalysis` | NGRA generic AST/token infrastructure, transitioning toward ε-NFA sequence analysis. Text Regex and its source generator are archived under `Legacy` and excluded from compilation. | | `Nitload.UserInterface` | State-oriented user-interface framework and its incremental source generator. | | `Nitload.TerminalDialogue` | Terminal user-interface components and an interactive terminal application. | | `Nitload.Benchmark` | BenchmarkDotNet workloads for measuring selected runtime and allocation behavior. | diff --git a/test/README.md b/test/README.md index 167dcd3e..7dc11bae 100644 --- a/test/README.md +++ b/test/README.md @@ -16,36 +16,28 @@ Run the active debug harness directly when investigating a subsystem: dotnet run --project test/StgSharpDebug/StgSharpDebug.csproj ``` -## NGRA generation preview +## Archived Text Regex diagnostics -`StgSharpDebug/TestRegexGenerate.cs` contains a standalone debugging routine that calls the -existing NGRA emitter without Roslyn attribute binding. Select it from the debug `Main`: +The Text Regex analyzer and source-generator preview are frozen. Their original source files +are retained under [StgSharpDebug/Legacy/TextRegex](StgSharpDebug/Legacy/TextRegex) and excluded +from compilation, packaging, and build/publish copying. `Main` no longer calls them, and +`--regex-preview` is not an active entry point. -```powershell -dotnet run --project test/StgSharpDebug/StgSharpDebug.csproj -- --regex-preview -``` +The corresponding runtime, text-specific AST/parser, and generator are archived under +[src/Nitload.RegularAnalysis/Legacy](../src/Nitload.RegularAnalysis/Legacy/README.md). +Generic AST and token infrastructure remain active. See the +[NGRA transformation plan](../future/NGRA_Transformation_Plan.md) for the new NFA direction. -This branch returns before `World` or native-memory initialization. Without the switch, the -existing debug flow is unchanged. You can also call `TestRegexGenerate.Run()` directly from -`Main` while debugging; pass a pattern to `Run("abc")` for a single preview. -The switch only selects runtime behavior; building the debug project still builds its normal -project references, including Graphics and Mathematics, and prepares the native library with -CMake. `BuildNativeBeforeManaged` defaults to true; Debug selects `clang-debug`, and other -configurations select `clang-release`. `RequireNativeLibrary` reports a missing DLL at build -time instead of allowing the default runtime path to fail later. +## Native builds -For managed-only preview work without a native toolchain or DLL, add -`-p:BuildNativeBeforeManaged=false -p:RequireNativeLibrary=false` before `--` in the run command. +Building the debug project still builds its normal project references, including Graphics and +Mathematics, and prepares the native library with CMake. `BuildNativeBeforeManaged` defaults +to true; Debug selects `clang-debug`, and other configurations select `clang-release`. +`RequireNativeLibrary` reports a missing DLL at build time instead of allowing execution to +fail later. -Enter raw regular expressions without C# string delimiters. The routine prints only the emitted -code, without status headers or file output. Exceptions propagate directly to the caller/debugger. -Use `:exit` or end-of-input to leave the loop. Empty input is an empty regex. For one-shot mode: - -```powershell -dotnet run --project test/StgSharpDebug/StgSharpDebug.csproj -- --regex-preview 'a{2,3}' -``` - -Set breakpoints in `TestRegexGenerate.cs` or any existing `Generate*` method. The output is a -method-body fragment; this routine does not compile or execute it. +For a managed-only build without a native toolchain or DLL, pass +`-p:BuildNativeBeforeManaged=false -p:RequireNativeLibrary=false`. +This does not make debug routines that use native services runnable without the native library. New repeatable tests should use a test project and avoid machine-specific paths, external mutable data, and dependencies on previous test execution. Keep one-off investigations in `SingleFile` or the manual debug harness until their expected behavior is stable enough for automation. diff --git a/test/StgSharpDebug/TestRegexAnalyse.cs b/test/StgSharpDebug/Legacy/TextRegex/TestRegexAnalyse.cs similarity index 100% rename from test/StgSharpDebug/TestRegexAnalyse.cs rename to test/StgSharpDebug/Legacy/TextRegex/TestRegexAnalyse.cs diff --git a/test/StgSharpDebug/TestRegexGenerate.cs b/test/StgSharpDebug/Legacy/TextRegex/TestRegexGenerate.cs similarity index 100% rename from test/StgSharpDebug/TestRegexGenerate.cs rename to test/StgSharpDebug/Legacy/TextRegex/TestRegexGenerate.cs diff --git a/test/StgSharpDebug/Program.cs b/test/StgSharpDebug/Program.cs index 05d101de..4e99d3d8 100644 --- a/test/StgSharpDebug/Program.cs +++ b/test/StgSharpDebug/Program.cs @@ -37,8 +37,7 @@ namespace Nitload // .UseModule() .InitializeModule(); - TestRegexAnalyse.Run(); - _ = TestRegexGenerate.Run("abc1{1,2}23"); + // Text Regex diagnostics are frozen under Legacy/TextRegex and excluded from compilation. // TestHlsf(); diff --git a/test/StgSharpDebug/StgSharpDebug.csproj b/test/StgSharpDebug/StgSharpDebug.csproj index 70048fe6..badee4b0 100644 --- a/test/StgSharpDebug/StgSharpDebug.csproj +++ b/test/StgSharpDebug/StgSharpDebug.csproj @@ -14,8 +14,10 @@ - - + + + + From 6e6bd7794b7ebb1647d705034a5ffb3a296964a7 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Thu, 17 Sep 2026 18:25:10 +0800 Subject: [PATCH 6/7] add doc --- .../NGRA_TENSOR_MEMORY.md | 288 ++++++++++++++++++ 1 file changed, 288 insertions(+) create mode 100644 src/Nitload.RegularAnalysis/NGRA_TENSOR_MEMORY.md diff --git a/src/Nitload.RegularAnalysis/NGRA_TENSOR_MEMORY.md b/src/Nitload.RegularAnalysis/NGRA_TENSOR_MEMORY.md new file mode 100644 index 00000000..c2799dc7 --- /dev/null +++ b/src/Nitload.RegularAnalysis/NGRA_TENSOR_MEMORY.md @@ -0,0 +1,288 @@ +# NGRA-Tensor 设计讨论记忆 + +更新日期：2026-09-17 +目标项目：`D:\platform-dev\src\Nitload.RegularAnalysis` +来源：关于正则分析、NFA、Transformer、GPU 执行与 NGRA 双算子设计的讨论。 + +> 本文件记录设计意图与讨论结果，不代表项目现有代码已经实现这些功能。本次仅检查了目标目录及祖先目录的 AGENTS.md，没有审计源码。下文区分用户已提出的设计决定、辅助建议与待验证问题。 + +## 1. 项目目标与讨论方式 + +NGRA-Tensor 试图结合结构化关系、模糊特征识别、多候选解释及模式预测。出发点包含正则模式分析与 NFA，但不应预设必须逐节点模拟 NFA 的执行方式。 + +核心工程矛盾：细粒度、动态、自组织的关系网络表达灵活，却容易产生大量小操作、串行依赖、零碎访存和复杂调度，影响 GPU 利用率与训练。 + +用户希望学习“如何构造能运行且高效的系统”，而不只是阅读既有模型的前向公式或事后解释。后续讨论应说明：需求 → 设计选择 → 数学表达 → 执行方式 → 代价 → 验证方法。不要用 Transformer 也存在串行步骤，来回避 NGRA 的节点过小与调度问题。 + +“Tr”在不影响理解时作为 Transformer 的简称。 + +## 2. 数学符号约定 + +| 符号 | 含义 | +| --- | --- | +| `Scal` | 标量 | +| `Vec_d` | d 维向量，本文使用列形式表示分量 | +| `Covec_d` | 对向量进行线性求值的余向量 | +| `Mat_x_y` | 每行 x 个元素、每列 y 个元素，即 y 行、x 列；列数在前 | +| `Tens_u_v` | 接收 u 个 Vec 和 v 个 Covec、输出 Scal 的多重线性对象 | + +注意： + +- 数组的存储维数不等于 `Tens_u_v` 的槽位数。ONNX 等工具中的 tensor 通常只是多维数组的名称。 +- 线性映射 L 可通过 `T(x, φ) = φ(Lx)` 对应 `Tens_1_1`；输入向量与输出余向量可以属于不同维度的空间。 +- 点积所对应的度量可视为 `Tens_2_0`。将向量与余向量识别需要相应的度量约定，不能只凭行列形状混为一谈。 +- 带偏置映射一般为仿射映射；softmax、ReLU、LayerNorm 等不是多重线性运算。完整神经网络不能直接按上述定义称为一个 `Tens_u_v`。 + +## 3. 用户已提出的核心架构决定 + +### 3.1 两种组合算子 + +**relation：评估一定数量的 notation 按顺序串接形成的关系。** + +```text +R = Relation(N1, N2, ..., Np) +``` + +port 表达有序关系，交换 port 输入通常会改变含义。此前使用过 `R_6` 表示 64 口 relation 的约定；64 指串行关系中的输入数量，不是特征向量维数。早期等级与端口数使用过 `2^L` 的联系，但它与新架构传播层级的准确对应仍须正式定义。 + +**notation（亦讨论为 idea）：组织若干并列 relation。** + +```text +N = Notation{R1, R2, ..., Rk} +``` + +这为多种可能结构提供显式位置，避免把并列候选强行塞进 relation 的串行 port。 + +- 两者底层数据结构可以形态一致，但必须保留类型及不同的运算语义。 +- 并列候选不自动等于语义独立，也不自动等于互斥。互斥解释与可以同时成立的特征，需要不同的组合规则。 +- 候选评估、候选权重、候选混合是可区分的操作；不必一生成候选就压成一个 Vec。 + +### 3.2 双向分层与反复传播 + +用户提出的方向： + +```text +relation 方向：L0 → L1 → ... → L6 +notation 方向：N6 → N5 → ... → N0 +一次推理可能经历多轮上述传播。 +``` + +这是当前设计意图，尚不是完整的状态更新规范。L/N 等级的含义、跨级连接是否合法、上下两个方向如何连接，都需要进一步明确。 + +讨论中提出的一种解释（待落实）： + +- 上行：由局部证据评估更大的组合与解释。 +- 下行：较高层的解释、约束和关注影响局部候选的保留与活跃程度。 +- notation 的候选评估仍须汇集，才能为高层提供依据；“notation 向下传播”不能遗漏候选证据如何上传。 +- 区分来自输入的证据与来自上层的反馈，防止反馈被反复当成独立证据累加，产生自我强化。 + +### 3.3 统一机器字典与索引空间 + +用户决定让 token、relation、notation 位于同一个字典和索引空间。“字”本身也是一种原子理念。 + +- token 使用预存字典提供初始表示。 +- relation、notation 的实例“词义”通过节点计算特征向量得到。 +- 字典可以保存结构定义、参数及引用；运行时计算得到本次语境中的实例特征。 +- 预存 token 特征不必永久固定为运行时最终特征，可以接受上下文更新。 + +建议明确区分两层身份： + +| 层面 | 内容 | +| --- | --- | +| 字典条目 | token/关系/概念的定义、类型、结构与参数 | +| 运行时实例 | 引用哪个条目、出现在何处、连接哪些实例、本轮状态与得分 | + +同一个字在文本中出现两次，可以共享字典 ID，但不能因此共享同一份位置相关状态。可采用“实例 ID + 字典引用”的实现，不要求推翻统一字典。 + +## 4. 每轮输出：面向下一轮的规范形 + +用户提出：一次“小 node → 大 node → 小 node”的传播后，输出一种较工整、供下一轮继续处理的状态。 + +示例： + +```text +输入：吃饭了吗，您？ +初步规范表达：我，提问，你，吃饭，状态 +``` + +各状态同时具有特征向量，用于表达语气等细节。 + +这里建议把“规范形”理解为结构统一的语义中间表示，暂不要求第一轮得到唯一、最终正确的解释。 + +### 4.1 必须保留角色绑定 + +仅有上述扁平序列，不能明确区分“我问你是否吃饭”和“我问你我是否吃饭”。一个可参考的结构为： + +```text +提问 +├─ 发起者：当前说话者（来自会话上下文） +├─ 接收者：您所指的人 +└─ 内容：进食事件是否已经发生 + ├─ 行为者：您所指的人 + ├─ 行为：吃饭 + └─ 发生状态：待确认 +``` + +这是讨论中的建议示例，不是已决定的 IR 格式。 + +- “我”没有直接出现在原句，应记录其上下文来源。 +- 被询问的事件状态不能在规范化时变成已成立的事实。 +- 敬称、语气等细节可通过特征表示；关键角色关系应有明确结构。 +- 返回小节点时，单位可以是实体引用、行为、状态变量或关系实例，不必恢复成原始字符。 + +### 4.2 保留候选与来源 + +“询问进食情况”和“以此寒暄”可能是候选解释，也可能兼有。notation 应能保留候选身份、得分与结构，而不是强制提前选唯一结果。 + +建议保留输入片段或上下文依据，区分：显式内容、上下文补足、模型推断、待确认状态。 + +### 4.3 轮次与稳定性 + +可用以下关系描述一次传播，但具体更新函数尚未确定： + +```text +S(t) → 组合与解释评估 → H(t) → 角色绑定、反馈与整理 → S(t+1) +``` + +建议每轮读取上一轮状态并集中形成下一轮状态，避免执行顺序无意改变语义。最终应定义最大轮数、变化阈值或其他停止条件。 + +期望已稳定的表示在没有新证据时满足 `F(S*) ≈ S*`；这只是验证目标，不代表目前已证明收敛。 + +## 5. 构建、剪枝与训练：尚未完成的设计 + +用户明确指出，当前“葫芦形”传播还不够明确，需要剪枝和额外构建规则。 + +优先需要确定： + +1. relation 各 port 的角色、输入类型、顺序与允许的连接范围。 +2. notation 中哪些候选回答同一个解释问题，是否覆盖相同区间或具有相容的输出类型。 +3. 候选是互斥、可并存还是具有其他依赖；采用何种打分与归一化。 +4. 构建网络、选择路径与更新特征，哪些是离散决策，哪些可微、哪些需要额外训练机制。 +5. 如何限制候选数量，并测量剪枝造成的正确解释丢失。 +6. 如何防止重复证据、自我支持、循环振荡及无上限的关系扩张。 +7. 如何保留原始词序、实体身份、否定、疑问、范围及来源。 + +简单 regex/NFA 的布尔关系可以用向量、矩阵及相应的逻辑运算表达；转成浮点矩阵后，必须重新定义多路径的组合含义。布尔可达性、概率、模糊兼容度与特征混合不可直接互换。 + +目前没有得到“任意一组 NFA 能以常数总计算量执行”的结论。并行深度、总工作量、状态规模和存储量必须分别分析。 + +## 6. GPU 执行路线 + +### 6.1 语义粒度与执行粒度分离 + +讨论中形成的总体判断： + +- 常规稠密 Tr 从规则的大计算出发，再为硬件分块、融合与安排数据复用。 +- NGRA 从细粒度的关系与候选出发，再把已经就绪、形状相容的操作合并为批次。 + +不必让每个小 relation 对应一次 GPU 调用，也不必为批处理把语义上的多个 relation 合并成一个大 relation。 + +| 方法 | 收益与限制 | +| --- | --- | +| 批处理独立实例 | 可保持实例语义，同时提高单次执行规模 | +| 融合有依赖的操作 | 可减少启动及中间搬运；不会自动消除数据依赖 | +| 按层级、轮次、形状分组 | 可以简化调度；具体规则尚待验证 | +| 稳定的数据布局与参数分组 | 有机会提高复用并减少碎片化读取 | + +早期低阶、小 port 关系的串行活动密集，是重点瓶颈。把 L0/L1 放在 CPU、把短模式预筛编译成矩阵运算等，均只是候选方向，尚未通过基准测试决定。 + +### 6.2 按需加载模型内容 + +用户提出的潜在优势：根据激活情况，只加载本次需要的 relation/notation 参数，降低显存工作集。 + +成立条件与待验证因素： + +- 用廉价、常驻的索引或路由信息判断加载对象，避免必须先读完完整参数才能决定不用它。 +- 测量当前活跃集合、跨轮复用、加载粒度、预取能力及批次内激活集合的并集。 +- 同时核算参数、活跃状态、候选记录、运行缓冲与传输缓冲。 +- 省显存不自动等于提速；零碎且依赖刚完成计算的加载可能使 GPU 等待。 +- 区分 GPU 内部缓存未命中与数据不驻留显存、需要从主存迁入。 + +Tr 也能参数卸载、逐层加载或使用 MoE，故不能把“必须所有权重常驻显存”当作 Tr 的数学要求。NGRA 的优势仍需在质量相当时实测。 + +## 7. Tr 对照：已经澄清的关键概念 + +### 7.1 block、上下文与执行分块不是一回事 + +- Tr block 通常指深度方向的一层计算，不是固定 4096 口的文本单元。 +- 4096 是本讨论的示例长度，也可能是某个模型的上下文配置；不是 Tr 固有 I/O 宽度。 +- 同一层的投影权重作用于特征轴，通常不含序列长度 n 这个参数维度。 +- 支持长上下文还涉及位置机制、训练与资源，不能据此宣称任意短上下文模型可直接可靠处理 1M。 +- 彼此隔离的文本窗口不会自动互通。共享模型权重不等于保留上一段输入。 + +### 7.2 输入与输出 + +以 512 维、8 个头、每头 64 维的示例来说： + +```text +X: Mat_n_512 → 一个完整 Tr block → Y: Mat_n_512 +``` + +每个输出列仍对应一个位置，但携带上下文整合后的特征。输出通常不是显式 relation 表，也不是 vocabulary 概率。只有第一层直接接收 embedding 与初始位置表示，后续层接收前层输出。 + +token embedding 提供初始向量，不能把后续位置向量理解成始终只有原始 token 的静态词义。 + +### 7.3 本文列向量约定下的主要运算 + +| 运算 | 形状或说明 | +| --- | --- | +| `Q = WQ X`, `K = WK X`, `V = WV X` | W 为 `Mat_512_64`；Q/K/V 为 `Mat_n_64` | +| `S = QᵀK / sqrt(64) + M` | `Mat_n_n`；行是查询，列是候选 | +| `A = softmax_rows(S)` | 对每个查询的允许候选统一归一化 | +| `H = V Aᵀ` | `Mat_n_64`，按位置混合内容 | +| 8 个 H 沿特征轴拼接 | `Mat_n_512` | +| 输出投影、残差、Norm、FFN、残差、Norm | 最终仍为 `Mat_n_512`；此顺序是原论文 post-norm 示例 | + +参数 WQ/WK/WV 通常分层、分头拥有，并在所有位置复用。ONNX 的图级 initializer 存储位置不代表各层共享同一组参数。Q/K/V/A 是本次输入产生的中间结果，不是查表得到的固定权重。 + +### 7.4 混合不等于重排 + +- `W X`：特征投影，不移动 token 列。 +- `V Aᵀ`：根据输入计算权重，允许多个输出读取同一来源。 +- `X Π`：用置换矩阵重排位置。 +- 标准注意力不默认执行“聚类相关 token → 打包 → 分组路由”。若需要这种行为，须明确设计路由过程。 +- 两个局部组的输出可以按 `[B00h B01h]`、`[B00l B01l]` 重组后交给下一层。若 h/l 表示 token 子集，这属于改变窗口分组；若表示特征通道子集，则是另一种操作。 +- 重组本身不做语义融合，下游计算才融合；需要保留位置身份与因果约束。 + +### 7.5 1M 上下文与跨计算块通信 + +全局注意力可以逻辑上覆盖整个上下文，执行时按块读取 K/V。对于两个分块 A/B，跨块关系直接来自 `QAᵀ KB` 与 `QBᵀ KA`；因果掩码限制允许的方向。 + +一次处理 4096 个查询，不代表这些查询只能访问 4096 个候选。它们可以分批访问全部允许的 K/V，并维护统一的 softmax 累加状态。 + +不能对各候选块独立 softmax 后简单平均。FlashAttention 使用分块与在线归一化执行精确注意力；跨设备方案如 Ring Attention 可以传递 K/V 块完成跨分片交互。 + +这不要求先分配完整的巨大注意力矩阵再把它切开。 + +### 7.6 成本与能力边界 + +- 全局注意力的配对计算为 `O(n²d)`；序列长度翻倍，该部分工作量约四倍，不代表端到端推理时间必然四倍。 +- 固定窗口宽度 w 的局部注意力为 `O(nwd)`；固定层数的间接信息覆盖范围有限。 +- 使用 KV cache 生成一个新 token 时，该 token 对历史的注意力计算随历史长度线性增长，而非重新计算完整历史的所有配对。 +- KV cache 不会自动把上下文容量变成无限，且需要区分缓存、压缩记忆与原始历史可访问性。 +- 普通前馈网络与关闭随机操作的 Tr 都可确定性执行。“DNN”通常表示深度神经网络，不能用来与 Tr 作“确定/非确定”分类。 +- 普通网络通常训练权重与偏置，而非修改固定 ReLU 等激活函数本身。 +- 注意力混合不自动保留完整候选结构，也不等于显式逻辑关系或可解释的置信度。 + +## 8. 下一阶段建议产物（尚未执行） + +1. 明确统一字典条目与运行时实例的最小 schema。 +2. 明确 relation/notation 的输入输出类型、候选组合与角色绑定规则。 +3. 写出一个完整传播轮次的状态读取、写入与反馈语义。 +4. 用“吃饭了吗，您？”制作可逐节点检查的示例，包括歧义保留与下一轮稳定性。 +5. 定义候选数量上限、剪枝指标、最大轮数与停止条件。 +6. 先建立语义正确的参考执行，再比较逐节点、批处理、融合和按需加载的性能。 +7. 训练目标、路由学习方式、收敛性质以及相对 Tr 的性能优势均保持待验证状态。 + +## 9. 讨论中核对过的参考资料 + +- [Attention Is All You Need](https://arxiv.org/html/1706.03762v7)：架构、注意力、复杂度与参数作用。 +- [Neural Machine Translation by Jointly Learning to Align and Translate](https://arxiv.org/abs/1409.0473)：按当前需要读取输入位置的早期注意力设计。 +- [FlashAttention](https://arxiv.org/abs/2205.14135)：分块、IO 优化与精确注意力。 +- [Ring Attention](https://arxiv.org/abs/2310.01889)：跨设备传递 K/V 块与长上下文。 +- [Swin Transformer](https://arxiv.org/abs/2103.14030)：局部窗口之间通过改变划分建立联系的实例。 +- [Switch Transformers](https://arxiv.org/abs/2101.03961)：选择性激活专家参数。 +- [ZeRO-Inference](https://www.deepspeed.ai/2022/09/09/zero-inference.html)：参数卸载、逐层加载与预取。 +- [ONNX Concepts](https://onnx.ai/onnx/intro/concepts.html)：计算图、节点与 initializer。 + +上述资料作为对照，不意味着 NGRA 已被证明等价于其中某种架构。 From 044b15346307020dbde9a3874fdeb9bc5cd45c22 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Fri, 18 Sep 2026 18:28:09 +0800 Subject: [PATCH 7/7] add doc --- .../NGRA_TENSOR_MEMORY.md | 55 ++- .../regex_node_transaction.md | 374 ++++++++++++++++++ 2 files changed, 428 insertions(+), 1 deletion(-) diff --git a/src/Nitload.RegularAnalysis/NGRA_TENSOR_MEMORY.md b/src/Nitload.RegularAnalysis/NGRA_TENSOR_MEMORY.md index c2799dc7..9e621585 100644 --- a/src/Nitload.RegularAnalysis/NGRA_TENSOR_MEMORY.md +++ b/src/Nitload.RegularAnalysis/NGRA_TENSOR_MEMORY.md @@ -1,11 +1,13 @@ # NGRA-Tensor 设计讨论记忆 -更新日期：2026-09-17 +更新日期：2026-09-18 目标项目：`D:\platform-dev\src\Nitload.RegularAnalysis` 来源：关于正则分析、NFA、Transformer、GPU 执行与 NGRA 双算子设计的讨论。 > 本文件记录设计意图与讨论结果，不代表项目现有代码已经实现这些功能。本次仅检查了目标目录及祖先目录的 AGENTS.md，没有审计源码。下文区分用户已提出的设计决定、辅助建议与待验证问题。 +最新设计补充（2026-09-18，详见第 10 节）：NGRA 使用持久状态上的增量流输入/输出；以连接占用作为主要容量指标的候选方案；容量不足时丢弃低激活连接。**字典本身是结构的一部分，结构是记忆的一部分。** 字典与运行时实例的区分属于身份与生命周期管理，不表示字典位于记忆系统之外。 + ## 1. 项目目标与讨论方式 NGRA-Tensor 试图结合结构化关系、模糊特征识别、多候选解释及模式预测。出发点包含正则模式分析与 NFA，但不应预设必须逐节点模拟 NFA 的执行方式。 @@ -286,3 +288,54 @@ token embedding 提供初始向量，不能把后续位置向量理解成始终 - [ONNX Concepts](https://onnx.ai/onnx/intro/concepts.html)：计算图、节点与 initializer。 上述资料作为对照，不意味着 NGRA 已被证明等价于其中某种架构。 + +## 10. 2026-09-18：持久结构、增量流与连接容量 + +### 10.1 用户明确提出的设计方向 + +- 从正则引擎的流式增量输入思路演进：输入、输出都在上一次运行结果所保留的状态上继续进行。 +- 不以每次重新输入完整历史文本作为基本执行语义；历史对当前的影响由已保留的结构及其状态承接。 +- 容量很可能按连接数量或连接占用评估，而不是按 token 数量评估；具体度量尚未定稿。 +- 容量不足时，通过丢弃低激活连接释放容量。这是已提出的方向，评分、阈值、删除语义与正确性规则尚未确定。 +- 字典本身也是结构，结构本身也是记忆。记忆不只包含特征向量数值，还包括对象身份、连接、组合方式、候选与参数化定义。 + +### 10.2 建议的抽象接口（非已实现 API） + +```text +(S[t+1], ΔO[t]) = Step(S[t], ΔI[t]) +``` + +- `S[t]`：上次保留的结构、字典相关状态、实例特征、候选、待执行依赖及流位置等。 +- `ΔI[t]`：新增输入；是否包含撤回、修改和外部反馈，需进一步定义。 +- `ΔO[t]`：本次新增输出；若允许修订先前解释，需要明确输出更新或撤回协议。 +- `S[t+1]`：完成必要传播与容量管理后保留的状态。 + +输入流步 t 与一次输入触发的内部上下传播轮次应分开编号；一次增量输入可能触发多轮关系更新。 + +### 10.3 增量失效规则 + +Roslyn 增量管线与因果 Tr 的缓存复用提供了讨论类比：依赖没有改变的结果可以复用。但 NGRA 必须按自己的依赖图定义失效，而不能只检查原始文字是否变化。 + +- 下游输入、上层反馈、候选集合、相关字典结构或参数的变化，都可能使实例状态失效。 +- 新输入可能激活此前尚未选择的连接，因此只记录已经活跃的依赖可能不够；候选发现与再激活机制待设计。 +- 双向反馈可以扩大受影响区域，增量输入不保证只需要少量增量计算。 +- 纯函数结果复用、依赖版本与参数版本是实现候选，不是目前已经确定的数据结构。 + +### 10.4 连接预算与遗忘：需要进一步决定的内容 + +1. 预算计算活跃连接、全部保留连接、驻留显存的连接，还是三者分别统计？ +2. 不同连接携带的特征、参数、元数据大小可能不同；是否采用连接数量加权或真实字节数作为物理预算？ +3. “丢弃”表示暂停执行、移出显存、删除当前实例边，还是删除可持久化的字典结构？这些行为对记忆的影响不同。 +4. 当前激活低不必然意味着未来无用或结构可删除；角色绑定、串行 port、待完成模式等所需连接应有明确的完整性规则。 +5. 连接移除后，哪些实例需要重新评估？候选如何重新发现？还保留哪些历史依据？ +6. 遗忘顺序不必等同于时间顺序；连接重要性的估计方法和低激活持续时间阈值尚待实验。 + +这些是辅助建议与待决问题，不覆盖用户已提出的“容量不足时优先丢弃低激活连接”方向。 + +### 10.5 容量与智能的关系：待验证假设 + +用户提出“越智能的模型可能上下文更快耗尽”：在相同连接预算下，若模型为同一输入识别并保留更多关系、候选和依赖，容量可能更快用尽。 + +这不是已证明的必然关系。更好的抽象、共享、合并与剪枝也可能降低连接需求。后续应测量单位输入的新增连接、保留连接、复用率、候选召回与任务质量，而不是直接把智能程度等同于连接消耗。 + +补充参考：[Roslyn Incremental Generators Cookbook](https://github.com/dotnet/roslyn/blob/main/docs/features/incremental-generators.cookbook.md#pipeline-model-design)。该资料只支持增量管线类比，不证明 NGRA 的依赖追踪或遗忘方案正确。 diff --git a/src/Nitload.RegularAnalysis/regex_node_transaction.md b/src/Nitload.RegularAnalysis/regex_node_transaction.md index ab70da42..3314bdf8 100644 --- a/src/Nitload.RegularAnalysis/regex_node_transaction.md +++ b/src/Nitload.RegularAnalysis/regex_node_transaction.md @@ -14,11 +14,385 @@ left boarder-> match group start //push stack, add group index -> match group end //pop stack, set group end, build sub sequence +--- + +# NGRA / NGRA-Tensor 设计讨论纪要 + +记录日期：2026-09-18。本文压缩本次讨论，作为研究与设计记录；未检查现有实现是否符合这些描述。以下区分设计意图、候选实现和待验证问题，不将讨论建议视为已经确定或验证的架构。无歧义时以 `tr` 指 Transformer。 + +## 1. NGRA 的背景与当前重点 + +- NGRA 的基础方向是把正则从字符序列推广到一般符号及结构化属性序列。 +- 基本有限正则 DLR（Deterministic Limited Regex）：面向定长输入元素，重复具有上限，不采用断言及前后瞻，处理选择、重复、组合。 +- DLR-CP：加入 case 优先级，可通过后处理器或节点信息处理。case 排序与模糊匹配是不同问题。 +- 属性正则（讨论中称 ALR）：输入是含有限个属性的 Struct，通过属性布尔判定 λ 进行匹配。有限属性个数不自动意味着有限取值空间。 +- 编译器前端可采用增量管线：字符 → token → 属性处理 → 更高层序列/region → AST；通过递归提取“岛屿”region，再交给下级 matcher。 +- 当前重点是 NGRA-Tensor：以关联程度处理模糊序列，研究如何根据新输入建立、调整和组合 relation。模糊包括关键特征之间存在低关联内容，不仅是 case 优先级或字符编辑距离。 +- DFA 自发现与这一方向有联系，但布尔自动机学习不能未经修改就用于任意加权、上下文相关的关系网络。 + +## 2. 统一词汇与理念系统：核心设计意图 + +字、词、短语、语义理念在计算接口上平等，均可存在于统一字典中，并通过向量表示；序列允许混合不同粒度的节点。 + +示例： + +```text +原句：吃饭了么，您？ +基本词序列：吃饭，了，么，逗号，您，问号 +候选语义标准型：我，询问，你的，吃饭，状态 +``` + +标准型是设计示例，不是已确定的规范语法或唯一正确解释。理念的可读标签不必直接等同于自然语言词义。需要能表达角色、顺序、指代、时间、否定与上下文条件；不能把理念集合当作无结构的词袋。 + +应区分字典条目与本次出现的实例：相同词条可以在不同位置、事件和角色中出现。统一字典不要求把这些实例状态合并。 + +需要区分三件事： + +1. 词条/理念的长期向量和参数。 +2. 本次出现的上下文状态、绑定与激活程度。 +3. 本轮计算后创建的新实例，以及长期学习时登记的新词条。 + +创建下一波次实例不必扩充词典。将稳定、反复复用的组合固化为新理念，属于独立的结构学习问题。 + +## 3. relation 与 mixer 交替执行 + +用户澄清：relation 判定器与 mixer 都有很多；mixer 是分布式、可学习的局部计算单元，不是一个集中式全局裁决器。 + +```text +有序理念实例 + → 多个 relation 计算关系响应 + → 多个 mixer 选择、融合、抑制、重表示 + → 新的有序理念实例 + → 下一计算波次 +``` + +- mixer 可保留输入、融合成抽象节点、降低响应或使其不再进入下一波。 +- 关系响应除数值外必须有对象身份或结构对应关系；裸分数不能唯一确定输出含义。这些对应可以由参数及数据布局实现，不要求全部采用显式符号提案。 +- relation 可以独立作为计算结构，不必属于词汇表；这是可选架构方向，尚未最终确定。 +- 表面词可退出传播，功能由抽象节点接替。例如“吗”可转为疑问功能；但将“了吗”全部只压成疑问可能丢失“是否已发生”的区别。 +- 大量单元提供容量；训练目标仍须约束应保留哪些区别、合并哪些等价表达。 +- 确定的执行算法不等于正确或唯一的语义推理。可以确定地保留多个解释；不能通过强制选择来宣称消除了歧义。 + +之前讨论过“relation 输出带证据提案、mixer 做约束组装”的方案，它可作局部实现参考，但不是用户最终确认的集中式架构。 + +## 4. 波次调度、依赖与候选发现 + +既有思路：下游节点编译后知道依赖及调用约束，通过 GPU 加载并激活。动态架构下，需要补充运行时激活发现。 + +区分： + +- 依赖拉取：已经决定执行某个 relation，需要哪些输入与参数。 +- 激活发现：当前结果出现后，哪些下游 relation 值得成为候选。 +- 就绪判定：候选是否已绑定足够、相容的输入。 + +编译期可以保留 relation 模板、输入槽位、参数需求与先后约束；运行时决定实例绑定、候选组合与执行波次。反向依赖索引可通知受影响的候选，relation 自身的依赖描述负责取数，两者可以并存。 + +调度候选时需区分实例/region，避免跨语境错误组合；上游激活不代表下游立即执行。预算应同时约束活跃连接、候选展开和部分匹配状态，而不只限制最终执行数。 + +GPU 原型可按运算类型批处理任务数组，批量构造下一波任务；逻辑节点激活不必对应独立 kernel 启动。稀疏算法需要计入索引、去重、同步和随机访存成本。 + +## 5. 记忆与预测性淘汰 + +目标是持续更新语义状态，以活跃节点/连接控制工作规模，而不是一直保存全部原始 token。 + +三类操作应分开： + +1. 已由高层表示承载含义的中间状态：可以释放。 +2. 当前低激活但仍可能有用的信息：可移入持久记忆，未来召回。 +3. 永久删除：接受由此失去某些未来回答能力。 + +用户明确接受遗忘：不要求无限保留，缺少信息时可以承认无法确认。系统需避免以相似候选替代缺失事实；完全遗忘后未必知道自己曾知道，不宜总声称“我忘了”。 + +淘汰策略可以借鉴预测缓存，综合估计未来使用概率、遗漏损失、保留成本、召回成本与成功率。当前激活低不代表未来价值低。 + +持续抽象可能优于临时文本摘要，但摘要本身也可以承载抽象；关键区别是状态能否局部更新、按需召回、保存关系与来源。tr 也存在内部状态记忆/压缩变体，不能把文本摘要当成所有 tr 的唯一记忆方式。 + +## 6. 小型 tr 作为局部计算模块：候选方向 + +考虑由 NGRA 管理字典、记忆、候选和波次，小型注意力模块计算局部 relation/mix。多个实例可共享参数，通过任务向量和角色信息区分，不必每条关系配一套网络。 + +```text +当前状态 → 路由/召回候选 → 配对的 K/V → 局部计算 → 输出映射 → 下一波 +``` + +潜在强链接可提前组织候选，但不能仅允许历史强关系进入，否则新关系难以被发现。候选组织是独立难点，局部 attention 不会自动解决。 + +若每轮有 m 个包、每包 k 个实例、维度 d，局部稠密注意力部分约为 O(m k² d)；总成本还包括投影、前馈网络、路由、访存、包间通信和波次数。 + +必须明确局部 V 是输入证据，还是候选输出理念。前者混合后仍需输出映射；后者已把输出候选发现交给上游。 + +## 7. 已澄清的 Transformer 计算原理 + +标准自注意力并不把整个字典放入 V，也不要求上下文窗口大于词表。 + +设词表大小 N、上下文长度 L、隐藏维度 d： + +| 对象 | 形状/作用 | +|---|---| +| 词嵌入表 E | N × d，所有词条的基础表示 | +| 上下文状态 H | L × d，各位置的上下文表示 | +| Q、K、V | 由上下文状态投影；自注意力中行对应位置 | +| 注意力权重 A | 每头在数学上为 L × L，受掩码约束 | +| 最终预测状态 h | d 维，生成时通常使用最后位置 | +| 输出矩阵 W_out | N × d，覆盖全词表 | + +```text +A = softmax(Q Kᵀ / sqrt(d_k)) # 决定从各位置读取多少 +O = A V # 已经实际读取并混合信息 +多层注意力、前馈、残差等处理 → h +logits = W_out h + b # 对整个词表打分 +p = softmax(logits) # 下一 token 概率 +``` + +两个 softmax 作用不同：注意力 softmax 面向可访问的上下文位置；输出 softmax 面向词表。注意力不是仅安排“以后如何注意”，AV 当场执行读取。 + +上下文未出现的词仍在 W_out 中有对应行，所以可以获得分数。训练同时调整前面的网络与输出映射，使合适上下文提高正确输出的概率。输入、输出嵌入可以共享权重，但不意味着输出层等于注意力 V。 + +上下文是生成条件，不是随机种子；随机采样种子影响如何从分布选择。新 token 加入上下文后，后续步骤再利用它。 + +有限字典不意味着无限长度序列的集合有限；有限属性个数不意味着各属性取值有限。生成未出现符号的能力来自可学习输出映射，不依赖“窗口覆盖字典”或“符号空间封闭”的解释。 + +注意力权重是当前层/头的相对读取权重，不宜直接视为永久关系强度或事实可信度。模型能力也分布在前馈、投影等参数中。上下文状态、KV 缓存与注意力激活通常不是可训练参数。 + +注意力不会保证全面、充分、正确地理解全部联系。多个 block 通常具有各自参数，不能无条件视为同一计算的循环。 + +## 8. 训练、生成与事实约束 + +从“相关”到“可推导”需要关系类型、方向、角色和组合规则。强边不自动具有传递性；向量检索适合召回候选，不能独自证明结论。 + +用户的事实定义：训练/前文中被接受为真，并经逻辑检验支持的信息。实现时仍需区分前提来源、适用条件与推导；内部一致不等于外部真实。 + +生成不是理解过程的简单求逆。可探索：问题与记忆 → 回答标准型/内容计划 → 混合理念与词的逐步展开 → 自然语言。需要跟踪角色、时态、否定、覆盖与依据，避免风格润色新增事实。 + +内容规划、语义到文本生成、忠实性检查是可分开训练和评估的任务。这是候选方案，不要求必须使用符号规划器。回译标准型可检查一致性，但共享错误使其不能成为唯一验证。 + +训练至少涉及： + +- 表示与预测：下一节点、解释序列或其他明确任务。 +- 参数学习：relation/mixer 的匹配和组合行为。 +- 结构学习：新增、拆分、合并、删除关系及理念。 +- 路由学习：有限预算下召回有用候选，并为新关系保留发现机会。 +- 记忆学习：保留、休眠、淘汰与召回。 +- 反例训练：错配人物、日期、否定、确定程度以及缺乏依据的组合。 + +tr 的进步同时依赖数据质量与规模、容量、后训练、可验证奖励、推理计算和工具；不能将连贯性或事实性提升仅归因于 QKV 或语料量。流畅、自洽与事实正确需要分开评价。 + +## 9. 当前判断与待验证事项 + +NGRA-Tensor 的组成思想有研究基础，但完整架构尚未被证明能训练通用 LLM。也不能据此断言它只适合典型模式，或必然缺少长程理解与自然交互能力。 + +优先做小型闭环：输入 → relation/mixer → 输出理念/序列 → 损失 → 更新；验证未见组合泛化、候选遗漏、信息遗忘与实际效率。 + +关键开放问题： + +1. relation 与 mixer 的精确定义、参数共享和输出接口。 +2. 多个 mixer 输出的顺序、实例绑定、冲突及重复处理。 +3. 活跃集合外的新关系如何获得训练机会。 +4. 向量如何映射到已有理念，何时登记新词条。 +5. 离散激活/淘汰/结构变更的训练方式。 +6. 弱证据如何积累，休眠信息如何被后续线索唤起。 +7. 在同等质量下，端到端速度、显存和训练成本是否优于基线。 + +## 10. 讨论中相关文献 + +以下是可借鉴的组成部分，不表示这些系统已实现完整 NGRA-Tensor： + +- [Transformer 原论文](https://arxiv.org/html/1706.03762v7)：QKV、前馈网络与词表输出层。 +- [SoPa](https://aclanthology.org/P18-1028/)：可学习软序列模式；原始模式拓扑受限。 +- [Rational Recurrences](https://aclanthology.org/D18-1152/)：加权自动机与循环状态更新、语言建模。 +- [流式概率自动机学习](https://borjaballe.github.io/papers/preprint-bcg13.pdf)：增量状态发现与合并。 +- [TensorLog](https://arxiv.org/abs/1605.06523)、[Neural LP](https://arxiv.org/abs/1702.08367)：可微关系推理与规则学习。 +- [NBFNet](https://arxiv.org/abs/2106.06935)：路径上的关系组合和多路径汇总。 +- [RetoMaton](https://arxiv.org/abs/2201.12431)：通过自动机连接辅助语言模型检索。 +- [Step-by-Step](https://aclanthology.org/N19-1236/)：内容规划与语言实现分离。 +- [DataTuner](https://aclanthology.org/2020.coling-main.218/)：生成与语义忠实性重排序。 +- [Transformer-XL](https://aclanthology.org/P19-1285/)、[Compressive Transformer](https://arxiv.org/abs/1911.05507)：跨片段状态与内部记忆压缩。 +- [Mamba](https://arxiv.org/abs/2312.00752)、[RWKV](https://arxiv.org/abs/2305.13048)：其他可训练序列架构。 +- [FlashAttention](https://arxiv.org/abs/2205.14135)：精确注意力的访存优化，与减少候选关系不同。 +## 11. 自动机理论：从 DLR 到可学习关系计算 +本节整理讨论中的理论联系，并补齐理解公式所需的定义；不把这些形式化直接认作 NGRA 当前实现。 +### 11.1 有限语言、正则语言与有限状态不是同一个概念 +设有限字母表为 Σ，普通 DFA 的转移为 δ: Q × Σ → Q，识别语言 L ⊆ Σ*。DFA 状态有限，但识别的语言可以包含无限多个串，例如 a*。 +若 DLR 同时满足有限输入字母表、所有重复有上限，且没有额外引入无界递归，则允许的串长具有有限上限 B，语言也是有限集： +```text +L ⊆ Σ^{≤B} +|Σ^{≤B}| = Σ_{k=0}^{B} |Σ|^k +``` + +这不代表状态数或学习成本必然小。递归 region 是另一层机制：局部有限匹配器配合可增长的栈/递归结构后，整体不能不加说明地继续视为一个固定有限状态机。 + +属性匹配可把字母表推广为 Struct，并用谓词检查输入。有限个字段不意味着有限个可能的 Struct；若要归约到普通有限字母表，需要有限取值或可用的有限等价分类。UTF-16 code unit 是定长单元，但不等于每个 Unicode 字符或字素都只占一个单元。 + +原文件顶部的 lookaround 草稿被保留作历史内容；它与本次 DLR“不采用断言/前后瞻”的约束不应未经核实就合并为一份最终规范。 + +### 11.2 自动机自发现的关键是区分后续行为 + +对于语言 L，两个历史前缀 u、v 在所有后缀上具有相同接受行为时，可以属于同一右同余类： + +```text +u ~_L v ⇔ 对所有 z∈Σ*，uz∈L 当且仅当 vz∈L +``` + +这提供了“何时需要不同状态”的理论视角。样本学习只能观察有限后缀，因此是否合并还依赖样本覆盖、查询能力、统计假设或归纳偏好。 + +- RPNI 类方法：利用正反例，尝试合并前缀树状态，并避免与已知标签冲突。 +- ALERGIA/流式 PDFA 学习：比较后续概率行为的统计兼容性，决定区分或合并。 +- 从有限样本得到一个一致模型，不代表已经唯一恢复真实规则，也不代表所有未知组合都可靠。 + +对 NGRA 的启示：新序列不是必然要求新增 relation；需要判断现有结构能否表达新的行为，以及扩展结构是否改善泛化而非只记住样例。 + +### 11.3 加权计算需要定义路径组合与路径汇总 + +一个一般的加权路径计算可以示意为： + +```text +f(x) = ⊕_{π 为消费 x 的接受路径} [初始权重 ⊗ 路径各边权重 ⊗ 终止权重] +``` + +其中 ⊗ 表示同一路径中如何组合，⊕ 表示不同路径之间如何汇总。不同运算意味着不同语义： + +| 组合与汇总 | 含义示例 | +|---|---| +| AND / OR | 是否存在接受路径 | +| 乘法 / 求和 | 汇总路径权重；满足概率约束时可解释为概率计算 | +| 乘法 / max | 最强路径，常见于非负权重下的最佳路径计算 | +| 加法 / max | 累积路径得分，再选择最佳路径 | +| min / max | 一种模糊隶属度组合方式 | + +半环等代数条件使动态规划与路径汇总能够一致地组合，但任意设计的 relation/mixer 并不自动满足这些条件。数值在 [0,1] 内，也不自动是概率。 + +用户的“case 优先级加权”只是加权模型的一种用途；文献中的 weighted automaton 还包括路径得分、代价及概率。模糊匹配也不必只保留最高分路径。 + +### 11.4 实数加权自动机与谱学习 + +不含 ε 转移的实数加权自动机可写成： + +```text +f(x₁…x_T) = αᵀ A_{x₁} … A_{x_T} ω +H[u,v] = f(uv) # Hankel 矩阵 +``` + +在实数域的相应线性表示理论中，有限 Hankel 秩对应有限维加权自动机表示，最小表示维数由该秩刻画。谱学习通过有限子矩阵的估计和分解恢复模型，仍依赖数据与可识别性条件。 + +潜在维度不保证是可命名的语义关系，实数分解也不自动产生合法概率或隶属度。该理论不能直接移植到任意非线性、动态增删结构或任意 max/min 计算。 + +参考：[模糊正则到模糊自动机](https://arxiv.org/abs/1105.6190)、[实数加权自动机谱学习](https://cs.nyu.edu/~mohri/pub/swa.pdf)、[符号加权自动机查询学习](https://proceedings.mlr.press/v153/suzuki21a.html)。 + +## 12. Transformer 数学补充：为什么能输出上下文外的词 + +### 12.1 两次打分具有不同的候选范围 + +以列向量 h 表示最终位置状态，一个标准预测过程为： + +```text +输入 token ID → 词嵌入与位置信息 → 多个 block → 最终状态 h + +block 中： +Q = H W_Q，K = H W_K，V = H W_V +A = softmax(QKᵀ / √d_k + mask) # 沿允许的 key 位置归一化 +O = A V # 混合上下文信息 +随后还有输出投影、残差、归一化、前馈网络等；顺序取决于变体 + +词表输出： +s_i = w_iᵀ h + b_i # i 遍历整个词表 +p_i = exp(s_i) / Σ_j exp(s_j) +``` + +前一个 softmax 回答“读哪些位置、各读多少”；后一个回答“输出哪个词表符号”。标准 V 的行数由被读取序列长度决定，不由词表大小决定。交叉注意力也可读取另一段序列，但仍不能把它默认等同于全词表。 + +### 12.2 本次讨论中的数值例子 + +忽略偏置，设 h=[2,1,-1]，输出表有四行： + +| token | 输出权重 w_i | 点积得分 | +|---|---|---:| +| 我 | [0,1,0] | 1 | +| 吃饭 | [1,0,0] | 2 | +| 灯塔 | [1,1,-1] | 4 | +| 苹果 | [-1,0,1] | -3 | + +即使“灯塔”没有出现在输入里，它对应的输出参数行仍参与点积。此例仅说明计算过程，向量坐标不是人为规定的真实语义属性。 + +训练时，目标 token 为 y，交叉熵为： + +```text +loss = -log p_y +∂loss/∂s_i = p_i - 1[i=y] # 标准全词表 softmax 下 +``` + +因此目标输出与其他输出的得分共同参与训练，梯度也会通过 h 传回前面的网络。模型能输出未在当前上下文出现的词，来自这个训练过的映射，不需要先将目标词放进注意力 V。 + +### 12.3 条件、参数与随机采样 + +```text +Pθ(x₁…x_T) = ∏_t Pθ(x_t | x_{N。 +- 数学上的注意力得分矩阵为每头 L×L；高效实现不一定完整物化它。 +- 稠密注意力的主要位置交互计算约为 O(L²d)；投影、前馈网络及全词表输出另有成本。 +- 给单个位置做标准全词表输出投影约为 O(Nd)，不因输入里只有少数词而自动缩小。 +- 增长的 KV 缓存等是运行时状态，不等于新增训练参数。 + +对 NGRA：输入候选路由与输出理念选择是两个问题。小型 mixer 产生 h 后仍可映射到大字典；若也要避免全字典输出计算，需增加输出候选检索/路由，而非要求字典都进入局部 V。 + +## 13. 数学与研究判断的边界 + +### 13.1 必须分别验证的性质 + +| 性质 | 问题 | +|---|---| +| 表达能力 | 是否存在能表达目标行为的参数/结构？ | +| 可学习性 | 给定数据与反馈，能否找到这种结构？ | +| 执行确定性 | 固定输入、状态及选择规则，是否输出一致？ | +| 逻辑可靠性 | 结论是否由有效规则和适用前提支持？ | +| 唯一解释 | 信息是否足以排除其他候选？ | +| 工程效率 | 把路由、访存、同步算进去后是否实际更快？ | + +不能从“确定地取最大值”推出语义无歧义，也不能从“存在理论表示”推出可高效训练。减少活跃张量不自动减少全局参数，更不自动提高 GPU 吞吐。 + +### 13.2 关系强度不等于关系组合规则 + +“位于某城市”与“城市位于某国家”可在适当语义下组合；“喜欢”不能因两条边都强就自动传递。需要保留关系类型、方向、角色和条件。不同证据路径还可能重复使用同一事实，简单累加可能制造虚假支持。 + +同时，长距离不等于弱关系；当前低分不等于未来无用；稀疏不等于必然缺少长程能力。潜在难点是低显著度线索被过早删除，或语义压缩丢失后来需要的区别。 + +### 13.3 联合学习与预测淘汰的候选形式 + +讨论中的结构/参数学习可概括为研究目标： + +```text +min_{G,θ} Σ_(x,y) loss(f_{G,θ}(x),y) + λ C(G) +``` + +G 为关系结构，θ 为数值参数，C(G) 惩罚结构复杂度。这只是问题定义，不提供自动增删关系或反向传播穿过离散操作的算法。 + +预测淘汰可用启发式价值表达： + +```text +保留价值 ≈ 未来使用概率 × 遗忘损失 − 保留成本 +``` + +实际还应计入召回成功率、召回成本和时间范围；这些量需要学习或明确估计，不能把当前激活值直接代入所有项。 + +### 13.4 关于“封闭”的最终澄清 + +固定有限词表 D 的长度至多 L 的序列集合有限；D* 在非空字表、长度无上限时无限。有限维实向量空间在数学上无限，机器有限精度不使实际学习与枚举问题自动变简单。 + +输入和输出共享词表，只说明可在相同符号系统中表达，不证明推理正确，也不构成预测未出现符号的充分解释。真正需要明确的是输入状态如何经可学习函数映射为输出候选及其得分。