From 5a294461e8e07def6536ecc07271c1aef5b0ef6d Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Tue, 3 Nov 2026 23:16:26 +0800 Subject: [PATCH] fix(copilot): make the step budget real, configurable and visible The graph passed remaining_steps=20 (50 for injection) in the run inputs. RemainingSteps is a LangGraph managed value derived from recursion_limit, so the input was ignored and the real budget was 25 (100 for injection). No one could see those numbers, and a turn that hit the cap ended on a bare tool result with no explanation. - Remove the dead remaining_steps input. - Add resolve_recursion_limit(): max_steps from the LLM model config wins; otherwise 25 teaching_assistant, 50 lab_automation_assistant, 100 troubleshooting_injection. - When fewer than 4 steps remain after a tool run, route to the new budget_wrapup node. The next llm_call binds no tools and gets a wrap-up instruction, so the turn ends with a short summary of what was done and what remains. - should_continue skips the title call after a wrap-up (no step left). - budget_exhausted is reset at the start of every user turn. - Add max_steps to the LLM config schemas. The config is a JSON column, so no migration is needed. - Tests: 16 new tests cover the resolver, the routing, the tool-free wrap-up call and the removed input. Docs updated. --- docs/gns3-copilot/implemented/chat-api.md | 7 +- .../implemented/llm-model-configs.md | 4 +- .../agent/gns3_copilot/agent/gns3_copilot.py | 118 ++++++++++++++--- .../agent/gns3_copilot/agent_service.py | 30 ++--- .../schemas/controller/llm_model_configs.py | 40 ++++++ tests/agent/test_step_budget.py | 120 ++++++++++++++++++ 6 files changed, 282 insertions(+), 37 deletions(-) create mode 100644 tests/agent/test_step_budget.py diff --git a/docs/gns3-copilot/implemented/chat-api.md b/docs/gns3-copilot/implemented/chat-api.md index 104357677..a3c3e2d76 100644 --- a/docs/gns3-copilot/implemented/chat-api.md +++ b/docs/gns3-copilot/implemented/chat-api.md @@ -821,15 +821,16 @@ OpenAI-compatible message model. 3. First interaction without title → `title_generator_node` 4. Otherwise → `END` -- **recursion_limit_continue** (after `tool_node`): Routes to `llm_call` or `END` based on: +- **recursion_limit_continue** (after `tool_node`): Routes to `llm_call`, `budget_wrapup` or `END` based on: 1. Check abort flag → `END` - 2. Remaining steps < 4 → `END` (prevent infinite loops) + 2. Remaining steps < 4 → `budget_wrapup` (one final summary call, no tools) 3. Otherwise → `llm_call` **State** (`MessagesState`): - `messages`: Conversation messages (cumulative with `operator.add`) - `llm_calls`: LLM invocation counter -- `remaining_steps`: Recursion depth tracker (initial: 20) +- `remaining_steps`: LangGraph managed value (`recursion_limit - step`); the limit is the per-mode default or `max_steps` from the LLM config. Passing it in inputs has no effect +- `budget_exhausted`: Set by `budget_wrapup`, reset at the start of every user turn - `conversation_title`: Auto-generated title - `topology_info`: GNS3 project topology data - `session_id`: Session identifier (for abort tracking) diff --git a/docs/gns3-copilot/implemented/llm-model-configs.md b/docs/gns3-copilot/implemented/llm-model-configs.md index 5e6bc580a..15d02f98e 100644 --- a/docs/gns3-copilot/implemented/llm-model-configs.md +++ b/docs/gns3-copilot/implemented/llm-model-configs.md @@ -225,6 +225,7 @@ For a complete list of LangChain-supported providers, see: https://python.langch | `max_tokens` | integer | **⚠️ Reserved for future use, not currently implemented** | | `context_strategy` | string | Context trimming strategy: "conservative" (60%), "balanced" (75%), "aggressive" (85%). Default: "balanced" | | `copilot_mode` | string | GNS3-Copilot mode: "teaching_assistant" (diagnostics only, default) or "lab_automation_assistant" (full configuration access) | +| `max_steps` | integer | Step budget of one copilot turn, as the LangGraph recursion limit (1 LLM call + 1 tool run = 2 steps). Defaults: 25 teaching_assistant, 50 lab_automation_assistant, 100 troubleshooting_injection | | `is_default` | boolean | Set as default (default: false) | **Important Notes:** @@ -253,6 +254,7 @@ For a complete list of LangChain-supported providers, see: https://python.langch | `context_strategy` | string (optional) | Context trimming strategy | | `is_default` | boolean (optional) | Default flag | | `copilot_mode` | string (optional) | GNS3-Copilot mode: "teaching_assistant" or "lab_automation_assistant" | +| `max_steps` | integer (optional) | Step budget of one copilot turn (LangGraph recursion limit). Leave empty for the per-mode default | | `expected_version` | integer (optional) | **Optimistic locking version** | **Note:** When using `expected_version`, the API will verify the version hasn't changed since you read the data. If it has, you'll receive a 409 Conflict error. @@ -261,7 +263,7 @@ For a complete list of LangChain-supported providers, see: https://python.langch - `max_tokens`: Reserved for future implementation. Currently not used by the system. The maximum output tokens are controlled automatically by the LLM provider based on the model and input size. **Update Limitations:** -- `context_strategy` and `copilot_mode` can be set to new values but **cannot be cleared to null** via update. This is because the API filters out null values before processing. +- `context_strategy`, `copilot_mode` and `max_steps` can be set to new values but **cannot be cleared to null** via update. This is because the API filters out null values before processing. ### LLMModelConfigResponse diff --git a/gns3server/agent/gns3_copilot/agent/gns3_copilot.py b/gns3server/agent/gns3_copilot/agent/gns3_copilot.py index ac65101b6..aa610f742 100644 --- a/gns3server/agent/gns3_copilot/agent/gns3_copilot.py +++ b/gns3server/agent/gns3_copilot/agent/gns3_copilot.py @@ -64,6 +64,7 @@ from gns3server.agent.gns3_copilot.agent.context_manager import ( create_pre_model_hook, ) from gns3server.agent.gns3_copilot.agent.model_factory import ( + create_base_model, create_base_model_with_tools, create_title_model, ) @@ -187,6 +188,44 @@ TROUBLESHOOTING_INJECTION_MODE_TOOLS = [ # Default tools (legacy support - will be overridden by mode-specific tools) tools = LAB_AUTOMATION_ASSISTANT_MODE_TOOLS +# LangGraph recursion limit per copilot mode: the hard cap on super-steps +# per user turn. One LLM call plus one tool execution costs two steps, so +# 25 is about 11 tool rounds; lab mode needs more room (topology build, +# configure, verify), fault injection more still. +MODE_RECURSION_LIMITS = { + "teaching_assistant": 25, + "lab_automation_assistant": 50, + "troubleshooting_injection": 100, +} +DEFAULT_RECURSION_LIMIT = 25 + + +def resolve_recursion_limit(llm_config: dict | None) -> int: + """Resolve the LangGraph recursion limit for one user turn. + + ``max_steps`` in the LLM model config wins when set to a positive + integer; otherwise the per-mode default applies. The value is a hard + safety cap, not a quality target: without it a tool loop or a routing + bug can spin forever and burn tokens. + """ + config = llm_config or {} + mode = (config.get("copilot_mode") or "teaching_assistant").lower() + default = MODE_RECURSION_LIMITS.get(mode, DEFAULT_RECURSION_LIMIT) + try: + configured = int(config.get("max_steps")) + except (TypeError, ValueError): + return default + return configured if configured > 0 else default + + +# Injected before the final LLM call when the step budget runs out. The +# model gets no tools in that call, so it can only produce text. +BUDGET_WRAPUP_INSTRUCTION = ( + "The tool-step budget for this turn is exhausted. Do not call any tools. " + "Reply with a short summary: what you completed, what is still unfinished, " + "and what the user should do next (for example, ask to continue)." +) + # Create combined tool lookup for tool_node (supports all modes) # tool_node will receive tool calls based on mode-specific tools bound to the # model. The registry must cover every mode: a tool bound to the model but @@ -260,7 +299,11 @@ class MessagesState(TypedDict): operator.add llm_calls: Counter for tracking the number of LLM invocations remaining_steps: Is automatically managed by LangGraph's RemainingSteps - to track and limit recursion depth. + to track and limit recursion depth. The value comes from + the recursion_limit in the run config, not from inputs. + budget_exhausted: True once the step budget ran out and the graph + switched to the final summary call (no tools bound). + Reset to False at the start of every user turn. conversation_title: Optional conversation title for session identification and management topology_info: Dictionary containing GNS3 project topology information @@ -274,6 +317,9 @@ class MessagesState(TypedDict): remaining_steps: RemainingSteps + # True when the step budget is used up and the turn ends with a summary + budget_exhausted: bool + # Optional conversation title conversation_title: str | None @@ -384,15 +430,26 @@ def llm_call(state: dict, config: RunnableConfig | None = None): # token estimation ) - # Create fresh model with tools for each LLM call - logger.debug( - "Creating model with tools: provider=%s, model=%s, mode=%s, tools=%d", - llm_config.get("provider"), - llm_config.get("model"), - copilot_mode, - len(mode_tools), - ) - model_with_tools = create_base_model_with_tools(mode_tools, llm_config=llm_config) + # When the step budget ran out, this is the final summary call: no tools + # are bound, so the model can only answer in text, and an instruction + # tells it to wrap up instead of planning more work. + budget_exhausted = bool(state.get("budget_exhausted")) + if budget_exhausted: + logger.warning( + "Step budget exhausted; making the final summary call without tools (provider=%s, model=%s)", + llm_config.get("provider"), + llm_config.get("model"), + ) + model_to_call = create_base_model(llm_config) + else: + logger.debug( + "Creating model with tools: provider=%s, model=%s, mode=%s, tools=%d", + llm_config.get("provider"), + llm_config.get("model"), + copilot_mode, + len(mode_tools), + ) + model_to_call = create_base_model_with_tools(mode_tools, llm_config=llm_config) # Call pre_hook directly to prepare messages (topology injection + # trimming) @@ -403,8 +460,11 @@ def llm_call(state: dict, config: RunnableConfig | None = None): prepared_messages = prepared_state["messages"] logger.info("Messages prepared: %d → %d", len(messages), len(prepared_messages)) + if budget_exhausted: + prepared_messages = [*prepared_messages, SystemMessage(content=BUDGET_WRAPUP_INSTRUCTION)] + # Invoke model with prepared messages - response = model_with_tools.invoke(prepared_messages) + response = model_to_call.invoke(prepared_messages) # Add metadata with created_at timestamp to AI response if hasattr(response, "metadata"): @@ -640,6 +700,11 @@ def should_continue( if last_message.tool_calls: return "tool_node" + # Final summary after the step budget ran out: end the turn, skip the + # title call (it would need one more super-step we may not have) + if state.get("budget_exhausted"): + return END + # First full interaction completed and title not yet generated if current_title in [None, "New Conversation"]: return "title_generator_node" @@ -648,23 +713,41 @@ def should_continue( return END +def budget_wrapup(state: MessagesState) -> dict: + """Switch the graph to the final summary call after the step budget ran out. + + The next llm_call binds no tools and appends BUDGET_WRAPUP_INSTRUCTION, + so the turn ends with a text answer that tells the user where the work + stopped instead of ending silently on a tool result. + """ + logger.warning( + "Step budget nearly exhausted (remaining_steps=%s); switching to the summary call", + state.get("remaining_steps"), + ) + return {"budget_exhausted": True, "session_id": state.get("session_id")} + + # Routing logic after the tool node, Check remaining_steps -def recursion_limit_continue(state: MessagesState) -> Literal["llm_call", END]: +def recursion_limit_continue(state: MessagesState) -> Literal["llm_call", "budget_wrapup", END]: """ Routing logic after tool execution to prevent infinite recursion. - Determines whether to continue with another LLM call or end the - conversation based on remaining steps and message type. + Determines whether to continue with another LLM call, switch to the + final summary call, or end the conversation based on remaining steps and + message type. Args: state: Current conversation state with messages and remaining steps Returns: - "llm_call" to continue processing, END to terminate conversation + "llm_call" to continue processing, "budget_wrapup" for the final + summary call, END to terminate the conversation. Logic: - If abort flag is set → end the conversation - If the last message is ToolMessage and steps >= 4: continue to LLM + - If the last message is ToolMessage and steps < 4: wrap up with one + final summary call (no tools bound) - Otherwise: end the conversation to prevent infinite loops """ # Check abort flag first @@ -675,7 +758,7 @@ def recursion_limit_continue(state: MessagesState) -> Literal["llm_call", END]: last_message = state["messages"][-1] if isinstance(last_message, ToolMessage): if state["remaining_steps"] < 4: - return END + return "budget_wrapup" return "llm_call" return END @@ -688,6 +771,7 @@ agent_builder = StateGraph(MessagesState) # Add nodes agent_builder.add_node("llm_call", llm_call) agent_builder.add_node("tool_node", tool_node) +agent_builder.add_node("budget_wrapup", budget_wrapup) agent_builder.add_node("title_generator_node", generate_title) agent_builder.add_node("abort_handler_node", abort_handler_node) @@ -716,9 +800,11 @@ agent_builder.add_conditional_edges( { "llm_call": "llm_call", # Continue to LLM if tools executed and steps # remain + "budget_wrapup": "budget_wrapup", # Final summary call, no tools END: END, # End conversation to prevent infinite loops }, ) +agent_builder.add_edge("budget_wrapup", "llm_call") agent_builder.add_edge("title_generator_node", END) agent_builder.add_edge("abort_handler_node", END) diff --git a/gns3server/agent/gns3_copilot/agent_service.py b/gns3server/agent/gns3_copilot/agent_service.py index 5c58cd4e1..483e55f01 100644 --- a/gns3server/agent/gns3_copilot/agent_service.py +++ b/gns3server/agent/gns3_copilot/agent_service.py @@ -43,7 +43,10 @@ import aiosqlite from langchain_core.messages import HumanMessage from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver -from gns3server.agent.gns3_copilot.agent.gns3_copilot import agent_builder +from gns3server.agent.gns3_copilot.agent.gns3_copilot import ( + agent_builder, + resolve_recursion_limit, +) from gns3server.agent.gns3_copilot.chat_sessions_repository import ( ChatSessionsRepository, ) @@ -282,13 +285,13 @@ class AgentService: ) # Build config - only thread-safe identifiers - # Determine recursion_limit based on copilot_mode + # The recursion limit is the step budget of one user turn: the + # per-mode default, or max_steps from the LLM model config when set. + # RemainingSteps is a managed value derived from this limit; passing + # it in inputs has no effect. copilot_mode = (llm_config or {}).get("copilot_mode", "teaching_assistant").lower() - if copilot_mode == "troubleshooting_injection": - recursion_limit = 100 # Need more recursion depth for fault injection workflow - log.debug("Using extended recursion_limit for troubleshooting_injection mode: 100") - else: - recursion_limit = 25 # Default recursion limit + recursion_limit = resolve_recursion_limit(llm_config) + log.debug("Using recursion_limit %d for copilot_mode %s", recursion_limit, copilot_mode) config = { "configurable": { @@ -302,15 +305,8 @@ class AgentService: "recursion_limit": recursion_limit, } - # Build inputs - # Determine remaining_steps based on copilot_mode - # Troubleshooting injection mode needs more steps for multi-tool workflow - if copilot_mode == "troubleshooting_injection": - remaining_steps = 50 # Need more steps for fault injection workflow - log.debug("Using extended remaining_steps for troubleshooting_injection mode: 50") - else: - remaining_steps = 20 # Default for other modes - + # Build inputs. budget_exhausted is reset per user turn: the + # checkpoint keeps it from a previous turn that ended on a summary. inputs = { "messages": [ HumanMessage( @@ -320,7 +316,7 @@ class AgentService: ) ], "llm_calls": 0, - "remaining_steps": remaining_steps, + "budget_exhausted": False, "session_id": session_id, "abort": False, } diff --git a/gns3server/schemas/controller/llm_model_configs.py b/gns3server/schemas/controller/llm_model_configs.py index 78381d6d8..80dacbc98 100644 --- a/gns3server/schemas/controller/llm_model_configs.py +++ b/gns3server/schemas/controller/llm_model_configs.py @@ -53,6 +53,16 @@ class LLMModelConfigData(BaseModel): copilot_mode: Optional[str] = Field( None, description="GNS3-Copilot mode: 'teaching_assistant' or 'lab_automation_assistant'" ) + max_steps: Optional[int] = Field( + None, + gt=0, + le=500, + description=( + "Max LangGraph super-steps per copilot turn " + "(1 LLM call + 1 tool run = 2 steps). Defaults: 25 teaching_assistant, " + "50 lab_automation_assistant, 100 troubleshooting_injection" + ), + ) # Allow extra fields for extensibility # Ensure all fields are included in serialization, even if None @@ -82,6 +92,16 @@ class LLMModelConfigCreate(BaseModel): copilot_mode: Optional[str] = Field( None, description="GNS3-Copilot mode: 'teaching_assistant' or 'lab_automation_assistant'" ) + max_steps: Optional[int] = Field( + None, + gt=0, + le=500, + description=( + "Max LangGraph super-steps per copilot turn " + "(1 LLM call + 1 tool run = 2 steps). Defaults: 25 teaching_assistant, " + "50 lab_automation_assistant, 100 troubleshooting_injection" + ), + ) # Allow extra config fields model_config = ConfigDict(extra="allow") @@ -112,6 +132,16 @@ class LLMModelConfigUpdate(BaseModel): copilot_mode: Optional[str] = Field( None, description="GNS3-Copilot mode: 'teaching_assistant' or 'lab_automation_assistant'" ) + max_steps: Optional[int] = Field( + None, + gt=0, + le=500, + description=( + "Max LangGraph super-steps per copilot turn " + "(1 LLM call + 1 tool run = 2 steps). Defaults: 25 teaching_assistant, " + "50 lab_automation_assistant, 100 troubleshooting_injection" + ), + ) # Allow extra config fields model_config = ConfigDict(extra="allow") @@ -149,6 +179,16 @@ class LLMModelConfigDataWithoutSecret(BaseModel): copilot_mode: Optional[str] = Field( None, description="GNS3-Copilot mode: 'teaching_assistant' or 'lab_automation_assistant'" ) + max_steps: Optional[int] = Field( + None, + gt=0, + le=500, + description=( + "Max LangGraph super-steps per copilot turn " + "(1 LLM call + 1 tool run = 2 steps). Defaults: 25 teaching_assistant, " + "50 lab_automation_assistant, 100 troubleshooting_injection" + ), + ) # Allow extra fields for extensibility model_config = ConfigDict(extra="allow", populate_by_name=True) diff --git a/tests/agent/test_step_budget.py b/tests/agent/test_step_budget.py new file mode 100644 index 000000000..43fe02587 --- /dev/null +++ b/tests/agent/test_step_budget.py @@ -0,0 +1,120 @@ +""" +Tests for the copilot step budget. + +The budget is the LangGraph recursion limit of one user turn. It defaults +per copilot mode and can be overridden with max_steps in the LLM model +config. When the budget runs out the graph makes one final summary call +without tools instead of ending silently on a tool result. +""" + +import pytest +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage + +from gns3server.agent.gns3_copilot.agent import gns3_copilot as mod + + +class TestResolveRecursionLimit: + def test_mode_defaults(self): + assert mod.resolve_recursion_limit(None) == 25 + assert mod.resolve_recursion_limit({}) == 25 + assert mod.resolve_recursion_limit({"copilot_mode": "teaching_assistant"}) == 25 + assert mod.resolve_recursion_limit({"copilot_mode": "lab_automation_assistant"}) == 50 + assert mod.resolve_recursion_limit({"copilot_mode": "troubleshooting_injection"}) == 100 + + def test_unknown_mode_falls_back(self): + assert mod.resolve_recursion_limit({"copilot_mode": "something_new"}) == 25 + + def test_max_steps_overrides_the_mode_default(self): + assert mod.resolve_recursion_limit({"copilot_mode": "lab_automation_assistant", "max_steps": 40}) == 40 + assert mod.resolve_recursion_limit({"copilot_mode": "teaching_assistant", "max_steps": "60"}) == 60 + + @pytest.mark.parametrize("value", [0, -5, "abc", None, [1]]) + def test_invalid_max_steps_falls_back_to_the_default(self, value): + assert mod.resolve_recursion_limit({"copilot_mode": "lab_automation_assistant", "max_steps": value}) == 50 + + +def _tool_message() -> ToolMessage: + return ToolMessage(content='{"ok": true}', tool_call_id="call-1") + + +class TestRecursionLimitContinue: + def test_tools_done_with_budget_left_continue(self): + state = {"messages": [_tool_message()], "remaining_steps": 5, "session_id": None} + assert mod.recursion_limit_continue(state) == "llm_call" + + def test_tools_done_with_low_budget_wrap_up(self): + state = {"messages": [_tool_message()], "remaining_steps": 3, "session_id": None} + assert mod.recursion_limit_continue(state) == "budget_wrapup" + + def test_non_tool_message_ends(self): + state = {"messages": [AIMessage(content="done")], "remaining_steps": 5, "session_id": None} + assert mod.recursion_limit_continue(state) == mod.END + + def test_abort_ends(self): + mod.set_abort_flag("abort-session") + try: + state = {"messages": [_tool_message()], "remaining_steps": 5, "session_id": "abort-session"} + assert mod.recursion_limit_continue(state) == mod.END + finally: + mod.clear_abort_flag("abort-session") + + +class TestBudgetWrapup: + def test_wrapup_sets_the_flag(self): + out = mod.budget_wrapup({"messages": [_tool_message()], "remaining_steps": 3, "session_id": "s1"}) + assert out["budget_exhausted"] is True + assert out["session_id"] == "s1" + + def test_wrapup_call_binds_no_tools_and_injects_the_instruction(self, monkeypatch): + captured = {} + + class FakeModel: + def invoke(self, messages): + captured["messages"] = messages + return AIMessage(content="Summary: stopped before verification.") + + monkeypatch.setattr( + mod, + "get_current_llm_config", + lambda: {"provider": "fake", "model": "fake", "copilot_mode": "lab_automation_assistant"}, + ) + monkeypatch.setattr(mod, "load_system_prompt", lambda config: "system prompt") + monkeypatch.setattr(mod, "create_pre_model_hook", lambda **kwargs: lambda state: state) + monkeypatch.setattr(mod, "create_base_model", lambda config: FakeModel()) + + def fail_if_tools_bound(*args, **kwargs): + raise AssertionError("tools must not be bound for the wrap-up call") + + monkeypatch.setattr(mod, "create_base_model_with_tools", fail_if_tools_bound) + + state = { + "messages": [HumanMessage(content="build the lab")], + "budget_exhausted": True, + "llm_calls": 3, + "session_id": "s1", + } + out = mod.llm_call(state) + + assert out["messages"][0].content == "Summary: stopped before verification." + assert out["llm_calls"] == 4 + assert captured["messages"][-1].content == mod.BUDGET_WRAPUP_INSTRUCTION + + def test_should_continue_ends_without_title_after_wrapup(self): + state = { + "messages": [AIMessage(content="Summary")], + "budget_exhausted": True, + "conversation_title": None, + "session_id": None, + } + assert mod.should_continue(state) == mod.END + + +def test_stream_inputs_do_not_carry_the_dead_remaining_steps(): + """RemainingSteps is a managed value: an input for it is ignored. + + Passing it looked like a second budget knob but never did anything. + """ + from pathlib import Path + + source = Path(mod.__file__).parent.parent / "agent_service.py" + assert '"remaining_steps"' not in source.read_text()