From fd7594f62e51cc70b366b2a1a75b8340b5fcb238 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Tue, 18 Aug 2026 01:36:52 +0800 Subject: [PATCH] docker: give the docker_exec console a tall default PTY geometry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The docker_exec console resized its exec PTY to 80x24 until a client sent NAWS. CLIs that page on the PTY window size instead of the terminal length (the IOS-XR pager) therefore parked long output at --More-- for clients that never negotiate NAWS — netmiko, bare telnet — making copilot device commands time out on XRd. Default the exec to 511x10000 instead (511 matches netmiko's own 'terminal width 511' convention): no paging and no hard wrapping for non-NAWS clients, while real NAWS clients keep resizing to their actual geometry as before. Also updates the project memory record with the confirmed root cause and the fix. --- .claude/memory/MEMORY.md | 2 +- .claude/memory/xrd-console-more-pager-bug.md | 77 ++++++++----------- gns3server/compute/docker/vendor_docker_vm.py | 9 ++- tests/compute/docker/test_vendor_docker_vm.py | 11 +++ 4 files changed, 52 insertions(+), 47 deletions(-) diff --git a/.claude/memory/MEMORY.md b/.claude/memory/MEMORY.md index a9b9e74f2..2b757063e 100644 --- a/.claude/memory/MEMORY.md +++ b/.claude/memory/MEMORY.md @@ -30,7 +30,7 @@ - **[Docker Container Stop Delay](./docker-container-stop-delay.md)** - Some containers take ~5s to stop because they don't handle SIGTERM (AlpiNet, OstinatoWireshark) ### Device Console / Copilot Known Bugs -- **[XRd Console --More-- Pager Bug](./xrd-console-more-pager-bug.md)** - XRd console (docker exec PTY, 24 rows) pager bypasses `terminal length 0`, long-output commands consistently hit netmiko ReadTimeout; `| no-more` does not exist on XR; agreed fix design: copilot tail-anchored `--More--` auto-answer + reconnect before retry + session_log (designed, not yet implemented) +- **[XRd Console --More-- Pager Bug](./xrd-console-more-pager-bug.md)** - FIXED: root cause was our own 80x24 initial PTY geometry for the docker_exec console (the XR pager reads PTY rows, not `terminal length`); initial geometry is now 511x10000. Copilot reconnect-before-retry + session_log still open; `--More--` auto-answer kept as fallback design ### MCP Service - **[MCP Service Design](./mcp-service-design.md)** - MCP (Model Context Protocol) service architecture using FastMCP with SSE transport, JWT auth, 29 tools across 5 domains diff --git a/.claude/memory/xrd-console-more-pager-bug.md b/.claude/memory/xrd-console-more-pager-bug.md index 1a7e2e300..d796d3a4a 100644 --- a/.claude/memory/xrd-console-more-pager-bug.md +++ b/.claude/memory/xrd-console-more-pager-bug.md @@ -1,71 +1,62 @@ -# XRd Console --More-- Pager Bug (PTY 24-row window ignores terminal length 0) +# XRd Console --More-- Pager Bug (PTY window size paging vs terminal length 0) ## Background -Copilot commands with long output consistently fail against XRd (IOS XRv 9000 container) nodes: `device_show_run` running `show ipv4 interface brief` always reports `netmiko_multiline (failed)` (even as a single command); short-output commands like `show ipv4 interface ` and `show running-config interface ` work fine. +Copilot commands with long output consistently failed against XRd (IOS XRv 9000 container) nodes: `device_show_run` running `show ipv4 interface brief` always reported `netmiko_multiline (failed)` (even as a single command); short-output commands like `show ipv4 interface ` worked fine. Failure sequence (from the 2026-08-18 logs): -1. First failure: `ReadTimeout: Pattern not detected: 'RP/0/RP0/CPU0:ios\#'` — the command echo already matched, but the prompt never appeared within 60s -2. The copilot's single-retry reuses the same netmiko session (nornir caches it on the host object) with half-consumed output in the buffer → the second failure dies earlier in `command_echo_read` (`Pattern not detected: 'show\ ipv4\ interface\ brief'`) — a follow-on effect of the dirty session, not an independent fault +1. First failure: `ReadTimeout: Pattern not detected: 'RP/0/RP0/CPU0:ios\#'` — the command echo matched, but the prompt never appeared within 60s +2. The copilot's single-retry reused the same netmiko session (nornir caches it on the host object) with half-consumed output in the buffer → the second failure died earlier in `command_echo_read` — a follow-on effect of the dirty session, not an independent fault -## Root Cause (confirmed by live testing) +## Root Cause (fully traced) -1. The GNS3 XRd console reaches the container CLI via **docker exec**; Docker allocates the exec PTY with a default **24-row** window -2. The XR pager (at least for table-engine/TABLAST-style commands) reads the **PTY window size (TIOCGWINSZ)**, not the CLI-level `terminal length` setting -3. Evidence: `show terminal` reports `Length: 0 lines, Width: 511 columns` (so `terminal length 0` IS in effect), yet `show ipv4 interface brief` pages after **exactly 24 lines** (timestamp + blank line + header + 21 interface rows) with `--More--` -4. The device parks at `--More--` waiting for a keypress → the channel goes silent → netmiko ReadTimeout. Netmiko never presses space; it assumes paging is disabled +The XRd node uses the `docker_exec` console type (`gns3server/compute/docker/vendor_docker_vm.py`, `_LazyExecTelnetServer`): a telnet TCP server whose backend is a `docker exec` PTY. Three facts combine: + +1. The XR pager (at least for table-engine/TABLAST-style commands) pages on the **PTY window size (TIOCGWINSZ)**, not the CLI-level `terminal length` — `show terminal` happily reports `Length: 0 lines` while output still pages +2. `_LazyExecTelnetServer.client_connected_hook` explicitly resized the exec PTY to **80×24** "before NAWS" (`await self._on_naws(80, 24)`) — the 24 rows were **our own initial geometry, not a Docker default**. Live test: `show ipv4 interface brief` paged after exactly 24 lines +3. netmiko's telnetlib never negotiates **NAWS**, so the initial geometry is never corrected for copilot/bare-telnet clients. Real NAWS clients resize to their own geometry via the existing `_on_naws` → `POST /exec/{id}/resize` wiring ### Dead ends (do not retry) -- `show ipv4 interface brief | no-more` → `% Invalid input detected`. `| no-more` is a **Junos** pipe modifier; IOS-XR does not have it (the CRS 4.1 doc covers filtering *at* the --More-- prompt, not this) +- `show ipv4 interface brief | no-more` → `% Invalid input detected`. `| no-more` is a **Junos** pipe modifier; IOS-XR does not have it - `terminal length 0`: takes effect at the CLI layer, but this pager ignores it -- The XR CLI has no command to change PTY rows, and from inside the container you cannot reach another session's PTY +- The XR CLI has no command to change PTY rows -### Container-layer levers (deferred, high risk) +### Related fact: paramiko vs docker_exec -Changing the console plumbing in `gns3server/compute/docker/`: run the exec with `tty=false` (over pipes the CLI would likely fall back to the configured length = 0, i.e. no paging), or resize the exec PTY via the Docker API after creation. Both are XRd special-cases on a shared path used by every docker node's console — not accepted. +paramiko (SSH) cannot connect to a `docker_exec` console — the client-side endpoint is a plain telnet server (`AsyncioTelnetServer`); only `console_type: ssh` (standard attach path, `AsyncioSSHServer`) speaks SSH. The copilot correctly uses netmiko `*_telnet` drivers (netmiko's vendored `_telnetlib`, stdlib-free on Python 3.13). -## Decision/Implementation (design agreed, NOT yet implemented) +## Decision/Implementation -Planned branch `feat/copilot-xrd-more-handling` (based on `feat/copilot-node-default-credentials`), three parts: +### Fix (implemented 2026-08-18, branch `feat/docker-exec-default-pty-geometry`) -### 1. Display tool: for `cisco_xr*` platforms, replace netmiko_multiline with a channel-level loop that answers `--More--` +Change the initial exec geometry in `vendor_docker_vm.py` `client_connected_hook` from 80×24 to **511×10000** (`await self._on_naws(511, 10000)`): -```python -conn.write_channel(cmd + "\n") -buf = "" -while not past_deadline: - buf += conn.read_channel() - if prompt_re.search(buf): # normal path: prompt only - break - if re.search(r"--More--\s*$", buf): # tail-anchored check only - if still_quiet_after_one_poll: # ~150ms quiet double-check - conn.write_channel(" ") # page forward (space, not q — q truncates) - sleep(0.2) -``` +- Tall/wide default so CLIs that page on PTY rows never hit `--More--` for clients that never send NAWS (netmiko, bare telnet) +- Width 511 matches netmiko's `terminal width 511` convention +- Real NAWS clients still resize to their actual geometry right after connecting (existing `_on_naws` path unchanged) +- Test: `test_first_connect_sets_tall_default_pty_geometry` in `tests/compute/docker/test_vendor_docker_vm.py` -**Key design constraint (user-flagged)**: never put `--More--` into netmiko's `expect_string` — expect does a `re.search` over accumulated output, so banner/description text containing "More" would false-trigger. The fundamental distinction used: **the real pager's `--More--` is the last byte of the stream (no newline, nothing follows until a keypress); a literal `--More--` in content is always followed by more arriving bytes**. Three safeguards: normal path never looks for More + tail anchoring + quiet double-confirmation. +### Fallback design (NOT implemented — keep if the pager ever resurfaces on another console type) -### 2. Reconnect before retrying +Channel-level loop answering `--More--` in the copilot display tool for `cisco_xr*`: prompt regex breaks; **tail-anchored** `re.search(r"--More--\s*$", buf)` with a ~150ms quiet double-confirmation before `write_channel(" ")`. Never put `--More--` into netmiko's `expect_string` — expect `re.search`es accumulated output, so content containing "More" would false-trigger. -The retry branch of `_run_all_device_configs_with_single_retry` must call `task.host.close_connection("netmiko")` first — otherwise the retry is doomed by the dirty session (the second error in the logs proves it). The config tool has the same retry structure and needs the same fix. +### Still-open copilot improvements (agreed, not yet implemented) -### 3. Persist session_log - -Add `session_log_file` to `connection_options.netmiko.extras` in hosts_data (a native netmiko ConnectHandler parameter, passed through by nornir_netmiko). The copilot currently has no session_log anywhere, which made this bug a guessing game. +1. Reconnect before retry: `_run_all_device_configs_with_single_retry` (display tool) and the config tool's retry should `task.host.close_connection("netmiko")` first — retrying on a dirty session is doomed +2. `session_log_file` in hosts_data netmiko extras — the copilot has no session_log anywhere, which made this bug a guessing game ## Rationale -- Device side is unsolvable (CLI layer) or high-risk (docker plumbing layer); answering `--More--` at the copilot layer is a generic fix with zero plumbing risk: whatever the PTY row count, whatever odd console produces a pager, sending space on More heals it -- Tail-anchored detection drives the false-positive probability down to "content happens to be TCP-segment-split right after `--More--` AND stays silent" — which the quiet double-check then covers +Resizing the PTY at exec creation attacks the root (geometry is fixed before the CLI outputs anything); it covers every consumer of the docker_exec console (copilot, MCP, bare telnet) with a one-line change, while the `--More--` auto-answer design remains as a generic fallback for consoles without a resize path. ## Related Files -- `gns3server/agent/gns3_copilot/tools_v2/display_tools_nornir.py:284-341` — `_run_all_device_configs_with_single_retry` (the dirty-session retry and the loop replacement point) -- `gns3server/agent/gns3_copilot/tools_v2/config_tools_nornir.py` — same retry structure to fix -- `gns3server/agent/gns3_copilot/utils/get_gns3_device_port.py` — hosts_data construction (session_log extras insertion point) -- `venv/.../netmiko/cisco/cisco_xr.py:14-22` — `session_preparation` (sends `terminal width 511` + `disable_paging`, both ineffective against this bug; note the comment at line 16: "IOS-XR has an issue where it echoes the command even though it hasn't returned the prompt") +- `gns3server/compute/docker/vendor_docker_vm.py` — `_LazyExecTelnetServer`: `_on_naws` (exec resize), `_create_exec` (Tty=True, TERM=xterm), `client_connected_hook` (initial geometry — the fix) +- `gns3server/compute/docker/docker_vm.py:1074-1087` — standard console path: NAWS → `containers/{cid}/resize` +- `gns3server/agent/gns3_copilot/tools_v2/display_tools_nornir.py:284-341` — dirty-session retry (open item) +- `gns3server/agent/gns3_copilot/utils/get_gns3_device_port.py` — hosts_data (session_log extras insertion point) ## Examples @@ -73,13 +64,9 @@ Live console transcript (2026-08-17): ``` RP/0/RP0/CPU0:ios#show terminal -Line "vty2", Location "", Type "VTY" Length: 0 lines, Width: 511 columns <- CLI-layer setting in effect -RP/0/RP0/CPU0:ios#show ipv4 interface brief | no-more -% Invalid input detected at '^' marker. <- pipe does not exist on XR - RP/0/RP0/CPU0:ios#show ipv4 interface brief (exactly 24 lines: timestamp + blank + header + 21 interface rows) - --More-- <- the 24-row PTY is paging + --More-- <- the 80x24 initial exec geometry was paging ``` diff --git a/gns3server/compute/docker/vendor_docker_vm.py b/gns3server/compute/docker/vendor_docker_vm.py index 800d46603..dbbdf340c 100644 --- a/gns3server/compute/docker/vendor_docker_vm.py +++ b/gns3server/compute/docker/vendor_docker_vm.py @@ -504,7 +504,14 @@ class _LazyExecTelnetServer(AsyncioTelnetServer): log.warning(f"{self._log_name}: failed to create exec: {exc}", exc_info=True) raise try: - await self._on_naws(80, 24) # initial size before NAWS + # Tall/wide default geometry before any NAWS arrives: a + # 24-row PTY makes CLIs that page on the PTY window size + # (e.g. the IOS-XR pager) park at --More-- for clients + # that never negotiate NAWS (netmiko, bare telnet). + # Width 511 matches netmiko's 'terminal width 511'. + # Real NAWS clients resize to their own geometry right + # after connecting. + await self._on_naws(511, 10000) except Exception: pass else: diff --git a/tests/compute/docker/test_vendor_docker_vm.py b/tests/compute/docker/test_vendor_docker_vm.py index 3619d0b10..691adf1dd 100644 --- a/tests/compute/docker/test_vendor_docker_vm.py +++ b/tests/compute/docker/test_vendor_docker_vm.py @@ -531,6 +531,17 @@ async def test_first_connect_creates_exec(compute_project, manager): srv._create_exec.assert_called_once() +@pytest.mark.asyncio +async def test_first_connect_sets_tall_default_pty_geometry(compute_project, manager): + """The exec PTY must start tall/wide: a 24-row initial geometry makes CLIs + that page on the PTY window size (IOS-XR pager) park at --More-- for + clients that never send NAWS (netmiko, bare telnet).""" + + srv = _make_lazy_server(compute_project, manager) + await srv.client_connected_hook() + srv._on_naws.assert_called_once_with(511, 10000) + + @pytest.mark.asyncio async def test_reconnect_live_exec_not_recreated(compute_project, manager): """Reconnecting while the exec is alive must NOT recreate it."""