From 52c19002f2225ab215fdd36c47e711f5822f1717 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Tue, 3 Nov 2026 23:33:45 +0800 Subject: [PATCH] fix: bound graceful shutdown and stop notification WebSockets on exit By default uvicorn waits for open connections forever. A long-lived streaming response keeps the server waiting after Ctrl+C. One common case is the MCP Streamable HTTP transport: the client opens a standalone GET request and the server answers with an SSE stream (not the old SSE transport). A second Ctrl+C sets force_exit. Uvicorn then skips the lifespan shutdown, so project and controller cleanup does not run. Add timeout_graceful_shutdown. After the timeout uvicorn cancels the remaining tasks and runs the lifespan shutdown. Also make the four notification WebSocket handlers stop when app.state.exiting is set. The HTTP stream handlers already do this. --- gns3server/api/routes/compute/__init__.py | 1 + .../api/routes/compute/notifications.py | 2 +- .../api/routes/controller/controller.py | 2 +- gns3server/api/routes/controller/projects.py | 4 +- gns3server/api/server.py | 2 + gns3server/server.py | 34 +++++++++---- .../api/routes/compute/test_notifications.py | 37 +++++++------- .../routes/controller/test_notifications.py | 51 +++++++++++++++++++ tests/test_server.py | 8 ++- 9 files changed, 110 insertions(+), 31 deletions(-) diff --git a/gns3server/api/routes/compute/__init__.py b/gns3server/api/routes/compute/__init__.py index f99de579c..9f5abb11c 100644 --- a/gns3server/api/routes/compute/__init__.py +++ b/gns3server/api/routes/compute/__init__.py @@ -67,6 +67,7 @@ compute_api = FastAPI( ) compute_api.state.controller_host = None +compute_api.state.exiting = False @compute_api.exception_handler(ComputeError) diff --git a/gns3server/api/routes/compute/notifications.py b/gns3server/api/routes/compute/notifications.py index 24f8ee059..f429bb42f 100644 --- a/gns3server/api/routes/compute/notifications.py +++ b/gns3server/api/routes/compute/notifications.py @@ -44,7 +44,7 @@ async def project_ws_notifications(websocket: Union[None, WebSocket] = Depends(w log.info(f"New client {client} has connected to compute WebSocket") try: with NotificationManager.instance().queue() as queue: - while True: + while not websocket.app.state.exiting: notification = await queue.get_json(5) await websocket.send_text(notification) except (ConnectionClosed, WebSocketDisconnect): diff --git a/gns3server/api/routes/controller/controller.py b/gns3server/api/routes/controller/controller.py index abd08026d..79d08f45a 100644 --- a/gns3server/api/routes/controller/controller.py +++ b/gns3server/api/routes/controller/controller.py @@ -308,7 +308,7 @@ async def controller_ws_notifications( log.info(f"New client {client} has connected to controller WebSocket") try: with Controller.instance().notification.controller_queue() as queue: - while True: + while not websocket.app.state.exiting: notification = await queue.get_json(5) await websocket.send_text(notification) except (ConnectionClosed, WebSocketDisconnect): diff --git a/gns3server/api/routes/controller/projects.py b/gns3server/api/routes/controller/projects.py index 32992a090..17d1e36a4 100644 --- a/gns3server/api/routes/controller/projects.py +++ b/gns3server/api/routes/controller/projects.py @@ -659,7 +659,7 @@ async def project_ws_notifications( log.info(f"New client has connected to the notification stream for project ID '{project.id}' (WebSocket method)") try: with controller.notification.project_queue(project.id) as queue: - while True: + while not websocket.app.state.exiting: notification = await queue.get_json(5) await websocket.send_text(notification) except (ConnectionClosed, WebSocketDisconnect): @@ -701,7 +701,7 @@ async def project_marker_ws_notifications( ) try: with controller.notification.project_marker_queue(project.id) as queue: - while True: + while not websocket.app.state.exiting: notification = await queue.get_json(5) await websocket.send_text(notification) except (ConnectionClosed, WebSocketDisconnect): diff --git a/gns3server/api/server.py b/gns3server/api/server.py index 1b5c3dc0d..98eedcdab 100644 --- a/gns3server/api/server.py +++ b/gns3server/api/server.py @@ -134,6 +134,8 @@ unicorn_exit_handler = UvicornServer.handle_exit def handle_exit(*args, **kwargs): app.state.exiting = True + # The compute API is a mounted sub-app with its own State object. + compute_api.state.exiting = True unicorn_exit_handler(*args, **kwargs) diff --git a/gns3server/server.py b/gns3server/server.py index 099696f55..e3d5209e9 100644 --- a/gns3server/server.py +++ b/gns3server/server.py @@ -42,6 +42,12 @@ from gns3server.version import __version__ log = logging.getLogger(__name__) +# Uvicorn waits for open connections forever by default. A long-lived +# streaming response (for example the MCP GET/SSE stream) would then block +# Ctrl+C indefinitely. Bound the wait so remaining tasks are cancelled and +# the lifespan shutdown (controller and project cleanup) always runs. +GRACEFUL_SHUTDOWN_TIMEOUT = 10 + class Server: _stream_handler = None @@ -222,6 +228,24 @@ class Server: log.critical("Can't write pid file %s: %s", path, str(e)) sys.exit(1) + @staticmethod + def _create_uvicorn_config(app, host: str, port: int, access_log: bool) -> uvicorn.Config: + """ + Build the uvicorn configuration. + """ + + server_config = Config.instance().settings.Server + return uvicorn.Config( + app, + host=host, + port=port, + access_log=access_log, + ssl_certfile=server_config.certfile, + ssl_keyfile=server_config.certkey, + lifespan="on", + timeout_graceful_shutdown=GRACEFUL_SHUTDOWN_TIMEOUT, + ) + async def run(self, parser, args): self._setup_logging(args) @@ -301,15 +325,7 @@ class Server: if config.Server.enable_ssl: log.info("SSL is enabled") - config = uvicorn.Config( - app, - host=host, - port=port, - access_log=access_log, - ssl_certfile=config.Server.certfile, - ssl_keyfile=config.Server.certkey, - lifespan="on", - ) + config = self._create_uvicorn_config(app, host, port, access_log) # overwrite uvicorn loggers with our own logger for uvicorn_logger_name in ("uvicorn", "uvicorn.error"): diff --git a/tests/api/routes/compute/test_notifications.py b/tests/api/routes/compute/test_notifications.py index 752afdaf8..59e87134c 100644 --- a/tests/api/routes/compute/test_notifications.py +++ b/tests/api/routes/compute/test_notifications.py @@ -15,23 +15,26 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock -# @pytest.mark.asyncio -# async def test_notification_ws(compute_api): -# -# # FIXME: how to test websockets -# pass +import pytest +from fastapi import FastAPI -# with compute_api.ws("/notifications/ws") as ws: +from gns3server.api.routes.compute import compute_api +from gns3server.api.routes.compute.notifications import project_ws_notifications -# answer = await ws.receive_text() -# print(answer) -# answer = json.loads(answer) -# -# assert answer["action"] == "ping" -# -# NotificationManager.instance().emit("test", {}) -# -# answer = await ws.receive_text() -# answer = json.loads(answer) -# assert answer["action"] == "test" + +@pytest.mark.asyncio +async def test_compute_ws_stops_when_app_is_exiting(app: FastAPI, monkeypatch) -> None: + + monkeypatch.setattr(compute_api.state, "exiting", True) + websocket = MagicMock() + websocket.app = compute_api + websocket.client = SimpleNamespace(host="127.0.0.1", port=1000) + websocket.send_text = AsyncMock() + + await asyncio.wait_for(project_ws_notifications(websocket=websocket), timeout=1) + + websocket.send_text.assert_not_awaited() diff --git a/tests/api/routes/controller/test_notifications.py b/tests/api/routes/controller/test_notifications.py index ff6209e19..c21ab6af1 100644 --- a/tests/api/routes/controller/test_notifications.py +++ b/tests/api/routes/controller/test_notifications.py @@ -14,14 +14,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import asyncio import json import uuid +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import FastAPI from pydantic import ValidationError from gns3server import schemas +from gns3server.api.routes.controller.controller import controller_ws_notifications +from gns3server.api.routes.controller.projects import ( + project_marker_ws_notifications, + project_ws_notifications, +) from gns3server.utils.notification_queue import NotificationQueue STREAM_PATHS = ["/v3/notifications", "/v3/projects/{project_id}/notifications"] @@ -86,3 +94,46 @@ class TestNotificationSchema: notification = schemas.Notification.model_validate(message) assert notification.action == schemas.NotificationAction.log_warning assert notification.compute_id == "local" + + +def _fake_websocket(app: FastAPI) -> MagicMock: + + websocket = MagicMock() + websocket.app = app + websocket.client = SimpleNamespace(host="127.0.0.1", port=1000) + websocket.send_text = AsyncMock() + return websocket + + +class TestNotificationWebSocketShutdown: + """Notification WebSockets must stop when the server is shutting down. + + Otherwise uvicorn has to wait for them during graceful shutdown. + """ + + @pytest.mark.asyncio + async def test_controller_ws_stops_when_app_is_exiting(self, app: FastAPI, monkeypatch) -> None: + + monkeypatch.setattr(app.state, "exiting", True) + websocket = _fake_websocket(app) + await asyncio.wait_for(controller_ws_notifications(websocket, current_user=MagicMock()), timeout=1) + websocket.send_text.assert_not_awaited() + + @pytest.mark.asyncio + async def test_project_ws_stops_when_app_is_exiting(self, app: FastAPI, project, monkeypatch) -> None: + + project.auto_close = False + monkeypatch.setattr(app.state, "exiting", True) + websocket = _fake_websocket(app) + await asyncio.wait_for(project_ws_notifications(project.id, websocket, current_user=MagicMock()), timeout=1) + websocket.send_text.assert_not_awaited() + + @pytest.mark.asyncio + async def test_marker_ws_stops_when_app_is_exiting(self, app: FastAPI, project, monkeypatch) -> None: + + monkeypatch.setattr(app.state, "exiting", True) + websocket = _fake_websocket(app) + await asyncio.wait_for( + project_marker_ws_notifications(project.id, websocket, current_user=MagicMock()), timeout=1 + ) + websocket.send_text.assert_not_awaited() diff --git a/tests/test_server.py b/tests/test_server.py index 95f644503..1d5f944d3 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -23,7 +23,13 @@ import pytest from gns3server.config import Config from gns3server.main import parse_arguments as parse -from gns3server.server import Server +from gns3server.server import GRACEFUL_SHUTDOWN_TIMEOUT, Server + + +def test_uvicorn_config_bounds_graceful_shutdown(config): + + uvicorn_config = Server._create_uvicorn_config(None, "127.0.0.1", 3080, False) + assert uvicorn_config.timeout_graceful_shutdown == GRACEFUL_SHUTDOWN_TIMEOUT def test_locale_check():