diff --git a/gns3server/api/routes/compute/__init__.py b/gns3server/api/routes/compute/__init__.py index f99de579c..9f5abb11c 100644 --- a/gns3server/api/routes/compute/__init__.py +++ b/gns3server/api/routes/compute/__init__.py @@ -67,6 +67,7 @@ compute_api = FastAPI( ) compute_api.state.controller_host = None +compute_api.state.exiting = False @compute_api.exception_handler(ComputeError) diff --git a/gns3server/api/routes/compute/notifications.py b/gns3server/api/routes/compute/notifications.py index 24f8ee059..f429bb42f 100644 --- a/gns3server/api/routes/compute/notifications.py +++ b/gns3server/api/routes/compute/notifications.py @@ -44,7 +44,7 @@ async def project_ws_notifications(websocket: Union[None, WebSocket] = Depends(w log.info(f"New client {client} has connected to compute WebSocket") try: with NotificationManager.instance().queue() as queue: - while True: + while not websocket.app.state.exiting: notification = await queue.get_json(5) await websocket.send_text(notification) except (ConnectionClosed, WebSocketDisconnect): diff --git a/gns3server/api/routes/controller/controller.py b/gns3server/api/routes/controller/controller.py index abd08026d..79d08f45a 100644 --- a/gns3server/api/routes/controller/controller.py +++ b/gns3server/api/routes/controller/controller.py @@ -308,7 +308,7 @@ async def controller_ws_notifications( log.info(f"New client {client} has connected to controller WebSocket") try: with Controller.instance().notification.controller_queue() as queue: - while True: + while not websocket.app.state.exiting: notification = await queue.get_json(5) await websocket.send_text(notification) except (ConnectionClosed, WebSocketDisconnect): diff --git a/gns3server/api/routes/controller/projects.py b/gns3server/api/routes/controller/projects.py index 32992a090..17d1e36a4 100644 --- a/gns3server/api/routes/controller/projects.py +++ b/gns3server/api/routes/controller/projects.py @@ -659,7 +659,7 @@ async def project_ws_notifications( log.info(f"New client has connected to the notification stream for project ID '{project.id}' (WebSocket method)") try: with controller.notification.project_queue(project.id) as queue: - while True: + while not websocket.app.state.exiting: notification = await queue.get_json(5) await websocket.send_text(notification) except (ConnectionClosed, WebSocketDisconnect): @@ -701,7 +701,7 @@ async def project_marker_ws_notifications( ) try: with controller.notification.project_marker_queue(project.id) as queue: - while True: + while not websocket.app.state.exiting: notification = await queue.get_json(5) await websocket.send_text(notification) except (ConnectionClosed, WebSocketDisconnect): diff --git a/gns3server/api/server.py b/gns3server/api/server.py index 1b5c3dc0d..98eedcdab 100644 --- a/gns3server/api/server.py +++ b/gns3server/api/server.py @@ -134,6 +134,8 @@ unicorn_exit_handler = UvicornServer.handle_exit def handle_exit(*args, **kwargs): app.state.exiting = True + # The compute API is a mounted sub-app with its own State object. + compute_api.state.exiting = True unicorn_exit_handler(*args, **kwargs) diff --git a/gns3server/server.py b/gns3server/server.py index 099696f55..e3d5209e9 100644 --- a/gns3server/server.py +++ b/gns3server/server.py @@ -42,6 +42,12 @@ from gns3server.version import __version__ log = logging.getLogger(__name__) +# Uvicorn waits for open connections forever by default. A long-lived +# streaming response (for example the MCP GET/SSE stream) would then block +# Ctrl+C indefinitely. Bound the wait so remaining tasks are cancelled and +# the lifespan shutdown (controller and project cleanup) always runs. +GRACEFUL_SHUTDOWN_TIMEOUT = 10 + class Server: _stream_handler = None @@ -222,6 +228,24 @@ class Server: log.critical("Can't write pid file %s: %s", path, str(e)) sys.exit(1) + @staticmethod + def _create_uvicorn_config(app, host: str, port: int, access_log: bool) -> uvicorn.Config: + """ + Build the uvicorn configuration. + """ + + server_config = Config.instance().settings.Server + return uvicorn.Config( + app, + host=host, + port=port, + access_log=access_log, + ssl_certfile=server_config.certfile, + ssl_keyfile=server_config.certkey, + lifespan="on", + timeout_graceful_shutdown=GRACEFUL_SHUTDOWN_TIMEOUT, + ) + async def run(self, parser, args): self._setup_logging(args) @@ -301,15 +325,7 @@ class Server: if config.Server.enable_ssl: log.info("SSL is enabled") - config = uvicorn.Config( - app, - host=host, - port=port, - access_log=access_log, - ssl_certfile=config.Server.certfile, - ssl_keyfile=config.Server.certkey, - lifespan="on", - ) + config = self._create_uvicorn_config(app, host, port, access_log) # overwrite uvicorn loggers with our own logger for uvicorn_logger_name in ("uvicorn", "uvicorn.error"): diff --git a/tests/api/routes/compute/test_notifications.py b/tests/api/routes/compute/test_notifications.py index 752afdaf8..59e87134c 100644 --- a/tests/api/routes/compute/test_notifications.py +++ b/tests/api/routes/compute/test_notifications.py @@ -15,23 +15,26 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock -# @pytest.mark.asyncio -# async def test_notification_ws(compute_api): -# -# # FIXME: how to test websockets -# pass +import pytest +from fastapi import FastAPI -# with compute_api.ws("/notifications/ws") as ws: +from gns3server.api.routes.compute import compute_api +from gns3server.api.routes.compute.notifications import project_ws_notifications -# answer = await ws.receive_text() -# print(answer) -# answer = json.loads(answer) -# -# assert answer["action"] == "ping" -# -# NotificationManager.instance().emit("test", {}) -# -# answer = await ws.receive_text() -# answer = json.loads(answer) -# assert answer["action"] == "test" + +@pytest.mark.asyncio +async def test_compute_ws_stops_when_app_is_exiting(app: FastAPI, monkeypatch) -> None: + + monkeypatch.setattr(compute_api.state, "exiting", True) + websocket = MagicMock() + websocket.app = compute_api + websocket.client = SimpleNamespace(host="127.0.0.1", port=1000) + websocket.send_text = AsyncMock() + + await asyncio.wait_for(project_ws_notifications(websocket=websocket), timeout=1) + + websocket.send_text.assert_not_awaited() diff --git a/tests/api/routes/controller/test_notifications.py b/tests/api/routes/controller/test_notifications.py index ff6209e19..c21ab6af1 100644 --- a/tests/api/routes/controller/test_notifications.py +++ b/tests/api/routes/controller/test_notifications.py @@ -14,14 +14,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import asyncio import json import uuid +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import FastAPI from pydantic import ValidationError from gns3server import schemas +from gns3server.api.routes.controller.controller import controller_ws_notifications +from gns3server.api.routes.controller.projects import ( + project_marker_ws_notifications, + project_ws_notifications, +) from gns3server.utils.notification_queue import NotificationQueue STREAM_PATHS = ["/v3/notifications", "/v3/projects/{project_id}/notifications"] @@ -86,3 +94,46 @@ class TestNotificationSchema: notification = schemas.Notification.model_validate(message) assert notification.action == schemas.NotificationAction.log_warning assert notification.compute_id == "local" + + +def _fake_websocket(app: FastAPI) -> MagicMock: + + websocket = MagicMock() + websocket.app = app + websocket.client = SimpleNamespace(host="127.0.0.1", port=1000) + websocket.send_text = AsyncMock() + return websocket + + +class TestNotificationWebSocketShutdown: + """Notification WebSockets must stop when the server is shutting down. + + Otherwise uvicorn has to wait for them during graceful shutdown. + """ + + @pytest.mark.asyncio + async def test_controller_ws_stops_when_app_is_exiting(self, app: FastAPI, monkeypatch) -> None: + + monkeypatch.setattr(app.state, "exiting", True) + websocket = _fake_websocket(app) + await asyncio.wait_for(controller_ws_notifications(websocket, current_user=MagicMock()), timeout=1) + websocket.send_text.assert_not_awaited() + + @pytest.mark.asyncio + async def test_project_ws_stops_when_app_is_exiting(self, app: FastAPI, project, monkeypatch) -> None: + + project.auto_close = False + monkeypatch.setattr(app.state, "exiting", True) + websocket = _fake_websocket(app) + await asyncio.wait_for(project_ws_notifications(project.id, websocket, current_user=MagicMock()), timeout=1) + websocket.send_text.assert_not_awaited() + + @pytest.mark.asyncio + async def test_marker_ws_stops_when_app_is_exiting(self, app: FastAPI, project, monkeypatch) -> None: + + monkeypatch.setattr(app.state, "exiting", True) + websocket = _fake_websocket(app) + await asyncio.wait_for( + project_marker_ws_notifications(project.id, websocket, current_user=MagicMock()), timeout=1 + ) + websocket.send_text.assert_not_awaited() diff --git a/tests/test_server.py b/tests/test_server.py index 95f644503..1d5f944d3 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -23,7 +23,13 @@ import pytest from gns3server.config import Config from gns3server.main import parse_arguments as parse -from gns3server.server import Server +from gns3server.server import GRACEFUL_SHUTDOWN_TIMEOUT, Server + + +def test_uvicorn_config_bounds_graceful_shutdown(config): + + uvicorn_config = Server._create_uvicorn_config(None, "127.0.0.1", 3080, False) + assert uvicorn_config.timeout_graceful_shutdown == GRACEFUL_SHUTDOWN_TIMEOUT def test_locale_check():