fix: bound graceful shutdown and stop notification WebSockets on exit

By default uvicorn waits for open connections forever. A long-lived
streaming response keeps the server waiting after Ctrl+C. One common
case is the MCP Streamable HTTP transport: the client opens a
standalone GET request and the server answers with an SSE stream
(not the old SSE transport). A second Ctrl+C sets force_exit. Uvicorn
then skips the lifespan shutdown, so project and controller cleanup
does not run.

Add timeout_graceful_shutdown. After the timeout uvicorn cancels the
remaining tasks and runs the lifespan shutdown. Also make the four
notification WebSocket handlers stop when app.state.exiting is set.
The HTTP stream handlers already do this.
This commit is contained in:
YueGuobin committed 2026-11-04 00:32:34 +08:00
1 parent bfe25641fc
commit 52c19002f2
9 files changed
+110 -31

No files matched your search

+20 -17
View File
@@ -15,23 +15,26 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
# @pytest.mark.asyncio
# async def test_notification_ws(compute_api):
#
# # FIXME: how to test websockets
# pass
import pytest
from fastapi import FastAPI
# with compute_api.ws("/notifications/ws") as ws:
from gns3server.api.routes.compute import compute_api
from gns3server.api.routes.compute.notifications import project_ws_notifications
# answer = await ws.receive_text()
# print(answer)
# answer = json.loads(answer)
#
# assert answer["action"] == "ping"
#
# NotificationManager.instance().emit("test", {})
#
# answer = await ws.receive_text()
# answer = json.loads(answer)
# assert answer["action"] == "test"
@pytest.mark.asyncio
async def test_compute_ws_stops_when_app_is_exiting(app: FastAPI, monkeypatch) -> None:
monkeypatch.setattr(compute_api.state, "exiting", True)
websocket = MagicMock()
websocket.app = compute_api
websocket.client = SimpleNamespace(host="127.0.0.1", port=1000)
websocket.send_text = AsyncMock()
await asyncio.wait_for(project_ws_notifications(websocket=websocket), timeout=1)
websocket.send_text.assert_not_awaited()
@@ -14,14 +14,22 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import asyncio
import json
import uuid
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
from fastapi import FastAPI
from pydantic import ValidationError
from gns3server import schemas
from gns3server.api.routes.controller.controller import controller_ws_notifications
from gns3server.api.routes.controller.projects import (
project_marker_ws_notifications,
project_ws_notifications,
)
from gns3server.utils.notification_queue import NotificationQueue
STREAM_PATHS = ["/v3/notifications", "/v3/projects/{project_id}/notifications"]
@@ -86,3 +94,46 @@ class TestNotificationSchema:
notification = schemas.Notification.model_validate(message)
assert notification.action == schemas.NotificationAction.log_warning
assert notification.compute_id == "local"
def _fake_websocket(app: FastAPI) -> MagicMock:
websocket = MagicMock()
websocket.app = app
websocket.client = SimpleNamespace(host="127.0.0.1", port=1000)
websocket.send_text = AsyncMock()
return websocket
class TestNotificationWebSocketShutdown:
"""Notification WebSockets must stop when the server is shutting down.
Otherwise uvicorn has to wait for them during graceful shutdown.
"""
@pytest.mark.asyncio
async def test_controller_ws_stops_when_app_is_exiting(self, app: FastAPI, monkeypatch) -> None:
monkeypatch.setattr(app.state, "exiting", True)
websocket = _fake_websocket(app)
await asyncio.wait_for(controller_ws_notifications(websocket, current_user=MagicMock()), timeout=1)
websocket.send_text.assert_not_awaited()
@pytest.mark.asyncio
async def test_project_ws_stops_when_app_is_exiting(self, app: FastAPI, project, monkeypatch) -> None:
project.auto_close = False
monkeypatch.setattr(app.state, "exiting", True)
websocket = _fake_websocket(app)
await asyncio.wait_for(project_ws_notifications(project.id, websocket, current_user=MagicMock()), timeout=1)
websocket.send_text.assert_not_awaited()
@pytest.mark.asyncio
async def test_marker_ws_stops_when_app_is_exiting(self, app: FastAPI, project, monkeypatch) -> None:
monkeypatch.setattr(app.state, "exiting", True)
websocket = _fake_websocket(app)
await asyncio.wait_for(
project_marker_ws_notifications(project.id, websocket, current_user=MagicMock()), timeout=1
)
websocket.send_text.assert_not_awaited()