From 8c98142d26150ebf53e35d52234467d817af8ca1 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 6 Nov 2026 15:03:34 +0800 Subject: [PATCH] feat: report Docker pull progress and add an idle timeout A pull now reports docker.image.pull notifications on the controller stream: started, throttled downloading events with the layer counts and the latest daemon message, then completed or failed. An optional progress_id is echoed in every event so a client can correlate them with its own request. Image operations belong to a compute, not to a project, so the events go to the controller stream and reach clients that have no project open. The pull also has an idle socket read timeout of 60 seconds. Without it an unreachable registry left the pull, the compute request and the client request hanging forever with no feedback at all. The timeout is idle time, not total time: the daemon streams progress during the whole pull, so a minute without data means the registry is gone. When it fires, the daemon connection closes and Docker cancels the pull. Docker.http_query now accepts an aiohttp ClientTimeout object so a caller can ask for a policy finer than a total timeout. --- gns3server/api/routes/compute/images.py | 49 ++++++++++++++- gns3server/api/routes/controller/computes.py | 14 ++++- gns3server/compute/docker/__init__.py | 51 ++++++++++++---- gns3server/controller/compute.py | 4 +- gns3server/controller/notification.py | 6 ++ tests/api/routes/compute/test_images.py | 64 ++++++++++++++++++-- tests/api/routes/controller/test_computes.py | 26 +++++++- tests/compute/docker/test_docker.py | 51 +++++++++++++++- tests/controller/test_notification.py | 28 +++++++++ 9 files changed, 266 insertions(+), 27 deletions(-) diff --git a/gns3server/api/routes/compute/images.py b/gns3server/api/routes/compute/images.py index 2d631b84b..4a34bf794 100644 --- a/gns3server/api/routes/compute/images.py +++ b/gns3server/api/routes/compute/images.py @@ -19,6 +19,7 @@ API routes for images. """ import os +import time import urllib.parse from typing import List, Optional @@ -26,10 +27,16 @@ from fastapi import APIRouter, Body, HTTPException, Query, Request, status from fastapi.responses import FileResponse from gns3server.compute.docker import Docker +from gns3server.compute.docker.docker_error import DockerError from gns3server.compute.dynamips import Dynamips from gns3server.compute.iou import IOU +from gns3server.compute.notification_manager import NotificationManager from gns3server.compute.qemu import Qemu +# Daemon progress lines arrive dozens of times per second per layer: emit at +# most one downloading event per second per pull. +PULL_PROGRESS_INTERVAL = 1.0 + router = APIRouter() @@ -44,13 +51,51 @@ async def get_docker_images() -> List[dict]: @router.post("/docker/images/pull", status_code=status.HTTP_204_NO_CONTENT) -async def pull_docker_image(image: str = Body(..., embed=True, min_length=1, pattern=r"^\S+$")) -> None: +async def pull_docker_image( + image: str = Body(..., embed=True, min_length=1, pattern=r"^\S+$"), + progress_id: Optional[str] = Query( + None, pattern=r"^\S+$", description="Client supplied id to correlate progress notifications" + ), +) -> None: """ Pull or update a Docker image. + + The pull progress is reported as docker.image.pull notifications: a + started event, throttled downloading events and a terminal completed or + failed event. The optional progress_id is echoed in every event so a + client can correlate them with its own request. """ docker_manager = Docker.instance() - await docker_manager.pull_image(image, force=True) + + def emit(state: str, message: Optional[str] = None, progress: Optional[dict] = None) -> None: + event = {"image": image, "state": state} + if message is not None: + event["message"] = message + if progress is not None: + event["layers_total"] = progress["layers_total"] + event["layers_done"] = progress["layers_done"] + if progress_id is not None: + event["progress_id"] = progress_id + NotificationManager.instance().emit("docker.image.pull", event) + + last_emit = 0.0 + + def progress_callback(progress: dict) -> None: + nonlocal last_emit + now = time.monotonic() + if now - last_emit < PULL_PROGRESS_INTERVAL: + return + last_emit = now + emit("downloading", progress["message"], progress) + + emit("started", f"Pulling '{image}' from Docker repository") + try: + await docker_manager.pull_image(image, progress_callback=progress_callback, force=True) + except DockerError as e: + emit("failed", str(e)) + raise + emit("completed", f"Success pulling image {image}") @router.post("/docker/images/load", status_code=status.HTTP_204_NO_CONTENT) diff --git a/gns3server/api/routes/controller/computes.py b/gns3server/api/routes/controller/computes.py index 90bd51486..51c6f51d2 100644 --- a/gns3server/api/routes/controller/computes.py +++ b/gns3server/api/routes/controller/computes.py @@ -215,16 +215,26 @@ async def docker_get_images(compute_id: Union[str, UUID]) -> List[schemas.Comput dependencies=[Depends(has_privilege("Compute.Modify"))], ) async def docker_pull_image( - compute_id: Union[str, UUID], image: str = Body(..., embed=True, min_length=1, pattern=r"^\S+$") + compute_id: Union[str, UUID], + image: str = Body(..., embed=True, min_length=1, pattern=r"^\S+$"), + progress_id: Optional[str] = Query( + None, pattern=r"^\S+$", description="Client supplied id to correlate progress notifications" + ), ) -> None: """ Pull or update a Docker image on a compute. + The pull progress is reported on the controller notification stream as + docker.image.pull events (started, downloading, completed or failed). The + optional progress_id is echoed in every event so a client can correlate + them with its own request. + Required privilege: Compute.Modify """ compute = Controller.instance().get_compute(str(compute_id)) - await compute.forward("POST", "docker", "images/pull", data={"image": image}) + params = {"progress_id": progress_id} if progress_id is not None else None + await compute.forward("POST", "docker", "images/pull", data={"image": image}, params=params) @router.post( diff --git a/gns3server/compute/docker/__init__.py b/gns3server/compute/docker/__init__.py index f71141ab3..c75f2a86c 100644 --- a/gns3server/compute/docker/__init__.py +++ b/gns3server/compute/docker/__init__.py @@ -51,6 +51,11 @@ DOCKER_MINIMUM_VERSION = "19.03.8" DOCKER_PREFERRED_API_VERSION = "1.44" CHUNK_SIZE = 1024 * 8 # 8KB +# A pull streams progress from the daemon for its whole duration: when no data +# arrives for this many seconds the registry is unreachable and the pull would +# otherwise hang forever. This is an idle timeout, not a total time limit. +PULL_IDLE_TIMEOUT = 60 + class Docker(BaseManager): _NODE_CLASS = DockerVM @@ -291,7 +296,8 @@ class Docker(BaseManager): :param path: Endpoint in API :param data: Dictionary with the body. Will be transformed to a JSON :param params: Parameters added as a query arg - :param timeout: Timeout + :param timeout: Total timeout in seconds, an aiohttp ClientTimeout object + for a finer policy (e.g. an idle socket read timeout), or None for no timeout :returns: HTTP response """ @@ -302,7 +308,7 @@ class Docker(BaseManager): # not a dict (e.g. a Docker image tar stream): let aiohttp stream the raw body headers = {"content-type": "application/x-tar"} if timeout is None: - timeout = 60 * 60 * 24 * 31 # One month timeout + timeout = aiohttp.ClientTimeout(total=None) if path == "version": url = "http://docker/" + path @@ -361,7 +367,8 @@ class Docker(BaseManager): Pulls an image from the Docker repository :params image: Image name - :params progress_callback: A function that receive a log message about image download progress + :params progress_callback: A function that receives a dict with the + overall progress of the pull: layer counts and the latest daemon message :params force: Pull the image even if it is already available locally """ @@ -372,10 +379,23 @@ class Docker(BaseManager): except DockerHttp404Error: pass - if progress_callback: - progress_callback(f"Pulling '{image}' from Docker repository") + layers_total = 0 + layers_done = 0 + + def report(message): + if progress_callback: + progress_callback({"layers_total": layers_total, "layers_done": layers_done, "message": message}) + try: - response = await self.http_query("POST", "images/create", params={"fromImage": image}, timeout=None) + # The daemon streams progress lines for the whole pull: an idle + # socket read for a minute means the registry is unreachable and + # the pull would otherwise hang forever. + response = await self.http_query( + "POST", + "images/create", + params={"fromImage": image}, + timeout=aiohttp.ClientTimeout(total=None, sock_read=PULL_IDLE_TIMEOUT), + ) except DockerError as e: raise DockerError( f"Could not pull the '{image}' image from Docker repository, " @@ -392,7 +412,11 @@ class Docker(BaseManager): f"Disconnected while pulling Docker image '{image}' from Docker repository" ) from e except TimeoutError as e: - raise DockerError(f"Timeout while pulling Docker image '{image}' from Docker repository") from e + raise DockerError( + f"Timeout while pulling Docker image '{image}' from Docker repository: " + f"no data received for {PULL_IDLE_TIMEOUT} seconds, " + f"please check your Internet connection or Docker registry mirror" + ) from e if not chunk: break content += chunk.decode("utf-8") @@ -409,8 +433,14 @@ class Docker(BaseManager): error = error_detail.get("message") if error: raise DockerError(error) - if "progress" in answer and progress_callback: - progress_callback("Pulling image {}:{}: {}".format(image, answer["id"], answer["progress"])) + status = answer.get("status", "") + if status == "Pulling fs layer": + layers_total += 1 + elif status in ("Pull complete", "Already exists"): + layers_done += 1 + message = answer.get("progress") or status + if message: + report(message) content = content[index:] except ValueError: # Partial JSON pass @@ -420,9 +450,6 @@ class Docker(BaseManager): finally: response.close() - if progress_callback: - progress_callback(f"Success pulling image {image}") - @locking async def load_image(self, stream, image=None, digest=None, progress_callback=None): """ diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index 8a9e6fe87..be1e0fdba 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -733,13 +733,13 @@ class Compute: async def delete(self, path, **kwargs): return await self.http_query("DELETE", path, **kwargs) - async def forward(self, method, type, path, data=None): + async def forward(self, method, type, path, data=None, params=None): """ Forward a call to the emulator on compute """ try: action = f"/{type}/{path}" - res = await self.http_query(method, action, data=data, timeout=None) + res = await self.http_query(method, action, data=data, timeout=None, params=params) except aiohttp.ServerDisconnectedError: raise ControllerError(f"Connection lost to {self._id} during {method} {action}") return res.json diff --git a/gns3server/controller/notification.py b/gns3server/controller/notification.py index 711fabcc2..6eee15572 100644 --- a/gns3server/controller/notification.py +++ b/gns3server/controller/notification.py @@ -126,6 +126,12 @@ class Notification: elif action == "ping": event["compute_id"] = compute_id self.project_emit(action, event) + elif action.startswith("docker.image."): + # Image operations are scoped to a compute, not to a project, and + # clients that are not in a project still need them: emit on the + # controller stream, like compute.updated. + event["compute_id"] = compute_id + self.controller_emit(action, event) elif action.startswith("marker."): self.marker_emit(action, event, project_id) else: diff --git a/tests/api/routes/compute/test_images.py b/tests/api/routes/compute/test_images.py index 041a55702..ae3a22e4b 100644 --- a/tests/api/routes/compute/test_images.py +++ b/tests/api/routes/compute/test_images.py @@ -18,6 +18,7 @@ import pytest from fastapi import FastAPI, status from httpx import AsyncClient +from gns3server.compute.docker.docker_error import DockerError from tests.utils import asyncio_patch pytestmark = pytest.mark.asyncio @@ -27,11 +28,64 @@ class TestImagesRoutes: async def test_pull_docker_image(self, app: FastAPI, compute_client: AsyncClient) -> None: with asyncio_patch("gns3server.compute.docker.Docker.pull_image") as mock: - response = await compute_client.post( - app.url_path_for("compute:pull_docker_image"), json={"image": "nginx:latest"} - ) - mock.assert_called_once_with("nginx:latest", force=True) - assert response.status_code == status.HTTP_204_NO_CONTENT + with asyncio_patch("gns3server.compute.notification_manager.NotificationManager.emit") as emit_mock: + response = await compute_client.post( + app.url_path_for("compute:pull_docker_image"), json={"image": "nginx:latest"} + ) + assert mock.call_args[0] == ("nginx:latest",) + assert mock.call_args[1]["force"] is True + assert callable(mock.call_args[1]["progress_callback"]) + # started and completed events are emitted to the notification stream + assert [(c.args[0], c.args[1]["state"]) for c in emit_mock.call_args_list] == [ + ("docker.image.pull", "started"), + ("docker.image.pull", "completed"), + ] + assert response.status_code == status.HTTP_204_NO_CONTENT + + async def test_pull_docker_image_progress_id(self, app: FastAPI, compute_client: AsyncClient) -> None: + + with asyncio_patch("gns3server.compute.docker.Docker.pull_image"): + with asyncio_patch("gns3server.compute.notification_manager.NotificationManager.emit") as emit_mock: + response = await compute_client.post( + app.url_path_for("compute:pull_docker_image"), + json={"image": "nginx:latest"}, + params={"progress_id": "abc123"}, + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + assert all(c.args[1]["progress_id"] == "abc123" for c in emit_mock.call_args_list) + + async def test_pull_docker_image_reports_downloading(self, app: FastAPI, compute_client: AsyncClient) -> None: + + async def fake_pull(image, progress_callback=None, force=False): + progress_callback({"layers_total": 2, "layers_done": 1, "message": "[==> ] 40/100"}) + + with asyncio_patch("gns3server.compute.docker.Docker.pull_image", side_effect=fake_pull): + with asyncio_patch("gns3server.compute.notification_manager.NotificationManager.emit") as emit_mock: + response = await compute_client.post( + app.url_path_for("compute:pull_docker_image"), json={"image": "nginx:latest"} + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + states = [c.args[1]["state"] for c in emit_mock.call_args_list] + assert states == ["started", "downloading", "completed"] + downloading = emit_mock.call_args_list[1].args[1] + assert downloading["layers_total"] == 2 + assert downloading["layers_done"] == 1 + assert downloading["message"] == "[==> ] 40/100" + + async def test_pull_docker_image_emits_failed(self, app: FastAPI, compute_client: AsyncClient) -> None: + + async def fake_pull(image, progress_callback=None, force=False): + raise DockerError("no data received for 60 seconds") + + with asyncio_patch("gns3server.compute.docker.Docker.pull_image", side_effect=fake_pull): + with asyncio_patch("gns3server.compute.notification_manager.NotificationManager.emit") as emit_mock: + response = await compute_client.post( + app.url_path_for("compute:pull_docker_image"), json={"image": "nginx:latest"} + ) + assert response.status_code == status.HTTP_409_CONFLICT + failed = emit_mock.call_args_list[-1].args[1] + assert failed["state"] == "failed" + assert "no data received" in failed["message"] @pytest.mark.parametrize("image", ["", " ", "nginx latest"]) async def test_pull_docker_image_rejects_invalid_name( diff --git a/tests/api/routes/controller/test_computes.py b/tests/api/routes/controller/test_computes.py index c32b245e8..c38b70592 100644 --- a/tests/api/routes/controller/test_computes.py +++ b/tests/api/routes/controller/test_computes.py @@ -197,7 +197,31 @@ class TestComputeFeatures: app.url_path_for("docker_pull_image", compute_id=test_compute.compute_id), json={"image": "nginx:latest"}, ) - compute.forward.assert_called_with("POST", "docker", "images/pull", data={"image": "nginx:latest"}) + compute.forward.assert_called_with( + "POST", "docker", "images/pull", data={"image": "nginx:latest"}, params=None + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + + async def test_compute_pull_docker_image_progress_id( + self, app: FastAPI, client: AsyncClient, test_compute: Compute + ) -> None: + + compute = MagicMock() + compute.forward = AsyncioMagicMock(return_value={}) + with patch("gns3server.api.routes.controller.computes.Controller.instance") as controller: + controller.return_value.get_compute.return_value = compute + response = await client.post( + app.url_path_for("docker_pull_image", compute_id=test_compute.compute_id), + json={"image": "nginx:latest"}, + params={"progress_id": "abc123"}, + ) + compute.forward.assert_called_with( + "POST", + "docker", + "images/pull", + data={"image": "nginx:latest"}, + params={"progress_id": "abc123"}, + ) assert response.status_code == status.HTTP_204_NO_CONTENT async def test_compute_delete_docker_image(self, app: FastAPI, client: AsyncClient, test_compute: Compute) -> None: diff --git a/tests/compute/docker/test_docker.py b/tests/compute/docker/test_docker.py index 8699d6337..71636d04d 100644 --- a/tests/compute/docker/test_docker.py +++ b/tests/compute/docker/test_docker.py @@ -21,7 +21,12 @@ from unittest.mock import MagicMock, patch import pytest import pytest_asyncio -from gns3server.compute.docker import DOCKER_MINIMUM_API_VERSION, DOCKER_PREFERRED_API_VERSION, Docker +from gns3server.compute.docker import ( + DOCKER_MINIMUM_API_VERSION, + DOCKER_PREFERRED_API_VERSION, + PULL_IDLE_TIMEOUT, + Docker, +) from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error from tests.utils import AsyncioMagicMock, asyncio_patch @@ -168,7 +173,44 @@ async def test_pull_image(): with asyncio_patch("gns3server.compute.docker.Docker.query", side_effect=DockerHttp404Error("404")): with asyncio_patch("gns3server.compute.docker.Docker.http_query", return_value=mock_query) as mock: await Docker.instance().pull_image("ubuntu") - mock.assert_called_with("POST", "images/create", params={"fromImage": "ubuntu"}, timeout=None) + args, kwargs = mock.call_args + assert args == ("POST", "images/create") + assert kwargs["params"] == {"fromImage": "ubuntu"} + # no total timeout (a pull can take as long as it needs) but an + # idle socket read timeout, so an unreachable registry fails + assert kwargs["timeout"].total is None + assert kwargs["timeout"].sock_read == PULL_IDLE_TIMEOUT + + +@pytest.mark.asyncio +async def test_pull_image_reports_aggregated_progress(): + + class Content: + def __init__(self): + self._chunks = [ + b'{"status": "Pulling fs layer", "id": "sha256:a"}\n' + b'{"status": "Pulling fs layer", "id": "sha256:b"}\n' + b'{"status": "Downloading", "id": "sha256:a", "progress": "[==> ] 40/100"}\n' + b'{"status": "Pull complete", "id": "sha256:a"}\n', + b'{"status": "Pull complete", "id": "sha256:b"}\n', + b"", + ] + + async def read(self, size): + return self._chunks.pop(0) + + response = MagicMock() + response.content = Content() + + progress = [] + with asyncio_patch("gns3server.compute.docker.Docker.query", side_effect=DockerHttp404Error("404")): + with asyncio_patch("gns3server.compute.docker.Docker.http_query", return_value=response): + await Docker.instance().pull_image("ubuntu", progress_callback=progress.append) + # the callback sees the running totals of layers, not raw daemon lines + assert progress[0] == {"layers_total": 1, "layers_done": 0, "message": "Pulling fs layer"} + assert progress[1] == {"layers_total": 2, "layers_done": 0, "message": "Pulling fs layer"} + assert {"layers_total": 2, "layers_done": 0, "message": "[==> ] 40/100"} in progress + assert progress[-1] == {"layers_total": 2, "layers_done": 2, "message": "Pull complete"} @pytest.mark.asyncio @@ -191,7 +233,10 @@ async def test_force_pull_image(): with asyncio_patch("gns3server.compute.docker.Docker.http_query", return_value=response) as pull_mock: await Docker.instance().pull_image("ubuntu", force=True) query_mock.assert_not_called() - pull_mock.assert_called_with("POST", "images/create", params={"fromImage": "ubuntu"}, timeout=None) + args, kwargs = pull_mock.call_args + assert args == ("POST", "images/create") + assert kwargs["params"] == {"fromImage": "ubuntu"} + assert kwargs["timeout"].sock_read == PULL_IDLE_TIMEOUT @pytest.mark.asyncio diff --git a/tests/controller/test_notification.py b/tests/controller/test_notification.py index 53e24f1f2..82aa996b6 100644 --- a/tests/controller/test_notification.py +++ b/tests/controller/test_notification.py @@ -119,6 +119,34 @@ async def test_dispatch_node_updated(controller, node, project): assert event["properties"]["startup_config"] == "ip 192" +@pytest.mark.asyncio +async def test_dispatch_docker_image_routed_to_controller_stream(controller, project): + """ + docker.image.* events are scoped to a compute, not to a project: they must + reach the controller stream even when no project is open, and not the + project stream. + """ + + notif = controller.notification + with notif.controller_queue() as controller_q, notif.project_queue(project.id) as project_q: + await controller_q.get(0.1) # ping + await project_q.get(0.1) # ping + + await notif.dispatch( + "docker.image.pull", {"image": "debian", "state": "started"}, project_id=None, compute_id="local" + ) + + msg = await controller_q.get(5) + assert msg == ( + "docker.image.pull", + {"image": "debian", "state": "started", "compute_id": "local"}, + {}, + ) + # the project stream only carries its periodic ping + msg = await project_q.get(0.2) + assert msg[0] == "ping" + + @pytest.mark.asyncio async def test_dispatch_marker_routed_to_marker_channel(controller, project): """