diff --git a/gns3server/api/routes/compute/images.py b/gns3server/api/routes/compute/images.py index 2d631b84b..4a34bf794 100644 --- a/gns3server/api/routes/compute/images.py +++ b/gns3server/api/routes/compute/images.py @@ -19,6 +19,7 @@ API routes for images. """ import os +import time import urllib.parse from typing import List, Optional @@ -26,10 +27,16 @@ from fastapi import APIRouter, Body, HTTPException, Query, Request, status from fastapi.responses import FileResponse from gns3server.compute.docker import Docker +from gns3server.compute.docker.docker_error import DockerError from gns3server.compute.dynamips import Dynamips from gns3server.compute.iou import IOU +from gns3server.compute.notification_manager import NotificationManager from gns3server.compute.qemu import Qemu +# Daemon progress lines arrive dozens of times per second per layer: emit at +# most one downloading event per second per pull. +PULL_PROGRESS_INTERVAL = 1.0 + router = APIRouter() @@ -44,13 +51,51 @@ async def get_docker_images() -> List[dict]: @router.post("/docker/images/pull", status_code=status.HTTP_204_NO_CONTENT) -async def pull_docker_image(image: str = Body(..., embed=True, min_length=1, pattern=r"^\S+$")) -> None: +async def pull_docker_image( + image: str = Body(..., embed=True, min_length=1, pattern=r"^\S+$"), + progress_id: Optional[str] = Query( + None, pattern=r"^\S+$", description="Client supplied id to correlate progress notifications" + ), +) -> None: """ Pull or update a Docker image. + + The pull progress is reported as docker.image.pull notifications: a + started event, throttled downloading events and a terminal completed or + failed event. The optional progress_id is echoed in every event so a + client can correlate them with its own request. """ docker_manager = Docker.instance() - await docker_manager.pull_image(image, force=True) + + def emit(state: str, message: Optional[str] = None, progress: Optional[dict] = None) -> None: + event = {"image": image, "state": state} + if message is not None: + event["message"] = message + if progress is not None: + event["layers_total"] = progress["layers_total"] + event["layers_done"] = progress["layers_done"] + if progress_id is not None: + event["progress_id"] = progress_id + NotificationManager.instance().emit("docker.image.pull", event) + + last_emit = 0.0 + + def progress_callback(progress: dict) -> None: + nonlocal last_emit + now = time.monotonic() + if now - last_emit < PULL_PROGRESS_INTERVAL: + return + last_emit = now + emit("downloading", progress["message"], progress) + + emit("started", f"Pulling '{image}' from Docker repository") + try: + await docker_manager.pull_image(image, progress_callback=progress_callback, force=True) + except DockerError as e: + emit("failed", str(e)) + raise + emit("completed", f"Success pulling image {image}") @router.post("/docker/images/load", status_code=status.HTTP_204_NO_CONTENT) diff --git a/gns3server/api/routes/controller/computes.py b/gns3server/api/routes/controller/computes.py index 90bd51486..51c6f51d2 100644 --- a/gns3server/api/routes/controller/computes.py +++ b/gns3server/api/routes/controller/computes.py @@ -215,16 +215,26 @@ async def docker_get_images(compute_id: Union[str, UUID]) -> List[schemas.Comput dependencies=[Depends(has_privilege("Compute.Modify"))], ) async def docker_pull_image( - compute_id: Union[str, UUID], image: str = Body(..., embed=True, min_length=1, pattern=r"^\S+$") + compute_id: Union[str, UUID], + image: str = Body(..., embed=True, min_length=1, pattern=r"^\S+$"), + progress_id: Optional[str] = Query( + None, pattern=r"^\S+$", description="Client supplied id to correlate progress notifications" + ), ) -> None: """ Pull or update a Docker image on a compute. + The pull progress is reported on the controller notification stream as + docker.image.pull events (started, downloading, completed or failed). The + optional progress_id is echoed in every event so a client can correlate + them with its own request. + Required privilege: Compute.Modify """ compute = Controller.instance().get_compute(str(compute_id)) - await compute.forward("POST", "docker", "images/pull", data={"image": image}) + params = {"progress_id": progress_id} if progress_id is not None else None + await compute.forward("POST", "docker", "images/pull", data={"image": image}, params=params) @router.post( diff --git a/gns3server/compute/docker/__init__.py b/gns3server/compute/docker/__init__.py index f71141ab3..c75f2a86c 100644 --- a/gns3server/compute/docker/__init__.py +++ b/gns3server/compute/docker/__init__.py @@ -51,6 +51,11 @@ DOCKER_MINIMUM_VERSION = "19.03.8" DOCKER_PREFERRED_API_VERSION = "1.44" CHUNK_SIZE = 1024 * 8 # 8KB +# A pull streams progress from the daemon for its whole duration: when no data +# arrives for this many seconds the registry is unreachable and the pull would +# otherwise hang forever. This is an idle timeout, not a total time limit. +PULL_IDLE_TIMEOUT = 60 + class Docker(BaseManager): _NODE_CLASS = DockerVM @@ -291,7 +296,8 @@ class Docker(BaseManager): :param path: Endpoint in API :param data: Dictionary with the body. Will be transformed to a JSON :param params: Parameters added as a query arg - :param timeout: Timeout + :param timeout: Total timeout in seconds, an aiohttp ClientTimeout object + for a finer policy (e.g. an idle socket read timeout), or None for no timeout :returns: HTTP response """ @@ -302,7 +308,7 @@ class Docker(BaseManager): # not a dict (e.g. a Docker image tar stream): let aiohttp stream the raw body headers = {"content-type": "application/x-tar"} if timeout is None: - timeout = 60 * 60 * 24 * 31 # One month timeout + timeout = aiohttp.ClientTimeout(total=None) if path == "version": url = "http://docker/" + path @@ -361,7 +367,8 @@ class Docker(BaseManager): Pulls an image from the Docker repository :params image: Image name - :params progress_callback: A function that receive a log message about image download progress + :params progress_callback: A function that receives a dict with the + overall progress of the pull: layer counts and the latest daemon message :params force: Pull the image even if it is already available locally """ @@ -372,10 +379,23 @@ class Docker(BaseManager): except DockerHttp404Error: pass - if progress_callback: - progress_callback(f"Pulling '{image}' from Docker repository") + layers_total = 0 + layers_done = 0 + + def report(message): + if progress_callback: + progress_callback({"layers_total": layers_total, "layers_done": layers_done, "message": message}) + try: - response = await self.http_query("POST", "images/create", params={"fromImage": image}, timeout=None) + # The daemon streams progress lines for the whole pull: an idle + # socket read for a minute means the registry is unreachable and + # the pull would otherwise hang forever. + response = await self.http_query( + "POST", + "images/create", + params={"fromImage": image}, + timeout=aiohttp.ClientTimeout(total=None, sock_read=PULL_IDLE_TIMEOUT), + ) except DockerError as e: raise DockerError( f"Could not pull the '{image}' image from Docker repository, " @@ -392,7 +412,11 @@ class Docker(BaseManager): f"Disconnected while pulling Docker image '{image}' from Docker repository" ) from e except TimeoutError as e: - raise DockerError(f"Timeout while pulling Docker image '{image}' from Docker repository") from e + raise DockerError( + f"Timeout while pulling Docker image '{image}' from Docker repository: " + f"no data received for {PULL_IDLE_TIMEOUT} seconds, " + f"please check your Internet connection or Docker registry mirror" + ) from e if not chunk: break content += chunk.decode("utf-8") @@ -409,8 +433,14 @@ class Docker(BaseManager): error = error_detail.get("message") if error: raise DockerError(error) - if "progress" in answer and progress_callback: - progress_callback("Pulling image {}:{}: {}".format(image, answer["id"], answer["progress"])) + status = answer.get("status", "") + if status == "Pulling fs layer": + layers_total += 1 + elif status in ("Pull complete", "Already exists"): + layers_done += 1 + message = answer.get("progress") or status + if message: + report(message) content = content[index:] except ValueError: # Partial JSON pass @@ -420,9 +450,6 @@ class Docker(BaseManager): finally: response.close() - if progress_callback: - progress_callback(f"Success pulling image {image}") - @locking async def load_image(self, stream, image=None, digest=None, progress_callback=None): """ diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index 8a9e6fe87..be1e0fdba 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -733,13 +733,13 @@ class Compute: async def delete(self, path, **kwargs): return await self.http_query("DELETE", path, **kwargs) - async def forward(self, method, type, path, data=None): + async def forward(self, method, type, path, data=None, params=None): """ Forward a call to the emulator on compute """ try: action = f"/{type}/{path}" - res = await self.http_query(method, action, data=data, timeout=None) + res = await self.http_query(method, action, data=data, timeout=None, params=params) except aiohttp.ServerDisconnectedError: raise ControllerError(f"Connection lost to {self._id} during {method} {action}") return res.json diff --git a/gns3server/controller/notification.py b/gns3server/controller/notification.py index 711fabcc2..6eee15572 100644 --- a/gns3server/controller/notification.py +++ b/gns3server/controller/notification.py @@ -126,6 +126,12 @@ class Notification: elif action == "ping": event["compute_id"] = compute_id self.project_emit(action, event) + elif action.startswith("docker.image."): + # Image operations are scoped to a compute, not to a project, and + # clients that are not in a project still need them: emit on the + # controller stream, like compute.updated. + event["compute_id"] = compute_id + self.controller_emit(action, event) elif action.startswith("marker."): self.marker_emit(action, event, project_id) else: diff --git a/tests/api/routes/compute/test_images.py b/tests/api/routes/compute/test_images.py index 041a55702..ae3a22e4b 100644 --- a/tests/api/routes/compute/test_images.py +++ b/tests/api/routes/compute/test_images.py @@ -18,6 +18,7 @@ import pytest from fastapi import FastAPI, status from httpx import AsyncClient +from gns3server.compute.docker.docker_error import DockerError from tests.utils import asyncio_patch pytestmark = pytest.mark.asyncio @@ -27,11 +28,64 @@ class TestImagesRoutes: async def test_pull_docker_image(self, app: FastAPI, compute_client: AsyncClient) -> None: with asyncio_patch("gns3server.compute.docker.Docker.pull_image") as mock: - response = await compute_client.post( - app.url_path_for("compute:pull_docker_image"), json={"image": "nginx:latest"} - ) - mock.assert_called_once_with("nginx:latest", force=True) - assert response.status_code == status.HTTP_204_NO_CONTENT + with asyncio_patch("gns3server.compute.notification_manager.NotificationManager.emit") as emit_mock: + response = await compute_client.post( + app.url_path_for("compute:pull_docker_image"), json={"image": "nginx:latest"} + ) + assert mock.call_args[0] == ("nginx:latest",) + assert mock.call_args[1]["force"] is True + assert callable(mock.call_args[1]["progress_callback"]) + # started and completed events are emitted to the notification stream + assert [(c.args[0], c.args[1]["state"]) for c in emit_mock.call_args_list] == [ + ("docker.image.pull", "started"), + ("docker.image.pull", "completed"), + ] + assert response.status_code == status.HTTP_204_NO_CONTENT + + async def test_pull_docker_image_progress_id(self, app: FastAPI, compute_client: AsyncClient) -> None: + + with asyncio_patch("gns3server.compute.docker.Docker.pull_image"): + with asyncio_patch("gns3server.compute.notification_manager.NotificationManager.emit") as emit_mock: + response = await compute_client.post( + app.url_path_for("compute:pull_docker_image"), + json={"image": "nginx:latest"}, + params={"progress_id": "abc123"}, + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + assert all(c.args[1]["progress_id"] == "abc123" for c in emit_mock.call_args_list) + + async def test_pull_docker_image_reports_downloading(self, app: FastAPI, compute_client: AsyncClient) -> None: + + async def fake_pull(image, progress_callback=None, force=False): + progress_callback({"layers_total": 2, "layers_done": 1, "message": "[==> ] 40/100"}) + + with asyncio_patch("gns3server.compute.docker.Docker.pull_image", side_effect=fake_pull): + with asyncio_patch("gns3server.compute.notification_manager.NotificationManager.emit") as emit_mock: + response = await compute_client.post( + app.url_path_for("compute:pull_docker_image"), json={"image": "nginx:latest"} + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + states = [c.args[1]["state"] for c in emit_mock.call_args_list] + assert states == ["started", "downloading", "completed"] + downloading = emit_mock.call_args_list[1].args[1] + assert downloading["layers_total"] == 2 + assert downloading["layers_done"] == 1 + assert downloading["message"] == "[==> ] 40/100" + + async def test_pull_docker_image_emits_failed(self, app: FastAPI, compute_client: AsyncClient) -> None: + + async def fake_pull(image, progress_callback=None, force=False): + raise DockerError("no data received for 60 seconds") + + with asyncio_patch("gns3server.compute.docker.Docker.pull_image", side_effect=fake_pull): + with asyncio_patch("gns3server.compute.notification_manager.NotificationManager.emit") as emit_mock: + response = await compute_client.post( + app.url_path_for("compute:pull_docker_image"), json={"image": "nginx:latest"} + ) + assert response.status_code == status.HTTP_409_CONFLICT + failed = emit_mock.call_args_list[-1].args[1] + assert failed["state"] == "failed" + assert "no data received" in failed["message"] @pytest.mark.parametrize("image", ["", " ", "nginx latest"]) async def test_pull_docker_image_rejects_invalid_name( diff --git a/tests/api/routes/controller/test_computes.py b/tests/api/routes/controller/test_computes.py index c32b245e8..c38b70592 100644 --- a/tests/api/routes/controller/test_computes.py +++ b/tests/api/routes/controller/test_computes.py @@ -197,7 +197,31 @@ class TestComputeFeatures: app.url_path_for("docker_pull_image", compute_id=test_compute.compute_id), json={"image": "nginx:latest"}, ) - compute.forward.assert_called_with("POST", "docker", "images/pull", data={"image": "nginx:latest"}) + compute.forward.assert_called_with( + "POST", "docker", "images/pull", data={"image": "nginx:latest"}, params=None + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + + async def test_compute_pull_docker_image_progress_id( + self, app: FastAPI, client: AsyncClient, test_compute: Compute + ) -> None: + + compute = MagicMock() + compute.forward = AsyncioMagicMock(return_value={}) + with patch("gns3server.api.routes.controller.computes.Controller.instance") as controller: + controller.return_value.get_compute.return_value = compute + response = await client.post( + app.url_path_for("docker_pull_image", compute_id=test_compute.compute_id), + json={"image": "nginx:latest"}, + params={"progress_id": "abc123"}, + ) + compute.forward.assert_called_with( + "POST", + "docker", + "images/pull", + data={"image": "nginx:latest"}, + params={"progress_id": "abc123"}, + ) assert response.status_code == status.HTTP_204_NO_CONTENT async def test_compute_delete_docker_image(self, app: FastAPI, client: AsyncClient, test_compute: Compute) -> None: diff --git a/tests/compute/docker/test_docker.py b/tests/compute/docker/test_docker.py index 8699d6337..71636d04d 100644 --- a/tests/compute/docker/test_docker.py +++ b/tests/compute/docker/test_docker.py @@ -21,7 +21,12 @@ from unittest.mock import MagicMock, patch import pytest import pytest_asyncio -from gns3server.compute.docker import DOCKER_MINIMUM_API_VERSION, DOCKER_PREFERRED_API_VERSION, Docker +from gns3server.compute.docker import ( + DOCKER_MINIMUM_API_VERSION, + DOCKER_PREFERRED_API_VERSION, + PULL_IDLE_TIMEOUT, + Docker, +) from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error from tests.utils import AsyncioMagicMock, asyncio_patch @@ -168,7 +173,44 @@ async def test_pull_image(): with asyncio_patch("gns3server.compute.docker.Docker.query", side_effect=DockerHttp404Error("404")): with asyncio_patch("gns3server.compute.docker.Docker.http_query", return_value=mock_query) as mock: await Docker.instance().pull_image("ubuntu") - mock.assert_called_with("POST", "images/create", params={"fromImage": "ubuntu"}, timeout=None) + args, kwargs = mock.call_args + assert args == ("POST", "images/create") + assert kwargs["params"] == {"fromImage": "ubuntu"} + # no total timeout (a pull can take as long as it needs) but an + # idle socket read timeout, so an unreachable registry fails + assert kwargs["timeout"].total is None + assert kwargs["timeout"].sock_read == PULL_IDLE_TIMEOUT + + +@pytest.mark.asyncio +async def test_pull_image_reports_aggregated_progress(): + + class Content: + def __init__(self): + self._chunks = [ + b'{"status": "Pulling fs layer", "id": "sha256:a"}\n' + b'{"status": "Pulling fs layer", "id": "sha256:b"}\n' + b'{"status": "Downloading", "id": "sha256:a", "progress": "[==> ] 40/100"}\n' + b'{"status": "Pull complete", "id": "sha256:a"}\n', + b'{"status": "Pull complete", "id": "sha256:b"}\n', + b"", + ] + + async def read(self, size): + return self._chunks.pop(0) + + response = MagicMock() + response.content = Content() + + progress = [] + with asyncio_patch("gns3server.compute.docker.Docker.query", side_effect=DockerHttp404Error("404")): + with asyncio_patch("gns3server.compute.docker.Docker.http_query", return_value=response): + await Docker.instance().pull_image("ubuntu", progress_callback=progress.append) + # the callback sees the running totals of layers, not raw daemon lines + assert progress[0] == {"layers_total": 1, "layers_done": 0, "message": "Pulling fs layer"} + assert progress[1] == {"layers_total": 2, "layers_done": 0, "message": "Pulling fs layer"} + assert {"layers_total": 2, "layers_done": 0, "message": "[==> ] 40/100"} in progress + assert progress[-1] == {"layers_total": 2, "layers_done": 2, "message": "Pull complete"} @pytest.mark.asyncio @@ -191,7 +233,10 @@ async def test_force_pull_image(): with asyncio_patch("gns3server.compute.docker.Docker.http_query", return_value=response) as pull_mock: await Docker.instance().pull_image("ubuntu", force=True) query_mock.assert_not_called() - pull_mock.assert_called_with("POST", "images/create", params={"fromImage": "ubuntu"}, timeout=None) + args, kwargs = pull_mock.call_args + assert args == ("POST", "images/create") + assert kwargs["params"] == {"fromImage": "ubuntu"} + assert kwargs["timeout"].sock_read == PULL_IDLE_TIMEOUT @pytest.mark.asyncio diff --git a/tests/controller/test_notification.py b/tests/controller/test_notification.py index 53e24f1f2..82aa996b6 100644 --- a/tests/controller/test_notification.py +++ b/tests/controller/test_notification.py @@ -119,6 +119,34 @@ async def test_dispatch_node_updated(controller, node, project): assert event["properties"]["startup_config"] == "ip 192" +@pytest.mark.asyncio +async def test_dispatch_docker_image_routed_to_controller_stream(controller, project): + """ + docker.image.* events are scoped to a compute, not to a project: they must + reach the controller stream even when no project is open, and not the + project stream. + """ + + notif = controller.notification + with notif.controller_queue() as controller_q, notif.project_queue(project.id) as project_q: + await controller_q.get(0.1) # ping + await project_q.get(0.1) # ping + + await notif.dispatch( + "docker.image.pull", {"image": "debian", "state": "started"}, project_id=None, compute_id="local" + ) + + msg = await controller_q.get(5) + assert msg == ( + "docker.image.pull", + {"image": "debian", "state": "started", "compute_id": "local"}, + {}, + ) + # the project stream only carries its periodic ping + msg = await project_q.get(0.2) + assert msg[0] == "ping" + + @pytest.mark.asyncio async def test_dispatch_marker_routed_to_marker_channel(controller, project): """