diff --git a/gns3server/api/routes/compute/images.py b/gns3server/api/routes/compute/images.py index 5e5f6410b..87b2fea43 100644 --- a/gns3server/api/routes/compute/images.py +++ b/gns3server/api/routes/compute/images.py @@ -19,16 +19,24 @@ API routes for images. """ import os +import time import urllib.parse from typing import List, Optional -from fastapi import APIRouter, Body, HTTPException, Query, Request, status +from fastapi import APIRouter, Body, HTTPException, Path, Query, Request, status from fastapi.responses import FileResponse from gns3server.compute.docker import Docker +from gns3server.compute.docker.docker_error import DockerError from gns3server.compute.dynamips import Dynamips from gns3server.compute.iou import IOU +from gns3server.compute.notification_manager import NotificationManager from gns3server.compute.qemu import Qemu +from gns3server.utils.images import DOCKER_IMAGE_NAME_PATTERN, validate_docker_image_name + +# Daemon progress lines arrive dozens of times per second per layer: emit at +# most one downloading event per second per pull. +PULL_PROGRESS_INTERVAL = 1.0 router = APIRouter() @@ -44,13 +52,55 @@ async def get_docker_images() -> List[dict]: @router.post("/docker/images/pull", status_code=status.HTTP_204_NO_CONTENT) -async def pull_docker_image(image: str = Body(..., embed=True, min_length=1, pattern=r"^\S+$")) -> None: +async def pull_docker_image( + image: str = Body(..., embed=True, min_length=1, pattern=r"^\S+$"), + progress_id: Optional[str] = Query( + None, pattern=r"^\S+$", description="Client supplied id to correlate progress notifications" + ), +) -> None: """ Pull or update a Docker image. + + The pull progress is reported as docker.image.pull notifications: a + started event, throttled downloading events and a terminal completed or + failed event. The optional progress_id is echoed in every event so a + client can correlate them with its own request. """ docker_manager = Docker.instance() - await docker_manager.pull_image(image, force=True) + + def emit(state: str, message: Optional[str] = None, progress: Optional[dict] = None) -> None: + event = {"image": image, "state": state} + if message is not None: + event["message"] = message + if progress is not None: + event["layers_total"] = progress["layers_total"] + event["layers_done"] = progress["layers_done"] + if progress_id is not None: + event["progress_id"] = progress_id + NotificationManager.instance().emit("docker.image.pull", event) + + last_emit = 0.0 + + def progress_callback(progress: dict) -> None: + nonlocal last_emit + # the started event comes from the pull itself, once it owns the lock, + # and is never throttled + if progress.get("state") == "started": + emit("started", progress["message"]) + return + now = time.monotonic() + if now - last_emit < PULL_PROGRESS_INTERVAL: + return + last_emit = now + emit("downloading", progress["message"], progress) + + try: + await docker_manager.pull_image(image, progress_callback=progress_callback, force=True) + except DockerError as e: + emit("failed", str(e)) + raise + emit("completed", f"Success pulling image {image}") @router.post("/docker/images/load", status_code=status.HTTP_204_NO_CONTENT) @@ -73,6 +123,27 @@ async def load_docker_image( await docker_manager.load_image(request.stream(), image=image, digest=digest) +@router.delete("/docker/images/{image:path}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_docker_image( + image: str = Path(..., pattern=DOCKER_IMAGE_NAME_PATTERN), + force: bool = Query(False, description="Remove the image even if a container uses it"), +) -> None: + """ + Delete a Docker image from the Docker daemon. + + The daemon refuses to delete an image used by a container (even a stopped + one) and answers a conflict, unless force is set. + """ + + image = urllib.parse.unquote(image) + try: + validate_docker_image_name(image) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(e)) from e + docker_manager = Docker.instance() + await docker_manager.delete_image(image, force=force) + + @router.get("/dynamips/images") async def get_dynamips_images() -> List[dict]: """ diff --git a/gns3server/api/routes/controller/computes.py b/gns3server/api/routes/controller/computes.py index c8de19a55..b7cb04920 100644 --- a/gns3server/api/routes/controller/computes.py +++ b/gns3server/api/routes/controller/computes.py @@ -21,7 +21,7 @@ API routes for computes. from typing import Any, List, Optional, Union from uuid import UUID -from fastapi import APIRouter, Body, Depends, Query, status +from fastapi import APIRouter, Body, Depends, HTTPException, Path, Query, Request, status import gns3server.db.models as models from gns3server import schemas @@ -29,7 +29,13 @@ from gns3server.controller import Controller from gns3server.controller.controller_error import ControllerBadRequestError, ControllerError from gns3server.db.repositories.computes import ComputesRepository from gns3server.db.repositories.rbac import RbacRepository +from gns3server.db.repositories.templates import TemplatesRepository from gns3server.services.computes import ComputesService +from gns3server.utils.images import ( + DOCKER_IMAGE_NAME_PATTERN, + canonical_docker_image_name, + validate_docker_image_name, +) from .dependencies.database import get_repository from .dependencies.rbac import has_privilege @@ -215,16 +221,111 @@ async def docker_get_images(compute_id: Union[str, UUID]) -> List[schemas.Comput dependencies=[Depends(has_privilege("Compute.Modify"))], ) async def docker_pull_image( - compute_id: Union[str, UUID], image: str = Body(..., embed=True, min_length=1, pattern=r"^\S+$") + compute_id: Union[str, UUID], + image: str = Body(..., embed=True, min_length=1, pattern=r"^\S+$"), + progress_id: Optional[str] = Query( + None, pattern=r"^\S+$", description="Client supplied id to correlate progress notifications" + ), ) -> None: """ Pull or update a Docker image on a compute. + The pull progress is reported on the controller notification stream as + docker.image.pull events (started, downloading, completed or failed). The + optional progress_id is echoed in every event so a client can correlate + them with its own request. + Required privilege: Compute.Modify """ compute = Controller.instance().get_compute(str(compute_id)) - await compute.forward("POST", "docker", "images/pull", data={"image": image}) + params = {"progress_id": progress_id} if progress_id is not None else None + await compute.forward("POST", "docker", "images/pull", data={"image": image}, params=params) + + +@router.post( + "/{compute_id}/docker/images/load", + status_code=status.HTTP_204_NO_CONTENT, + dependencies=[Depends(has_privilege("Compute.Modify"))], +) +async def docker_load_image(compute_id: Union[str, UUID], request: Request) -> None: + """ + Load a Docker image on a compute from a docker save tar stream. + + The tar archive (plain or gzip compressed) is streamed to the compute + and loaded into its Docker daemon, without buffering the whole file + in the controller memory. + + Required privilege: Compute.Modify + """ + + compute = Controller.instance().get_compute(str(compute_id)) + await compute.http_query("POST", "/docker/images/load", data=request.stream(), timeout=None) + + +@router.delete( + "/{compute_id}/docker/images/{image:path}", + status_code=status.HTTP_204_NO_CONTENT, + dependencies=[Depends(has_privilege("Compute.Modify"))], +) +async def docker_delete_image( + compute_id: Union[str, UUID], + image: str = Path(..., pattern=DOCKER_IMAGE_NAME_PATTERN), + force: bool = Query(False, description="Remove the image even if a container uses it"), + templates_repo: TemplatesRepository = Depends(get_repository(TemplatesRepository)), +) -> None: + """ + Delete a Docker image on a compute. + + The deletion is refused while templates reference the image, unless force + is set. The compute also refuses an image used by a container (even a + stopped one) and answers a conflict naming the GNS3 nodes and projects + using it. + + After a successful deletion a docker.image.deleted notification is + emitted on the controller notification stream, with the templates that + referenced the image, so clients can refresh their image lists. + + Required privilege: Compute.Modify + """ + + try: + validate_docker_image_name(image) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(e)) from e + + canonical_image = canonical_docker_image_name(image) + referencing_templates = [ + str(template.name) + for template in await templates_repo.get_templates() + if isinstance(template, models.DockerTemplate) + and template.image + and canonical_docker_image_name(str(template.image)) == canonical_image + ] + + if referencing_templates and not force: + # refuse before touching the compute: the client can prompt the user + # and repeat with force once the user confirms + raise ControllerError( + f"Image '{image}' is referenced by template(s): {', '.join(referencing_templates)}. " + f"Delete the templates first or repeat with force=true.", + code="image_referenced_by_templates", + details={"referenced_by_templates": referencing_templates}, + ) + + compute = Controller.instance().get_compute(str(compute_id)) + await compute.http_query("DELETE", f"/docker/images/{image}", params={"force": str(force).lower()}) + # tell the clients the image changed on this compute: they can refresh + # their image lists, and the templates that referenced the image can no + # longer start their nodes + Controller.instance().notification.controller_emit( + "docker.image.deleted", + { + "compute_id": str(compute_id), + "image": image, + "referenced_by_templates": referencing_templates, + }, + ) @router.get("/{compute_id}/virtualbox/vms", response_model=List[schemas.ComputeVirtualBoxVM]) diff --git a/gns3server/compute/docker/__init__.py b/gns3server/compute/docker/__init__.py index fdb3eaf51..4c88a6407 100644 --- a/gns3server/compute/docker/__init__.py +++ b/gns3server/compute/docker/__init__.py @@ -35,12 +35,12 @@ from gns3server.compute.docker.docker_error import ( DockerHttp404Error, DockerHttp409Error, ) +from gns3server.compute.docker.docker_images import DockerImagesMixin from gns3server.compute.docker.docker_vm import DockerVM from gns3server.compute.docker.iol_docker_vm import IOLDockerVM from gns3server.compute.docker.vendor_docker_vm import VendorDockerVM from gns3server.config import Config from gns3server.utils import parse_version -from gns3server.utils.asyncio import locking log = logging.getLogger(__name__) @@ -49,10 +49,9 @@ log = logging.getLogger(__name__) DOCKER_MINIMUM_API_VERSION = "1.40" DOCKER_MINIMUM_VERSION = "19.03.8" DOCKER_PREFERRED_API_VERSION = "1.44" -CHUNK_SIZE = 1024 * 8 # 8KB -class Docker(BaseManager): +class Docker(DockerImagesMixin, BaseManager): _NODE_CLASS = DockerVM def __init__(self): @@ -291,7 +290,8 @@ class Docker(BaseManager): :param path: Endpoint in API :param data: Dictionary with the body. Will be transformed to a JSON :param params: Parameters added as a query arg - :param timeout: Timeout + :param timeout: Total timeout in seconds, an aiohttp ClientTimeout object + for a finer policy (e.g. an idle socket read timeout), or None for no timeout :returns: HTTP response """ @@ -302,7 +302,7 @@ class Docker(BaseManager): # not a dict (e.g. a Docker image tar stream): let aiohttp stream the raw body headers = {"content-type": "application/x-tar"} if timeout is None: - timeout = 60 * 60 * 24 * 31 # One month timeout + timeout = aiohttp.ClientTimeout(total=None) if path == "version": url = "http://docker/" + path @@ -354,177 +354,3 @@ class Docker(BaseManager): url = "http://docker/v" + self._api_version + "/" + path connection = await self._session.ws_connect(url, origin="http://docker", autoping=True) return connection - - @locking - async def pull_image(self, image, progress_callback=None, force=False): - """ - Pulls an image from the Docker repository - - :params image: Image name - :params progress_callback: A function that receive a log message about image download progress - :params force: Pull the image even if it is already available locally - """ - - if not force: - try: - await self.query("GET", f"images/{image}/json") - return # We already have the image skip the download - except DockerHttp404Error: - pass - - if progress_callback: - progress_callback(f"Pulling '{image}' from Docker repository") - try: - response = await self.http_query("POST", "images/create", params={"fromImage": image}, timeout=None) - except DockerError as e: - raise DockerError( - f"Could not pull the '{image}' image from Docker repository, " - f"please check your Internet connection (original error: {e})" - ) - # The pull api will stream status via an HTTP JSON stream - content = "" - try: - while True: - try: - chunk = await response.content.read(CHUNK_SIZE) - except aiohttp.ServerDisconnectedError as e: - raise DockerError( - f"Disconnected while pulling Docker image '{image}' from Docker repository" - ) from e - except TimeoutError as e: - raise DockerError(f"Timeout while pulling Docker image '{image}' from Docker repository") from e - if not chunk: - break - content += chunk.decode("utf-8") - - try: - while True: - content = content.lstrip(" \r\n\t") - answer, index = json.JSONDecoder().raw_decode(content) - if not isinstance(answer, dict): - raise DockerError(f"Invalid response while pulling Docker image '{image}'") - error_detail = answer.get("errorDetail") - error = answer.get("error") - if not error and isinstance(error_detail, dict): - error = error_detail.get("message") - if error: - raise DockerError(error) - if "progress" in answer and progress_callback: - progress_callback("Pulling image {}:{}: {}".format(image, answer["id"], answer["progress"])) - content = content[index:] - except ValueError: # Partial JSON - pass - - if content.strip(): - raise DockerError(f"Invalid response while pulling Docker image '{image}'") - finally: - response.close() - - if progress_callback: - progress_callback(f"Success pulling image {image}") - - @locking - async def load_image(self, stream, image=None, digest=None, progress_callback=None): - """ - Load a Docker image into the Docker daemon from a docker save tar stream - - :param stream: An async iterable of bytes (the tar produced by docker save) - :param image: Tag of the image in the archive. Together with digest, the - (controller digest, local id) pair is remembered so that node creation - can accept the digest the controller pinned for this image - :param digest: Image id as reported by the Docker daemon on the controller - host (the value the controller pins in the node create payload) - :param progress_callback: A function that receive a log message about image load progress - """ - - if progress_callback: - progress_callback("Loading Docker image from stream") - response = await self.http_query("POST", "images/load", data=stream, timeout=None) - # The load api will stream status via an HTTP JSON stream - content = "" - try: - while True: - try: - chunk = await response.content.read(CHUNK_SIZE) - except aiohttp.ServerDisconnectedError as e: - raise DockerError("Disconnected while loading Docker image") from e - except TimeoutError as e: - raise DockerError("Timeout while loading Docker image") from e - if not chunk: - break - content += chunk.decode("utf-8", errors="ignore") - - try: - while True: - content = content.lstrip(" \r\n\t") - answer, index = json.JSONDecoder().raw_decode(content) - if not isinstance(answer, dict): - raise DockerError("Invalid response while loading Docker image") - error_detail = answer.get("errorDetail") - error = answer.get("error") - if not error and isinstance(error_detail, dict): - error = error_detail.get("message") - if error: - raise DockerError(error) - if "stream" in answer and progress_callback: - progress_callback(answer["stream"].rstrip()) - content = content[index:] - except ValueError: # Partial JSON - pass - - if content.strip(): - raise DockerError("Invalid response while loading Docker image") - finally: - response.close() - - if progress_callback: - progress_callback("Docker image loaded") - - if image and digest: - await self._remember_image_id_translation(image, digest) - - async def _remember_image_id_translation(self, image, controller_digest): - """ - Remember how the local daemon identifies an image that was just loaded - from the controller. - - Docker ids are not comparable across image stores: for the same content - a classic store reports the config digest, while a containerd store (or - an image loaded from an OCI archive) reports the manifest digest. The - controller pins the id of its own daemon, so recording the pair lets - DockerVM accept that pin for the image it actually received instead of - asking for another full sync on every node creation. - """ - - try: - image_infos = await self.query("GET", f"images/{image}/json") - except DockerError as e: - log.warning(f"Image '{image}' has been loaded but cannot be inspected: {e}") - return - local_id = image_infos.get("Id") - if local_id: - self._image_id_translations[image] = (controller_digest, local_id) - - def image_id_translation(self, image): - """ - :returns: the (controller digest, local id) pair recorded the last time - this image was loaded from the controller, None otherwise - """ - - return self._image_id_translations.get(image) - - async def list_images(self): - """ - Gets Docker image list. - - :returns: list of dicts - :rtype: list - """ - - images = [] - for image in await self.query("GET", "images/json", params={"all": 0}): - if image["RepoTags"]: - for tag in image["RepoTags"]: - if tag != ":": - images.append({"image": tag}) - return sorted(images, key=lambda i: i["image"]) diff --git a/gns3server/compute/docker/docker_images.py b/gns3server/compute/docker/docker_images.py new file mode 100644 index 000000000..fb62e00e8 --- /dev/null +++ b/gns3server/compute/docker/docker_images.py @@ -0,0 +1,395 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Free Software Foundation as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +The image-management half of the Docker manager: pull, load, list and +delete images, plus the translation between the id the controller pins and +the id this daemon reports for the same content. Consumed by the Docker +manager as a mixin, so __init__.py stays focused on the daemon connection, +the HTTP layer and the node factory. +""" + +import json +import logging +import re +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any + +import aiohttp + +from gns3server.compute.compute_error import ComputeError +from gns3server.compute.docker.docker_error import ( + DockerError, + DockerHttp404Error, + DockerHttp409Error, +) +from gns3server.compute.project_manager import ProjectManager +from gns3server.utils.asyncio import locking + +log = logging.getLogger(__name__) + +CHUNK_SIZE = 1024 * 8 # 8KB + +# A pull streams progress from the daemon for its whole duration: when no data +# arrives for this many seconds the registry is unreachable and the pull would +# otherwise hang forever. This is an idle timeout, not a total time limit. +PULL_IDLE_TIMEOUT = 60 + +# Docker names GNS3 node containers "GNS3.{node name}.{project_id}", so a +# container name can be translated back into the node and project that block +# an image deletion. The node name may itself contain dots. +GNS3_CONTAINER_NAME_RE = re.compile( + r"^GNS3\.(?P.+)\.(?P[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12})$" +) +# The containers blocking an image deletion, as named by the daemon, e.g. +# 'container 6d59f00236ff is using its referenced image d529dd0c6e55' +CONFLICTING_CONTAINER_RE = re.compile(r"container ([0-9a-f]{12,64})\b") + +# Images built for reproducibility, such as the Ubuntu images assembled with +# rockcraft, carry no created timestamp in their config and Docker reports 0 +# (displayed as 1970-01-01). Their build tool writes the real time in this +# OCI label instead. +OCI_CREATED_LABEL = "org.opencontainers.image.created" + + +def image_created_timestamp(image: dict) -> int: + """ + Build time of an image as a Unix timestamp. + + The daemon timestamp wins when it is set. When it is zero, fall back to + the OCI created label, if present. + + :param image: an image summary as returned by the daemon images API + """ + + created = image.get("Created") or 0 + if created > 0: + return created + label = (image.get("Labels") or {}).get(OCI_CREATED_LABEL, "") + if not label: + return created + try: + built = datetime.fromisoformat(label) + except ValueError: + log.warning(f"Ignoring invalid {OCI_CREATED_LABEL} label: {label!r}") + return created + if built.tzinfo is None: + built = built.replace(tzinfo=UTC) + return int(built.timestamp()) + + +class DockerImagesMixin: + """ + Image management for the Docker manager. + + The host manager (Docker) provides the daemon connection: query(), + http_query() and the _image_id_translations dict filled in its __init__. + """ + + if TYPE_CHECKING: + _image_id_translations: dict[str, tuple[str, str]] + + def query(self, method: str, path: str, *args: Any, **kwargs: Any) -> Any: ... + + def http_query(self, method: str, path: str, *args: Any, **kwargs: Any) -> Any: ... + + @locking + async def pull_image(self, image, progress_callback=None, force=False): + """ + Pulls an image from the Docker repository + + :params image: Image name + :params progress_callback: A function that receives a dict with the + overall progress of the pull: layer counts and the latest daemon + message. The first call carries state="started" and happens once + the pull owns the compute pull lock, so a caller queued behind + another pull only sees it when the pull really begins. + :params force: Pull the image even if it is already available locally + """ + + layers_total = 0 + layers_done = 0 + + def report(message, state=None): + if progress_callback: + progress = {"layers_total": layers_total, "layers_done": layers_done, "message": message} + if state is not None: + progress["state"] = state + progress_callback(progress) + + report(f"Pulling '{image}' from Docker repository", state="started") + + if not force: + try: + await self.query("GET", f"images/{image}/json") + return # We already have the image skip the download + except DockerHttp404Error: + pass + + params = {"fromImage": image} + if "@" not in image: + # Without a tag or digest the daemon pulls every tag of the + # repository. Only a ':' after the last '/' is a tag separator: + # before it, it is a registry port (e.g. registry:5000/repo). + if ":" not in image.rpartition("/")[2]: + params["tag"] = "latest" + + try: + # The daemon streams progress lines for the whole pull: an idle + # socket read for a minute means the registry is unreachable and + # the pull would otherwise hang forever. + response = await self.http_query( + "POST", + "images/create", + params=params, + timeout=aiohttp.ClientTimeout(total=None, sock_read=PULL_IDLE_TIMEOUT), + ) + except DockerHttp404Error as e: + raise DockerError( + f"Could not pull the '{image}' image: it was not found in the registry. " + f"Check the image name or run 'docker login' on the compute host " + f"(original error: {e})" + ) + except DockerError as e: + raise DockerError( + f"Could not pull the '{image}' image from Docker repository. " + f"Check the image name, the registry authentication and your Internet connection " + f"(original error: {e})" + ) + # The pull api will stream status via an HTTP JSON stream + content = "" + try: + while True: + try: + chunk = await response.content.read(CHUNK_SIZE) + except aiohttp.ServerDisconnectedError as e: + raise DockerError( + f"Disconnected while pulling Docker image '{image}' from Docker repository" + ) from e + except TimeoutError as e: + raise DockerError( + f"Timeout while pulling Docker image '{image}' from Docker repository: " + f"no data received for {PULL_IDLE_TIMEOUT} seconds, " + f"please check your Internet connection or Docker registry mirror" + ) from e + if not chunk: + break + content += chunk.decode("utf-8") + + try: + while True: + content = content.lstrip(" \r\n\t") + answer, index = json.JSONDecoder().raw_decode(content) + if not isinstance(answer, dict): + raise DockerError(f"Invalid response while pulling Docker image '{image}'") + error_detail = answer.get("errorDetail") + error = answer.get("error") + if not error and isinstance(error_detail, dict): + error = error_detail.get("message") + if error: + raise DockerError(error) + status = answer.get("status", "") + if status == "Pulling fs layer": + layers_total += 1 + elif status in ("Pull complete", "Already exists"): + layers_done += 1 + message = answer.get("progress") or status + if message: + report(message) + content = content[index:] + except ValueError: # Partial JSON + pass + + if content.strip(): + raise DockerError(f"Invalid response while pulling Docker image '{image}'") + finally: + response.close() + + @locking + async def load_image(self, stream, image=None, digest=None, progress_callback=None): + """ + Load a Docker image into the Docker daemon from a docker save tar stream + + :param stream: An async iterable of bytes (the tar produced by docker save) + :param image: Tag of the image in the archive. Together with digest, the + (controller digest, local id) pair is remembered so that node creation + can accept the digest the controller pinned for this image + :param digest: Image id as reported by the Docker daemon on the controller + host (the value the controller pins in the node create payload) + :param progress_callback: A function that receive a log message about image load progress + """ + + if progress_callback: + progress_callback("Loading Docker image from stream") + response = await self.http_query("POST", "images/load", data=stream, timeout=None) + # The load api will stream status via an HTTP JSON stream + content = "" + try: + while True: + try: + chunk = await response.content.read(CHUNK_SIZE) + except aiohttp.ServerDisconnectedError as e: + raise DockerError("Disconnected while loading Docker image") from e + except TimeoutError as e: + raise DockerError("Timeout while loading Docker image") from e + if not chunk: + break + content += chunk.decode("utf-8", errors="ignore") + + try: + while True: + content = content.lstrip(" \r\n\t") + answer, index = json.JSONDecoder().raw_decode(content) + if not isinstance(answer, dict): + raise DockerError("Invalid response while loading Docker image") + error_detail = answer.get("errorDetail") + error = answer.get("error") + if not error and isinstance(error_detail, dict): + error = error_detail.get("message") + if error: + raise DockerError(error) + if "stream" in answer and progress_callback: + progress_callback(answer["stream"].rstrip()) + content = content[index:] + except ValueError: # Partial JSON + pass + + if content.strip(): + raise DockerError("Invalid response while loading Docker image") + finally: + response.close() + + if progress_callback: + progress_callback("Docker image loaded") + + if image and digest: + await self._remember_image_id_translation(image, digest) + + async def _remember_image_id_translation(self, image, controller_digest): + """ + Remember how the local daemon identifies an image that was just loaded + from the controller. + + Docker ids are not comparable across image stores: for the same content + a classic store reports the config digest, while a containerd store (or + an image loaded from an OCI archive) reports the manifest digest. The + controller pins the id of its own daemon, so recording the pair lets + DockerVM accept that pin for the image it actually received instead of + asking for another full sync on every node creation. + """ + + try: + image_infos = await self.query("GET", f"images/{image}/json") + except DockerError as e: + log.warning(f"Image '{image}' has been loaded but cannot be inspected: {e}") + return + local_id = image_infos.get("Id") + if local_id: + self._image_id_translations[image] = (controller_digest, local_id) + + def image_id_translation(self, image): + """ + :returns: the (controller digest, local id) pair recorded the last time + this image was loaded from the controller, None otherwise + """ + + return self._image_id_translations.get(image) + + async def list_images(self): + """ + Gets Docker image list. + + :returns: list of dicts + :rtype: list + """ + + images = [] + for image in await self.query("GET", "images/json", params={"all": 0}): + if image["RepoTags"]: + for tag in image["RepoTags"]: + if tag != ":": + images.append( + { + "image": tag, + "id": image.get("Id"), + "size": image.get("Size"), + "created": image_created_timestamp(image), + } + ) + return sorted(images, key=lambda i: i["image"]) + + async def delete_image(self, image, force=False): + """ + Delete a Docker image from the Docker daemon. + + When the daemon refuses because containers use the image, the error + names the GNS3 nodes and projects behind them instead of raw + container ids. + + :param image: Image name or id + :param force: Remove the image even if a container uses it + """ + + try: + await self.query("DELETE", f"images/{image}", params={"force": str(force).lower()}) + except DockerHttp409Error as e: + blockers = await self._describe_containers_using_image(str(e)) + if blockers: + raise DockerHttp409Error( + f"Cannot delete image '{image}': it is still used by {blockers}. " + f"Delete the node or container first, or repeat with force=true." + ) from e + raise + # the image id translation recorded when this image was synced from the + # controller no longer matches anything on this daemon + self._image_id_translations.pop(image, None) + + async def _describe_containers_using_image(self, daemon_error: str) -> str: + """ + Translate the container ids named in a daemon image deletion conflict + into node, project or container descriptions. + + :param daemon_error: the raw daemon conflict message + :returns: a comma separated description, empty when the message names + no container + """ + + descriptions = [] + for container_id in CONFLICTING_CONTAINER_RE.findall(daemon_error): + try: + info = await self.query("GET", f"containers/{container_id}/json") + except DockerError: + info = None + if not info: + # the container cannot be inspected (or already vanished) + descriptions.append(f"container '{container_id}'") + continue + name = info.get("Name", container_id).lstrip("/") + state = "running" if info.get("State", {}).get("Running") else "stopped" + match = GNS3_CONTAINER_NAME_RE.match(name) + if match: + node = match.group("node") + try: + project = ProjectManager.instance().get_project(match.group("project_id")) + except ComputeError: + # the project is closed on this compute: a leftover container + descriptions.append(f"leftover container of node '{node}' ({state}, its project is closed)") + else: + descriptions.append(f"node '{node}' in project '{project.name}' ({state})") + else: + # a container created outside GNS3 + descriptions.append(f"container '{name}' ({state})") + return ", ".join(descriptions) diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index 8a9e6fe87..be1e0fdba 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -733,13 +733,13 @@ class Compute: async def delete(self, path, **kwargs): return await self.http_query("DELETE", path, **kwargs) - async def forward(self, method, type, path, data=None): + async def forward(self, method, type, path, data=None, params=None): """ Forward a call to the emulator on compute """ try: action = f"/{type}/{path}" - res = await self.http_query(method, action, data=data, timeout=None) + res = await self.http_query(method, action, data=data, timeout=None, params=params) except aiohttp.ServerDisconnectedError: raise ControllerError(f"Connection lost to {self._id} during {method} {action}") return res.json diff --git a/gns3server/controller/notification.py b/gns3server/controller/notification.py index 711fabcc2..6eee15572 100644 --- a/gns3server/controller/notification.py +++ b/gns3server/controller/notification.py @@ -126,6 +126,12 @@ class Notification: elif action == "ping": event["compute_id"] = compute_id self.project_emit(action, event) + elif action.startswith("docker.image."): + # Image operations are scoped to a compute, not to a project, and + # clients that are not in a project still need them: emit on the + # controller stream, like compute.updated. + event["compute_id"] = compute_id + self.controller_emit(action, event) elif action.startswith("marker."): self.marker_emit(action, event, project_id) else: diff --git a/gns3server/schemas/controller/computes.py b/gns3server/schemas/controller/computes.py index 873228a70..e8df6658a 100644 --- a/gns3server/schemas/controller/computes.py +++ b/gns3server/schemas/controller/computes.py @@ -146,6 +146,9 @@ class ComputeDockerImage(BaseModel): """ image: str = Field(..., description="Docker image name") + id: Optional[str] = Field(None, description="Docker image id reported by the daemon") + size: Optional[int] = Field(None, description="Docker image size in bytes") + created: Optional[Union[int, str]] = Field(None, description="Docker image creation date (unix timestamp)") class AutoIdlePC(BaseModel): diff --git a/gns3server/services/image_reconciliation.py b/gns3server/services/image_reconciliation.py index 5411d6992..cbe1bb8ad 100644 --- a/gns3server/services/image_reconciliation.py +++ b/gns3server/services/image_reconciliation.py @@ -47,8 +47,34 @@ def utcnow(): return datetime.now(UTC).replace(tzinfo=None) +# Files that are never images: scripts, documentation and certificates users +# keep next to their images (e.g. the Cisco IOU keygen script). Skipping them +# keeps them out of the catalog and stops them from triggering rescans. +IGNORED_EXTENSIONS = { + ".cfg", + ".conf", + ".crt", + ".json", + ".key", + ".lic", + ".license", + ".log", + ".md", + ".pem", + ".py", + ".sh", + ".txt", + ".yaml", + ".yml", +} + + def ignored(name): - return name.startswith(".") or name.endswith((".tmp", ".md5sum")) + return ( + name.startswith(".") + or name.endswith((".tmp", ".md5sum")) + or os.path.splitext(name)[1].lower() in IGNORED_EXTENSIONS + ) def configured_roots(): diff --git a/gns3server/utils/images.py b/gns3server/utils/images.py index f61b80b61..512e570b0 100644 --- a/gns3server/utils/images.py +++ b/gns3server/utils/images.py @@ -53,6 +53,62 @@ log = logging.getLogger(__name__) # Include the first ISO 9660 volume descriptor in image type detection. IMAGE_HEADER_SIZE = 16 * 2048 + 7 +# A Docker image reference: name, optional registry, tag and digest. Callers +# must also reject ".." (not expressible here, the pydantic regex engine has no +# lookaround) so a name cannot escape the daemon API path it is interpolated +# into: see validate_docker_image_name. +DOCKER_IMAGE_NAME_PATTERN = r"^[a-zA-Z0-9][a-zA-Z0-9._/:@-]*$" + + +def validate_docker_image_name(image: str) -> None: + """ + Reject Docker image names that could escape the daemon API path they are + interpolated into. + + :raises ValueError: when the name is not safe + """ + + if ".." in image: + raise ValueError(f"Invalid Docker image name: {image}") + + +def canonical_docker_image_name(image: str) -> str: + """ + Normalize a Docker image reference for name comparisons. + + Registry hosts are dropped, Docker Hub's implicit "library/" namespace and + the implicit ":latest" tag are made explicit, so "alpine", + "library/alpine:latest" and "docker.io/library/alpine" all normalize to + the same string. A reference pinned by digest keeps its digest and loses + its tag. + """ + + reference = image.strip() + if "/" in reference: + registry, _, remainder = reference.partition("/") + if registry == "localhost" or "." in registry or ":" in registry: + # the first path element is a registry host, not a namespace + reference = remainder + if "@" in reference: + repository, _, digest = reference.rpartition("@") + if not repository: + return reference # a bare digest, nothing to normalize + # with a digest pinned the tag does not take part in the identity + repository = repository.partition(":")[0] + return f"{_canonical_docker_repository(repository)}@{digest}" + repository, separator, tag = reference.rpartition(":") + if not separator or not repository: + repository, tag = reference, "latest" + return f"{_canonical_docker_repository(repository)}:{tag or 'latest'}" + + +def _canonical_docker_repository(repository: str) -> str: + + if "/" not in repository: + # images without a namespace live in Docker Hub's "library/" namespace + return f"library/{repository}" + return repository + async def list_images(image_type): """ diff --git a/pyproject.toml b/pyproject.toml index 9b610023a..94d520003 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -180,7 +180,8 @@ extend-immutable-calls = [ "gns3server/api/routes/controller/templates.py" = ["S324"] "gns3server/api/routes/controller/users.py" = ["S105", "S106"] "gns3server/compute/base_node.py" = ["C901", "RUF019"] -"gns3server/compute/docker/__init__.py" = ["B006", "C901"] +"gns3server/compute/docker/__init__.py" = ["B006"] +"gns3server/compute/docker/docker_images.py" = ["C901"] "gns3server/compute/docker/docker_vm.py" = ["B006", "C901", "E731", "RUF003", "RUF006", "S108"] "gns3server/compute/docker/iol_docker_vm.py" = ["RUF002", "S108"] "gns3server/compute/docker/vendor_docker_vm.py" = ["RUF002", "S108"] diff --git a/tests/api/routes/compute/test_images.py b/tests/api/routes/compute/test_images.py index 8d4e0c5d5..a104b9474 100644 --- a/tests/api/routes/compute/test_images.py +++ b/tests/api/routes/compute/test_images.py @@ -18,6 +18,7 @@ import pytest from fastapi import FastAPI, status from httpx import AsyncClient +from gns3server.compute.docker.docker_error import DockerError from tests.utils import asyncio_patch pytestmark = pytest.mark.asyncio @@ -26,12 +27,75 @@ pytestmark = pytest.mark.asyncio class TestImagesRoutes: async def test_pull_docker_image(self, app: FastAPI, compute_client: AsyncClient) -> None: - with asyncio_patch("gns3server.compute.docker.Docker.pull_image") as mock: - response = await compute_client.post( - app.url_path_for("compute:pull_docker_image"), json={"image": "nginx:latest"} + async def fake_pull(image, progress_callback=None, force=False): + progress_callback( + {"layers_total": 0, "layers_done": 0, "message": "Pulling 'nginx:latest'", "state": "started"} ) - mock.assert_called_once_with("nginx:latest", force=True) - assert response.status_code == status.HTTP_204_NO_CONTENT + + with asyncio_patch("gns3server.compute.docker.Docker.pull_image", side_effect=fake_pull) as mock: + with asyncio_patch("gns3server.compute.notification_manager.NotificationManager.emit") as emit_mock: + response = await compute_client.post( + app.url_path_for("compute:pull_docker_image"), json={"image": "nginx:latest"} + ) + assert mock.call_args[0] == ("nginx:latest",) + assert mock.call_args[1]["force"] is True + assert callable(mock.call_args[1]["progress_callback"]) + # started and completed events are emitted to the notification stream + assert [(c.args[0], c.args[1]["state"]) for c in emit_mock.call_args_list] == [ + ("docker.image.pull", "started"), + ("docker.image.pull", "completed"), + ] + assert response.status_code == status.HTTP_204_NO_CONTENT + + async def test_pull_docker_image_progress_id(self, app: FastAPI, compute_client: AsyncClient) -> None: + + async def fake_pull(image, progress_callback=None, force=False): + progress_callback({"layers_total": 1, "layers_done": 0, "message": "Pulling", "state": "started"}) + progress_callback({"layers_total": 1, "layers_done": 0, "message": "[==> ] 10/100"}) + + with asyncio_patch("gns3server.compute.docker.Docker.pull_image", side_effect=fake_pull): + with asyncio_patch("gns3server.compute.notification_manager.NotificationManager.emit") as emit_mock: + response = await compute_client.post( + app.url_path_for("compute:pull_docker_image"), + json={"image": "nginx:latest"}, + params={"progress_id": "abc123"}, + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + assert all(c.args[1]["progress_id"] == "abc123" for c in emit_mock.call_args_list) + + async def test_pull_docker_image_reports_downloading(self, app: FastAPI, compute_client: AsyncClient) -> None: + + async def fake_pull(image, progress_callback=None, force=False): + progress_callback({"layers_total": 2, "layers_done": 0, "message": "Pulling", "state": "started"}) + progress_callback({"layers_total": 2, "layers_done": 1, "message": "[==> ] 40/100"}) + + with asyncio_patch("gns3server.compute.docker.Docker.pull_image", side_effect=fake_pull): + with asyncio_patch("gns3server.compute.notification_manager.NotificationManager.emit") as emit_mock: + response = await compute_client.post( + app.url_path_for("compute:pull_docker_image"), json={"image": "nginx:latest"} + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + states = [c.args[1]["state"] for c in emit_mock.call_args_list] + assert states == ["started", "downloading", "completed"] + downloading = emit_mock.call_args_list[1].args[1] + assert downloading["layers_total"] == 2 + assert downloading["layers_done"] == 1 + assert downloading["message"] == "[==> ] 40/100" + + async def test_pull_docker_image_emits_failed(self, app: FastAPI, compute_client: AsyncClient) -> None: + + async def fake_pull(image, progress_callback=None, force=False): + raise DockerError("no data received for 60 seconds") + + with asyncio_patch("gns3server.compute.docker.Docker.pull_image", side_effect=fake_pull): + with asyncio_patch("gns3server.compute.notification_manager.NotificationManager.emit") as emit_mock: + response = await compute_client.post( + app.url_path_for("compute:pull_docker_image"), json={"image": "nginx:latest"} + ) + assert response.status_code == status.HTTP_409_CONFLICT + failed = emit_mock.call_args_list[-1].args[1] + assert failed["state"] == "failed" + assert "no data received" in failed["message"] @pytest.mark.parametrize("image", ["", " ", "nginx latest"]) async def test_pull_docker_image_rejects_invalid_name( @@ -55,6 +119,33 @@ class TestImagesRoutes: assert hasattr(stream, "__aiter__") assert response.status_code == status.HTTP_204_NO_CONTENT + async def test_delete_docker_image(self, app: FastAPI, compute_client: AsyncClient) -> None: + + with asyncio_patch("gns3server.compute.docker.Docker.delete_image") as mock: + response = await compute_client.delete( + app.url_path_for("compute:delete_docker_image", image="ios-xr/xrd-control-plane:24.4.1") + ) + mock.assert_called_once_with("ios-xr/xrd-control-plane:24.4.1", force=False) + assert response.status_code == status.HTTP_204_NO_CONTENT + + async def test_delete_docker_image_rejects_path_traversal(self, app: FastAPI, compute_client: AsyncClient) -> None: + # percent encoded so the client does not normalize the ".." segments + # away before the request reaches the server + with asyncio_patch("gns3server.compute.docker.Docker.delete_image") as mock: + response = await compute_client.delete("/v3/compute/docker/images/a%2F..%2F..%2Fversion") + mock.assert_not_called() + assert response.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT + + async def test_delete_docker_image_force(self, app: FastAPI, compute_client: AsyncClient) -> None: + + with asyncio_patch("gns3server.compute.docker.Docker.delete_image") as mock: + response = await compute_client.delete( + app.url_path_for("compute:delete_docker_image", image="ubuntu:latest"), + params={"force": "true"}, + ) + mock.assert_called_once_with("ubuntu:latest", force=True) + assert response.status_code == status.HTTP_204_NO_CONTENT + async def test_load_docker_image_with_image_and_digest(self, app: FastAPI, compute_client: AsyncClient) -> None: digest = "sha256:" + "a" * 64 diff --git a/tests/api/routes/controller/test_computes.py b/tests/api/routes/controller/test_computes.py index 08e2ec7df..04359ed0b 100644 --- a/tests/api/routes/controller/test_computes.py +++ b/tests/api/routes/controller/test_computes.py @@ -182,7 +182,10 @@ class TestComputeFeatures: ) as mock: response = await client.get(app.url_path_for("docker_get_images", compute_id=compute_id)) mock.assert_called_with("GET", "docker", "images") - assert response.json() == [{"image": "docker1"}, {"image": "docker2"}] + assert response.json() == [ + {"image": "docker1", "id": None, "size": None, "created": None}, + {"image": "docker2", "id": None, "size": None, "created": None}, + ] async def test_compute_pull_docker_image(self, app: FastAPI, client: AsyncClient, test_compute: Compute) -> None: @@ -194,7 +197,188 @@ class TestComputeFeatures: app.url_path_for("docker_pull_image", compute_id=test_compute.compute_id), json={"image": "nginx:latest"}, ) - compute.forward.assert_called_with("POST", "docker", "images/pull", data={"image": "nginx:latest"}) + compute.forward.assert_called_with( + "POST", "docker", "images/pull", data={"image": "nginx:latest"}, params=None + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + + async def test_compute_pull_docker_image_progress_id( + self, app: FastAPI, client: AsyncClient, test_compute: Compute + ) -> None: + + compute = MagicMock() + compute.forward = AsyncioMagicMock(return_value={}) + with patch("gns3server.api.routes.controller.computes.Controller.instance") as controller: + controller.return_value.get_compute.return_value = compute + response = await client.post( + app.url_path_for("docker_pull_image", compute_id=test_compute.compute_id), + json={"image": "nginx:latest"}, + params={"progress_id": "abc123"}, + ) + compute.forward.assert_called_with( + "POST", + "docker", + "images/pull", + data={"image": "nginx:latest"}, + params={"progress_id": "abc123"}, + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + + async def test_compute_delete_docker_image(self, app: FastAPI, client: AsyncClient, test_compute: Compute) -> None: + + compute = MagicMock() + compute.http_query = AsyncioMagicMock(return_value={}) + with patch("gns3server.api.routes.controller.computes.Controller.instance") as controller: + controller.return_value.get_compute.return_value = compute + response = await client.delete( + app.url_path_for( + "docker_delete_image", + compute_id=test_compute.compute_id, + image="ios-xr/xrd-control-plane:24.4.1", + ), + params={"force": True}, + ) + compute.http_query.assert_called_once_with( + "DELETE", + "/docker/images/ios-xr/xrd-control-plane:24.4.1", + params={"force": "true"}, + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + + async def test_compute_delete_docker_image_rejects_path_traversal( + self, app: FastAPI, client: AsyncClient, test_compute: Compute + ) -> None: + # percent encoded so the client does not normalize the ".." segments + # away before the request reaches the server + compute = MagicMock() + compute.http_query = AsyncioMagicMock(return_value={}) + with patch("gns3server.api.routes.controller.computes.Controller.instance") as controller: + controller.return_value.get_compute.return_value = compute + response = await client.delete( + f"/v3/computes/{test_compute.compute_id}/docker/images/a%2F..%2F..%2Fversion" + ) + compute.http_query.assert_not_called() + assert response.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT + + async def test_compute_delete_docker_image_refused_while_referenced_by_templates( + self, app: FastAPI, client: AsyncClient, test_compute: Compute + ) -> None: + """ + A deletion is refused while a template references the image, matched + through the image reference spelling, and the compute is not touched. + """ + + new_template = { + "name": "Alpine conflict test", + "compute_id": "local", + # a different spelling of the deleted 'alpine:3' + "image": "docker.io/library/alpine:3", + "template_type": "docker", + } + response = await client.post(app.url_path_for("create_template"), json=new_template) + assert response.status_code == status.HTTP_201_CREATED + + compute = MagicMock() + compute.http_query = AsyncioMagicMock(return_value={}) + with patch("gns3server.api.routes.controller.computes.Controller.instance") as controller: + controller.return_value.get_compute.return_value = compute + response = await client.delete( + app.url_path_for("docker_delete_image", compute_id=test_compute.compute_id, image="alpine:3") + ) + assert response.status_code == status.HTTP_409_CONFLICT + body = response.json() + assert body["code"] == "image_referenced_by_templates" + assert body["details"]["referenced_by_templates"] == ["Alpine conflict test"] + assert "referenced by template(s): Alpine conflict test" in body["message"] + compute.http_query.assert_not_called() + + async def test_compute_delete_docker_image_force_overrides_template_reference( + self, app: FastAPI, client: AsyncClient, test_compute: Compute + ) -> None: + """ + force=true deletes an image referenced by templates and the deletion + notification reports them. + """ + + new_template = { + "name": "Alpine force test", + "compute_id": "local", + # an image no other test in this class references: the assertions + # below are exact and the class shares one database + "image": "gns3/force-delete-test:2", + "template_type": "docker", + } + response = await client.post(app.url_path_for("create_template"), json=new_template) + assert response.status_code == status.HTTP_201_CREATED + + compute = MagicMock() + compute.http_query = AsyncioMagicMock(return_value={}) + with patch("gns3server.api.routes.controller.computes.Controller.instance") as controller: + controller.return_value.get_compute.return_value = compute + response = await client.delete( + app.url_path_for( + "docker_delete_image", compute_id=test_compute.compute_id, image="gns3/force-delete-test:2" + ), + params={"force": True}, + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + compute.http_query.assert_called_once_with( + "DELETE", "/docker/images/gns3/force-delete-test:2", params={"force": "true"} + ) + controller.return_value.notification.controller_emit.assert_called_once_with( + "docker.image.deleted", + { + "compute_id": str(test_compute.compute_id), + "image": "gns3/force-delete-test:2", + "referenced_by_templates": ["Alpine force test"], + }, + ) + + async def test_compute_delete_docker_image_notifies_without_templates( + self, app: FastAPI, client: AsyncClient, test_compute: Compute + ) -> None: + """ + A deletion nothing references still emits a docker.image.deleted + notification, so clients can refresh their image lists. + """ + + compute = MagicMock() + compute.http_query = AsyncioMagicMock(return_value={}) + with patch("gns3server.api.routes.controller.computes.Controller.instance") as controller: + controller.return_value.get_compute.return_value = compute + response = await client.delete( + app.url_path_for( + "docker_delete_image", compute_id=test_compute.compute_id, image="gns3/delete-event-test:1" + ) + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + controller.return_value.notification.controller_emit.assert_called_once_with( + "docker.image.deleted", + { + "compute_id": str(test_compute.compute_id), + "image": "gns3/delete-event-test:1", + "referenced_by_templates": [], + }, + ) + + async def test_compute_load_docker_image(self, app: FastAPI, client: AsyncClient, test_compute: Compute) -> None: + + compute = MagicMock() + compute.http_query = AsyncioMagicMock(return_value={}) + with patch("gns3server.api.routes.controller.computes.Controller.instance") as controller: + controller.return_value.get_compute.return_value = compute + response = await client.post( + app.url_path_for("docker_load_image", compute_id=test_compute.compute_id), + content=b"docker save tar data", + headers={"Content-Type": "application/octet-stream"}, + ) + compute.http_query.assert_called_once() + args, kwargs = compute.http_query.call_args + assert args[0] == "POST" + assert args[1] == "/docker/images/load" + assert kwargs["timeout"] is None + # the request body must be streamed, not buffered + assert hasattr(kwargs["data"], "__aiter__") assert response.status_code == status.HTTP_204_NO_CONTENT async def test_compute_list_virtualbox_vms(self, app: FastAPI, client: AsyncClient) -> None: diff --git a/tests/compute/docker/test_docker.py b/tests/compute/docker/test_docker.py index 7fe316f67..27a41afad 100644 --- a/tests/compute/docker/test_docker.py +++ b/tests/compute/docker/test_docker.py @@ -16,13 +16,20 @@ # along with this program. If not, see . import asyncio +import uuid from unittest.mock import MagicMock, patch import pytest import pytest_asyncio -from gns3server.compute.docker import DOCKER_MINIMUM_API_VERSION, DOCKER_PREFERRED_API_VERSION, Docker -from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error +from gns3server.compute.docker import ( + DOCKER_MINIMUM_API_VERSION, + DOCKER_PREFERRED_API_VERSION, + Docker, +) +from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error, DockerHttp409Error +from gns3server.compute.docker.docker_images import PULL_IDLE_TIMEOUT +from gns3server.compute.project_manager import ProjectManager from tests.utils import AsyncioMagicMock, asyncio_patch @@ -132,11 +139,64 @@ async def test_list_images(): images = await Docker.instance().list_images() mock.assert_called_with("GET", "images/json", params={"all": 0}) assert len(images) == 5 - assert {"image": "ubuntu:12.04"} in images - assert {"image": "ubuntu:precise"} in images - assert {"image": "ubuntu:latest"} in images - assert {"image": "ubuntu:12.10"} in images - assert {"image": "ubuntu:quantal"} in images + by_tag = {image["image"]: image for image in images} + assert by_tag["ubuntu:12.04"]["id"] == "8dbd9e392a964056420e5d58ca5cc376ef18e2de93b5cc90e868a1bbc8318c1c" + assert by_tag["ubuntu:12.04"]["size"] == 131506275 + assert by_tag["ubuntu:12.04"]["created"] == 1365714795 + # tags of the same image share its id + assert by_tag["ubuntu:precise"]["id"] == by_tag["ubuntu:12.04"]["id"] + assert ":" not in by_tag + assert {"ubuntu:precise", "ubuntu:latest", "ubuntu:12.10", "ubuntu:quantal"} <= set(by_tag) + + +@pytest.mark.asyncio +async def test_list_images_created_fallback_to_oci_label(): + """ + Images built without a created timestamp (e.g. reproducible Ubuntu images) + report 0 from the daemon: the build time comes from the OCI created label. + """ + + response = [ + { + # zero timestamp, the label is used (Z suffix, nanosecond precision) + "RepoTags": ["ubuntu:latest"], + "Id": "febc1a2eec1f", + "Created": 0, + "Size": 78101592, + "Labels": {"org.opencontainers.image.created": "2026-10-25T10:26:07.538865123Z"}, + }, + { + # zero timestamp without a label stays at zero + "RepoTags": ["no-label:latest"], + "Id": "aaaaaa", + "Created": 0, + "Size": 1, + }, + { + # the daemon timestamp wins over the label when it is set + "RepoTags": ["normal:latest"], + "Id": "bbbbbb", + "Created": 1365714795, + "Size": 1, + "Labels": {"org.opencontainers.image.created": "2000-01-01T00:00:00Z"}, + }, + { + # a broken label is ignored, the timestamp stays at zero + "RepoTags": ["broken-label:latest"], + "Id": "cccccc", + "Created": 0, + "Size": 1, + "Labels": {"org.opencontainers.image.created": "not a date"}, + }, + ] + + with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response): + images = await Docker.instance().list_images() + by_tag = {image["image"]: image for image in images} + assert by_tag["ubuntu:latest"]["created"] == 1792923967 + assert by_tag["no-label:latest"]["created"] == 0 + assert by_tag["normal:latest"]["created"] == 1365714795 + assert by_tag["broken-label:latest"]["created"] == 0 @pytest.mark.asyncio @@ -165,7 +225,50 @@ async def test_pull_image(): with asyncio_patch("gns3server.compute.docker.Docker.query", side_effect=DockerHttp404Error("404")): with asyncio_patch("gns3server.compute.docker.Docker.http_query", return_value=mock_query) as mock: await Docker.instance().pull_image("ubuntu") - mock.assert_called_with("POST", "images/create", params={"fromImage": "ubuntu"}, timeout=None) + args, kwargs = mock.call_args + assert args == ("POST", "images/create") + assert kwargs["params"] == {"fromImage": "ubuntu", "tag": "latest"} + # no total timeout (a pull can take as long as it needs) but an + # idle socket read timeout, so an unreachable registry fails + assert kwargs["timeout"].total is None + assert kwargs["timeout"].sock_read == PULL_IDLE_TIMEOUT + + +@pytest.mark.asyncio +async def test_pull_image_reports_aggregated_progress(): + + class Content: + def __init__(self): + self._chunks = [ + b'{"status": "Pulling fs layer", "id": "sha256:a"}\n' + b'{"status": "Pulling fs layer", "id": "sha256:b"}\n' + b'{"status": "Downloading", "id": "sha256:a", "progress": "[==> ] 40/100"}\n' + b'{"status": "Pull complete", "id": "sha256:a"}\n', + b'{"status": "Pull complete", "id": "sha256:b"}\n', + b"", + ] + + async def read(self, size): + return self._chunks.pop(0) + + response = MagicMock() + response.content = Content() + + progress = [] + with asyncio_patch("gns3server.compute.docker.Docker.query", side_effect=DockerHttp404Error("404")): + with asyncio_patch("gns3server.compute.docker.Docker.http_query", return_value=response): + await Docker.instance().pull_image("ubuntu", progress_callback=progress.append) + # the callback sees the running totals of layers, not raw daemon lines + assert progress[0] == { + "layers_total": 0, + "layers_done": 0, + "message": "Pulling 'ubuntu' from Docker repository", + "state": "started", + } + assert progress[1] == {"layers_total": 1, "layers_done": 0, "message": "Pulling fs layer"} + assert progress[2] == {"layers_total": 2, "layers_done": 0, "message": "Pulling fs layer"} + assert {"layers_total": 2, "layers_done": 0, "message": "[==> ] 40/100"} in progress + assert progress[-1] == {"layers_total": 2, "layers_done": 2, "message": "Pull complete"} @pytest.mark.asyncio @@ -188,7 +291,40 @@ async def test_force_pull_image(): with asyncio_patch("gns3server.compute.docker.Docker.http_query", return_value=response) as pull_mock: await Docker.instance().pull_image("ubuntu", force=True) query_mock.assert_not_called() - pull_mock.assert_called_with("POST", "images/create", params={"fromImage": "ubuntu"}, timeout=None) + args, kwargs = pull_mock.call_args + assert args == ("POST", "images/create") + assert kwargs["params"] == {"fromImage": "ubuntu", "tag": "latest"} + assert kwargs["timeout"].sock_read == PULL_IDLE_TIMEOUT + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "image, expected_params", + [ + # without a tag or digest the daemon would pull every tag of the + # repository: default to latest so the pull targets a single tag + ("ubuntu", {"fromImage": "ubuntu", "tag": "latest"}), + ("gns3/ubuntu", {"fromImage": "gns3/ubuntu", "tag": "latest"}), + # the ':' before the last '/' is a registry port, not a tag + ("registry:5000/ubuntu", {"fromImage": "registry:5000/ubuntu", "tag": "latest"}), + ("registry:5000/gns3/ubuntu", {"fromImage": "registry:5000/gns3/ubuntu", "tag": "latest"}), + # an explicit tag or digest is left alone + ("ubuntu:24.04", {"fromImage": "ubuntu:24.04"}), + ("registry:5000/ubuntu:24.04", {"fromImage": "registry:5000/ubuntu:24.04"}), + ("ubuntu@sha256:8b2f", {"fromImage": "ubuntu@sha256:8b2f"}), + ], +) +async def test_pull_image_targets_a_single_tag(image, expected_params): + + response = MagicMock() + response.content.read = AsyncioMagicMock(return_value=b"") + + with asyncio_patch("gns3server.compute.docker.Docker.query", side_effect=DockerHttp404Error("404")): + with asyncio_patch("gns3server.compute.docker.Docker.http_query", return_value=response) as pull_mock: + await Docker.instance().pull_image(image) + args, kwargs = pull_mock.call_args + assert args == ("POST", "images/create") + assert kwargs["params"] == expected_params @pytest.mark.asyncio @@ -211,6 +347,24 @@ async def test_pull_image_error(): response.close.assert_called_once() +@pytest.mark.asyncio +async def test_pull_image_not_found_is_not_reported_as_a_network_error(): + """ + A 404 from the registry means the image name is wrong or the registry needs + authentication: telling the user to check the Internet connection sends + them in the wrong direction. + """ + + with asyncio_patch("gns3server.compute.docker.Docker.query", side_effect=DockerHttp404Error("404")): + with asyncio_patch( + "gns3server.compute.docker.Docker.http_query", + side_effect=DockerHttp404Error("Docker has returned an error: 404 pull access denied for debian13"), + ): + with pytest.raises(DockerError, match="not found in the registry") as error: + await Docker.instance().pull_image("debian13") + assert "Internet connection" not in str(error.value) + + @pytest.mark.asyncio async def test_pull_image_rejects_incomplete_response(): @@ -330,6 +484,125 @@ async def test_load_image_error(): response.close.assert_called_once() +@pytest.mark.asyncio +async def test_delete_image(): + + docker = Docker.instance() + docker._image_id_translations["ubuntu:latest"] = ("sha256:" + "a" * 64, "sha256:" + "b" * 64) + + with asyncio_patch("gns3server.compute.docker.Docker.query") as mock: + await docker.delete_image("ubuntu:latest", force=True) + # the daemon query string only accepts strings, ints and floats: a + # Python bool would make the aiohttp URL builder raise a TypeError + mock.assert_called_with("DELETE", "images/ubuntu:latest", params={"force": "true"}) + # deleting the image drops the id translation recorded for a previous sync + assert docker.image_id_translation("ubuntu:latest") is None + + +@pytest.mark.asyncio +async def test_delete_image_error_keeps_translation(): + """ + When the daemon refuses the deletion the image is still there, so the id + translation recorded for a previous sync stays valid. + """ + + docker = Docker.instance() + translation = ("sha256:" + "a" * 64, "sha256:" + "b" * 64) + docker._image_id_translations["ubuntu:latest"] = translation + + with asyncio_patch("gns3server.compute.docker.Docker.query", side_effect=DockerError("conflict: image is in use")): + with pytest.raises(DockerError, match="image is in use"): + await docker.delete_image("ubuntu:latest") + assert docker.image_id_translation("ubuntu:latest") == translation + + +DAEMON_DELETE_CONFLICT = ( + "Docker has returned an error: 409 conflict: unable to remove repository reference " + '"alpine:3" (must force) - container 6d59f00236ff is using its referenced image d529dd0c6e55' +) + + +@pytest.mark.asyncio +async def test_delete_image_conflict_names_the_node_and_project(): + """ + A container blocking the deletion is reported as the GNS3 node and project + it belongs to, not as a raw container id. + """ + + docker = Docker.instance() + project_id = str(uuid.uuid4()) + project = ProjectManager.instance().create_project(name="test-lab", project_id=project_id) + + async def query(method, path, **kwargs): + assert path.startswith("images/") or path.startswith("containers/") + if method == "DELETE": + raise DockerHttp409Error(DAEMON_DELETE_CONFLICT) + return {"Name": f"/GNS3.alpine-1.{project_id}", "State": {"Running": True}} + + try: + with asyncio_patch("gns3server.compute.docker.Docker.query", side_effect=query): + with pytest.raises(DockerHttp409Error) as excinfo: + await docker.delete_image("alpine:3") + message = str(excinfo.value) + assert "node 'alpine-1' in project 'test-lab' (running)" in message + assert "force=true" in message + assert "6d59f00236ff" not in message + finally: + ProjectManager.instance().remove_project(project.id) + + +@pytest.mark.asyncio +async def test_delete_image_conflict_reports_a_leftover_container(): + """ + When the project of the blocking container is closed on this compute the + container is reported as a leftover. + """ + + docker = Docker.instance() + + async def query(method, path, **kwargs): + if method == "DELETE": + raise DockerHttp409Error(DAEMON_DELETE_CONFLICT) + return {"Name": f"/GNS3.E2E-2.{uuid.uuid4()}", "State": {"Running": False}} + + with asyncio_patch("gns3server.compute.docker.Docker.query", side_effect=query): + with pytest.raises(DockerHttp409Error, match="leftover container of node 'E2E-2' \\(stopped"): + await docker.delete_image("alpine:3") + + +@pytest.mark.asyncio +async def test_delete_image_conflict_reports_a_non_gns3_container(): + """ + A container created outside GNS3 is reported by its plain container name. + """ + + docker = Docker.instance() + + async def query(method, path, **kwargs): + if method == "DELETE": + raise DockerHttp409Error(DAEMON_DELETE_CONFLICT) + return {"Name": "/my-own-container", "State": {"Running": True}} + + with asyncio_patch("gns3server.compute.docker.Docker.query", side_effect=query): + with pytest.raises(DockerHttp409Error, match="container 'my-own-container' \\(running\\)"): + await docker.delete_image("alpine:3") + + +@pytest.mark.asyncio +async def test_delete_image_conflict_without_containers_is_not_rewritten(): + """ + A daemon conflict naming no container (e.g. a multi repository reference) + is passed through unchanged. + """ + + docker = Docker.instance() + original = "Docker has returned an error: 409 conflict: image is referenced in multiple repositories" + + with asyncio_patch("gns3server.compute.docker.Docker.query", side_effect=DockerHttp409Error(original)): + with pytest.raises(DockerHttp409Error, match="multiple repositories"): + await docker.delete_image("alpine:3") + + @pytest.mark.asyncio async def test_docker_check_connection_docker_minimum_version(vm): diff --git a/tests/controller/test_notification.py b/tests/controller/test_notification.py index 53e24f1f2..82aa996b6 100644 --- a/tests/controller/test_notification.py +++ b/tests/controller/test_notification.py @@ -119,6 +119,34 @@ async def test_dispatch_node_updated(controller, node, project): assert event["properties"]["startup_config"] == "ip 192" +@pytest.mark.asyncio +async def test_dispatch_docker_image_routed_to_controller_stream(controller, project): + """ + docker.image.* events are scoped to a compute, not to a project: they must + reach the controller stream even when no project is open, and not the + project stream. + """ + + notif = controller.notification + with notif.controller_queue() as controller_q, notif.project_queue(project.id) as project_q: + await controller_q.get(0.1) # ping + await project_q.get(0.1) # ping + + await notif.dispatch( + "docker.image.pull", {"image": "debian", "state": "started"}, project_id=None, compute_id="local" + ) + + msg = await controller_q.get(5) + assert msg == ( + "docker.image.pull", + {"image": "debian", "state": "started", "compute_id": "local"}, + {}, + ) + # the project stream only carries its periodic ping + msg = await project_q.get(0.2) + assert msg[0] == "ping" + + @pytest.mark.asyncio async def test_dispatch_marker_routed_to_marker_channel(controller, project): """ diff --git a/tests/services/test_image_reconciliation.py b/tests/services/test_image_reconciliation.py index 57611b66b..f871a520e 100644 --- a/tests/services/test_image_reconciliation.py +++ b/tests/services/test_image_reconciliation.py @@ -222,6 +222,22 @@ async def test_external_symlinks_hidden_files_and_libraries_are_not_imported(inv assert outside.read_bytes() == QCOW +async def test_non_image_files_next_to_images_are_ignored(inventory, config): + """Files a user keeps next to images (scripts, notes, licenses) must not be + reported as invalid images nor turn the sync job partial.""" + + image_file(config, "IOU/CiscoIOUKeygen.py", b"#!/usr/bin/env python\ndef main(): ...") + image_file(config, "IOU/README.md", b"# IOU images") + image_file(config, "QEMU/notes.TXT", b"my qemu notes") + image_file(config, "IOS/license.lic", b"license text") + result = await scan(inventory) + assert result["counts"]["scanned"] == 0 + assert result["counts"]["invalid"] == 0 + assert result["counts"]["errors"] == 0 + assert result["errors"] == [] + assert result["status"] == "completed" + + @pytest.mark.parametrize( "name,data,expected", [ diff --git a/tests/utils/test_images.py b/tests/utils/test_images.py index b932ab59b..1008b8851 100644 --- a/tests/utils/test_images.py +++ b/tests/utils/test_images.py @@ -20,7 +20,13 @@ import threading import pytest from gns3server.utils import force_unix_path -from gns3server.utils.images import images_directories, list_images, md5sum, remove_checksum +from gns3server.utils.images import ( + canonical_docker_image_name, + images_directories, + list_images, + md5sum, + remove_checksum, +) def test_images_directories(tmpdir, config): @@ -108,6 +114,29 @@ def test_remove_checksum(tmpdir): remove_checksum(str(tmpdir / "not_exists")) +@pytest.mark.parametrize( + "reference, expected", + ( + ("alpine", "library/alpine:latest"), + ("alpine:3", "library/alpine:3"), + ("library/alpine:3", "library/alpine:3"), + ("docker.io/library/alpine:3", "library/alpine:3"), + ("index.docker.io/library/alpine:3", "library/alpine:3"), + ("localhost/alpine:3", "library/alpine:3"), + ("gns3/frr", "gns3/frr:latest"), + ("gns3/frr:latest", "gns3/frr:latest"), + ("quay.io/some/namespace/image:2.0", "some/namespace/image:2.0"), + ("alpine@sha256:" + "a" * 64, "library/alpine@sha256:" + "a" * 64), + ("alpine:3@sha256:" + "a" * 64, "library/alpine@sha256:" + "a" * 64), + ), +) +def test_canonical_docker_image_name(reference, expected): + + assert canonical_docker_image_name(reference) == expected + # the canonical form is stable: canonicalizing twice does not change it + assert canonical_docker_image_name(expected) == expected + + @pytest.mark.asyncio async def test_list_images(tmpdir, config):