feat: sync Docker images from the controller to remote computes

When a Docker node is created on a remote compute whose Docker daemon
does not have the image, the compute now raises ImageMissingError
instead of blindly pulling from the Docker repository. The controller
exports the image from the Docker daemon on its host (docker save
stream) and streams it to the compute which loads it, so locally built
or docker-loaded images work across computes. When the image is not
available on the controller host either, the compute is asked to pull
it from the Docker repository as a fallback.

- add a POST /docker/images/load compute endpoint that streams a
  docker save tar into the Docker daemon
- let Docker.http_query pass raw (non-dict) request bodies through so
  the tar can be streamed to the daemon
- drop the inline pull from DockerVM.create() and the now unused
  DockerVM.pull_image wrapper

gns3server/api/routes/compute/images.py

@@ -53,6 +53,16 @@ async def pull_docker_image(image: str = Body(..., embed=True, min_length=1, pat
     await docker_manager.pull_image(image, force=True)
 
 
+@router.post("/docker/images/load", status_code=status.HTTP_204_NO_CONTENT)
+async def load_docker_image(request: Request) -> None:
+    """
+    Load a Docker image into the Docker daemon from a docker save tar stream.
+    """
+
+    docker_manager = Docker.instance()
+    await docker_manager.load_image(request.stream())
+
+
 @router.get("/dynamips/images")
 async def get_dynamips_images() -> List[dict]:
     """

gns3server/compute/docker/__init__.py

@@ -283,7 +283,12 @@ class Docker(BaseManager):
         :returns: HTTP response
         """
 
-        data = json.dumps(data)
+        if isinstance(data, dict):
+            data = json.dumps(data)
+            headers = {"content-type": "application/json"}
+        else:
+            # not a dict (e.g. a Docker image tar stream): let aiohttp stream the raw body
+            headers = {"content-type": "application/x-tar"}
         if timeout is None:
             timeout = 60 * 60 * 24 * 31  # One month timeout
 
@@ -301,9 +306,7 @@ class Docker(BaseManager):
                 url,
                 params=params,
                 data=data,
-                headers={
-                    "content-type": "application/json",
-                },
+                headers=headers,
                 timeout=timeout,
             )
         except aiohttp.ClientError as e:
@@ -410,6 +413,58 @@ class Docker(BaseManager):
         if progress_callback:
             progress_callback(f"Success pulling image {image}")
 
+    @locking
+    async def load_image(self, stream, progress_callback=None):
+        """
+        Load a Docker image into the Docker daemon from a docker save tar stream
+
+        :param stream: An async iterable of bytes (the tar produced by docker save)
+        :param progress_callback: A function that receive a log message about image load progress
+        """
+
+        if progress_callback:
+            progress_callback("Loading Docker image from stream")
+        response = await self.http_query("POST", "images/load", data=stream, timeout=None)
+        # The load api will stream status via an HTTP JSON stream
+        content = ""
+        try:
+            while True:
+                try:
+                    chunk = await response.content.read(CHUNK_SIZE)
+                except aiohttp.ServerDisconnectedError as e:
+                    raise DockerError("Disconnected while loading Docker image") from e
+                except asyncio.TimeoutError as e:
+                    raise DockerError("Timeout while loading Docker image") from e
+                if not chunk:
+                    break
+                content += chunk.decode("utf-8", errors="ignore")
+
+                try:
+                    while True:
+                        content = content.lstrip(" \r\n\t")
+                        answer, index = json.JSONDecoder().raw_decode(content)
+                        if not isinstance(answer, dict):
+                            raise DockerError("Invalid response while loading Docker image")
+                        error_detail = answer.get("errorDetail")
+                        error = answer.get("error")
+                        if not error and isinstance(error_detail, dict):
+                            error = error_detail.get("message")
+                        if error:
+                            raise DockerError(error)
+                        if "stream" in answer and progress_callback:
+                            progress_callback(answer["stream"].rstrip())
+                        content = content[index:]
+                except ValueError:  # Partial JSON
+                    pass
+
+            if content.strip():
+                raise DockerError("Invalid response while loading Docker image")
+        finally:
+            response.close()
+
+        if progress_callback:
+            progress_callback("Docker image loaded")
+
     async def list_images(self):
         """
         Gets Docker image list.

gns3server/compute/docker/docker_vm.py

@@ -47,6 +47,7 @@ from ..base_node import BaseNode
 from ..adapters.ethernet_adapter import EthernetAdapter
 from ..nios.nio_udp import NIOUDP
 from .docker_error import DockerError, DockerHttp304Error, DockerHttp404Error, DockerHttp409Error
+from ..error import ImageMissingError
 
 import logging
 
@@ -574,9 +575,10 @@ class DockerVM(BaseNode):
         try:
             image_infos = await self._get_image_information()
         except DockerHttp404Error:
-            log.info("Image '{}' is missing, pulling it from Docker repository...".format(self._image))
-            await self.pull_image(self._image)
-            image_infos = await self._get_image_information()
+            # the image is not on the local Docker daemon: raise ImageMissingError so the
+            # controller can sync it (docker save -> load from the controller host, or pull)
+            # and retry the node creation
+            raise ImageMissingError(self._image)
 
         if image_infos is None:
             raise DockerError(f"Cannot get information for image '{self._image}', please try again.")
@@ -1943,16 +1945,6 @@ class DockerVM(BaseNode):
             )
         )
 
-    async def pull_image(self, image):
-        """
-        Pulls an image from Docker repository
-        """
-
-        def callback(msg):
-            self.project.emit("log.info", {"message": msg})
-
-        await self.manager.pull_image(image, progress_callback=callback)
-
     async def _start_ubridge_capture(self, adapter_number, output_file, port_number=0):
         """
         Starts a packet capture in uBridge.

gns3server/controller/node.py

@@ -770,6 +770,9 @@ class Node:
         if the image exists
         """
 
+        if self._node_type == "docker":
+            return await self._sync_missing_docker_image(img)
+
         for directory in images_directories(type):
             image = os.path.join(directory, img)
             if os.path.exists(image):
@@ -785,6 +788,43 @@ class Node:
                 return True
         return False
 
+    async def _sync_missing_docker_image(self, image):
+        """
+        Export a Docker image from the Docker daemon on the controller host and
+        stream it to the remote compute, or ask the compute to pull it when the
+        image is not available locally
+        """
+
+        from gns3server.compute.docker import Docker
+        from gns3server.compute.docker.docker_error import DockerError
+
+        try:
+            response = await Docker.instance().http_query("GET", f"images/{image}/get", timeout=None)
+        except DockerError:
+            # the image is not on the Docker daemon of the controller host: ask the
+            # compute to pull it from the Docker repository as a fallback
+            self.project.emit_notification(
+                "log.info",
+                {"message": f"Docker image '{image}' is not on the controller host, "
+                            f"asking compute '{self._compute.name}' to pull it"}
+            )
+            await self._compute.post("/docker/images/pull", data={"image": image}, timeout=None)
+            return True
+
+        self.project.emit_notification(
+            "log.info",
+            {"message": f"Syncing Docker image '{image}' to compute '{self._compute.name}'"}
+        )
+        try:
+            await self._compute.post("/docker/images/load", data=response.content, timeout=None)
+        finally:
+            response.close()
+        self.project.emit_notification(
+            "log.info",
+            {"message": f"Docker image '{image}' has been synced to compute '{self._compute.name}'"}
+        )
+        return True
+
     async def dynamips_auto_idlepc(self):
         """
         Compute the idle PC for a dynamips node