Allow projects containing missing images to open in a degraded state instead of failing to load

gns3server/controller/node.py

@@ -40,6 +40,26 @@ import logging
 log = logging.getLogger(__name__)
 
 
+# Image properties of a node, mapped by node type. The value is the image
+# "type" used by the image manager (see utils.images). Docker images are not
+# registered in the image database, they are matched by tag instead.
+IMAGE_PROPERTIES_BY_NODE_TYPE = {
+    "qemu": {
+        "hda_disk_image": "qemu",
+        "hdb_disk_image": "qemu",
+        "hdc_disk_image": "qemu",
+        "hdd_disk_image": "qemu",
+        "cdrom_image": "qemu",
+        "initrd": "qemu",
+        "kernel_image": "qemu",
+        "bios_image": "qemu",
+    },
+    "dynamips": {"image": "ios"},
+    "iou": {"path": "iou"},
+    "docker": {"image": "docker"},
+}
+
+
 def _extract_iol_startup_config_knob(environment):
     """
     Split the GNS3_IOL_STARTUP_CONFIG knob out of a Docker environment
@@ -143,6 +163,11 @@ class Node:
         self._netmiko_device_type = None
         self._default_username = None
         self._default_password = None
+        # Set when the node cannot be created on its compute because one or
+        # more of its images are missing. The node is kept on the controller
+        # (topology and links are preserved) until the user provides a
+        # compatible replacement image.
+        self._missing_images = []
 
         # This properties will be recomputed
         ignore_properties = ("width", "height", "hover_symbol")
@@ -187,6 +212,18 @@ class Node:
     def status(self):
         return self._status
 
+    @property
+    def missing_image(self):
+        """
+        :returns: True if at least one image required by this node is missing
+            and the node could therefore not be created on its compute.
+        """
+        return len(self._missing_images) > 0
+
+    @property
+    def missing_images(self):
+        return self._missing_images
+
     @property
     def template_id(self):
         return self._template_id
@@ -457,9 +494,14 @@ class Node:
     def links(self):
         return self._links
 
-    async def create(self):
+    async def create(self, allow_missing_image=False):
         """
         Create the node on the compute
+
+        :param allow_missing_image: when True, if an image is missing and
+            cannot be uploaded/synced, the node is not created on the compute
+            but is kept on the controller and flagged with the missing image(s)
+            instead of raising.
         """
         data = self._node_data()
         data["node_id"] = self._id
@@ -469,6 +511,8 @@ class Node:
         else:
             timeout = 1200
         trial = 0
+        last_missing_image = None
+        last_missing_error = None
         while trial != 6:
             try:
                 response = await self._compute.post(
@@ -477,17 +521,126 @@ class Node:
             except ComputeConflictError as e:
                 response = e.response()
                 if response.get("exception") == "ImageMissingError":
-                    res = await self._upload_missing_image(self._node_type, response["image"])
+                    last_missing_error = e
+                    last_missing_image = response.get("image")
+                    res = False
+                    if last_missing_image:
+                        try:
+                            res = await self._upload_missing_image(self._node_type, last_missing_image)
+                        except ControllerError as upload_error:
+                            # For Docker the image is pulled from a registry: a
+                            # failed pull (unknown tag, no network, private image)
+                            # must not abort the whole project. Treat it as a
+                            # missing image so the user can pick another one.
+                            if not allow_missing_image:
+                                raise
+                            log.warning(
+                                "Could not provide missing image '%s' for node '%s' [%s]: %s",
+                                last_missing_image, self._name, self._id, upload_error
+                            )
                     if not res:
+                        if allow_missing_image:
+                            missing_images = self._compute_missing_images(last_missing_image)
+                            # A degraded node must always identify at least one
+                            # missing image. Otherwise the controller would keep
+                            # a node that was never created on the compute while
+                            # exposing it as healthy.
+                            if not missing_images:
+                                raise e
+                            self._missing_images = missing_images
+                            log.warning(
+                                "Node '%s' [%s] is kept in degraded state, missing image(s): %s",
+                                self._name, self._id, ", ".join(m["image"] for m in self._missing_images)
+                            )
+                            return False
                         raise e
                 else:
                     raise e
             else:
                 await self.parse_node_response(response.json)
+                self._missing_images = []
                 return True
             trial += 1
+        if allow_missing_image:
+            # The image was repeatedly reported missing (e.g. the upload/sync
+            # seemed to succeed but did not). Keep the node in degraded state.
+            self._missing_images = self._compute_missing_images(last_missing_image)
+            log.warning(
+                "Node '%s' [%s] could not be created, missing image(s): %s",
+                self._name, self._id, ", ".join(m["image"] for m in self._missing_images)
+            )
+        elif last_missing_error is not None:
+            # Uploading appeared to succeed, but the compute rejected every
+            # retry. Do not let the caller register a controller-only node as
+            # healthy.
+            raise last_missing_error
         return False
 
+    def _image_available(self, image_type, image):
+        """
+        Check whether an image is available on the controller (and can
+        therefore be uploaded to the compute when needed).
+
+        :param image_type: image type (e.g. "qemu", "ios", "iou")
+        :param image: image filename or path
+        """
+
+        if not image:
+            return True
+        if image_type == "docker":
+            # Docker images are not stored on the controller filesystem
+            return True
+        try:
+            directories = images_directories(image_type)
+        except NotImplementedError:
+            return True
+        for directory in directories:
+            if os.path.exists(os.path.join(directory, image)):
+                return True
+        return False
+
+    def _compute_missing_images(self, fallback_image=None):
+        """
+        Build the list of images referenced by this node that are not
+        available on the controller.
+
+        :param fallback_image: image reported as missing by the compute, always
+            included (used for Docker images and images not directly mapped to
+            a node property).
+        """
+
+        missing = []
+        mapping = IMAGE_PROPERTIES_BY_NODE_TYPE.get(self._node_type, {})
+        properties = self._properties or {}
+        for prop, image_type in mapping.items():
+            value = properties.get(prop)
+            if not value:
+                continue
+            if not self._image_available(image_type, value):
+                missing.append({"property": prop, "image": value, "image_type": image_type})
+        if fallback_image:
+            prop = next((p for p in mapping if properties.get(p) == fallback_image), None)
+            if prop is None:
+                fallback_basename = os.path.basename(fallback_image)
+                prop = next(
+                    (
+                        p
+                        for p in mapping
+                        if properties.get(p) and os.path.basename(properties[p]) == fallback_basename
+                    ),
+                    None,
+                )
+            if prop is None and mapping:
+                prop = next(iter(mapping))
+            already_reported = any(
+                m["image"] == fallback_image or (prop is not None and m["property"] == prop)
+                for m in missing
+            )
+            if not already_reported:
+                image_type = mapping.get(prop, self._node_type)
+                missing.append({"property": prop, "image": fallback_image, "image_type": image_type})
+        return missing
+
     async def update(self, **kwargs):
         """
         Update the node on the compute
@@ -499,6 +652,9 @@ class Node:
         update_compute = False
         old_json = self.asdict()
         old_name = self._name
+        old_properties = copy.deepcopy(self._properties)
+        old_custom_adapters = copy.deepcopy(self._custom_adapters)
+        old_missing_images = copy.deepcopy(self._missing_images)
 
         compute_properties = None
         # Update node properties with additional elements
@@ -522,8 +678,32 @@ class Node:
         if compute_properties and "custom_adapters" in compute_properties:
             # we need to check custom adapters to update the custom port names
             self.custom_adapters = compute_properties["custom_adapters"]
+        if self.missing_image and compute_properties is not None:
+            # The node was never created on the compute (missing image). Apply
+            # the new properties locally so create() can use them.
+            self._properties = compute_properties
         self._list_ports()
         if update_compute:
+            if self.missing_image:
+                # Try to create the node on the compute now that an image may
+                # have been provided. If it is still missing the node simply
+                # remains in its degraded state.
+                try:
+                    created = await self.create(allow_missing_image=True)
+                except Exception:
+                    # create() can reject the replacement for reasons other
+                    # than a missing image. Keep the controller state aligned
+                    # with the node that is still absent from the compute.
+                    self._properties = old_properties
+                    self._custom_adapters = old_custom_adapters
+                    self._missing_images = old_missing_images
+                    self._list_ports()
+                    raise
+                if created:
+                    await self.project.restore_deferred_links(self)
+                self.project.emit_notification("node.updated", self.asdict())
+                self.project.dump()
+                return
             data = self._node_data(properties=compute_properties)
             try:
                 response = await self.put(None, data=data)
@@ -666,6 +846,25 @@ class Node:
         """
         Start a node
         """
+        if self.missing_image:
+            images = ", ".join(m["image"] for m in self._missing_images)
+            raise ControllerError(
+                f"Cannot start node '{self._name}': the required image(s) ({images}) "
+                f"are missing. Please provide a compatible image."
+            )
+        deferred_links = [link for link in self.links if link.deferred]
+        if deferred_links:
+            await self.project.restore_deferred_links(self)
+            failed_links = [
+                link
+                for link in deferred_links
+                if link.deferred and not any(endpoint["node"].missing_image for endpoint in link._nodes)
+            ]
+            if failed_links:
+                raise ControllerError(
+                    f"Cannot start node '{self._name}': {len(failed_links)} deferred link(s) "
+                    "could not be restored. Please try again."
+                )
         try:
             # For IOU: we need to send the licence everytime we start a node
             if self.node_type == "iou":
@@ -681,6 +880,8 @@ class Node:
         """
         Stop a node
         """
+        if self.missing_image:
+            return
         try:
             await self.post("/stop", timeout=240, dont_connect=True)
         # We don't care if a node is down at this step
@@ -756,6 +957,10 @@ class Node:
         """
         HTTP post on the node
         """
+        if self.missing_image and path is None:
+            # The node was never created on the compute (missing image). Any
+            # partial object there is cleaned up when the project is closed.
+            return None
         if path is None:
             return await self._compute.delete(
                 f"/projects/{self._project.id}/{self._node_type}/nodes/{self._id}", **kwargs
@@ -1018,7 +1223,9 @@ class Node:
             "status": self._status,
             "console_host": str(self._compute.console_host),
             "node_directory": self._node_directory,
-            "ports": [port.asdict() for port in self.ports]
+            "ports": [port.asdict() for port in self.ports],
+            "missing_image": self.missing_image,
+            "missing_images": self._missing_images,
         }
         topology.update(additional_data)
         return topology

gns3server/controller/project.py

@@ -632,7 +632,7 @@ class Project:
         node = await self.add_node(compute, name, node_id, node_type=node_type, **template)
         return node
 
-    async def _create_node(self, compute, name, node_id, node_type=None, **kwargs):
+    async def _create_node(self, compute, name, node_id, node_type=None, allow_missing_image=False, **kwargs):
 
         node = Node(self, compute, name, node_id=node_id, node_type=node_type, **kwargs)
         # Hold the lock across the check + POST + register so that concurrent
@@ -651,17 +651,21 @@ class Project:
                 await compute.post("/projects", data=data)
                 self._project_created_on_compute.add(compute)
 
-        await node.create()
+        await node.create(allow_missing_image=allow_missing_image)
         self._nodes[node.id] = node
 
         return node
 
     @open_required
-    async def add_node(self, compute, name, node_id, dump=True, node_type=None, **kwargs):
+    async def add_node(
+        self, compute, name, node_id, dump=True, node_type=None, allow_missing_image=False, **kwargs
+    ):
         """
         Create a node or return an existing node
 
         :param dump: Dump topology to disk
+        :param allow_missing_image: Keep the node on the controller in a
+            degraded state instead of failing when its image is missing
         :param kwargs: See the documentation of node
         """
 
@@ -683,7 +687,9 @@ class Project:
                     )
                 elif "application_id" not in kwargs.keys() and not kwargs.get("properties"):
                     kwargs["application_id"] = get_next_application_id(self._controller.projects, self._computes)
-                node = await self._create_node(compute, name, node_id, node_type, **kwargs)
+                node = await self._create_node(
+                    compute, name, node_id, node_type, allow_missing_image=allow_missing_image, **kwargs
+                )
         elif node_type == "docker" and _is_iol_docker_kwargs(kwargs):
             # IOL Docker nodes derive interface MACs from the application ID
             # exactly like IOU; they draw from the disjoint upper half of the
@@ -700,9 +706,13 @@ class Project:
                     kwargs["application_id"] = get_next_application_id(
                         self._controller.projects, self._computes, iol_docker=True
                     )
-                node = await self._create_node(compute, name, node_id, node_type, **kwargs)
+                node = await self._create_node(
+                    compute, name, node_id, node_type, allow_missing_image=allow_missing_image, **kwargs
+                )
         else:
-            node = await self._create_node(compute, name, node_id, node_type, **kwargs)
+            node = await self._create_node(
+                compute, name, node_id, node_type, allow_missing_image=allow_missing_image, **kwargs
+            )
         self.emit_notification("node.created", node.asdict())
         if dump:
             self.dump()
@@ -975,6 +985,20 @@ class Project:
             # a link should have 2 attached nodes, this can happen with corrupted projects
             await self.delete_link(link.id, force_delete=True)
             return None
+        if any(n["node"].missing_image for n in link._nodes):
+            # One of the endpoints could not be created on its compute because
+            # an image is missing. Keep the link on the controller (so the
+            # topology is preserved) but defer the NIO creation until the
+            # missing image is resolved.
+            for n in link._nodes:
+                n["node"].add_link(link)
+                n["port"].link = link
+            link._deferred = True
+            log.info(
+                "Project '%s' [%s]: deferring link %s until missing image(s) are resolved",
+                self._name, self._id, link.id,
+            )
+            return None
         # Apply project-level marker definitions onto the link's memory
         # (memory_only) before _prepare() so the inherited markers ride the
         # batch NIO dispatch — zero extra HTTP round-trips.  The final
@@ -987,6 +1011,33 @@ class Project:
         entries = await link._prepare()
         return (link, entries)
 
+    async def restore_deferred_links(self, node):
+        """
+        Create on the computes the NIOs of the links that were deferred while
+        one of their endpoints had a missing image. Called once the node has
+        been successfully created.
+
+        :param node: node that has just been created on its compute
+        """
+
+        restored = []
+        for link in list(node.links):
+            if not link.deferred:
+                continue
+            if any(n["node"].missing_image for n in link._nodes):
+                # the other endpoint is still missing an image
+                continue
+            try:
+                await link.create()
+                link._deferred = False
+                restored.append(link)
+            except Exception as e:
+                log.exception("Could not restore deferred link %s: %s", link.id, e)
+        for link in restored:
+            self.emit_notification("link.updated", link.asdict())
+        if restored:
+            self.dump()
+
     @open_required
     async def add_link(self, link_id=None, dump=True):
         """
@@ -1867,7 +1918,15 @@ class Project:
             log.info("Project '%s' [%s]: loading %d nodes...", self._name, self._id, len(nodes_to_create))
             pool = Pool(concurrency=100)
             for compute, name, node_id, node_data in nodes_to_create:
-                pool.append(self.add_node, compute, name, node_id, dump=False, **node_data)
+                pool.append(
+                    self.add_node,
+                    compute,
+                    name,
+                    node_id,
+                    dump=False,
+                    allow_missing_image=True,
+                    **node_data,
+                )
             await pool.join()
             log.info("Project '%s' [%s]: loaded %d nodes", self._name, self._id, len(nodes_to_create))
             # Pre-allocate UDP ports for all links in batch to reduce HTTP round-trips
@@ -1875,9 +1934,12 @@ class Project:
             for link_data in topology.get("links", []):
                 if "link_id" not in link_data.keys():
                     continue
-                for node_link in link_data.get("nodes", []):
-                    node = self._nodes.get(node_link["node_id"])
-                    if node:
+                link_nodes = [self._nodes.get(nl["node_id"]) for nl in link_data.get("nodes", [])]
+                if any(node is not None and node.missing_image for node in link_nodes):
+                    # the link will be deferred, no NIO/port will be created now
+                    continue
+                for node in link_nodes:
+                    if node is not None:
                         ports_per_compute[node.compute.id] = ports_per_compute.get(node.compute.id, 0) + 1
             for compute in self.computes:
                 count = ports_per_compute.get(compute.id, 0)
@@ -2215,7 +2277,7 @@ class Project:
         """
         Start all nodes (except always-running types like Ethernet switch, Cloud, NAT, etc.)
         """
-        nodes_to_start = [n for n in self.nodes.values() if not n.is_always_running()]
+        nodes_to_start = [n for n in self.nodes.values() if not n.is_always_running() and not n.missing_image]
         if not nodes_to_start:
             return
         log.info("Project '%s' [%s]: starting %d nodes...", self._name, self._id, len(nodes_to_start))
@@ -2230,7 +2292,7 @@ class Project:
         """
         Stop all nodes (except always-running types like Ethernet switch, Cloud, NAT, etc.)
         """
-        nodes_to_stop = [n for n in self.nodes.values() if not n.is_always_running()]
+        nodes_to_stop = [n for n in self.nodes.values() if not n.is_always_running() and not n.missing_image]
         if not nodes_to_stop:
             return
         log.info("Project '%s' [%s]: stopping %d nodes...", self._name, self._id, len(nodes_to_stop))
@@ -2247,6 +2309,8 @@ class Project:
         """
         pool = Pool(concurrency=50)
         for node in self.nodes.values():
+            if node.missing_image:
+                continue
             pool.append(node.suspend)
         await pool.join()
 
@@ -2258,6 +2322,8 @@ class Project:
 
         pool = Pool(concurrency=3)
         for node in self.nodes.values():
+            if node.missing_image:
+                continue
             pool.append(node.reset_console)
         await pool.join()
 
@@ -2347,4 +2413,3 @@ class Project:
 
     def __repr__(self):
         return f"<gns3server.controller.Project {self._name} {self._id}>"
-

gns3server/controller/udp_link.py

@@ -42,6 +42,10 @@ class UDPLink(Link):
         super().__init__(project, link_id=link_id)
         self._created = False
         self._link_data = []
+        # True when the link could not be created on the computes because one
+        # of its endpoints has a missing image. The NIO creation is retried
+        # once the image is resolved.
+        self._deferred = False
 
     @property
     def debug_link_data(self):
@@ -49,6 +53,11 @@ class UDPLink(Link):
         Use for the debug exports
         """
         return self._link_data
+
+    @property
+    def deferred(self):
+        """Whether NIO creation is waiting for missing node images."""
+        return self._deferred
     
     def _get_node_filters(self, node1, node2):
         """
@@ -258,6 +267,10 @@ class UDPLink(Link):
         Delete the link and free the resources
         """
         if not self._created:
+            if self._deferred:
+                # There is no NIO on the computes to delete, but the local
+                # back-references created while loading must be cleared.
+                await super().delete()
             return
         try:
             node1 = self._nodes[0]["node"]