Allow projects containing missing images to open in a degraded state instead of failing to load

gns3server/controller/node.py

@@ -40,6 +40,26 @@ import logging
 log = logging.getLogger(__name__)
 
 
+# Image properties of a node, mapped by node type. The value is the image
+# "type" used by the image manager (see utils.images). Docker images are not
+# registered in the image database, they are matched by tag instead.
+IMAGE_PROPERTIES_BY_NODE_TYPE = {
+    "qemu": {
+        "hda_disk_image": "qemu",
+        "hdb_disk_image": "qemu",
+        "hdc_disk_image": "qemu",
+        "hdd_disk_image": "qemu",
+        "cdrom_image": "qemu",
+        "initrd": "qemu",
+        "kernel_image": "qemu",
+        "bios_image": "qemu",
+    },
+    "dynamips": {"image": "ios"},
+    "iou": {"path": "iou"},
+    "docker": {"image": "docker"},
+}
+
+
 def _extract_iol_startup_config_knob(environment):
     """
     Split the GNS3_IOL_STARTUP_CONFIG knob out of a Docker environment
@@ -143,6 +163,11 @@ class Node:
         self._netmiko_device_type = None
         self._default_username = None
         self._default_password = None
+        # Set when the node cannot be created on its compute because one or
+        # more of its images are missing. The node is kept on the controller
+        # (topology and links are preserved) until the user provides a
+        # compatible replacement image.
+        self._missing_images = []
 
         # This properties will be recomputed
         ignore_properties = ("width", "height", "hover_symbol")
@@ -187,6 +212,18 @@ class Node:
     def status(self):
         return self._status
 
+    @property
+    def missing_image(self):
+        """
+        :returns: True if at least one image required by this node is missing
+            and the node could therefore not be created on its compute.
+        """
+        return len(self._missing_images) > 0
+
+    @property
+    def missing_images(self):
+        return self._missing_images
+
     @property
     def template_id(self):
         return self._template_id
@@ -457,9 +494,14 @@ class Node:
     def links(self):
         return self._links
 
-    async def create(self):
+    async def create(self, allow_missing_image=False):
         """
         Create the node on the compute
+
+        :param allow_missing_image: when True, if an image is missing and
+            cannot be uploaded/synced, the node is not created on the compute
+            but is kept on the controller and flagged with the missing image(s)
+            instead of raising.
         """
         data = self._node_data()
         data["node_id"] = self._id
@@ -469,6 +511,8 @@ class Node:
         else:
             timeout = 1200
         trial = 0
+        last_missing_image = None
+        last_missing_error = None
         while trial != 6:
             try:
                 response = await self._compute.post(
@@ -477,17 +521,126 @@ class Node:
             except ComputeConflictError as e:
                 response = e.response()
                 if response.get("exception") == "ImageMissingError":
-                    res = await self._upload_missing_image(self._node_type, response["image"])
+                    last_missing_error = e
+                    last_missing_image = response.get("image")
+                    res = False
+                    if last_missing_image:
+                        try:
+                            res = await self._upload_missing_image(self._node_type, last_missing_image)
+                        except ControllerError as upload_error:
+                            # For Docker the image is pulled from a registry: a
+                            # failed pull (unknown tag, no network, private image)
+                            # must not abort the whole project. Treat it as a
+                            # missing image so the user can pick another one.
+                            if not allow_missing_image:
+                                raise
+                            log.warning(
+                                "Could not provide missing image '%s' for node '%s' [%s]: %s",
+                                last_missing_image, self._name, self._id, upload_error
+                            )
                     if not res:
+                        if allow_missing_image:
+                            missing_images = self._compute_missing_images(last_missing_image)
+                            # A degraded node must always identify at least one
+                            # missing image. Otherwise the controller would keep
+                            # a node that was never created on the compute while
+                            # exposing it as healthy.
+                            if not missing_images:
+                                raise e
+                            self._missing_images = missing_images
+                            log.warning(
+                                "Node '%s' [%s] is kept in degraded state, missing image(s): %s",
+                                self._name, self._id, ", ".join(m["image"] for m in self._missing_images)
+                            )
+                            return False
                         raise e
                 else:
                     raise e
             else:
                 await self.parse_node_response(response.json)
+                self._missing_images = []
                 return True
             trial += 1
+        if allow_missing_image:
+            # The image was repeatedly reported missing (e.g. the upload/sync
+            # seemed to succeed but did not). Keep the node in degraded state.
+            self._missing_images = self._compute_missing_images(last_missing_image)
+            log.warning(
+                "Node '%s' [%s] could not be created, missing image(s): %s",
+                self._name, self._id, ", ".join(m["image"] for m in self._missing_images)
+            )
+        elif last_missing_error is not None:
+            # Uploading appeared to succeed, but the compute rejected every
+            # retry. Do not let the caller register a controller-only node as
+            # healthy.
+            raise last_missing_error
         return False
 
+    def _image_available(self, image_type, image):
+        """
+        Check whether an image is available on the controller (and can
+        therefore be uploaded to the compute when needed).
+
+        :param image_type: image type (e.g. "qemu", "ios", "iou")
+        :param image: image filename or path
+        """
+
+        if not image:
+            return True
+        if image_type == "docker":
+            # Docker images are not stored on the controller filesystem
+            return True
+        try:
+            directories = images_directories(image_type)
+        except NotImplementedError:
+            return True
+        for directory in directories:
+            if os.path.exists(os.path.join(directory, image)):
+                return True
+        return False
+
+    def _compute_missing_images(self, fallback_image=None):
+        """
+        Build the list of images referenced by this node that are not
+        available on the controller.
+
+        :param fallback_image: image reported as missing by the compute, always
+            included (used for Docker images and images not directly mapped to
+            a node property).
+        """
+
+        missing = []
+        mapping = IMAGE_PROPERTIES_BY_NODE_TYPE.get(self._node_type, {})
+        properties = self._properties or {}
+        for prop, image_type in mapping.items():
+            value = properties.get(prop)
+            if not value:
+                continue
+            if not self._image_available(image_type, value):
+                missing.append({"property": prop, "image": value, "image_type": image_type})
+        if fallback_image:
+            prop = next((p for p in mapping if properties.get(p) == fallback_image), None)
+            if prop is None:
+                fallback_basename = os.path.basename(fallback_image)
+                prop = next(
+                    (
+                        p
+                        for p in mapping
+                        if properties.get(p) and os.path.basename(properties[p]) == fallback_basename
+                    ),
+                    None,
+                )
+            if prop is None and mapping:
+                prop = next(iter(mapping))
+            already_reported = any(
+                m["image"] == fallback_image or (prop is not None and m["property"] == prop)
+                for m in missing
+            )
+            if not already_reported:
+                image_type = mapping.get(prop, self._node_type)
+                missing.append({"property": prop, "image": fallback_image, "image_type": image_type})
+        return missing
+
     async def update(self, **kwargs):
         """
         Update the node on the compute
@@ -499,6 +652,9 @@ class Node:
         update_compute = False
         old_json = self.asdict()
         old_name = self._name
+        old_properties = copy.deepcopy(self._properties)
+        old_custom_adapters = copy.deepcopy(self._custom_adapters)
+        old_missing_images = copy.deepcopy(self._missing_images)
 
         compute_properties = None
         # Update node properties with additional elements
@@ -522,8 +678,32 @@ class Node:
         if compute_properties and "custom_adapters" in compute_properties:
             # we need to check custom adapters to update the custom port names
             self.custom_adapters = compute_properties["custom_adapters"]
+        if self.missing_image and compute_properties is not None:
+            # The node was never created on the compute (missing image). Apply
+            # the new properties locally so create() can use them.
+            self._properties = compute_properties
         self._list_ports()
         if update_compute:
+            if self.missing_image:
+                # Try to create the node on the compute now that an image may
+                # have been provided. If it is still missing the node simply
+                # remains in its degraded state.
+                try:
+                    created = await self.create(allow_missing_image=True)
+                except Exception:
+                    # create() can reject the replacement for reasons other
+                    # than a missing image. Keep the controller state aligned
+                    # with the node that is still absent from the compute.
+                    self._properties = old_properties
+                    self._custom_adapters = old_custom_adapters
+                    self._missing_images = old_missing_images
+                    self._list_ports()
+                    raise
+                if created:
+                    await self.project.restore_deferred_links(self)
+                self.project.emit_notification("node.updated", self.asdict())
+                self.project.dump()
+                return
             data = self._node_data(properties=compute_properties)
             try:
                 response = await self.put(None, data=data)
@@ -666,6 +846,25 @@ class Node:
         """
         Start a node
         """
+        if self.missing_image:
+            images = ", ".join(m["image"] for m in self._missing_images)
+            raise ControllerError(
+                f"Cannot start node '{self._name}': the required image(s) ({images}) "
+                f"are missing. Please provide a compatible image."
+            )
+        deferred_links = [link for link in self.links if link.deferred]
+        if deferred_links:
+            await self.project.restore_deferred_links(self)
+            failed_links = [
+                link
+                for link in deferred_links
+                if link.deferred and not any(endpoint["node"].missing_image for endpoint in link._nodes)
+            ]
+            if failed_links:
+                raise ControllerError(
+                    f"Cannot start node '{self._name}': {len(failed_links)} deferred link(s) "
+                    "could not be restored. Please try again."
+                )
         try:
             # For IOU: we need to send the licence everytime we start a node
             if self.node_type == "iou":
@@ -681,6 +880,8 @@ class Node:
         """
         Stop a node
         """
+        if self.missing_image:
+            return
         try:
             await self.post("/stop", timeout=240, dont_connect=True)
         # We don't care if a node is down at this step
@@ -756,6 +957,10 @@ class Node:
         """
         HTTP post on the node
         """
+        if self.missing_image and path is None:
+            # The node was never created on the compute (missing image). Any
+            # partial object there is cleaned up when the project is closed.
+            return None
         if path is None:
             return await self._compute.delete(
                 f"/projects/{self._project.id}/{self._node_type}/nodes/{self._id}", **kwargs
@@ -1018,7 +1223,9 @@ class Node:
             "status": self._status,
             "console_host": str(self._compute.console_host),
             "node_directory": self._node_directory,
-            "ports": [port.asdict() for port in self.ports]
+            "ports": [port.asdict() for port in self.ports],
+            "missing_image": self.missing_image,
+            "missing_images": self._missing_images,
         }
         topology.update(additional_data)
         return topology