Allow projects containing missing images to open in a degraded state instead of failing to load

tests/api/routes/controller/test_nodes.py

@@ -103,7 +103,20 @@ class TestNodeRoutes:
         response = await client.get(app.url_path_for("get_nodes", project_id=project.id))
         assert response.status_code == status.HTTP_200_OK
         assert response.json()[0]["name"] == "test"
+    async def test_list_node_with_missing_image(
+        self, app: FastAPI, client: AsyncClient, project: Project, node: Node
+    ) -> None:
+        node._missing_images = [
+            {"property": "path", "image": "i86bi-linux-l2.bin", "image_type": "iou"}
+        ]
 
+        response = await client.get(app.url_path_for("get_nodes", project_id=project.id))
+        assert response.status_code == status.HTTP_200_OK
+        data = response.json()[0]
+        assert data["missing_image"] is True
+        assert data["missing_images"] == [
+            {"property": "path", "image": "i86bi-linux-l2.bin", "image_type": "iou"}
+        ]
 
     @pytest.mark.parametrize(
         "tags, expected_match",

tests/controller/test_node.py

@@ -26,6 +26,7 @@ from gns3server.compute.docker.docker_error import DockerError
 
 from gns3server.controller.node import Node
 from gns3server.controller.project import Project
+from gns3server.controller.controller_error import ComputeConflictError, ControllerError
 
 
 @pytest.fixture
@@ -234,7 +235,9 @@ def test_json(node, compute):
                 "port_number": 0,
                 "short_name": "e0"
             }
-        ]
+        ],
+        "missing_image": False,
+        "missing_images": []
     }
 
     assert node.asdict(topology_dump=True) == {
@@ -319,6 +322,314 @@ async def test_create_image_missing(node, compute):
     #assert node._upload_missing_image.called is True
 
 
+@pytest.mark.asyncio
+async def test_create_image_missing_kept_in_degraded_state(project, compute, tmpdir, config):
+    """
+    With allow_missing_image=True a node whose image cannot be provided is kept
+    on the controller and flagged instead of aborting the whole project open.
+    """
+
+    config.settings.Server.images_path = str(tmpdir)
+    node = Node(project, compute, "r1",
+                node_id=str(uuid.uuid4()),
+                node_type="qemu",
+                properties={"hda_disk_image": "missing.qcow2", "ram": 256})
+
+    async def resp(*args, **kwargs):
+        raise ComputeConflictError(
+            "/projects/{}/qemu/nodes".format(project.id),
+            {"message": "The image is missing", "image": "missing.qcow2", "exception": "ImageMissingError"}
+        )
+
+    compute.post = AsyncioMagicMock(side_effect=resp)
+    node._upload_missing_image = AsyncioMagicMock(return_value=False)
+
+    assert await node.create(allow_missing_image=True) is False
+    assert node.missing_image is True
+    assert node.missing_images == [
+        {"property": "hda_disk_image", "image": "missing.qcow2", "image_type": "qemu"}
+    ]
+
+
+@pytest.mark.asyncio
+async def test_create_image_missing_raises_by_default(project, compute, tmpdir, config):
+    """
+    Without allow_missing_image the historical behaviour is preserved: the
+    ImageMissingError is re-raised.
+    """
+
+    config.settings.Server.images_path = str(tmpdir)
+    node = Node(project, compute, "r1",
+                node_id=str(uuid.uuid4()),
+                node_type="qemu",
+                properties={"hda_disk_image": "missing.qcow2", "ram": 256})
+
+    async def resp(*args, **kwargs):
+        raise ComputeConflictError(
+            "/projects/{}/qemu/nodes".format(project.id),
+            {"message": "The image is missing", "image": "missing.qcow2", "exception": "ImageMissingError"}
+        )
+
+    compute.post = AsyncioMagicMock(side_effect=resp)
+    node._upload_missing_image = AsyncioMagicMock(return_value=False)
+
+    with pytest.raises(ComputeConflictError):
+        await node.create()
+    assert node.missing_image is False
+
+
+@pytest.mark.asyncio
+async def test_create_image_missing_raises_after_upload_retries(project, compute):
+    node = Node(
+        project,
+        compute,
+        "r1",
+        node_id=str(uuid.uuid4()),
+        node_type="qemu",
+        properties={"hda_disk_image": "missing.qcow2", "ram": 256},
+    )
+
+    async def resp(*args, **kwargs):
+        raise ComputeConflictError(
+            f"/projects/{project.id}/qemu/nodes",
+            {"message": "missing", "image": "missing.qcow2", "exception": "ImageMissingError"},
+        )
+
+    compute.post = AsyncioMagicMock(side_effect=resp)
+    node._upload_missing_image = AsyncioMagicMock(return_value=True)
+
+    with pytest.raises(ComputeConflictError):
+        await node.create()
+    assert compute.post.call_count == 6
+    assert node.missing_image is False
+
+
+@pytest.mark.asyncio
+async def test_create_image_missing_without_image_name_is_not_degraded(project, compute):
+    """A malformed conflict must not leave a controller-only node marked healthy."""
+
+    node = Node(
+        project,
+        compute,
+        "r1",
+        node_id=str(uuid.uuid4()),
+        node_type="qemu",
+        properties={"hda_disk_image": "present.qcow2", "ram": 256},
+    )
+
+    async def resp(*args, **kwargs):
+        raise ComputeConflictError(
+            f"/projects/{project.id}/qemu/nodes",
+            {"message": "The image is missing", "exception": "ImageMissingError"},
+        )
+
+    compute.post = AsyncioMagicMock(side_effect=resp)
+    node._image_available = MagicMock(return_value=True)
+
+    with pytest.raises(ComputeConflictError):
+        await node.create(allow_missing_image=True)
+    assert node.missing_image is False
+
+
+def test_compute_missing_images_matches_reported_basename_to_property(project, compute):
+    node = Node(
+        project,
+        compute,
+        "r1",
+        node_id=str(uuid.uuid4()),
+        node_type="qemu",
+        properties={
+            "hda_disk_image": "/images/qemu/disk.qcow2",
+            "cdrom_image": "/images/qemu/installer.iso",
+            "ram": 256,
+        },
+    )
+    node._image_available = MagicMock(return_value=True)
+
+    assert node._compute_missing_images("installer.iso") == [
+        {"property": "cdrom_image", "image": "installer.iso", "image_type": "qemu"}
+    ]
+
+
+def test_compute_missing_images_deduplicates_path_and_reported_basename(project, compute):
+    node = Node(
+        project,
+        compute,
+        "r1",
+        node_id=str(uuid.uuid4()),
+        node_type="qemu",
+        properties={"cdrom_image": "/images/qemu/installer.iso", "ram": 256},
+    )
+    node._image_available = MagicMock(return_value=False)
+
+    assert node._compute_missing_images("installer.iso") == [
+        {
+            "property": "cdrom_image",
+            "image": "/images/qemu/installer.iso",
+            "image_type": "qemu",
+        }
+    ]
+
+
+@pytest.mark.asyncio
+async def test_create_docker_image_missing_after_failed_pull(project, compute):
+    """
+    A Docker image that cannot be pulled from the registry keeps the node in a
+    degraded state instead of aborting the project open.
+    """
+
+    node = Node(project, compute, "web",
+                node_id=str(uuid.uuid4()),
+                node_type="docker",
+                properties={"image": "ghost:latest", "adapters": 1})
+
+    async def resp(*args, **kwargs):
+        raise ComputeConflictError(
+            "/projects/{}/docker/nodes".format(project.id),
+            {"message": "The image is missing", "image": "ghost:latest", "exception": "ImageMissingError"}
+        )
+
+    compute.post = AsyncioMagicMock(side_effect=resp)
+    node._upload_missing_image = AsyncioMagicMock(
+        side_effect=ControllerError("Failed to pull Docker image 'ghost:latest'")
+    )
+
+    assert await node.create(allow_missing_image=True) is False
+    assert node.missing_image is True
+    assert node.missing_images == [
+        {"property": "image", "image": "ghost:latest", "image_type": "docker"}
+    ]
+
+
+@pytest.mark.asyncio
+async def test_create_docker_image_missing_pull_error_raises_by_default(project, compute):
+    """
+    Without allow_missing_image a failed Docker pull is still surfaced.
+    """
+
+    node = Node(project, compute, "web",
+                node_id=str(uuid.uuid4()),
+                node_type="docker",
+                properties={"image": "ghost:latest", "adapters": 1})
+
+    async def resp(*args, **kwargs):
+        raise ComputeConflictError(
+            "/projects/{}/docker/nodes".format(project.id),
+            {"message": "The image is missing", "image": "ghost:latest", "exception": "ImageMissingError"}
+        )
+
+    compute.post = AsyncioMagicMock(side_effect=resp)
+    node._upload_missing_image = AsyncioMagicMock(
+        side_effect=ControllerError("Failed to pull Docker image 'ghost:latest'")
+    )
+
+    with pytest.raises(ControllerError):
+        await node.create()
+
+
+def test_compute_missing_images_lists_all_unavailable_slots(project, compute, tmpdir, config):
+    """
+    All the image slots of a multi-image node are reported, not only the first
+    one the compute complained about.
+    """
+
+    config.settings.Server.images_path = str(tmpdir)
+    node = Node(project, compute, "r1",
+                node_id=str(uuid.uuid4()),
+                node_type="qemu",
+                properties={
+                    "hda_disk_image": "present.qcow2",
+                    "hdc_disk_image": "missing.qcow2",
+                    "initrd": "missing.initrd",
+                    "ram": 256,
+                })
+    # make one image available on the controller
+    os.makedirs(os.path.join(str(tmpdir), "QEMU"), exist_ok=True)
+    with open(os.path.join(str(tmpdir), "QEMU", "present.qcow2"), "w") as f:
+        f.write("x")
+
+    missing = node._compute_missing_images()
+    assert {m["property"] for m in missing} == {"hdc_disk_image", "initrd"}
+    assert all(m["image_type"] == "qemu" for m in missing)
+
+
+@pytest.mark.asyncio
+async def test_start_missing_image(node):
+
+    node._missing_images = [
+        {"property": "hda_disk_image", "image": "missing.qcow2", "image_type": "qemu"}
+    ]
+    with pytest.raises(ControllerError):
+        await node.start()
+
+
+@pytest.mark.asyncio
+async def test_update_recreates_missing_image_node(project, compute, node):
+    """
+    Updating the image of a degraded node creates it on the compute and clears
+    the missing image state.
+    """
+
+    node._missing_images = [
+        {"property": "image", "image": "missing.image", "image_type": "ios"}
+    ]
+    response = MagicMock()
+    response.json = {"console": 2048}
+    compute.post = AsyncioMagicMock(return_value=response)
+    project.restore_deferred_links = AsyncioMagicMock()
+
+    await node.update(properties={"image": "available.image"})
+    assert node.missing_image is False
+    assert node.missing_images == []
+    assert project.restore_deferred_links.called is True
+
+
+@pytest.mark.asyncio
+async def test_update_missing_image_node_rolls_back_after_create_error(project, compute, node):
+    original_properties = {"image": "missing.image"}
+    node._properties = original_properties.copy()
+    node._missing_images = [
+        {"property": "image", "image": "missing.image", "image_type": "ios"}
+    ]
+    compute.post = AsyncioMagicMock(side_effect=ControllerError("create failed"))
+
+    with pytest.raises(ControllerError):
+        await node.update(properties={"image": "invalid.image"})
+
+    assert node.properties == original_properties
+    assert node.missing_images == [
+        {"property": "image", "image": "missing.image", "image_type": "ios"}
+    ]
+
+
+@pytest.mark.asyncio
+async def test_start_retries_ready_deferred_links(node, project):
+    peer = MagicMock()
+    peer.missing_image = False
+    link = MagicMock()
+    link.deferred = True
+    link._nodes = [{"node": node}, {"node": peer}]
+    node._links.add(link)
+    project.restore_deferred_links = AsyncioMagicMock()
+
+    with pytest.raises(ControllerError, match="deferred link"):
+        await node.start()
+
+    project.restore_deferred_links.assert_called_once_with(node)
+
+
+@pytest.mark.asyncio
+async def test_stop_missing_image_node_does_not_contact_compute(node, compute):
+    node._missing_images = [
+        {"property": "image", "image": "missing.image", "image_type": "ios"}
+    ]
+    compute.post = AsyncioMagicMock()
+
+    await node.stop()
+
+    compute.post.assert_not_called()
+
+
 @pytest.mark.asyncio
 async def test_create_base_script(node, config, compute, tmpdir):
 

tests/controller/test_project.py

@@ -17,6 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import json
 import uuid
 import pytest
 import pytest_asyncio
@@ -28,7 +29,12 @@ from uuid import uuid4
 from gns3server.controller.project import Project
 from gns3server.controller.node import Node
 from gns3server.controller.ports.ethernet_port import EthernetPort
-from gns3server.controller.controller_error import ControllerError, ControllerNotFoundError, ControllerForbiddenError
+from gns3server.controller.controller_error import (
+    ControllerError,
+    ControllerNotFoundError,
+    ControllerForbiddenError,
+    ComputeConflictError,
+)
 from gns3server.config import Config
 
 
@@ -1006,6 +1012,19 @@ async def test_stop_all(project):
     assert len(compute.post.call_args_list) == 10
 
 
+@pytest.mark.asyncio
+async def test_stop_all_skips_nodes_with_missing_images(project):
+    node = MagicMock()
+    node.is_always_running.return_value = False
+    node.missing_image = True
+    node.stop = AsyncioMagicMock()
+    project._nodes[node.id] = node
+
+    await project.stop_all()
+
+    node.stop.assert_not_called()
+
+
 @pytest.mark.asyncio
 async def test_suspend_all(project):
 
@@ -1023,6 +1042,18 @@ async def test_suspend_all(project):
     assert len(compute.post.call_args_list) == 10
 
 
+@pytest.mark.asyncio
+async def test_suspend_all_skips_nodes_with_missing_images(project):
+    node = MagicMock()
+    node.missing_image = True
+    node.suspend = AsyncioMagicMock()
+    project._nodes[node.id] = node
+
+    await project.suspend_all()
+
+    node.suspend.assert_not_called()
+
+
 @pytest.mark.asyncio
 async def test_console_reset_all(project):
 
@@ -1040,6 +1071,18 @@ async def test_console_reset_all(project):
     assert len(compute.post.call_args_list) == 10
 
 
+@pytest.mark.asyncio
+async def test_console_reset_all_skips_nodes_with_missing_images(project):
+    node = MagicMock()
+    node.missing_image = True
+    node.reset_console = AsyncioMagicMock()
+    project._nodes[node.id] = node
+
+    await project.reset_console_all()
+
+    node.reset_console.assert_not_called()
+
+
 @pytest.mark.asyncio
 async def test_node_name(project):
 
@@ -1085,3 +1128,201 @@ async def test_duplicate_node(project):
         })
     new_node = await project.duplicate_node(original, 42, 10, 11)
     assert new_node.x == 42
+
+
+@pytest.mark.asyncio
+async def test_add_node_missing_image_kept_in_degraded_state(controller):
+    """
+    A node whose image is missing is kept on the controller with the missing
+    image list instead of failing, when allow_missing_image is set.
+    """
+
+    compute = MagicMock()
+    compute.id = "local"
+    compute.connected = True
+    project = Project(controller=controller, name="Test")
+    project.emit_notification = MagicMock()
+
+    async def post(url, data=None, **kwargs):
+        if url.endswith("/qemu/nodes"):
+            raise ComputeConflictError(
+                url, {"message": "missing", "image": "missing.qcow2", "exception": "ImageMissingError"}
+            )
+        response = MagicMock()
+        response.json = {}
+        return response
+
+    compute.post = AsyncioMagicMock(side_effect=post)
+
+    node = await project.add_node(
+        compute,
+        "r1",
+        None,
+        node_type="qemu",
+        allow_missing_image=True,
+        properties={"hda_disk_image": "missing.qcow2", "ram": 256},
+    )
+    assert node.missing_image is True
+    assert node.id in project._nodes
+    assert node.missing_images == [
+        {"property": "hda_disk_image", "image": "missing.qcow2", "image_type": "qemu"}
+    ]
+
+
+@pytest.mark.asyncio
+async def test_open_with_missing_image_defers_links(controller, projects_dir):
+    """
+    Opening a project with a missing image succeeds, keeps the degraded node
+    and its links in the topology, but does not create the NIOs.
+    """
+
+    project_id = "3c1be6f9-b4ba-4737-b209-63c47c23359f"
+    qemu_id = "11111111-1111-1111-1111-111111111111"
+    iou_id = "33333333-3333-3333-3333-333333333333"
+    vpcs_id = "22222222-2222-2222-2222-222222222222"
+    link_id = "5a3e3a64-e853-4055-9503-4a14e01290f1"
+
+    topology = {
+        "auto_close": True,
+        "auto_open": False,
+        "auto_start": False,
+        "name": "demo",
+        "project_id": project_id,
+        "revision": 5,
+        "topology": {
+            "computes": [],
+            "drawings": [],
+            "links": [
+                {
+                    "link_id": link_id,
+                    "nodes": [
+                        {"adapter_number": 0, "node_id": qemu_id, "port_number": 0},
+                        {"adapter_number": 0, "node_id": vpcs_id, "port_number": 0},
+                    ],
+                }
+            ],
+            "nodes": [
+                {
+                    "compute_id": "local",
+                    "name": "R1",
+                    "node_id": qemu_id,
+                    "node_type": "qemu",
+                    "properties": {"hda_disk_image": "missing.qcow2", "adapters": 1, "ram": 256},
+                    "symbol": ":/symbols/router.svg",
+                    "x": 0,
+                    "y": 0,
+                },
+                {
+                    "compute_id": "local",
+                    "name": "IOU1",
+                    "node_id": iou_id,
+                    "node_type": "iou",
+                    "properties": {"path": "missing.bin"},
+                    "symbol": ":/symbols/router.svg",
+                    "x": 200,
+                    "y": 0,
+                },
+                {
+                    "compute_id": "local",
+                    "name": "PC1",
+                    "node_id": vpcs_id,
+                    "node_type": "vpcs",
+                    "properties": {},
+                    "symbol": ":/symbols/computer.svg",
+                    "x": 100,
+                    "y": 0,
+                },
+            ],
+        },
+        "type": "topology",
+        "version": "2.0.0",
+    }
+
+    project_dir = os.path.join(projects_dir, "demo")
+    os.makedirs(project_dir, exist_ok=True)
+    with open(os.path.join(project_dir, "demo.gns3"), "w+") as f:
+        json.dump(topology, f)
+
+    compute = MagicMock()
+    compute.id = "local"
+    compute.connected = True
+    compute.name = "local"
+    compute.console_host = "127.0.0.1"
+    controller._computes["local"] = compute
+
+    async def post(url, data=None, **kwargs):
+        if url.endswith("/qemu/nodes"):
+            raise ComputeConflictError(
+                url, {"message": "missing", "image": "missing.qcow2", "exception": "ImageMissingError"}
+            )
+        if url.endswith("/iou/nodes"):
+            raise ComputeConflictError(
+                url, {"message": "missing", "image": "missing.bin", "exception": "ImageMissingError"}
+            )
+        response = MagicMock()
+        if "ports/udp/batch" in url:
+            response.json = {"udp_ports": [20000]}
+        else:
+            response.json = {}
+        return response
+
+    compute.post = AsyncioMagicMock(side_effect=post)
+
+    project = Project(
+        name="demo",
+        project_id=project_id,
+        path=project_dir,
+        controller=controller,
+        filename="demo.gns3",
+        status="closed",
+    )
+
+    await project.open()
+    assert project.status == "opened"
+
+    qemu_node = project.get_node(qemu_id)
+    assert qemu_node.missing_image is True
+    assert qemu_node.missing_images[0]["image"] == "missing.qcow2"
+
+    iou_node = project.get_node(iou_id)
+    assert iou_node.missing_image is True
+    assert iou_node.missing_images[0]["image"] == "missing.bin"
+    assert iou_node.missing_images[0]["image_type"] == "iou"
+
+    link = project._links[link_id]
+    assert link._deferred is True
+    assert link.created is False
+    assert link in qemu_node.links
+
+
+@pytest.mark.asyncio
+async def test_restore_deferred_links(project):
+    """
+    Once a degraded node is created, the deferred links are created on the
+    computes and no longer flagged as deferred.
+    """
+
+    compute = MagicMock()
+    compute.id = "local"
+    response = MagicMock()
+    response.json = {"console": 2048}
+    compute.post = AsyncioMagicMock(return_value=response)
+
+    node1 = await project.add_node(compute, "n1", None, node_type="vpcs", properties={})
+    node2 = await project.add_node(compute, "n2", None, node_type="vpcs", properties={})
+
+    link = await project.add_link()
+    await link.add_node(node1, 0, 0, batch=True)
+    await link.add_node(node2, 0, 0, batch=True)
+    link._nodes[0]["port"].link = link
+    link._nodes[1]["port"].link = link
+    node1.add_link(link)
+    node2.add_link(link)
+    link._deferred = True
+    link.create = AsyncioMagicMock()
+
+    project.emit_notification = MagicMock()
+    await project.restore_deferred_links(node1)
+
+    assert link._deferred is False
+    assert link.create.called is True

tests/controller/test_udp_link.py

@@ -187,6 +187,44 @@ async def test_delete(project):
     compute2.delete.assert_any_call("/projects/{}/vpcs/nodes/{}/adapters/3/ports/1/nio".format(project.id, node2.id), timeout=120)
 
 
+@pytest.mark.asyncio
+async def test_delete_deferred_link_clears_local_references(project):
+    node1 = Node(project, MagicMock(), "node1", node_type="vpcs")
+    node1._ports = [EthernetPort("E0", 0, 0, 0)]
+    node2 = Node(project, MagicMock(), "node2", node_type="vpcs")
+    node2._ports = [EthernetPort("E0", 0, 0, 0)]
+    link = UDPLink(project)
+
+    await link.add_node(node1, 0, 0, batch=True)
+    await link.add_node(node2, 0, 0, batch=True)
+    for entry in link._nodes:
+        entry["node"].add_link(link)
+        entry["port"].link = link
+    link._deferred = True
+
+    await link.delete()
+
+    assert link not in node1.links
+    assert link not in node2.links
+    assert node1.get_port(0, 0).link is None
+    assert node2.get_port(0, 0).link is None
+
+
+@pytest.mark.asyncio
+async def test_delete_uncreated_non_deferred_link_preserves_existing_behavior(project):
+    node = Node(project, MagicMock(), "node1", node_type="vpcs")
+    node._ports = [EthernetPort("E0", 0, 0, 0)]
+    link = UDPLink(project)
+    await link.add_node(node, 0, 0, batch=True)
+    node.add_link(link)
+    node.get_port(0, 0).link = link
+
+    await link.delete()
+
+    assert link in node.links
+    assert node.get_port(0, 0).link is link
+
+
 @pytest.mark.asyncio
 async def test_reset(project):
     """