feat(qemu): add PCIe network layout support with adapter limits and compatibility checks

gns3server/compute/qemu/qemu_vm.py

@@ -74,6 +74,10 @@ HYPERV_CLOCKSOURCES = ("hyperv_clocksource_tsc_page", "hyperv_clocksource_msr")
 # guest CPU features turned off on such hosts
 KVMCLOCK_OFF_FEATURES = ("kvmclock", "tsc-deadline")
 
+PCIE_PLATFORMS = ("aarch64", "i386", "x86_64")
+PCIE_ADAPTER_TYPES = ("e1000e", "igb", "virtio-net-pci", "vmxnet3")
+MAX_PCIE_ADAPTERS = 232
+
 
 def host_clocksource():
     """
@@ -2066,6 +2070,29 @@ class QemuVM(KernelDatapathMixin, BaseNode):
         else:
             return []
 
+    def _machine_options(self, additional_options):
+
+        if self.platform not in PCIE_PLATFORMS:
+            return []
+        for index, option in enumerate(additional_options):
+            if option in ("-machine", "-M") and index + 1 < len(additional_options):
+                for part in additional_options[index + 1].split(","):
+                    machine_type = part.removeprefix("type=")
+                    if "=" in machine_type:
+                        continue
+                    if self.platform == "aarch64":
+                        compatible = machine_type == "virt" or machine_type.startswith("virt-")
+                    else:
+                        compatible = machine_type == "q35" or machine_type.startswith("pc-q35-")
+                    if not compatible:
+                        raise QemuError(
+                            f"Machine type '{machine_type}' is incompatible with the PCIe network layout; "
+                            f"use {'virt' if self.platform == 'aarch64' else 'q35'}"
+                        )
+        if self.platform == "aarch64":
+            return self._arm64_options(shlex.join(additional_options))
+        return ["-machine", "q35"]
+
     def _serial_options(self, internal_console_port, external_console_port):
 
         if external_console_port:
@@ -2674,6 +2701,35 @@ class QemuVM(KernelDatapathMixin, BaseNode):
             ["-net", "none"]
         )  # we do not want any user networking back-end if no adapter is connected.
 
+        custom_adapters = [self._get_custom_adapter_settings(n) for n in range(self.adapters)]
+        adapter_types = [settings.get("adapter_type", self._adapter_type) for settings in custom_adapters]
+        pcie_adapters = {
+            n
+            for n, adapter_type in enumerate(adapter_types)
+            if self.platform in PCIE_PLATFORMS and adapter_type in PCIE_ADAPTER_TYPES
+        }
+        if len(pcie_adapters) > MAX_PCIE_ADAPTERS:
+            raise QemuError(f"At most {MAX_PCIE_ADAPTERS} PCIe network adapters are supported")
+        if self.platform in PCIE_PLATFORMS:
+            io_ports = sum(adapter_types[n] in ("e1000e", "igb") for n in pcie_adapters)
+            legacy_bridges = (self.adapters - len(pcie_adapters) + 31) // 32
+            if io_ports + legacy_bridges > 9:
+                raise QemuError(
+                    "The network layout exceeds PCI I/O space: at most 9 I/O-bearing root ports "
+                    "and legacy PCI bridge groups are supported in total; use virtio-net-pci or vmxnet3"
+                )
+        # Eight root-port functions per slot, leaving slots 0, 1 and 31 for platform devices.
+        for port_number, adapter_number in enumerate(sorted(pcie_adapters)):
+            slot, function = 2 + port_number // 8, port_number % 8
+            network_options.extend(
+                [
+                    "-device",
+                    f"pcie-root-port,id=gns3-pcie-{adapter_number},bus=pcie.0,"
+                    f"addr=0x{slot:x}.{function},multifunction=on,chassis={port_number + 1},hotplug=off",
+                ]
+            )
+
+        # Legacy PCI NICs share bridges to avoid exhausting the 64 KiB I/O space.
         # Each 32 PCI device we need to add a PCI bridge with max 9 bridges
         # Reserve 32 devices on root pci_bridge,
         # since the number of devices used by templates may differ significantly
@@ -2687,8 +2743,8 @@ class QemuVM(KernelDatapathMixin, BaseNode):
 
             nio = adapter.get_nio(0)
 
-            custom_adapter = self._get_custom_adapter_settings(adapter_number)
-            adapter_type = custom_adapter.get("adapter_type", self._adapter_type)
+            custom_adapter = custom_adapters[adapter_number]
+            adapter_type = adapter_types[adapter_number]
             custom_mac_address = custom_adapter.get("mac_address")
             if custom_mac_address:
                 mac = int_to_macaddress(macaddress_to_int(custom_mac_address))
@@ -2696,8 +2752,10 @@ class QemuVM(KernelDatapathMixin, BaseNode):
             device_string = f"{adapter_type},mac={mac}"
             if adapter_type == "virtio-net-pci":
                 device_string = f"{device_string},speed=10000,duplex=full"
-            bridge_id = math.floor(pci_device_id / 32)
-            if bridge_id > 0:
+            if adapter_number in pcie_adapters:
+                device_string = f"{device_string},bus=gns3-pcie-{adapter_number},addr=0x00"
+            else:
+                bridge_id = math.floor(pci_device_id / 32)
                 if pci_bridges_created < bridge_id:
                     network_options.extend(["-device", f"i82801b11-bridge,id=dmi_pci_bridge{bridge_id}"])
                     network_options.extend(
@@ -2709,7 +2767,7 @@ class QemuVM(KernelDatapathMixin, BaseNode):
                     pci_bridges_created += 1
                 addr = pci_device_id % 32
                 device_string = f"{device_string},bus=pci-bridge{bridge_id},addr=0x{addr:02x}"
-            pci_device_id += 1
+                pci_device_id += 1
             # Every adapter is backed by its own TAP (created before launch,
             # see _create_taps): the netdev is a kernel-side interface the
             # link attaches to, not a UDP socket into uBridge. A legacy
@@ -2919,6 +2977,16 @@ class QemuVM(KernelDatapathMixin, BaseNode):
         additional_options = additional_options.replace("%guest-cid%", str(self._guest_cid))
         if self._console_type != "none" and self._console:
             additional_options = additional_options.replace("%console-port%", str(self._console))
+        try:
+            additional_opt_list = shlex.split(additional_options)
+        except ValueError as e:
+            raise QemuError(f"Invalid additional options: {additional_options} error {e}")
+        allow_unsafe_options = self.manager.config.settings.Qemu.allow_unsafe_options
+        if allow_unsafe_options is False:
+            for opt in additional_opt_list:
+                if opt in FORBIDDEN_OPTIONS:
+                    raise QemuError(f"Forbidden additional option: {opt}")
+        machine_options = self._machine_options(additional_opt_list)
         command = [self.qemu_path]
         command.extend(["-name", vm_name])
         command.extend(["-m", f"{self._ram}M"])
@@ -2927,6 +2995,7 @@ class QemuVM(KernelDatapathMixin, BaseNode):
         if self._cpus > maxcpus:
             maxcpus = self._cpus
         command.extend(["-smp", f"cpus={self._cpus},maxcpus={maxcpus},sockets=1"])
+        command.extend(machine_options)
         kvm_enabled = False
         if await self._run_with_hardware_acceleration(self.qemu_path, self._options):
             command.extend(["-machine", "accel=kvm"])
@@ -2939,6 +3008,8 @@ class QemuVM(KernelDatapathMixin, BaseNode):
                 },
             )
         command.extend(["-boot", f"order={self._boot_priority}"])
+        # Reserve root-port addresses before other devices receive automatic PCI addresses.
+        command.extend(await self._network_options())
         command.extend(self._cdrom_option())
         command.extend(await self._disk_options())
         command.extend(self._linux_boot_options())
@@ -2947,7 +3018,6 @@ class QemuVM(KernelDatapathMixin, BaseNode):
         command.extend(self._console_options())
         command.extend(self._aux_options())
         command.extend(self._monitor_options())
-        command.extend(await self._network_options())
         # bios options must be last to have predictable NIC numbering, see https://github.com/GNS3/gns3-server/issues/2838
         command.extend(self._bios_option())
         if self.on_close != "save_vm_state":
@@ -2958,18 +3028,6 @@ class QemuVM(KernelDatapathMixin, BaseNode):
             command.extend(await self._disable_graphics())
         if self._tpm:
             command.extend(self._tpm_options())
-        additional_opt_list = []
-        if additional_options:
-            try:
-                additional_opt_list = shlex.split(additional_options)
-            except ValueError as e:
-                raise QemuError(f"Invalid additional options: {additional_options} error {e}")
-            allow_unsafe_options = self.manager.config.settings.Qemu.allow_unsafe_options
-            if allow_unsafe_options is False:
-                for opt in additional_opt_list:
-                    if opt in FORBIDDEN_OPTIONS:
-                        raise QemuError(f"Forbidden additional option: {opt}")
-        command.extend(self._arm64_options(additional_options))
         if kvm_enabled:
             kvmclock_off_reason = self._kvmclock_off_reason()
             if kvmclock_off_reason:

tests/compute/qemu/test_qemu_vm.py

@@ -729,12 +729,10 @@ async def test_build_command(vm, fake_qemu_binary):
             "256M",
             "-smp",
             "cpus=1,maxcpus=1,sockets=1",
+            "-machine",
+            "q35",
             "-boot",
             "order=c",
-            "-uuid",
-            vm.id,
-            "-serial",
-            f"telnet:127.0.0.1:{vm._internal_console_port},server,nowait",
             "-net",
             "none",
             "-device",
@@ -745,6 +743,10 @@ async def test_build_command(vm, fake_qemu_binary):
             f"e1000,mac={vm._mac_address},bus=pci-bridge1,addr=0x00,netdev=gns3-0",
             "-netdev",
             f"socket,id=gns3-0,udp=127.0.0.1:{nio.rport},localaddr=127.0.0.1:{nio.lport}",
+            "-uuid",
+            vm.id,
+            "-serial",
+            f"telnet:127.0.0.1:{vm._internal_console_port},server,nowait",
             "-display",
             "none",
         ]
@@ -823,12 +825,10 @@ async def test_build_command_two_adapters(vm, fake_qemu_binary):
             "256M",
             "-smp",
             "cpus=1,maxcpus=1,sockets=1",
+            "-machine",
+            "q35",
             "-boot",
             "order=c",
-            "-uuid",
-            vm.id,
-            "-serial",
-            f"telnet:127.0.0.1:{vm._internal_console_port},server,nowait",
             "-net",
             "none",
             "-device",
@@ -843,6 +843,10 @@ async def test_build_command_two_adapters(vm, fake_qemu_binary):
             f"e1000,mac={int_to_macaddress(macaddress_to_int(vm._mac_address) + 1)},bus=pci-bridge1,addr=0x01,netdev=gns3-1",
             "-netdev",
             f"socket,id=gns3-1,udp=127.0.0.1:{nio2.rport},localaddr=127.0.0.1:{nio2.lport}",
+            "-uuid",
+            vm.id,
+            "-serial",
+            f"telnet:127.0.0.1:{vm._internal_console_port},server,nowait",
             "-display",
             "none",
         ]
@@ -919,7 +923,154 @@ async def test_build_command_with_virtio_net_pci_adapter(vm):
     vm._adapter_type = "virtio-net-pci"
     with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()):
         cmd = await vm._build_command()
-    assert "virtio-net-pci,mac=00:00:ab:0e:0f:09,speed=10000,duplex=full,bus=pci-bridge1,addr=0x00,netdev=gns3-0" in cmd
+    assert "virtio-net-pci,mac=00:00:ab:0e:0f:09,speed=10000,duplex=full,bus=gns3-pcie-0,addr=0x00,netdev=gns3-0" in cmd
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("guest_platform", ["aarch64", "i386", "x86_64"])
+@pytest.mark.parametrize("adapter_type", ["e1000e", "igb", "virtio-net-pci", "vmxnet3"])
+async def test_network_options_pcie_root_ports(vm, guest_platform, adapter_type):
+
+    vm._platform = guest_platform
+    vm.adapter_type = adapter_type
+    vm.adapters = 9
+    options = await vm._network_options()
+    assert not any("pci-bridge" in option for option in options)
+    for n in range(9):
+        assert (
+            f"pcie-root-port,id=gns3-pcie-{n},bus=pcie.0,addr=0x{2 + n // 8:x}.{n % 8},"
+            f"multifunction=on,chassis={n + 1},hotplug=off"
+        ) in options
+        assert any(f",bus=gns3-pcie-{n},addr=0x00,netdev=gns3-{n}" in option for option in options)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("count", [0, 100, 232])
+async def test_network_options_pcie_adapter_count(vm, count):
+
+    vm.adapter_type = "virtio-net-pci"
+    vm.adapters = count
+    options = await vm._network_options()
+    assert sum(option.startswith("pcie-root-port,") for option in options) == count
+    assert sum(option.startswith("virtio-net-pci,") for option in options) == count
+    if count:
+        assert f",bus=gns3-pcie-{count - 1},addr=0x00,netdev=gns3-{count - 1}" in options[-3]
+
+
+@pytest.mark.asyncio
+async def test_network_options_too_many_pcie_adapters(vm):
+
+    vm.adapter_type = "virtio-net-pci"
+    vm.adapters = 233
+    with pytest.raises(QemuError, match="At most 232 PCIe network adapters"):
+        await vm._network_options()
+    assert vm._local_udp_tunnels == {}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("adapter_type", ["e1000e", "igb"])
+async def test_network_options_pcie_io_space_limit(vm, adapter_type):
+
+    vm.adapter_type = adapter_type
+    vm.adapters = 10
+    with pytest.raises(QemuError, match="exceeds PCI I/O space"):
+        await vm._network_options()
+
+
+@pytest.mark.asyncio
+async def test_network_options_mixed_io_space_limit(vm):
+
+    vm.adapters = 41
+    vm.custom_adapters = [{"adapter_number": n, "adapter_type": "e1000e"} for n in range(8)]
+    with pytest.raises(QemuError, match="exceeds PCI I/O space"):
+        await vm._network_options()
+
+
+@pytest.mark.asyncio
+async def test_network_options_mixed_pci_and_pcie(vm):
+
+    vm.adapters = 3
+    vm.custom_adapters = [{"adapter_number": 1, "adapter_type": "virtio-net-pci", "mac_address": "00:00:ab:0e:0f:09"}]
+    options = await vm._network_options()
+    assert sum(option.startswith("pcie-root-port,") for option in options) == 1
+    assert "pcie-root-port,id=gns3-pcie-1,bus=pcie.0,addr=0x2.0,multifunction=on,chassis=1,hotplug=off" in options
+    assert (
+        "virtio-net-pci,mac=00:00:ab:0e:0f:09,speed=10000,duplex=full,bus=gns3-pcie-1,addr=0x00,netdev=gns3-1"
+        in options
+    )
+    assert any(",bus=pci-bridge1,addr=0x00,netdev=gns3-0" in option for option in options)
+    assert any(",bus=pci-bridge1,addr=0x01,netdev=gns3-2" in option for option in options)
+
+
+@pytest.mark.asyncio
+async def test_network_options_other_platform_preserves_pci(vm):
+
+    vm._platform = "ppc"
+    vm.adapter_type = "virtio-net-pci"
+    options = await vm._network_options()
+    assert not any(option.startswith("pcie-root-port,") for option in options)
+    assert any(",bus=pci-bridge1,addr=0x00,netdev=gns3-0" in option for option in options)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("count", [32, 33, 275])
+async def test_network_options_legacy_bridge_capacity(vm, count):
+
+    vm.adapters = count
+    options = await vm._network_options()
+    assert sum(option.startswith("pci-bridge,") for option in options) == (count + 31) // 32
+    assert sum(option.startswith("e1000,") for option in options) == count
+    assert not any(option.startswith("pcie-root-port,") for option in options)
+
+
+@pytest.mark.parametrize("guest_platform", ["i386", "x86_64"])
+@pytest.mark.parametrize("options", [[], ["-machine", "q35"], ["-M", "pc-q35-6.2"], ["-machine", "type=q35,accel=tcg"]])
+def test_machine_options_q35(vm, guest_platform, options):
+
+    vm._platform = guest_platform
+    assert vm._machine_options(options) == ["-machine", "q35"]
+
+
+@pytest.mark.parametrize(
+    ("guest_platform", "machine"),
+    [("x86_64", "pc"), ("i386", "type=pc-i440fx-6.2"), ("aarch64", "virtually-not-virt")],
+)
+def test_machine_options_incompatible_machine(vm, guest_platform, machine):
+
+    vm._platform = guest_platform
+    with pytest.raises(QemuError, match="incompatible with the PCIe network layout"):
+        vm._machine_options(["-machine", machine])
+
+
+def test_machine_options_arm64(vm):
+
+    vm._platform = "aarch64"
+    with patch("gns3server.compute.qemu.qemu_vm.platform.machine", return_value="x86_64"):
+        assert vm._machine_options(["-machine", "type=virt-6.2", "-cpu", "cortex-a72"]) == [
+            "-machine",
+            "virt,gic-version=3",
+        ]
+
+
+def test_machine_options_other_platform(vm):
+
+    vm._platform = "ppc"
+    assert vm._machine_options(["-machine", "mac99"]) == []
+
+
+@pytest.mark.asyncio
+async def test_build_command_pcie_ports_precede_other_devices(vm):
+
+    vm.adapter_type = "virtio-net-pci"
+    disk_device = "virtio-blk-pci,id=test-disk"
+    with (
+        asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()),
+        patch.object(vm, "_disk_options", new=AsyncioMagicMock(return_value=["-device", disk_device])),
+    ):
+        cmd = await vm._build_command()
+    root_port = next(option for option in cmd if option.startswith("pcie-root-port,"))
+    assert cmd.index(root_port) < cmd.index(disk_device)
+    assert cmd.index(root_port) < cmd.index("-serial")
 
 
 @pytest.mark.asyncio